From a9733a32d35205274d6e808cd846688576ab0b6a Mon Sep 17 00:00:00 2001 From: Isis Lovecruft Date: Sat, 24 Feb 2018 02:16:28 +0000 Subject: [PATCH 0001/2557] build: Revert "Use -lresolv in LIBS with rust on OSX." This reverts commit 2ac9734bd21f3eecb20560cde5c4037230029157. * FIXES #25341: https://bugs.torproject.org/25341 Signed-off-by: Isis Lovecruft --- Makefile.am | 3 +-- configure.ac | 11 ----------- 2 files changed, 1 insertion(+), 13 deletions(-) diff --git a/Makefile.am b/Makefile.am index 04ca88a236..02ccbd91b5 100644 --- a/Makefile.am +++ b/Makefile.am @@ -26,8 +26,7 @@ TESTING_TOR_BINARY=$(top_builddir)/src/or/tor$(EXEEXT) endif if USE_RUST -rust_ldadd=$(top_builddir)/src/rust/target/release/@TOR_RUST_STATIC_NAME@ \ - @TOR_RUST_EXTRA_LIBS@ +rust_ldadd=$(top_builddir)/src/rust/target/release/@TOR_RUST_UTIL_STATIC_NAME@ else rust_ldadd= endif diff --git a/configure.ac b/configure.ac index 22c8a56137..d00e1f74be 100644 --- a/configure.ac +++ b/configure.ac @@ -468,15 +468,6 @@ if test "x$enable_rust" = "xyes"; then fi fi - dnl This is a workaround for #46797 - dnl (a.k.a https://github.com/rust-lang/rust/issues/46797 ). Once the - dnl upstream bug is fixed, we can remove this workaround. - case "$host_os" in - darwin*) - TOR_RUST_EXTRA_LIBS="-lresolv" - ;; - esac - dnl For now both MSVC and MinGW rust libraries will output static libs with dnl the MSVC naming convention. if test "$bwin32" = "true"; then @@ -503,8 +494,6 @@ if test "x$enable_rust" = "xyes"; then AC_MSG_RESULT([$RUSTC_VERSION]) fi -AC_SUBST(TOR_RUST_EXTRA_LIBS) - AC_SEARCH_LIBS(socket, [socket network]) AC_SEARCH_LIBS(gethostbyname, [nsl]) AC_SEARCH_LIBS(dlopen, [dl]) From af3e5bed1084fa321e90eaf87e5dc9a7256dd920 Mon Sep 17 00:00:00 2001 From: Isis Lovecruft Date: Sat, 24 Feb 2018 02:17:36 +0000 Subject: [PATCH 0002/2557] build: Fix Makefile.am TOR_RUST_STATIC_NAME variable name after revert. * FIXES #25341: https://bugs.torproject.org/25341 --- Makefile.am | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile.am b/Makefile.am index 02ccbd91b5..2c6de35743 100644 --- a/Makefile.am +++ b/Makefile.am @@ -26,7 +26,7 @@ TESTING_TOR_BINARY=$(top_builddir)/src/or/tor$(EXEEXT) endif if USE_RUST -rust_ldadd=$(top_builddir)/src/rust/target/release/@TOR_RUST_UTIL_STATIC_NAME@ +rust_ldadd=$(top_builddir)/src/rust/target/release/@TOR_RUST_STATIC_NAME@ else rust_ldadd= endif From 306ab8ae17b8f39a6b83bcb24fe02afd8389dc13 Mon Sep 17 00:00:00 2001 From: Colin Childs Date: Mon, 25 Jun 2018 14:52:40 -0500 Subject: [PATCH 0003/2557] Update fallback lists to match #24805 --- scripts/maint/fallback.blacklist | 3 +++ scripts/maint/fallback.whitelist | 40 ++++++++++++++++---------------- 2 files changed, 23 insertions(+), 20 deletions(-) diff --git a/scripts/maint/fallback.blacklist b/scripts/maint/fallback.blacklist index a118cb5919..52ef4f0ec5 100644 --- a/scripts/maint/fallback.blacklist +++ b/scripts/maint/fallback.blacklist @@ -230,3 +230,6 @@ # Email sent directly to teor 212.51.156.193:995 orport=110 id=32E7AAF1F602814D699BEF6761AD03E387758D49 ipv6=[2a02:168:4a01::49]:110 + +# Was moved from blacklist to whitelist by mistake (https://trac.torproject.org/projects/tor/ticket/24805#comment:6) +85.230.184.93:9030 orport=443 id=855BC2DABE24C861CD887DB9B2E950424B49FC34 diff --git a/scripts/maint/fallback.whitelist b/scripts/maint/fallback.whitelist index e9158e1280..819fa48f46 100644 --- a/scripts/maint/fallback.whitelist +++ b/scripts/maint/fallback.whitelist @@ -56,7 +56,6 @@ # https://lists.torproject.org/pipermail/tor-relays/2015-December/008379.html # Email sent directly to teor, verified using relay contact info 91.121.84.137:4951 orport=4051 id=6DE61A6F72C1E5418A66BFED80DFB63E4C77668F -91.121.84.137:4952 orport=4052 id=9FBEB75E8BC142565F12CBBE078D63310236A334 # https://lists.torproject.org/pipermail/tor-relays/2015-December/008381.html # Sent additional emails to teor with updated relays @@ -295,9 +294,6 @@ # Email sent directly to teor, verified using relay contact info 151.80.42.103:9030 orport=9001 id=9007C1D8E4F03D506A4A011B907A9E8D04E3C605 ipv6=[2001:41d0:e:f67::114]:9001 -# Email sent directly to teor, verified using relay contact info -5.39.92.199:80 orport=443 id=0BEA4A88D069753218EAAAD6D22EA87B9A1319D6 ipv6=[2001:41d0:8:b1c7::1]:443 - # Email sent directly to teor, verified using relay contact info 176.31.159.231:80 orport=443 id=D5DBCC0B4F029F80C7B8D33F20CF7D97F0423BB1 176.31.159.230:80 orport=443 id=631748AFB41104D77ADBB7E5CD4F8E8AE876E683 @@ -377,10 +373,6 @@ # Email sent directly to teor, verified using relay contact info 91.219.237.229:80 orport=443 id=1ECD73B936CB6E6B3CD647CC204F108D9DF2C9F7 -# Email sent directly to teor, verified using relay contact info -46.101.151.222:80 orport=443 id=1DBAED235E3957DE1ABD25B4206BE71406FB61F8 -178.62.60.37:80 orport=443 id=175921396C7C426309AB03775A9930B6F611F794 - # Email sent directly to teor, verified using relay contact info 178.62.197.82:80 orport=443 id=0D3EBA17E1C78F1E9900BABDB23861D46FCAF163 @@ -405,9 +397,6 @@ # Email sent directly to teor, verified using relay contact info 5.199.142.236:9030 orport=9001 id=F4C0EDAA0BF0F7EC138746F8FEF1CE26C7860265 -# Email sent directly to teor -188.166.133.133:9030 orport=9001 id=774555642FDC1E1D4FDF2E0C31B7CA9501C5C9C7 ipv6=[2a03:b0c0:2:d0::26c0:1]:9001 # dropsy - # Email sent directly to teor, verified using relay contact info 46.8.249.10:80 orport=443 id=31670150090A7C3513CB7914B9610E786391A95D @@ -563,7 +552,6 @@ 185.100.84.82:80 orport=443 id=7D05A38E39FC5D29AFE6BE487B9B4DC9E635D09E # Email sent directly to teor, verified using relay contact info -164.132.77.175:9030 orport=9001 id=3B33F6FCA645AD4E91428A3AF7DC736AD9FB727B 78.24.75.53:9030 orport=9001 id=DEB73705B2929AE9BE87091607388939332EF123 # Email sent directly to teor, verified using relay contact info @@ -684,6 +672,7 @@ # Email sent directly to teor, verified using relay contact info # Assume details update is permanent 188.40.128.246:9030 orport=9001 id=AD19490C7DBB26D3A68EFC824F67E69B0A96E601 ipv6=[2a01:4f8:221:1ac1:dead:beef:7005:9001]:9001 # sputnik +129.13.131.140:80 orport=443 id=F2DFE5FA1E4CF54F8E761A6D304B9B4EC69BDAE8 ipv6=[2a00:1398:5:f604:cafe:cafe:cafe:9001]:443 # AlleKochenKaffee # Email sent directly to teor, verified using relay contact info 88.198.253.13:9030 orport=9001 id=DF924196D69AAE3C00C115A9CCDF7BB62A175310 ipv6=[2a01:4f8:11a:b1f::2]:9001 @@ -695,8 +684,8 @@ # Email sent directly to teor, verified using relay contact info 176.10.104.240:80 orport=443 id=0111BA9B604669E636FFD5B503F382A4B7AD6E80 176.10.104.240:8080 orport=8443 id=AD86CD1A49573D52A7B6F4A35750F161AAD89C88 -176.10.104.243:80 orport=443 id=88487BDD980BF6E72092EE690E8C51C0AA4A538C 176.10.104.243:8080 orport=8443 id=95DA61AEF23A6C851028C1AA88AD8593F659E60F +94.230.208.147:80 orport=443 id=9AA3FF35E7A549D2337E962333D366E102FE4D50 ipv6=[2a02:418:6017::147]:443 # Email sent directly to teor, verified using relay contact info 107.170.101.39:9030 orport=443 id=30973217E70AF00EBE51797FF6D9AA720A902EAA @@ -743,7 +732,8 @@ 51.254.101.242:9002 orport=9001 id=4CC9CC9195EC38645B699A33307058624F660CCF # Emails sent directly to teor, verified using relay contact info -85.214.62.48:80 orport=443 id=6A7551EEE18F78A9813096E82BF84F740D32B911 +# Updated IP https://trac.torproject.org/projects/tor/ticket/24805#comment:16 +94.130.186.5:80 orport=443 id=6A7551EEE18F78A9813096E82BF84F740D32B911 # Email sent directly to teor, verified using relay contact info 173.255.245.116:9030 orport=9001 id=91E4015E1F82DAF0121D62267E54A1F661AB6DC7 @@ -942,10 +932,13 @@ 199.249.223.66:80 orport=443 id=C5A53BCC174EF8FD0DCB223E4AA929FA557DEDB2 # Quintex17 # https://lists.torproject.org/pipermail/tor-relays/2017-December/013914.html +# https://lists.torproject.org/pipermail/tor-relays/2018-January/014063.html 5.196.23.64:9030 orport=9001 id=775B0FAFDE71AADC23FFC8782B7BEB1D5A92733E # Aerodynamik01 217.182.75.181:9030 orport=9001 id=EFEACD781604EB80FBC025EDEDEA2D523AEAAA2F # Aerodynamik02 193.70.43.76:9030 orport=9001 id=484A10BA2B8D48A5F0216674C8DD50EF27BC32F3 # Aerodynamik03 149.56.141.138:9030 orport=9001 id=1938EBACBB1A7BFA888D9623C90061130E63BB3F # Aerodynamik04 +54.37.73.111:9030 orport=9001 id=92412EA1B9AA887D462B51D816777002F4D58907 # Aerodynamik05 +54.37.17.235:9030 orport=9001 id=360CBA08D1E24F513162047BDB54A1015E531534 # Aerodynamik06 # https://lists.torproject.org/pipermail/tor-relays/2017-December/013917.html 104.200.20.46:80 orport=9001 id=78E2BE744A53631B4AAB781468E94C52AB73968B # bynumlawtor @@ -959,16 +952,10 @@ # Email sent directly to teor 62.210.254.132:80 orport=443 id=8456DFA94161CDD99E480C2A2992C366C6564410 # turingmachine -# Email sent directly to teor -80.127.117.180:80 orport=443 id=328E54981C6DDD7D89B89E418724A4A7881E3192 ipv6=[2001:985:e77:10::4]:443 # sjc01 - # https://lists.torproject.org/pipermail/tor-relays/2017-December/013960.html 51.15.205.214:9030 orport=9001 id=8B6556601612F1E2AFCE2A12FFFAF8482A76DD1F ipv6=[2001:bc8:4400:2500::5:b07]:9001 # titania1 51.15.205.214:9031 orport=9002 id=5E363D72488276160D062DDD2DFA25CFEBAF5EA9 ipv6=[2001:bc8:4400:2500::5:b07]:9002 # titania2 -# Email sent directly to teor -185.129.249.124:9030 orport=9001 id=1FA8F638298645BE58AC905276680889CB795A94 # treadstone - # https://lists.torproject.org/pipermail/tor-relays/2017-December/014000.html 24.117.231.229:34175 orport=45117 id=CE24412AD69444954B4015E293AE53DDDAFEA3D6 # Anosognosia @@ -994,3 +981,16 @@ # https://lists.torproject.org/pipermail/tor-relays/2018-January/014024.html 82.161.212.209:9030 orport=9001 id=4E8CE6F5651E7342C1E7E5ED031E82078134FB0D ipv6=[2001:980:d7ed:1:ff:b0ff:fe00:d0b]:9001 # ymkeo + +# https://lists.torproject.org/pipermail/tor-relays/2018-January/014055.html +37.157.255.35:9030 orport=9090 id=361D33C96D0F161275EE67E2C91EE10B276E778B # cxx4freedom + +# https://lists.torproject.org/pipermail/tor-relays/2018-January/014064.html +87.118.122.120:80 orport=443 id=A2A6616723B511D8E068BB71705191763191F6B2 # otheontelth + +# https://lists.torproject.org/pipermail/tor-relays/2018-January/014069.html +185.100.86.182:9030 orport=8080 id=E51620B90DCB310138ED89EDEDD0A5C361AAE24E # NormalCitizen + +# https://lists.torproject.org/pipermail/tor-relays/2018-January/014267.html +51.15.72.211:80 orport=9001 id=D122094E396DF8BA560843E7B983B0EA649B7DF9 ipv6=[2001:bc8:4700:2300::1b:f09]:9001 # gjtorrelay + From 23d9ffcdf4877f574278b0f0b39ce511072898af Mon Sep 17 00:00:00 2001 From: Colin Childs Date: Thu, 28 Jun 2018 15:22:11 -0500 Subject: [PATCH 0004/2557] Update whitelist to match feedback from operators --- scripts/maint/fallback.whitelist | 37 +++++++++++++++++++++++++++++--- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/scripts/maint/fallback.whitelist b/scripts/maint/fallback.whitelist index 819fa48f46..0cca5f29cb 100644 --- a/scripts/maint/fallback.whitelist +++ b/scripts/maint/fallback.whitelist @@ -319,10 +319,14 @@ 185.66.250.141:9030 orport=9001 id=B1726B94885CE3AC3910CA8B60622B97B98E2529 # Email sent directly to teor, verified using relay contact info +# Email sent directly to Phoul 185.104.120.7:9030 orport=443 id=445F1C853966624FB3CF1E12442570DC553CC2EC ipv6=[2a06:3000::120:7]:443 -185.104.120.2:9030 orport=21 id=518FF8708698E1DA09C823C36D35DF89A2CAD956 -185.104.120.4:9030 orport=9001 id=F92B3CB9BBE0CB22409843FB1AE4DBCD5EFAC835 -185.104.120.3:9030 orport=21 id=707C1B61AC72227B34487B56D04BAA3BA1179CE8 ipv6=[2a06:3000::120:3]:21 +185.104.120.2:9030 orport=21 id=518FF8708698E1DA09C823C36D35DF89A2CAD956 ipv6=[2a06:3000::120:2]:443 +185.104.120.4:9030 orport=9001 id=F92B3CB9BBE0CB22409843FB1AE4DBCD5EFAC835 ipv6=[2a06:3000::120:4]:443 +185.104.120.3:9030 orport=21 id=707C1B61AC72227B34487B56D04BAA3BA1179CE8 ipv6=[2a06:3000::120:3]:443 +185.104.120.5:80 orport=443 id=3EBDF84DE3B16F0EBF7D51450F07913A02EFDA6C ipv6=[2a06:3000::120:5]:443 +185.104.120.60:80 orport=443 id=D05C9C7068EB5A45F670D5E38A14907EE6223141 ipv6=[2a06:3000::120:60]:443 + # Email sent directly to teor, verified using relay contact info 37.187.102.186:9030 orport=9001 id=489D94333DF66D57FFE34D9D59CC2D97E2CB0053 ipv6=[2001:41d0:a:26ba::1]:9001 @@ -752,6 +756,7 @@ # Email sent directly to teor, verified using relay contact info 5.9.158.75:80 orport=443 id=1AF72E8906E6C49481A791A6F8F84F8DFEBBB2BA ipv6=[2a01:4f8:190:514a::2]:443 +5.9.158.75:9030 orport=9001 id=D11D11877769B9E617537B4B46BFB92B443DE33D ipv6=[2a01:4f8:190:514a::2]:9001 # Email sent directly to teor, verified using relay contact info 46.101.169.151:9030 orport=9001 id=D760C5B436E42F93D77EF2D969157EEA14F9B39C ipv6=[2a03:b0c0:3:d0::74f:a001]:9001 @@ -823,9 +828,12 @@ 195.154.122.54:80 orport=443 id=64E99CB34C595A02A3165484BD1215E7389322C6 # Email sent directly to teor, verified using relay contact info +# Email sent directly to Phoul 185.100.86.128:9030 orport=9001 id=9B31F1F1C1554F9FFB3455911F82E818EF7C7883 185.100.85.101:9030 orport=9001 id=4061C553CA88021B8302F0814365070AAE617270 31.171.155.108:9030 orport=9001 id=D3E5EDDBE5159388704D6785BE51930AAFACEC6F +31.171.155.131:80 orport=443 id=516D1B9E22484202322828D8CAC30325030017E2 + # Email sent directly to teor, verified using relay contact info 89.163.247.43:9030 orport=9001 id=BC7ACFAC04854C77167C7D66B7E471314ED8C410 ipv6=[2001:4ba0:fff7:25::5]:9001 @@ -994,3 +1002,26 @@ # https://lists.torproject.org/pipermail/tor-relays/2018-January/014267.html 51.15.72.211:80 orport=9001 id=D122094E396DF8BA560843E7B983B0EA649B7DF9 ipv6=[2001:bc8:4700:2300::1b:f09]:9001 # gjtorrelay +# Email sent directly to Phoul +185.34.33.2:9265 orport=31415 id=D71B1CA1C9DC7E8CA64158E106AD770A21160FEE # lqdn + +# Email sent directly to Phoul +78.156.110.135:9091 orport=9090 id=F48FD1AED068496D51D1384BC7497C04E4985DA6 # SkynetC2 + +# Email sent directly to Phoul +5.200.21.144:80 orport=443 id=0C039F35C2E40DCB71CD8A07E97C7FD7787D42D6 # libel +64.79.152.132:80 orport=443 id=375DCBB2DBD94E5263BC0C015F0C9E756669617E # ebola + +# https://lists.torproject.org/pipermail/tor-relays/2018-June/015524.html +132.248.241.5:9030 orport=9001 id=4661DE96D3F8E923994B05218F23760C8D7935A4 + +# https://lists.torproject.org/pipermail/tor-relays/2018-June/015522.html +96.253.78.108:80 orport=442 id=924B24AFA7F075D059E8EEB284CC400B33D3D036 + +# Email sent directly to Phoul +163.172.218.10:9030 orport=9001 id=78809B6C50CB6491DB3A72C60EC39DC85BF72D1F ipv6=[2001:bc8:3f23:1100::1]:9001 +163.172.218.10:9130 orport=9101 id=B247BA9E0AEA93E6D7BF4080CFBB964034AF2B28 ipv6=[2001:bc8:3f23:1100::1]:9101 + +# Email sent directly to Phoul +158.255.212.178:8080 orport=8443 id=D941D380E5228E7B4D372AF4D484629A96DC48B9 ipv6=[2a03:f80:ed15:158:255:212:178:2]:8443 + From d7e7c0d20d15d90882bc0b963f897a99b5d7585a Mon Sep 17 00:00:00 2001 From: Colin Childs Date: Tue, 10 Jul 2018 13:14:20 -0500 Subject: [PATCH 0005/2557] Updating whitelist to match operator feedback --- scripts/maint/fallback.whitelist | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/scripts/maint/fallback.whitelist b/scripts/maint/fallback.whitelist index 0cca5f29cb..1ea3874451 100644 --- a/scripts/maint/fallback.whitelist +++ b/scripts/maint/fallback.whitelist @@ -1025,3 +1025,17 @@ # Email sent directly to Phoul 158.255.212.178:8080 orport=8443 id=D941D380E5228E7B4D372AF4D484629A96DC48B9 ipv6=[2a03:f80:ed15:158:255:212:178:2]:8443 +# Email sent directly to Phoul +45.79.108.130:9030 orport=9001 id=AEDAC7081AE14B8D241ECF0FF17A2858AB4383D0 ipv6=[2600:3c01:e000:131::8000:0]:9001 + +# Email sent directly to Phoul +51.254.147.57:80 orport=443 id=EB80A8D52F07238B576C42CEAB98ADD084EE075E +217.182.51.248:80 orport=443 id=D6BA940D3255AB40DC5EE5B0B285FA143E1F9865 + +# https://lists.torproject.org/pipermail/tor-relays/2018-June/015541.html +195.191.81.7:9030 orport=9001 id=41A3C16269C7B63DB6EB741DBDDB4E1F586B1592 ipv6=[2a00:1908:fffc:ffff:c0a6:ccff:fe62:e1a1]:9001 +51.254.96.208:9030 orport=9001 id=8101421BEFCCF4C271D5483C5AABCAAD245BBB9D ipv6=[2001:41d0:401:3100::30dc]:9001 +163.172.154.162:9030 orport=9001 id=F741E5124CB12700DA946B78C9B2DD175D6CD2A1 ipv6=[2001:bc8:4400:2100::17:419]:9001 +51.15.78.0:9030 orport=9001 id=15BE17C99FACE24470D40AF782D6A9C692AB36D6 ipv6=[2001:bc8:4700:2300::16:c0b]:9001 +54.37.139.118:9030 orport=9001 id=90A5D1355C4B5840E950EB61E673863A6AE3ACA1 ipv6=[2001:41d0:601:1100::1b8]:9001 +51.38.65.160:9030 orport=90013CB4193EF4E239FCEDC4DC43468E0B0D6B67ACC3 ipv6=[2001:41d0:801:2000::f6e]:9001 From cbdf6a04b10a27721b51df45daf4e3d8cd084287 Mon Sep 17 00:00:00 2001 From: Colin Childs Date: Tue, 10 Jul 2018 13:17:22 -0500 Subject: [PATCH 0006/2557] Fixing spacing issue in whitelist --- scripts/maint/fallback.whitelist | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/maint/fallback.whitelist b/scripts/maint/fallback.whitelist index 1ea3874451..7097d8911a 100644 --- a/scripts/maint/fallback.whitelist +++ b/scripts/maint/fallback.whitelist @@ -1038,4 +1038,4 @@ 163.172.154.162:9030 orport=9001 id=F741E5124CB12700DA946B78C9B2DD175D6CD2A1 ipv6=[2001:bc8:4400:2100::17:419]:9001 51.15.78.0:9030 orport=9001 id=15BE17C99FACE24470D40AF782D6A9C692AB36D6 ipv6=[2001:bc8:4700:2300::16:c0b]:9001 54.37.139.118:9030 orport=9001 id=90A5D1355C4B5840E950EB61E673863A6AE3ACA1 ipv6=[2001:41d0:601:1100::1b8]:9001 -51.38.65.160:9030 orport=90013CB4193EF4E239FCEDC4DC43468E0B0D6B67ACC3 ipv6=[2001:41d0:801:2000::f6e]:9001 +51.38.65.160:9030 orport=9001 id=3CB4193EF4E239FCEDC4DC43468E0B0D6B67ACC3 ipv6=[2001:41d0:801:2000::f6e]:9001 From 855feab70d35dd5093aae1f27d6cf19ee6cb3338 Mon Sep 17 00:00:00 2001 From: Colin Childs Date: Fri, 13 Jul 2018 13:17:42 -0500 Subject: [PATCH 0007/2557] Adding 3 more fallback mirrors --- scripts/maint/fallback.whitelist | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/scripts/maint/fallback.whitelist b/scripts/maint/fallback.whitelist index 7097d8911a..e60b29304c 100644 --- a/scripts/maint/fallback.whitelist +++ b/scripts/maint/fallback.whitelist @@ -1039,3 +1039,12 @@ 51.15.78.0:9030 orport=9001 id=15BE17C99FACE24470D40AF782D6A9C692AB36D6 ipv6=[2001:bc8:4700:2300::16:c0b]:9001 54.37.139.118:9030 orport=9001 id=90A5D1355C4B5840E950EB61E673863A6AE3ACA1 ipv6=[2001:41d0:601:1100::1b8]:9001 51.38.65.160:9030 orport=9001 id=3CB4193EF4E239FCEDC4DC43468E0B0D6B67ACC3 ipv6=[2001:41d0:801:2000::f6e]:9001 + +# Email sent directly to Phoul +54.37.138.138:8080 orport=993 id=1576BE143D8727745BB2BCDDF183291B3C3EFEFC + +# Email sent directly to Phoul +67.215.255.140:9030 orport=9001 id=23917BB3F3994BC61F0C9D7AD19B069F9E150D26 + +# Email sent directly to Phoul +195.154.105.170:9030 orport=9001 id=E947C029087FA1C3499BEF5D4372947C51223D8F From 54cf36400669c555a3af84b167fd79f19b9ebbe3 Mon Sep 17 00:00:00 2001 From: Colin Childs Date: Mon, 16 Jul 2018 12:21:46 -0500 Subject: [PATCH 0008/2557] Add mirrors from EmeraldOnion --- scripts/maint/fallback.whitelist | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/scripts/maint/fallback.whitelist b/scripts/maint/fallback.whitelist index e60b29304c..7d9d136780 100644 --- a/scripts/maint/fallback.whitelist +++ b/scripts/maint/fallback.whitelist @@ -12,7 +12,8 @@ # To replace this list with the hard-coded fallback list (for testing), use # a command similar to: -# cat src/or/fallback_dirs.inc | grep \" | grep -v weight | tr -d '\n' | \ +# cat src/app/config/fallback_dirs.inc | grep \" | grep -v weight | \ +# tr -d '\n' | \ # sed 's/"" / /g' | sed 's/""/"/g' | tr \" '\n' | grep -v '^$' \ # > scripts/maint/fallback.whitelist # @@ -1048,3 +1049,8 @@ # Email sent directly to Phoul 195.154.105.170:9030 orport=9001 id=E947C029087FA1C3499BEF5D4372947C51223D8F + +# Email sent directly to Phoul +23.129.64.101:80 orport=443 id=2EB20285FE55927B7AECC47BB94F22534FBC3941 ipv6=[2620:18c:0:1001::101]:443 +23.129.64.102:80 orport=443 id=CA9739E2805A3CD73CF75BBCB6785C32394240E3 ipv6=[2620:18c:0:1001::102]:443 +23.129.64.103:80 orport=443 id=8ED84B53BD9556CCBB036073A1AD508EC27CBE52 ipv6=[2620:18c:0:1001::103]:443 From 04de9443fef50a27036dd2e8550fde87731d2adc Mon Sep 17 00:00:00 2001 From: Colin Childs Date: Mon, 25 Jun 2018 14:52:40 -0500 Subject: [PATCH 0009/2557] Update fallback lists to match #24805 --- scripts/maint/fallback.whitelist | 40 ++++++++++++++++---------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/scripts/maint/fallback.whitelist b/scripts/maint/fallback.whitelist index 79551948c6..ff7eadda88 100644 --- a/scripts/maint/fallback.whitelist +++ b/scripts/maint/fallback.whitelist @@ -57,7 +57,6 @@ # https://lists.torproject.org/pipermail/tor-relays/2015-December/008379.html # Email sent directly to teor, verified using relay contact info 91.121.84.137:4951 orport=4051 id=6DE61A6F72C1E5418A66BFED80DFB63E4C77668F -91.121.84.137:4952 orport=4052 id=9FBEB75E8BC142565F12CBBE078D63310236A334 # https://lists.torproject.org/pipermail/tor-relays/2015-December/008381.html # Sent additional emails to teor with updated relays @@ -296,9 +295,6 @@ # Email sent directly to teor, verified using relay contact info 151.80.42.103:9030 orport=9001 id=9007C1D8E4F03D506A4A011B907A9E8D04E3C605 ipv6=[2001:41d0:e:f67::114]:9001 -# Email sent directly to teor, verified using relay contact info -5.39.92.199:80 orport=443 id=0BEA4A88D069753218EAAAD6D22EA87B9A1319D6 ipv6=[2001:41d0:8:b1c7::1]:443 - # Email sent directly to teor, verified using relay contact info 176.31.159.231:80 orport=443 id=D5DBCC0B4F029F80C7B8D33F20CF7D97F0423BB1 176.31.159.230:80 orport=443 id=631748AFB41104D77ADBB7E5CD4F8E8AE876E683 @@ -378,10 +374,6 @@ # Email sent directly to teor, verified using relay contact info 91.219.237.229:80 orport=443 id=1ECD73B936CB6E6B3CD647CC204F108D9DF2C9F7 -# Email sent directly to teor, verified using relay contact info -46.101.151.222:80 orport=443 id=1DBAED235E3957DE1ABD25B4206BE71406FB61F8 -178.62.60.37:80 orport=443 id=175921396C7C426309AB03775A9930B6F611F794 - # Email sent directly to teor, verified using relay contact info 178.62.197.82:80 orport=443 id=0D3EBA17E1C78F1E9900BABDB23861D46FCAF163 @@ -406,9 +398,6 @@ # Email sent directly to teor, verified using relay contact info 5.199.142.236:9030 orport=9001 id=F4C0EDAA0BF0F7EC138746F8FEF1CE26C7860265 -# Email sent directly to teor -188.166.133.133:9030 orport=9001 id=774555642FDC1E1D4FDF2E0C31B7CA9501C5C9C7 ipv6=[2a03:b0c0:2:d0::26c0:1]:9001 # dropsy - # Email sent directly to teor, verified using relay contact info 46.8.249.10:80 orport=443 id=31670150090A7C3513CB7914B9610E786391A95D @@ -564,7 +553,6 @@ 185.100.84.82:80 orport=443 id=7D05A38E39FC5D29AFE6BE487B9B4DC9E635D09E # Email sent directly to teor, verified using relay contact info -164.132.77.175:9030 orport=9001 id=3B33F6FCA645AD4E91428A3AF7DC736AD9FB727B 78.24.75.53:9030 orport=9001 id=DEB73705B2929AE9BE87091607388939332EF123 # Email sent directly to teor, verified using relay contact info @@ -685,6 +673,7 @@ # Email sent directly to teor, verified using relay contact info # Assume details update is permanent 188.40.128.246:9030 orport=9001 id=AD19490C7DBB26D3A68EFC824F67E69B0A96E601 ipv6=[2a01:4f8:221:1ac1:dead:beef:7005:9001]:9001 # sputnik +129.13.131.140:80 orport=443 id=F2DFE5FA1E4CF54F8E761A6D304B9B4EC69BDAE8 ipv6=[2a00:1398:5:f604:cafe:cafe:cafe:9001]:443 # AlleKochenKaffee # Email sent directly to teor, verified using relay contact info 88.198.253.13:9030 orport=9001 id=DF924196D69AAE3C00C115A9CCDF7BB62A175310 ipv6=[2a01:4f8:11a:b1f::2]:9001 @@ -696,8 +685,8 @@ # Email sent directly to teor, verified using relay contact info 176.10.104.240:80 orport=443 id=0111BA9B604669E636FFD5B503F382A4B7AD6E80 176.10.104.240:8080 orport=8443 id=AD86CD1A49573D52A7B6F4A35750F161AAD89C88 -176.10.104.243:80 orport=443 id=88487BDD980BF6E72092EE690E8C51C0AA4A538C 176.10.104.243:8080 orport=8443 id=95DA61AEF23A6C851028C1AA88AD8593F659E60F +94.230.208.147:80 orport=443 id=9AA3FF35E7A549D2337E962333D366E102FE4D50 ipv6=[2a02:418:6017::147]:443 # Email sent directly to teor, verified using relay contact info 107.170.101.39:9030 orport=443 id=30973217E70AF00EBE51797FF6D9AA720A902EAA @@ -744,7 +733,8 @@ 51.254.101.242:9002 orport=9001 id=4CC9CC9195EC38645B699A33307058624F660CCF # Emails sent directly to teor, verified using relay contact info -85.214.62.48:80 orport=443 id=6A7551EEE18F78A9813096E82BF84F740D32B911 +# Updated IP https://trac.torproject.org/projects/tor/ticket/24805#comment:16 +94.130.186.5:80 orport=443 id=6A7551EEE18F78A9813096E82BF84F740D32B911 # Email sent directly to teor, verified using relay contact info 173.255.245.116:9030 orport=9001 id=91E4015E1F82DAF0121D62267E54A1F661AB6DC7 @@ -943,10 +933,13 @@ 199.249.223.66:80 orport=443 id=C5A53BCC174EF8FD0DCB223E4AA929FA557DEDB2 # Quintex17 # https://lists.torproject.org/pipermail/tor-relays/2017-December/013914.html +# https://lists.torproject.org/pipermail/tor-relays/2018-January/014063.html 5.196.23.64:9030 orport=9001 id=775B0FAFDE71AADC23FFC8782B7BEB1D5A92733E # Aerodynamik01 217.182.75.181:9030 orport=9001 id=EFEACD781604EB80FBC025EDEDEA2D523AEAAA2F # Aerodynamik02 193.70.43.76:9030 orport=9001 id=484A10BA2B8D48A5F0216674C8DD50EF27BC32F3 # Aerodynamik03 149.56.141.138:9030 orport=9001 id=1938EBACBB1A7BFA888D9623C90061130E63BB3F # Aerodynamik04 +54.37.73.111:9030 orport=9001 id=92412EA1B9AA887D462B51D816777002F4D58907 # Aerodynamik05 +54.37.17.235:9030 orport=9001 id=360CBA08D1E24F513162047BDB54A1015E531534 # Aerodynamik06 # https://lists.torproject.org/pipermail/tor-relays/2017-December/013917.html 104.200.20.46:80 orport=9001 id=78E2BE744A53631B4AAB781468E94C52AB73968B # bynumlawtor @@ -960,16 +953,10 @@ # Email sent directly to teor 62.210.254.132:80 orport=443 id=8456DFA94161CDD99E480C2A2992C366C6564410 # turingmachine -# Email sent directly to teor -80.127.117.180:80 orport=443 id=328E54981C6DDD7D89B89E418724A4A7881E3192 ipv6=[2001:985:e77:10::4]:443 # sjc01 - # https://lists.torproject.org/pipermail/tor-relays/2017-December/013960.html 51.15.205.214:9030 orport=9001 id=8B6556601612F1E2AFCE2A12FFFAF8482A76DD1F ipv6=[2001:bc8:4400:2500::5:b07]:9001 # titania1 51.15.205.214:9031 orport=9002 id=5E363D72488276160D062DDD2DFA25CFEBAF5EA9 ipv6=[2001:bc8:4400:2500::5:b07]:9002 # titania2 -# Email sent directly to teor -185.129.249.124:9030 orport=9001 id=1FA8F638298645BE58AC905276680889CB795A94 # treadstone - # https://lists.torproject.org/pipermail/tor-relays/2017-December/014000.html 24.117.231.229:34175 orport=45117 id=CE24412AD69444954B4015E293AE53DDDAFEA3D6 # Anosognosia @@ -995,3 +982,16 @@ # https://lists.torproject.org/pipermail/tor-relays/2018-January/014024.html 82.161.212.209:9030 orport=9001 id=4E8CE6F5651E7342C1E7E5ED031E82078134FB0D ipv6=[2001:980:d7ed:1:ff:b0ff:fe00:d0b]:9001 # ymkeo + +# https://lists.torproject.org/pipermail/tor-relays/2018-January/014055.html +37.157.255.35:9030 orport=9090 id=361D33C96D0F161275EE67E2C91EE10B276E778B # cxx4freedom + +# https://lists.torproject.org/pipermail/tor-relays/2018-January/014064.html +87.118.122.120:80 orport=443 id=A2A6616723B511D8E068BB71705191763191F6B2 # otheontelth + +# https://lists.torproject.org/pipermail/tor-relays/2018-January/014069.html +185.100.86.182:9030 orport=8080 id=E51620B90DCB310138ED89EDEDD0A5C361AAE24E # NormalCitizen + +# https://lists.torproject.org/pipermail/tor-relays/2018-January/014267.html +51.15.72.211:80 orport=9001 id=D122094E396DF8BA560843E7B983B0EA649B7DF9 ipv6=[2001:bc8:4700:2300::1b:f09]:9001 # gjtorrelay + From 7479f75b95eaee2003a8bc9bb8ba6465cd8f9754 Mon Sep 17 00:00:00 2001 From: Colin Childs Date: Thu, 28 Jun 2018 15:22:11 -0500 Subject: [PATCH 0010/2557] Update whitelist to match feedback from operators --- scripts/maint/fallback.whitelist | 37 +++++++++++++++++++++++++++++--- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/scripts/maint/fallback.whitelist b/scripts/maint/fallback.whitelist index ff7eadda88..1077a0054b 100644 --- a/scripts/maint/fallback.whitelist +++ b/scripts/maint/fallback.whitelist @@ -320,10 +320,14 @@ 185.66.250.141:9030 orport=9001 id=B1726B94885CE3AC3910CA8B60622B97B98E2529 # Email sent directly to teor, verified using relay contact info +# Email sent directly to Phoul 185.104.120.7:9030 orport=443 id=445F1C853966624FB3CF1E12442570DC553CC2EC ipv6=[2a06:3000::120:7]:443 -185.104.120.2:9030 orport=21 id=518FF8708698E1DA09C823C36D35DF89A2CAD956 -185.104.120.4:9030 orport=9001 id=F92B3CB9BBE0CB22409843FB1AE4DBCD5EFAC835 -185.104.120.3:9030 orport=21 id=707C1B61AC72227B34487B56D04BAA3BA1179CE8 ipv6=[2a06:3000::120:3]:21 +185.104.120.2:9030 orport=21 id=518FF8708698E1DA09C823C36D35DF89A2CAD956 ipv6=[2a06:3000::120:2]:443 +185.104.120.4:9030 orport=9001 id=F92B3CB9BBE0CB22409843FB1AE4DBCD5EFAC835 ipv6=[2a06:3000::120:4]:443 +185.104.120.3:9030 orport=21 id=707C1B61AC72227B34487B56D04BAA3BA1179CE8 ipv6=[2a06:3000::120:3]:443 +185.104.120.5:80 orport=443 id=3EBDF84DE3B16F0EBF7D51450F07913A02EFDA6C ipv6=[2a06:3000::120:5]:443 +185.104.120.60:80 orport=443 id=D05C9C7068EB5A45F670D5E38A14907EE6223141 ipv6=[2a06:3000::120:60]:443 + # Email sent directly to teor, verified using relay contact info 37.187.102.186:9030 orport=9001 id=489D94333DF66D57FFE34D9D59CC2D97E2CB0053 ipv6=[2001:41d0:a:26ba::1]:9001 @@ -753,6 +757,7 @@ # Email sent directly to teor, verified using relay contact info 5.9.158.75:80 orport=443 id=1AF72E8906E6C49481A791A6F8F84F8DFEBBB2BA ipv6=[2a01:4f8:190:514a::2]:443 +5.9.158.75:9030 orport=9001 id=D11D11877769B9E617537B4B46BFB92B443DE33D ipv6=[2a01:4f8:190:514a::2]:9001 # Email sent directly to teor, verified using relay contact info 46.101.169.151:9030 orport=9001 id=D760C5B436E42F93D77EF2D969157EEA14F9B39C ipv6=[2a03:b0c0:3:d0::74f:a001]:9001 @@ -824,9 +829,12 @@ 195.154.122.54:80 orport=443 id=64E99CB34C595A02A3165484BD1215E7389322C6 # Email sent directly to teor, verified using relay contact info +# Email sent directly to Phoul 185.100.86.128:9030 orport=9001 id=9B31F1F1C1554F9FFB3455911F82E818EF7C7883 185.100.85.101:9030 orport=9001 id=4061C553CA88021B8302F0814365070AAE617270 31.171.155.108:9030 orport=9001 id=D3E5EDDBE5159388704D6785BE51930AAFACEC6F +31.171.155.131:80 orport=443 id=516D1B9E22484202322828D8CAC30325030017E2 + # Email sent directly to teor, verified using relay contact info 89.163.247.43:9030 orport=9001 id=BC7ACFAC04854C77167C7D66B7E471314ED8C410 ipv6=[2001:4ba0:fff7:25::5]:9001 @@ -995,3 +1003,26 @@ # https://lists.torproject.org/pipermail/tor-relays/2018-January/014267.html 51.15.72.211:80 orport=9001 id=D122094E396DF8BA560843E7B983B0EA649B7DF9 ipv6=[2001:bc8:4700:2300::1b:f09]:9001 # gjtorrelay +# Email sent directly to Phoul +185.34.33.2:9265 orport=31415 id=D71B1CA1C9DC7E8CA64158E106AD770A21160FEE # lqdn + +# Email sent directly to Phoul +78.156.110.135:9091 orport=9090 id=F48FD1AED068496D51D1384BC7497C04E4985DA6 # SkynetC2 + +# Email sent directly to Phoul +5.200.21.144:80 orport=443 id=0C039F35C2E40DCB71CD8A07E97C7FD7787D42D6 # libel +64.79.152.132:80 orport=443 id=375DCBB2DBD94E5263BC0C015F0C9E756669617E # ebola + +# https://lists.torproject.org/pipermail/tor-relays/2018-June/015524.html +132.248.241.5:9030 orport=9001 id=4661DE96D3F8E923994B05218F23760C8D7935A4 + +# https://lists.torproject.org/pipermail/tor-relays/2018-June/015522.html +96.253.78.108:80 orport=442 id=924B24AFA7F075D059E8EEB284CC400B33D3D036 + +# Email sent directly to Phoul +163.172.218.10:9030 orport=9001 id=78809B6C50CB6491DB3A72C60EC39DC85BF72D1F ipv6=[2001:bc8:3f23:1100::1]:9001 +163.172.218.10:9130 orport=9101 id=B247BA9E0AEA93E6D7BF4080CFBB964034AF2B28 ipv6=[2001:bc8:3f23:1100::1]:9101 + +# Email sent directly to Phoul +158.255.212.178:8080 orport=8443 id=D941D380E5228E7B4D372AF4D484629A96DC48B9 ipv6=[2a03:f80:ed15:158:255:212:178:2]:8443 + From eca58aec76ce87ea8eb56a8f7689945f79e0f319 Mon Sep 17 00:00:00 2001 From: Colin Childs Date: Tue, 10 Jul 2018 13:14:20 -0500 Subject: [PATCH 0011/2557] Updating whitelist to match operator feedback --- scripts/maint/fallback.whitelist | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/scripts/maint/fallback.whitelist b/scripts/maint/fallback.whitelist index 1077a0054b..0560d49108 100644 --- a/scripts/maint/fallback.whitelist +++ b/scripts/maint/fallback.whitelist @@ -1026,3 +1026,17 @@ # Email sent directly to Phoul 158.255.212.178:8080 orport=8443 id=D941D380E5228E7B4D372AF4D484629A96DC48B9 ipv6=[2a03:f80:ed15:158:255:212:178:2]:8443 +# Email sent directly to Phoul +45.79.108.130:9030 orport=9001 id=AEDAC7081AE14B8D241ECF0FF17A2858AB4383D0 ipv6=[2600:3c01:e000:131::8000:0]:9001 + +# Email sent directly to Phoul +51.254.147.57:80 orport=443 id=EB80A8D52F07238B576C42CEAB98ADD084EE075E +217.182.51.248:80 orport=443 id=D6BA940D3255AB40DC5EE5B0B285FA143E1F9865 + +# https://lists.torproject.org/pipermail/tor-relays/2018-June/015541.html +195.191.81.7:9030 orport=9001 id=41A3C16269C7B63DB6EB741DBDDB4E1F586B1592 ipv6=[2a00:1908:fffc:ffff:c0a6:ccff:fe62:e1a1]:9001 +51.254.96.208:9030 orport=9001 id=8101421BEFCCF4C271D5483C5AABCAAD245BBB9D ipv6=[2001:41d0:401:3100::30dc]:9001 +163.172.154.162:9030 orport=9001 id=F741E5124CB12700DA946B78C9B2DD175D6CD2A1 ipv6=[2001:bc8:4400:2100::17:419]:9001 +51.15.78.0:9030 orport=9001 id=15BE17C99FACE24470D40AF782D6A9C692AB36D6 ipv6=[2001:bc8:4700:2300::16:c0b]:9001 +54.37.139.118:9030 orport=9001 id=90A5D1355C4B5840E950EB61E673863A6AE3ACA1 ipv6=[2001:41d0:601:1100::1b8]:9001 +51.38.65.160:9030 orport=90013CB4193EF4E239FCEDC4DC43468E0B0D6B67ACC3 ipv6=[2001:41d0:801:2000::f6e]:9001 From 226816df558ccfc92b106bfefe129604448cd43a Mon Sep 17 00:00:00 2001 From: Colin Childs Date: Tue, 10 Jul 2018 13:17:22 -0500 Subject: [PATCH 0012/2557] Fixing spacing issue in whitelist --- scripts/maint/fallback.whitelist | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/maint/fallback.whitelist b/scripts/maint/fallback.whitelist index 0560d49108..30c5b3f707 100644 --- a/scripts/maint/fallback.whitelist +++ b/scripts/maint/fallback.whitelist @@ -1039,4 +1039,4 @@ 163.172.154.162:9030 orport=9001 id=F741E5124CB12700DA946B78C9B2DD175D6CD2A1 ipv6=[2001:bc8:4400:2100::17:419]:9001 51.15.78.0:9030 orport=9001 id=15BE17C99FACE24470D40AF782D6A9C692AB36D6 ipv6=[2001:bc8:4700:2300::16:c0b]:9001 54.37.139.118:9030 orport=9001 id=90A5D1355C4B5840E950EB61E673863A6AE3ACA1 ipv6=[2001:41d0:601:1100::1b8]:9001 -51.38.65.160:9030 orport=90013CB4193EF4E239FCEDC4DC43468E0B0D6B67ACC3 ipv6=[2001:41d0:801:2000::f6e]:9001 +51.38.65.160:9030 orport=9001 id=3CB4193EF4E239FCEDC4DC43468E0B0D6B67ACC3 ipv6=[2001:41d0:801:2000::f6e]:9001 From 831abc0e10ba19d7e33cdee8e328d39a1d849496 Mon Sep 17 00:00:00 2001 From: Colin Childs Date: Fri, 13 Jul 2018 13:17:42 -0500 Subject: [PATCH 0013/2557] Adding 3 more fallback mirrors --- scripts/maint/fallback.whitelist | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/scripts/maint/fallback.whitelist b/scripts/maint/fallback.whitelist index 30c5b3f707..53458ef6c0 100644 --- a/scripts/maint/fallback.whitelist +++ b/scripts/maint/fallback.whitelist @@ -1040,3 +1040,12 @@ 51.15.78.0:9030 orport=9001 id=15BE17C99FACE24470D40AF782D6A9C692AB36D6 ipv6=[2001:bc8:4700:2300::16:c0b]:9001 54.37.139.118:9030 orport=9001 id=90A5D1355C4B5840E950EB61E673863A6AE3ACA1 ipv6=[2001:41d0:601:1100::1b8]:9001 51.38.65.160:9030 orport=9001 id=3CB4193EF4E239FCEDC4DC43468E0B0D6B67ACC3 ipv6=[2001:41d0:801:2000::f6e]:9001 + +# Email sent directly to Phoul +54.37.138.138:8080 orport=993 id=1576BE143D8727745BB2BCDDF183291B3C3EFEFC + +# Email sent directly to Phoul +67.215.255.140:9030 orport=9001 id=23917BB3F3994BC61F0C9D7AD19B069F9E150D26 + +# Email sent directly to Phoul +195.154.105.170:9030 orport=9001 id=E947C029087FA1C3499BEF5D4372947C51223D8F From 5c2431a3436506862b2148a43821bf22f33c7269 Mon Sep 17 00:00:00 2001 From: Colin Childs Date: Mon, 16 Jul 2018 12:21:46 -0500 Subject: [PATCH 0014/2557] Add mirrors from EmeraldOnion --- scripts/maint/fallback.whitelist | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/maint/fallback.whitelist b/scripts/maint/fallback.whitelist index 53458ef6c0..7d9d136780 100644 --- a/scripts/maint/fallback.whitelist +++ b/scripts/maint/fallback.whitelist @@ -1049,3 +1049,8 @@ # Email sent directly to Phoul 195.154.105.170:9030 orport=9001 id=E947C029087FA1C3499BEF5D4372947C51223D8F + +# Email sent directly to Phoul +23.129.64.101:80 orport=443 id=2EB20285FE55927B7AECC47BB94F22534FBC3941 ipv6=[2620:18c:0:1001::101]:443 +23.129.64.102:80 orport=443 id=CA9739E2805A3CD73CF75BBCB6785C32394240E3 ipv6=[2620:18c:0:1001::102]:443 +23.129.64.103:80 orport=443 id=8ED84B53BD9556CCBB036073A1AD508EC27CBE52 ipv6=[2620:18c:0:1001::103]:443 From 0260b720053717cdc348a51ede1d7e51e7ea2741 Mon Sep 17 00:00:00 2001 From: Colin Childs Date: Fri, 3 Aug 2018 09:49:03 -0500 Subject: [PATCH 0015/2557] Adding relay from Edmond --- scripts/maint/fallback.whitelist | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/maint/fallback.whitelist b/scripts/maint/fallback.whitelist index 7d9d136780..20751b6e47 100644 --- a/scripts/maint/fallback.whitelist +++ b/scripts/maint/fallback.whitelist @@ -1054,3 +1054,7 @@ 23.129.64.101:80 orport=443 id=2EB20285FE55927B7AECC47BB94F22534FBC3941 ipv6=[2620:18c:0:1001::101]:443 23.129.64.102:80 orport=443 id=CA9739E2805A3CD73CF75BBCB6785C32394240E3 ipv6=[2620:18c:0:1001::102]:443 23.129.64.103:80 orport=443 id=8ED84B53BD9556CCBB036073A1AD508EC27CBE52 ipv6=[2620:18c:0:1001::103]:443 + +# Email sent directly to Phoul +37.139.8.104:9030 orport=9001 id=7088D485934E8A403B81531F8C90BDC75FA43C98 ipv6=[2a03:b0c0:0:1010::24c:1001]:9001 + From 9aceb928392ef8f3e2f737c14702a9cc330bd109 Mon Sep 17 00:00:00 2001 From: Colin Childs Date: Fri, 10 Aug 2018 15:19:03 -0500 Subject: [PATCH 0016/2557] Adding 2 new fallback mirrors --- scripts/maint/fallback.whitelist | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/maint/fallback.whitelist b/scripts/maint/fallback.whitelist index 20751b6e47..1b27cf7d29 100644 --- a/scripts/maint/fallback.whitelist +++ b/scripts/maint/fallback.whitelist @@ -1058,3 +1058,8 @@ # Email sent directly to Phoul 37.139.8.104:9030 orport=9001 id=7088D485934E8A403B81531F8C90BDC75FA43C98 ipv6=[2a03:b0c0:0:1010::24c:1001]:9001 +# Email sent directly to Phoul +178.254.7.88:9030 orpport=9001 id=85A885433E50B1874F11CEC9BE98451E24660976 + +# https://lists.torproject.org/pipermail/tor-relays/2018-August/015869.html +5.45.111.149:80 orport=443 id=D405FCCF06ADEDF898DF2F29C9348DCB623031BA ipv6=[2a03:4000:6:2388:df98:15f9:b34d:443]:443 From 51f8db1d92d64331d000d2847edb8b66e9fb4c0c Mon Sep 17 00:00:00 2001 From: Colin Childs Date: Thu, 16 Aug 2018 14:32:23 -0500 Subject: [PATCH 0017/2557] Drop relays from fallback list that were terminated by provider --- scripts/maint/fallback.whitelist | 3 --- 1 file changed, 3 deletions(-) diff --git a/scripts/maint/fallback.whitelist b/scripts/maint/fallback.whitelist index 1b27cf7d29..cd38e7f1fa 100644 --- a/scripts/maint/fallback.whitelist +++ b/scripts/maint/fallback.whitelist @@ -832,9 +832,6 @@ # Email sent directly to Phoul 185.100.86.128:9030 orport=9001 id=9B31F1F1C1554F9FFB3455911F82E818EF7C7883 185.100.85.101:9030 orport=9001 id=4061C553CA88021B8302F0814365070AAE617270 -31.171.155.108:9030 orport=9001 id=D3E5EDDBE5159388704D6785BE51930AAFACEC6F -31.171.155.131:80 orport=443 id=516D1B9E22484202322828D8CAC30325030017E2 - # Email sent directly to teor, verified using relay contact info 89.163.247.43:9030 orport=9001 id=BC7ACFAC04854C77167C7D66B7E471314ED8C410 ipv6=[2001:4ba0:fff7:25::5]:9001 From f0792537e9cf25e76b5cab16635de7b78bfbf8b7 Mon Sep 17 00:00:00 2001 From: Colin Childs Date: Tue, 28 Aug 2018 13:05:59 -0500 Subject: [PATCH 0018/2557] Adding new fallback --- scripts/maint/fallback.whitelist | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/maint/fallback.whitelist b/scripts/maint/fallback.whitelist index cd38e7f1fa..375ac8f748 100644 --- a/scripts/maint/fallback.whitelist +++ b/scripts/maint/fallback.whitelist @@ -1060,3 +1060,7 @@ # https://lists.torproject.org/pipermail/tor-relays/2018-August/015869.html 5.45.111.149:80 orport=443 id=D405FCCF06ADEDF898DF2F29C9348DCB623031BA ipv6=[2a03:4000:6:2388:df98:15f9:b34d:443]:443 + +# https://trac.torproject.org/projects/tor/ticket/27297 +37.252.185.182:9030 orport=8080 id=113143469021882C3A4B82F084F8125B08EE471E ipv6=[2a00:63c1:a:182::2]:8080 + From 23ed863da4f66d9e07f578118f328b9f1898fe52 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Fri, 31 Aug 2018 18:55:36 +0300 Subject: [PATCH 0019/2557] Improve bracket handling in tor_addr_parse() * Actually check for second bracket * Only attempt parsing IPv4 address when no brackets found --- src/lib/net/address.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/lib/net/address.c b/src/lib/net/address.c index 619fa13e9b..14c086a5b0 100644 --- a/src/lib/net/address.c +++ b/src/lib/net/address.c @@ -1187,14 +1187,19 @@ tor_addr_parse(tor_addr_t *addr, const char *src) int result; struct in_addr in_tmp; struct in6_addr in6_tmp; + int brackets_detected = 0; + tor_assert(addr && src); - if (src[0] == '[' && src[1]) + if (src[0] == '[' && src[1] && src[strlen(src)-1] == ']') { + brackets_detected = 1; src = tmp = tor_strndup(src+1, strlen(src)-2); + } if (tor_inet_pton(AF_INET6, src, &in6_tmp) > 0) { result = AF_INET6; tor_addr_from_in6(addr, &in6_tmp); - } else if (tor_inet_pton(AF_INET, src, &in_tmp) > 0) { + } else if (!brackets_detected && + tor_inet_pton(AF_INET, src, &in_tmp) > 0) { result = AF_INET; tor_addr_from_in(addr, &in_tmp); } else { From feae813e1bf32e7fc35a14d88ece4a9457cfd172 Mon Sep 17 00:00:00 2001 From: David Fifield Date: Wed, 2 Aug 2017 02:04:35 -0700 Subject: [PATCH 0020/2557] Add tests for tor_addr_parse, separate from tor_addr_port_parse. --- src/test/test_addr.c | 56 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 55 insertions(+), 1 deletion(-) diff --git a/src/test/test_addr.c b/src/test/test_addr.c index a9004048a5..1d97db52a6 100644 --- a/src/test/test_addr.c +++ b/src/test/test_addr.c @@ -723,7 +723,7 @@ test_addr_ip6_helpers(void *arg) ; } -/** Test tor_addr_port_parse(). */ +/** Test tor_addr_parse() and tor_addr_port_parse(). */ static void test_addr_parse(void *arg) { @@ -734,6 +734,60 @@ test_addr_parse(void *arg) /* Correct call. */ (void)arg; + r= tor_addr_parse(&addr, "192.0.2.1"); + tt_int_op(r,OP_EQ, AF_INET); + tor_addr_to_str(buf, &addr, sizeof(buf), 0); + tt_str_op(buf,OP_EQ, "192.0.2.1"); + + r= tor_addr_parse(&addr, "11:22::33:44"); + tt_int_op(r,OP_EQ, AF_INET6); + tor_addr_to_str(buf, &addr, sizeof(buf), 0); + tt_str_op(buf,OP_EQ, "11:22::33:44"); + + r= tor_addr_parse(&addr, "[11:22::33:44]"); + tt_int_op(r,OP_EQ, AF_INET6); + tor_addr_to_str(buf, &addr, sizeof(buf), 0); + tt_str_op(buf,OP_EQ, "11:22::33:44"); + + r= tor_addr_parse(&addr, "11:22:33:44:55:66:1.2.3.4"); + tt_int_op(r,OP_EQ, AF_INET6); + tor_addr_to_str(buf, &addr, sizeof(buf), 0); + tt_str_op(buf,OP_EQ, "11:22:33:44:55:66:102:304"); + + r= tor_addr_parse(&addr, "11:22::33:44:1.2.3.4"); + tt_int_op(r,OP_EQ, AF_INET6); + tor_addr_to_str(buf, &addr, sizeof(buf), 0); + tt_str_op(buf,OP_EQ, "11:22::33:44:102:304"); + + /* Empty string. */ + r= tor_addr_parse(&addr, ""); + tt_int_op(r,OP_EQ, -1); + + /* Square brackets around IPv4 address. */ + r= tor_addr_parse(&addr, "[192.0.2.1]"); + tt_int_op(r,OP_EQ, -1); + + /* Only left square bracket. */ + r= tor_addr_parse(&addr, "[11:22::33:44"); + tt_int_op(r,OP_EQ, -1); + + /* Only right square bracket. */ + r= tor_addr_parse(&addr, "11:22::33:44]"); + tt_int_op(r,OP_EQ, -1); + + /* Leading colon. */ + r= tor_addr_parse(&addr, ":11:22::33:44"); + tt_int_op(r,OP_EQ, -1); + + /* Trailing colon. */ + r= tor_addr_parse(&addr, "11:22::33:44:"); + tt_int_op(r,OP_EQ, -1); + + /* Too many hex words in IPv4-mapped IPv6 address. */ + r= tor_addr_parse(&addr, "11:22:33:44:55:66:77:88:1.2.3.4"); + tt_int_op(r,OP_EQ, -1); + + /* Correct call. */ r= tor_addr_port_parse(LOG_DEBUG, "192.0.2.1:1234", &addr, &port, -1); From 01eb164574bdcc938509810634dfb0348c484662 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Fri, 31 Aug 2018 19:34:14 +0300 Subject: [PATCH 0021/2557] Reject addresses with needless trailing colon --- src/lib/net/address.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/lib/net/address.c b/src/lib/net/address.c index 14c086a5b0..336693b464 100644 --- a/src/lib/net/address.c +++ b/src/lib/net/address.c @@ -1189,13 +1189,19 @@ tor_addr_parse(tor_addr_t *addr, const char *src) struct in6_addr in6_tmp; int brackets_detected = 0; + size_t len = strlen(src); + tor_assert(addr && src); - if (src[0] == '[' && src[1] && src[strlen(src)-1] == ']') { + if (src[0] == '[' && src[1] && src[len - 1] == ']') { brackets_detected = 1; src = tmp = tor_strndup(src+1, strlen(src)-2); + len -= 2; } - if (tor_inet_pton(AF_INET6, src, &in6_tmp) > 0) { + /* Reject if src has needless trailing ':'. */ + if (len > 2 && src[len - 1] == ':' && src[len - 2] != ':') { + result = -1; + } else if (tor_inet_pton(AF_INET6, src, &in6_tmp) > 0) { result = AF_INET6; tor_addr_from_in6(addr, &in6_tmp); } else if (!brackets_detected && From 1baa2703c36cf964d5fc1f07433196e8ccc0e55a Mon Sep 17 00:00:00 2001 From: rl1987 Date: Fri, 31 Aug 2018 20:05:54 +0300 Subject: [PATCH 0022/2557] Add changes file --- changes/bug23082 | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 changes/bug23082 diff --git a/changes/bug23082 b/changes/bug23082 new file mode 100644 index 0000000000..fc4b52c364 --- /dev/null +++ b/changes/bug23082 @@ -0,0 +1,4 @@ + o Minor bugfixes (networking): + - Introduce additional checks into tor_addr_parse() to + reject certain incorrect inputs that previously were + not detected. Fixes bug 23082; bugfix on 0.2.0.10-alpha. From 5595b212270215eaa020603cabbe2c7b3b34d624 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 11 Sep 2018 09:38:20 -0400 Subject: [PATCH 0023/2557] Consdiff: use lengths on inputs so they don't need NUL at the end This is part of #27244, so that we can safely mmap consensus documents. --- src/feature/dircache/consdiffmgr.c | 9 ++- src/feature/dircache/directory.c | 10 +++- src/feature/dircommon/consdiff.c | 42 +++++++------ src/feature/dircommon/consdiff.h | 15 ++--- src/feature/nodelist/routerparse.c | 13 +++-- src/feature/nodelist/routerparse.h | 4 +- src/test/bench.c | 6 +- src/test/fuzz/fuzz_diff.c | 15 +++-- src/test/fuzz/fuzz_diff_apply.c | 13 ++--- src/test/test_consdiff.c | 94 ++++++++++++++++++++---------- src/test/test_consdiffmgr.c | 25 ++++++-- 11 files changed, 156 insertions(+), 90 deletions(-) diff --git a/src/feature/dircache/consdiffmgr.c b/src/feature/dircache/consdiffmgr.c index 304b64f3b6..7999df08d5 100644 --- a/src/feature/dircache/consdiffmgr.c +++ b/src/feature/dircache/consdiffmgr.c @@ -1496,7 +1496,10 @@ consensus_diff_worker_threadfn(void *state_, void *work_) // XXXX ugh; this is going to calculate the SHA3 of both its // XXXX inputs again, even though we already have that. Maybe it's time // XXXX to change the API here? - consensus_diff = consensus_diff_generate(diff_from_nt, diff_to_nt); + consensus_diff = consensus_diff_generate(diff_from_nt, + strlen(diff_from_nt), + diff_to_nt, + strlen(diff_to_nt)); tor_free(diff_from_nt); tor_free(diff_to_nt); } @@ -1746,8 +1749,8 @@ consensus_compress_worker_threadfn(void *state_, void *work_) (const uint8_t *)consensus, bodylen); { const char *start, *end; - if (router_get_networkstatus_v3_signed_boundaries(consensus, - &start, &end) < 0) { + if (router_get_networkstatus_v3_signed_boundaries(consensus, bodylen, + &start, &end) < 0) { start = consensus; end = consensus+bodylen; } diff --git a/src/feature/dircache/directory.c b/src/feature/dircache/directory.c index de0bcdbfa7..8e5fc86836 100644 --- a/src/feature/dircache/directory.c +++ b/src/feature/dircache/directory.c @@ -2607,12 +2607,17 @@ handle_response_fetch_consensus(dir_connection_t *conn, /* First find our previous consensus. Maybe it's in ram, maybe not. */ cached_dir_t *cd = dirserv_get_consensus(flavname); const char *consensus_body; + size_t consensus_body_len; char *owned_consensus = NULL; if (cd) { consensus_body = cd->dir; + consensus_body_len = cd->dir_len; } else { owned_consensus = networkstatus_read_cached_consensus(flavname); - consensus_body = owned_consensus; + if (owned_consensus) { + consensus_body = owned_consensus; + consensus_body_len = strlen(consensus_body); + } } if (!consensus_body) { log_warn(LD_DIR, "Received a consensus diff, but we can't find " @@ -2622,7 +2627,8 @@ handle_response_fetch_consensus(dir_connection_t *conn, return -1; } - new_consensus = consensus_diff_apply(consensus_body, body); + new_consensus = consensus_diff_apply(consensus_body, consensus_body_len, + body, body_len); tor_free(owned_consensus); if (new_consensus == NULL) { log_warn(LD_DIR, "Could not apply consensus diff received from server " diff --git a/src/feature/dircommon/consdiff.c b/src/feature/dircommon/consdiff.c index 1823dc07fb..c67a118125 100644 --- a/src/feature/dircommon/consdiff.c +++ b/src/feature/dircommon/consdiff.c @@ -101,11 +101,11 @@ smartlist_add_linecpy(smartlist_t *lst, memarea_t *area, const char *s) /* This is a separate, mockable function so that we can override it when * fuzzing. */ MOCK_IMPL(STATIC int, -consensus_compute_digest,(const char *cons, +consensus_compute_digest,(const char *cons, size_t len, consensus_digest_t *digest_out)) { int r = crypto_digest256((char*)digest_out->sha3_256, - cons, strlen(cons), DIGEST_SHA3_256); + cons, len, DIGEST_SHA3_256); return r; } @@ -114,11 +114,11 @@ consensus_compute_digest,(const char *cons, /* This is a separate, mockable function so that we can override it when * fuzzing. */ MOCK_IMPL(STATIC int, -consensus_compute_digest_as_signed,(const char *cons, +consensus_compute_digest_as_signed,(const char *cons, size_t len, consensus_digest_t *digest_out)) { return router_get_networkstatus_v3_sha3_as_signed(digest_out->sha3_256, - cons); + cons, len); } /** Return true iff d1 and d2 contain the same digest */ @@ -1229,7 +1229,8 @@ consdiff_apply_diff(const smartlist_t *cons1, cons2_str = consensus_join_lines(cons2); consensus_digest_t cons2_digests; - if (consensus_compute_digest(cons2_str, &cons2_digests) < 0) { + if (consensus_compute_digest(cons2_str, strlen(cons2_str), + &cons2_digests) < 0) { /* LCOV_EXCL_START -- digest can't fail */ log_warn(LD_CONSDIFF, "Could not compute digests of the consensus " "resulting from applying a consensus diff."); @@ -1283,12 +1284,13 @@ consdiff_apply_diff(const smartlist_t *cons1, * generated cdlines will become invalid. */ STATIC int -consensus_split_lines(smartlist_t *out, const char *s, memarea_t *area) +consensus_split_lines(smartlist_t *out, + const char *s, size_t len, + memarea_t *area) { - const char *end_of_str = s + strlen(s); - tor_assert(*end_of_str == '\0'); + const char *end_of_str = s + len; - while (*s) { + while (s < end_of_str) { const char *eol = memchr(s, '\n', end_of_str - s); if (!eol) { /* File doesn't end with newline. */ @@ -1334,25 +1336,25 @@ consensus_join_lines(const smartlist_t *inp) * success, retun a newly allocated string containing that diff. On failure, * return NULL. */ char * -consensus_diff_generate(const char *cons1, - const char *cons2) +consensus_diff_generate(const char *cons1, size_t cons1len, + const char *cons2, size_t cons2len) { consensus_digest_t d1, d2; smartlist_t *lines1 = NULL, *lines2 = NULL, *result_lines = NULL; int r1, r2; char *result = NULL; - r1 = consensus_compute_digest_as_signed(cons1, &d1); - r2 = consensus_compute_digest(cons2, &d2); + r1 = consensus_compute_digest_as_signed(cons1, cons1len, &d1); + r2 = consensus_compute_digest(cons2, cons2len, &d2); if (BUG(r1 < 0 || r2 < 0)) return NULL; // LCOV_EXCL_LINE memarea_t *area = memarea_new(); lines1 = smartlist_new(); lines2 = smartlist_new(); - if (consensus_split_lines(lines1, cons1, area) < 0) + if (consensus_split_lines(lines1, cons1, cons1len, area) < 0) goto done; - if (consensus_split_lines(lines2, cons2, area) < 0) + if (consensus_split_lines(lines2, cons2, cons2len, area) < 0) goto done; result_lines = consdiff_gen_diff(lines1, lines2, &d1, &d2, area); @@ -1375,7 +1377,9 @@ consensus_diff_generate(const char *cons1, * consensus. On failure, return NULL. */ char * consensus_diff_apply(const char *consensus, - const char *diff) + size_t consensus_len, + const char *diff, + size_t diff_len) { consensus_digest_t d1; smartlist_t *lines1 = NULL, *lines2 = NULL; @@ -1383,15 +1387,15 @@ consensus_diff_apply(const char *consensus, char *result = NULL; memarea_t *area = memarea_new(); - r1 = consensus_compute_digest_as_signed(consensus, &d1); + r1 = consensus_compute_digest_as_signed(consensus, consensus_len, &d1); if (BUG(r1 < 0)) return NULL; // LCOV_EXCL_LINE lines1 = smartlist_new(); lines2 = smartlist_new(); - if (consensus_split_lines(lines1, consensus, area) < 0) + if (consensus_split_lines(lines1, consensus, consensus_len, area) < 0) goto done; - if (consensus_split_lines(lines2, diff, area) < 0) + if (consensus_split_lines(lines2, diff, diff_len, area) < 0) goto done; result = consdiff_apply_diff(lines1, lines2, &d1); diff --git a/src/feature/dircommon/consdiff.h b/src/feature/dircommon/consdiff.h index a5e4ba5cbf..eb7c9f9fe0 100644 --- a/src/feature/dircommon/consdiff.h +++ b/src/feature/dircommon/consdiff.h @@ -7,10 +7,10 @@ #include "core/or/or.h" -char *consensus_diff_generate(const char *cons1, - const char *cons2); -char *consensus_diff_apply(const char *consensus, - const char *diff); +char *consensus_diff_generate(const char *cons1, size_t cons1len, + const char *cons2, size_t cons2len); +char *consensus_diff_apply(const char *consensus, size_t consensus_len, + const char *diff, size_t diff_len); int looks_like_a_consensus_diff(const char *document, size_t len); @@ -78,7 +78,8 @@ STATIC int smartlist_slice_string_pos(const smartlist_slice_t *slice, STATIC void set_changed(bitarray_t *changed1, bitarray_t *changed2, const smartlist_slice_t *slice1, const smartlist_slice_t *slice2); -STATIC int consensus_split_lines(smartlist_t *out, const char *s, +STATIC int consensus_split_lines(smartlist_t *out, + const char *s, size_t len, struct memarea_t *area); STATIC void smartlist_add_linecpy(smartlist_t *lst, struct memarea_t *area, const char *s); @@ -86,10 +87,10 @@ STATIC int lines_eq(const cdline_t *a, const cdline_t *b); STATIC int line_str_eq(const cdline_t *a, const char *b); MOCK_DECL(STATIC int, - consensus_compute_digest,(const char *cons, + consensus_compute_digest,(const char *cons, size_t len, consensus_digest_t *digest_out)); MOCK_DECL(STATIC int, - consensus_compute_digest_as_signed,(const char *cons, + consensus_compute_digest_as_signed,(const char *cons, size_t len, consensus_digest_t *digest_out)); MOCK_DECL(STATIC int, consensus_digest_eq,(const uint8_t *d1, diff --git a/src/feature/nodelist/routerparse.c b/src/feature/nodelist/routerparse.c index 73d320de40..9c51799d9b 100644 --- a/src/feature/nodelist/routerparse.c +++ b/src/feature/nodelist/routerparse.c @@ -1024,10 +1024,11 @@ router_get_router_hash(const char *s, size_t s_len, char *digest) * -1. */ int router_get_networkstatus_v3_signed_boundaries(const char *s, + size_t len, const char **start_out, const char **end_out) { - return router_get_hash_impl_helper(s, strlen(s), + return router_get_hash_impl_helper(s, len, "network-status-version", "\ndirectory-signature", ' ', LOG_INFO, @@ -1039,12 +1040,13 @@ router_get_networkstatus_v3_signed_boundaries(const char *s, * signed portion can be identified. Return 0 on success, -1 on failure. */ int router_get_networkstatus_v3_sha3_as_signed(uint8_t *digest_out, - const char *s) + const char *s, size_t len) { const char *start, *end; - if (router_get_networkstatus_v3_signed_boundaries(s, &start, &end) < 0) { + if (router_get_networkstatus_v3_signed_boundaries(s, len, + &start, &end) < 0) { start = s; - end = s + strlen(s); + end = s + len; } tor_assert(start); tor_assert(end); @@ -3415,7 +3417,8 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, *eos_out = NULL; if (router_get_networkstatus_v3_hashes(s, &ns_digests) || - router_get_networkstatus_v3_sha3_as_signed(sha3_as_signed, s)<0) { + router_get_networkstatus_v3_sha3_as_signed(sha3_as_signed, + s, strlen(s))<0) { log_warn(LD_DIR, "Unable to compute digest of network-status"); goto err; } diff --git a/src/feature/nodelist/routerparse.h b/src/feature/nodelist/routerparse.h index 87c2a75aa5..be455984d1 100644 --- a/src/feature/nodelist/routerparse.h +++ b/src/feature/nodelist/routerparse.h @@ -32,11 +32,11 @@ int router_get_router_hash(const char *s, size_t s_len, char *digest); int router_get_dir_hash(const char *s, char *digest); int router_get_networkstatus_v3_hashes(const char *s, common_digests_t *digests); -int router_get_networkstatus_v3_signed_boundaries(const char *s, +int router_get_networkstatus_v3_signed_boundaries(const char *s, size_t len, const char **start_out, const char **end_out); int router_get_networkstatus_v3_sha3_as_signed(uint8_t *digest_out, - const char *s); + const char *s, size_t len); int router_get_extrainfo_hash(const char *s, size_t s_len, char *digest); #define DIROBJ_MAX_SIG_LEN 256 char *router_get_dirobj_signature(const char *digest, diff --git a/src/test/bench.c b/src/test/bench.c index 959d4374b1..9da1b46a1b 100644 --- a/src/test/bench.c +++ b/src/test/bench.c @@ -702,11 +702,13 @@ main(int argc, const char **argv) perror("X"); return 1; } + size_t f1len = strlen(f1); + size_t f2len = strlen(f2); for (i = 0; i < N; ++i) { - char *diff = consensus_diff_generate(f1, f2); + char *diff = consensus_diff_generate(f1, f1len, f2, f2len); tor_free(diff); } - char *diff = consensus_diff_generate(f1, f2); + char *diff = consensus_diff_generate(f1, f1len, f2, f2len); printf("%s", diff); tor_free(f1); tor_free(f2); diff --git a/src/test/fuzz/fuzz_diff.c b/src/test/fuzz/fuzz_diff.c index 1079856fdb..8966856be2 100644 --- a/src/test/fuzz/fuzz_diff.c +++ b/src/test/fuzz/fuzz_diff.c @@ -10,9 +10,11 @@ #include "test/fuzz/fuzzing.h" static int -mock_consensus_compute_digest_(const char *c, consensus_digest_t *d) +mock_consensus_compute_digest_(const char *c, size_t len, + consensus_digest_t *d) { (void)c; + (void)len; memset(d->sha3_256, 3, sizeof(d->sha3_256)); return 0; } @@ -42,14 +44,14 @@ fuzz_main(const uint8_t *stdin_buf, size_t data_size) if (! separator) return 0; size_t c1_len = separator - stdin_buf; - char *c1 = tor_memdup_nulterm(stdin_buf, c1_len); + const char *c1 = (const char *)stdin_buf; size_t c2_len = data_size - c1_len - SEPLEN; - char *c2 = tor_memdup_nulterm(separator + SEPLEN, c2_len); + const char *c2 = (const char *)separator + SEPLEN; - char *c3 = consensus_diff_generate(c1, c2); + char *c3 = consensus_diff_generate(c1, c1_len, c2, c2_len); if (c3) { - char *c4 = consensus_diff_apply(c1, c3); + char *c4 = consensus_diff_apply(c1, c1_len, c3, strlen(c3)); tor_assert(c4); if (strcmp(c2, c4)) { printf("%s\n", escaped(c1)); @@ -61,9 +63,6 @@ fuzz_main(const uint8_t *stdin_buf, size_t data_size) tor_free(c3); tor_free(c4); } - tor_free(c1); - tor_free(c2); return 0; } - diff --git a/src/test/fuzz/fuzz_diff_apply.c b/src/test/fuzz/fuzz_diff_apply.c index 165d0e6126..9b25185225 100644 --- a/src/test/fuzz/fuzz_diff_apply.c +++ b/src/test/fuzz/fuzz_diff_apply.c @@ -10,9 +10,11 @@ #include "test/fuzz/fuzzing.h" static int -mock_consensus_compute_digest_(const char *c, consensus_digest_t *d) +mock_consensus_compute_digest_(const char *c, size_t len, + consensus_digest_t *d) { (void)c; + (void)len; memset(d->sha3_256, 3, sizeof(d->sha3_256)); return 0; } @@ -50,16 +52,13 @@ fuzz_main(const uint8_t *stdin_buf, size_t data_size) if (! separator) return 0; size_t c1_len = separator - stdin_buf; - char *c1 = tor_memdup_nulterm(stdin_buf, c1_len); + const char *c1 = (const char *)stdin_buf; size_t c2_len = data_size - c1_len - SEPLEN; - char *c2 = tor_memdup_nulterm(separator + SEPLEN, c2_len); + const char *c2 = (const char *)separator + SEPLEN; - char *c3 = consensus_diff_apply(c1, c2); + char *c3 = consensus_diff_apply(c1, c1_len, c2, c2_len); - tor_free(c1); - tor_free(c2); tor_free(c3); return 0; } - diff --git a/src/test/test_consdiff.c b/src/test/test_consdiff.c index b836befd22..23e8f7167e 100644 --- a/src/test/test_consdiff.c +++ b/src/test/test_consdiff.c @@ -14,6 +14,39 @@ #define tt_str_eq_line(a,b) \ tt_assert(line_str_eq((b),(a))) +static int +consensus_split_lines_(smartlist_t *out, const char *s, memarea_t *area) +{ + size_t len = strlen(s); + return consensus_split_lines(out, s, len, area); +} + +static int +consensus_compute_digest_(const char *cons, + consensus_digest_t *digest_out) +{ + size_t len = strlen(cons); + char *tmp = tor_memdup(cons, len); + // We use memdup here to ensure that the input is NOT nul-terminated. + // This makes it likelier for us to spot bugs. + int r = consensus_compute_digest(tmp, len, digest_out); + tor_free(tmp); + return r; +} + +static int +consensus_compute_digest_as_signed_(const char *cons, + consensus_digest_t *digest_out) +{ + size_t len = strlen(cons); + char *tmp = tor_memdup(cons, len); + // We use memdup here to ensure that the input is NOT nul-terminated. + // This makes it likelier for us to spot bugs. + int r = consensus_compute_digest_as_signed(tmp, len, digest_out); + tor_free(tmp); + return r; +} + static void test_consdiff_smartlist_slice(void *arg) { @@ -58,7 +91,7 @@ test_consdiff_smartlist_slice_string_pos(void *arg) /* Create a regular smartlist. */ (void)arg; - consensus_split_lines(sl, "a\nd\nc\na\nb\n", area); + consensus_split_lines_(sl, "a\nd\nc\na\nb\n", area); /* See that smartlist_slice_string_pos respects the bounds of the slice. */ sls = smartlist_slice(sl, 2, 5); @@ -87,8 +120,8 @@ test_consdiff_lcs_lengths(void *arg) int e_lengths2[] = { 0, 1, 1, 2, 3, 4 }; (void)arg; - consensus_split_lines(sl1, "a\nb\nc\nd\ne\n", area); - consensus_split_lines(sl2, "a\nc\nd\ni\ne\n", area); + consensus_split_lines_(sl1, "a\nb\nc\nd\ne\n", area); + consensus_split_lines_(sl2, "a\nc\nd\ni\ne\n", area); sls1 = smartlist_slice(sl1, 0, -1); sls2 = smartlist_slice(sl2, 0, -1); @@ -119,10 +152,10 @@ test_consdiff_trim_slices(void *arg) memarea_t *area = memarea_new(); (void)arg; - consensus_split_lines(sl1, "a\nb\nb\nb\nd\n", area); - consensus_split_lines(sl2, "a\nc\nc\nc\nd\n", area); - consensus_split_lines(sl3, "a\nb\nb\nb\na\n", area); - consensus_split_lines(sl4, "c\nb\nb\nb\nc\n", area); + consensus_split_lines_(sl1, "a\nb\nb\nb\nd\n", area); + consensus_split_lines_(sl2, "a\nc\nc\nc\nd\n", area); + consensus_split_lines_(sl3, "a\nb\nb\nb\na\n", area); + consensus_split_lines_(sl4, "c\nb\nb\nb\nc\n", area); sls1 = smartlist_slice(sl1, 0, -1); sls2 = smartlist_slice(sl2, 0, -1); sls3 = smartlist_slice(sl3, 0, -1); @@ -165,8 +198,8 @@ test_consdiff_set_changed(void *arg) memarea_t *area = memarea_new(); (void)arg; - consensus_split_lines(sl1, "a\nb\na\na\n", area); - consensus_split_lines(sl2, "a\na\na\na\n", area); + consensus_split_lines_(sl1, "a\nb\na\na\n", area); + consensus_split_lines_(sl2, "a\na\na\na\n", area); /* Length of sls1 is 0. */ sls1 = smartlist_slice(sl1, 0, 0); @@ -240,8 +273,8 @@ test_consdiff_calc_changes(void *arg) memarea_t *area = memarea_new(); (void)arg; - consensus_split_lines(sl1, "a\na\na\na\n", area); - consensus_split_lines(sl2, "a\na\na\na\n", area); + consensus_split_lines_(sl1, "a\na\na\na\n", area); + consensus_split_lines_(sl2, "a\na\na\na\n", area); sls1 = smartlist_slice(sl1, 0, -1); sls2 = smartlist_slice(sl2, 0, -1); @@ -259,7 +292,7 @@ test_consdiff_calc_changes(void *arg) tt_assert(!bitarray_is_set(changed2, 3)); smartlist_clear(sl2); - consensus_split_lines(sl2, "a\nb\na\nb\n", area); + consensus_split_lines_(sl2, "a\nb\na\nb\n", area); tor_free(sls1); tor_free(sls2); sls1 = smartlist_slice(sl1, 0, -1); @@ -282,7 +315,7 @@ test_consdiff_calc_changes(void *arg) bitarray_clear(changed1, 3); smartlist_clear(sl2); - consensus_split_lines(sl2, "b\nb\nb\nb\n", area); + consensus_split_lines_(sl2, "b\nb\nb\nb\n", area); tor_free(sls1); tor_free(sls2); sls1 = smartlist_slice(sl1, 0, -1); @@ -610,8 +643,8 @@ test_consdiff_gen_ed_diff(void *arg) /* Test 'a', 'c' and 'd' together. See that it is done in reverse order. */ smartlist_clear(cons1); smartlist_clear(cons2); - consensus_split_lines(cons1, "A\nB\nC\nD\nE\n", area); - consensus_split_lines(cons2, "A\nC\nO\nE\nU\n", area); + consensus_split_lines_(cons1, "A\nB\nC\nD\nE\n", area); + consensus_split_lines_(cons2, "A\nC\nO\nE\nU\n", area); diff = gen_ed_diff(cons1, cons2, area); tt_ptr_op(NULL, OP_NE, diff); tt_int_op(7, OP_EQ, smartlist_len(diff)); @@ -627,8 +660,8 @@ test_consdiff_gen_ed_diff(void *arg) smartlist_clear(cons1); smartlist_clear(cons2); - consensus_split_lines(cons1, "B\n", area); - consensus_split_lines(cons2, "A\nB\n", area); + consensus_split_lines_(cons1, "B\n", area); + consensus_split_lines_(cons2, "A\nB\n", area); diff = gen_ed_diff(cons1, cons2, area); tt_ptr_op(NULL, OP_NE, diff); tt_int_op(3, OP_EQ, smartlist_len(diff)); @@ -656,7 +689,7 @@ test_consdiff_apply_ed_diff(void *arg) diff = smartlist_new(); setup_capture_of_logs(LOG_WARN); - consensus_split_lines(cons1, "A\nB\nC\nD\nE\n", area); + consensus_split_lines_(cons1, "A\nB\nC\nD\nE\n", area); /* Command without range. */ smartlist_add_linecpy(diff, area, "a"); @@ -829,7 +862,7 @@ test_consdiff_apply_ed_diff(void *arg) smartlist_clear(diff); /* Test appending text, 'a'. */ - consensus_split_lines(diff, "3a\nU\nO\n.\n0a\nV\n.\n", area); + consensus_split_lines_(diff, "3a\nU\nO\n.\n0a\nV\n.\n", area); cons2 = apply_ed_diff(cons1, diff, 0); tt_ptr_op(NULL, OP_NE, cons2); tt_int_op(8, OP_EQ, smartlist_len(cons2)); @@ -846,7 +879,7 @@ test_consdiff_apply_ed_diff(void *arg) smartlist_free(cons2); /* Test deleting text, 'd'. */ - consensus_split_lines(diff, "4d\n1,2d\n", area); + consensus_split_lines_(diff, "4d\n1,2d\n", area); cons2 = apply_ed_diff(cons1, diff, 0); tt_ptr_op(NULL, OP_NE, cons2); tt_int_op(2, OP_EQ, smartlist_len(cons2)); @@ -857,7 +890,7 @@ test_consdiff_apply_ed_diff(void *arg) smartlist_free(cons2); /* Test changing text, 'c'. */ - consensus_split_lines(diff, "4c\nT\nX\n.\n1,2c\nM\n.\n", area); + consensus_split_lines_(diff, "4c\nT\nX\n.\n1,2c\nM\n.\n", area); cons2 = apply_ed_diff(cons1, diff, 0); tt_ptr_op(NULL, OP_NE, cons2); tt_int_op(5, OP_EQ, smartlist_len(cons2)); @@ -871,7 +904,7 @@ test_consdiff_apply_ed_diff(void *arg) smartlist_free(cons2); /* Test 'a', 'd' and 'c' together. */ - consensus_split_lines(diff, "4c\nT\nX\n.\n2d\n0a\nM\n.\n", area); + consensus_split_lines_(diff, "4c\nT\nX\n.\n2d\n0a\nM\n.\n", area); cons2 = apply_ed_diff(cons1, diff, 0); tt_ptr_op(NULL, OP_NE, cons2); tt_int_op(6, OP_EQ, smartlist_len(cons2)); @@ -918,12 +951,12 @@ test_consdiff_gen_diff(void *arg) ); tt_int_op(0, OP_EQ, - consensus_compute_digest_as_signed(cons1_str, &digests1)); + consensus_compute_digest_as_signed_(cons1_str, &digests1)); tt_int_op(0, OP_EQ, - consensus_compute_digest(cons2_str, &digests2)); + consensus_compute_digest_(cons2_str, &digests2)); - consensus_split_lines(cons1, cons1_str, area); - consensus_split_lines(cons2, cons2_str, area); + consensus_split_lines_(cons1, cons1_str, area); + consensus_split_lines_(cons2, cons2_str, area); diff = consdiff_gen_diff(cons1, cons2, &digests1, &digests2, area); tt_ptr_op(NULL, OP_EQ, diff); @@ -937,9 +970,9 @@ test_consdiff_gen_diff(void *arg) "directory-signature foo bar\nbar\n" ); tt_int_op(0, OP_EQ, - consensus_compute_digest_as_signed(cons1_str, &digests1)); + consensus_compute_digest_as_signed_(cons1_str, &digests1)); smartlist_clear(cons1); - consensus_split_lines(cons1, cons1_str, area); + consensus_split_lines_(cons1, cons1_str, area); diff = consdiff_gen_diff(cons1, cons2, &digests1, &digests2, area); tt_ptr_op(NULL, OP_NE, diff); tt_int_op(11, OP_EQ, smartlist_len(diff)); @@ -991,8 +1024,8 @@ test_consdiff_apply_diff(void *arg) "directory-signature foo bar\nbar\n" ); tt_int_op(0, OP_EQ, - consensus_compute_digest(cons1_str, &digests1)); - consensus_split_lines(cons1, cons1_str, area); + consensus_compute_digest_(cons1_str, &digests1)); + consensus_split_lines_(cons1, cons1_str, area); /* diff doesn't have enough lines. */ cons2 = consdiff_apply_diff(cons1, diff, &digests1); @@ -1182,4 +1215,3 @@ struct testcase_t consdiff_tests[] = { CONSDIFF_LEGACY(apply_diff), END_OF_TESTCASES }; - diff --git a/src/test/test_consdiffmgr.c b/src/test/test_consdiffmgr.c index 6c0601b504..dc4fea7f6f 100644 --- a/src/test/test_consdiffmgr.c +++ b/src/test/test_consdiffmgr.c @@ -21,6 +21,21 @@ #include "test/test.h" #include "test/log_test_helpers.h" +static char * +consensus_diff_apply_(const char *c, const char *d) +{ + size_t c_len = strlen(c); + size_t d_len = strlen(d); + // We use memdup here to ensure that the input is NOT nul-terminated. + // This makes it likelier for us to spot bugs. + char *c_tmp = tor_memdup(c, c_len); + char *d_tmp = tor_memdup(d, d_len); + char *result = consensus_diff_apply(c_tmp, c_len, d_tmp, d_len); + tor_free(c_tmp); + tor_free(d_tmp); + return result; +} + // ============================== Setup/teardown the consdiffmgr // These functions get run before/after each test in this module @@ -153,7 +168,8 @@ lookup_diff_from(consensus_cache_entry_t **out, const char *str1) { uint8_t digest[DIGEST256_LEN]; - if (router_get_networkstatus_v3_sha3_as_signed(digest, str1)<0) { + if (router_get_networkstatus_v3_sha3_as_signed(digest, + str1, strlen(str1))<0) { TT_FAIL(("Unable to compute sha3-as-signed")); return CONSDIFF_NOT_FOUND; } @@ -181,7 +197,7 @@ lookup_apply_and_verify_diff(consensus_flavor_t flav, if (diff_string == NULL || r < 0) return -1; - char *applied = consensus_diff_apply(str1, diff_string); + char *applied = consensus_diff_apply_(str1, diff_string); tor_free(diff_string); if (applied == NULL) return -1; @@ -370,7 +386,8 @@ test_consdiffmgr_make_diffs(void *arg) ns = fake_ns_new(FLAV_MICRODESC, now-3600); md_ns_body = fake_ns_body_new(FLAV_MICRODESC, now-3600); r = consdiffmgr_add_consensus(md_ns_body, ns); - router_get_networkstatus_v3_sha3_as_signed(md_ns_sha3, md_ns_body); + router_get_networkstatus_v3_sha3_as_signed(md_ns_sha3, md_ns_body, + strlen(md_ns_body)); networkstatus_vote_free(ns); tt_int_op(r, OP_EQ, 0); @@ -414,7 +431,7 @@ test_consdiffmgr_make_diffs(void *arg) r = consensus_cache_entry_get_body(diff, &diff_body, &diff_size); tt_int_op(r, OP_EQ, 0); diff_text = tor_memdup_nulterm(diff_body, diff_size); - applied = consensus_diff_apply(md_ns_body, diff_text); + applied = consensus_diff_apply_(md_ns_body, diff_text); tt_assert(applied); tt_str_op(applied, OP_EQ, md_ns_body_2); From e014b72b73b2a299068f1ca5b7a22f2bea2f58f8 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 11 Sep 2018 10:09:12 -0400 Subject: [PATCH 0024/2557] Stop memcpy'ing uncompressed consensuses when making diffs --- src/feature/dircache/consdiffmgr.c | 50 ++++++++++++++++++------------ src/feature/dircache/consdiffmgr.h | 5 +-- src/test/fuzz/fuzz_diff.c | 17 +++++++--- src/test/test_consdiffmgr.c | 16 +++++----- 4 files changed, 56 insertions(+), 32 deletions(-) diff --git a/src/feature/dircache/consdiffmgr.c b/src/feature/dircache/consdiffmgr.c index 7999df08d5..bf3a0ef3cf 100644 --- a/src/feature/dircache/consdiffmgr.c +++ b/src/feature/dircache/consdiffmgr.c @@ -1387,19 +1387,21 @@ typedef struct consensus_diff_worker_job_t { } consensus_diff_worker_job_t; /** Given a consensus_cache_entry_t, check whether it has a label claiming - * that it was compressed. If so, uncompress its contents into out and - * set outlen to hold their size. If not, just copy the body into - * out and set outlen to its length. Return 0 on success, - * -1 on failure. - * - * In all cases, the output is nul-terminated. */ + * that it was compressed. If so, uncompress its contents into *out and + * set outlen to hold their size, and set *owned_out to a pointer + * that the caller will need to free. If not, just set *out and + * outlen to its extent in memory. Return 0 on success, -1 on failure. + **/ STATIC int -uncompress_or_copy(char **out, size_t *outlen, - consensus_cache_entry_t *ent) +uncompress_or_set_ptr(const char **out, size_t *outlen, + char **owned_out, + consensus_cache_entry_t *ent) { const uint8_t *body; size_t bodylen; + *owned_out = NULL; + if (consensus_cache_entry_get_body(ent, &body, &bodylen) < 0) return -1; @@ -1410,8 +1412,17 @@ uncompress_or_copy(char **out, size_t *outlen, if (lv_compression) method = compression_method_get_by_name(lv_compression); - return tor_uncompress(out, outlen, (const char *)body, bodylen, + int rv; + if (method == NO_METHOD) { + *out = (const char *)body; + *outlen = bodylen; + rv = 0; + } else { + rv = tor_uncompress(owned_out, outlen, (const char *)body, bodylen, method, 1, LOG_WARN); + *out = *owned_out; + } + return rv; } /** @@ -1478,16 +1489,17 @@ consensus_diff_worker_threadfn(void *state_, void *work_) char *consensus_diff; { - char *diff_from_nt = NULL, *diff_to_nt = NULL; + const char *diff_from_nt = NULL, *diff_to_nt = NULL; + char *owned1 = NULL, *owned2 = NULL; size_t diff_from_nt_len, diff_to_nt_len; - if (uncompress_or_copy(&diff_from_nt, &diff_from_nt_len, - job->diff_from) < 0) { + if (uncompress_or_set_ptr(&diff_from_nt, &diff_from_nt_len, &owned1, + job->diff_from) < 0) { return WQ_RPL_REPLY; } - if (uncompress_or_copy(&diff_to_nt, &diff_to_nt_len, - job->diff_to) < 0) { - tor_free(diff_from_nt); + if (uncompress_or_set_ptr(&diff_to_nt, &diff_to_nt_len, &owned2, + job->diff_to) < 0) { + tor_free(owned1); return WQ_RPL_REPLY; } tor_assert(diff_from_nt); @@ -1497,11 +1509,11 @@ consensus_diff_worker_threadfn(void *state_, void *work_) // XXXX inputs again, even though we already have that. Maybe it's time // XXXX to change the API here? consensus_diff = consensus_diff_generate(diff_from_nt, - strlen(diff_from_nt), + diff_from_nt_len, diff_to_nt, - strlen(diff_to_nt)); - tor_free(diff_from_nt); - tor_free(diff_to_nt); + diff_to_nt_len); + tor_free(owned1); + tor_free(owned2); } if (!consensus_diff) { /* Couldn't generate consensus; we'll leave the reply blank. */ diff --git a/src/feature/dircache/consdiffmgr.h b/src/feature/dircache/consdiffmgr.h index 66c3d65002..d6f273cc4e 100644 --- a/src/feature/dircache/consdiffmgr.h +++ b/src/feature/dircache/consdiffmgr.h @@ -68,8 +68,9 @@ STATIC consensus_cache_entry_t *cdm_cache_lookup_consensus( STATIC int cdm_entry_get_sha3_value(uint8_t *digest_out, consensus_cache_entry_t *ent, const char *label); -STATIC int uncompress_or_copy(char **out, size_t *outlen, - consensus_cache_entry_t *ent); +STATIC int uncompress_or_set_ptr(const char **out, size_t *outlen, + char **owned_out, + consensus_cache_entry_t *ent); #endif /* defined(CONSDIFFMGR_PRIVATE) */ #endif /* !defined(TOR_CONSDIFFMGR_H) */ diff --git a/src/test/fuzz/fuzz_diff.c b/src/test/fuzz/fuzz_diff.c index 8966856be2..64aecc8a64 100644 --- a/src/test/fuzz/fuzz_diff.c +++ b/src/test/fuzz/fuzz_diff.c @@ -48,18 +48,27 @@ fuzz_main(const uint8_t *stdin_buf, size_t data_size) size_t c2_len = data_size - c1_len - SEPLEN; const char *c2 = (const char *)separator + SEPLEN; + const char *cp = memchr(c1, 0, c1_len); + if (cp) + c1_len = cp - c1; + + cp = memchr(c2, 0, c2_len); + if (cp) + c2_len = cp - c2; + char *c3 = consensus_diff_generate(c1, c1_len, c2, c2_len); if (c3) { char *c4 = consensus_diff_apply(c1, c1_len, c3, strlen(c3)); tor_assert(c4); - if (strcmp(c2, c4)) { - printf("%s\n", escaped(c1)); - printf("%s\n", escaped(c2)); + int equal = (c2_len == strlen(c4)) && fast_memeq(c2, c4, c2_len); + if (! equal) { + //printf("%s\n", escaped(c1)); + //printf("%s\n", escaped(c2)); printf("%s\n", escaped(c3)); printf("%s\n", escaped(c4)); } - tor_assert(! strcmp(c2, c4)); + tor_assert(equal); tor_free(c3); tor_free(c4); } diff --git a/src/test/test_consdiffmgr.c b/src/test/test_consdiffmgr.c index dc4fea7f6f..4b49fdb6aa 100644 --- a/src/test/test_consdiffmgr.c +++ b/src/test/test_consdiffmgr.c @@ -191,14 +191,15 @@ lookup_apply_and_verify_diff(consensus_flavor_t flav, consensus_cache_entry_incref(ent); size_t size; - char *diff_string = NULL; - int r = uncompress_or_copy(&diff_string, &size, ent); + const char *diff_string = NULL; + char *diff_owned = NULL; + int r = uncompress_or_set_ptr(&diff_string, &size, &diff_owned, ent); consensus_cache_entry_decref(ent); if (diff_string == NULL || r < 0) return -1; - char *applied = consensus_diff_apply_(str1, diff_string); - tor_free(diff_string); + char *applied = consensus_diff_apply(str1, strlen(str1), diff_string, size); + tor_free(diff_owned); if (applied == NULL) return -1; @@ -298,7 +299,8 @@ test_consdiffmgr_add(void *arg) (void) arg; time_t now = approx_time(); - char *body = NULL; + const char *body = NULL; + char *body_owned = NULL; consensus_cache_entry_t *ent = NULL; networkstatus_t *ns_tmp = fake_ns_new(FLAV_NS, now); @@ -340,7 +342,7 @@ test_consdiffmgr_add(void *arg) tt_assert(ent); consensus_cache_entry_incref(ent); size_t s; - r = uncompress_or_copy(&body, &s, ent); + r = uncompress_or_set_ptr(&body, &s, &body_owned, ent); tt_int_op(r, OP_EQ, 0); tt_int_op(s, OP_EQ, 4); tt_mem_op(body, OP_EQ, "quux", 4); @@ -353,7 +355,7 @@ test_consdiffmgr_add(void *arg) networkstatus_vote_free(ns_tmp); teardown_capture_of_logs(); consensus_cache_entry_decref(ent); - tor_free(body); + tor_free(body_owned); } static void From abaca3fc8c6bc54408084e7514468fa2cd7b3edf Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 11 Sep 2018 10:32:17 -0400 Subject: [PATCH 0025/2557] Revise networkstatus parsing code to use lengths This way the networkstatus can be parsed without being NUL-terminated, so we can implement 27244 and mmap our consensus objects. --- src/feature/dirauth/dirvote.c | 15 ++++++---- src/feature/nodelist/networkstatus.c | 4 ++- src/feature/nodelist/routerparse.c | 44 ++++++++++++++++------------ src/feature/nodelist/routerparse.h | 6 ++-- src/test/fuzz/fuzz_consensus.c | 6 ++-- src/test/fuzz/fuzz_vrs.c | 16 +++++----- src/test/test_dir.c | 32 +++++++++++++++----- src/test/test_dir_common.c | 5 ++-- src/test/test_routerlist.c | 4 ++- 9 files changed, 85 insertions(+), 47 deletions(-) diff --git a/src/feature/dirauth/dirvote.c b/src/feature/dirauth/dirvote.c index 14357c770e..42821760c5 100644 --- a/src/feature/dirauth/dirvote.c +++ b/src/feature/dirauth/dirvote.c @@ -401,7 +401,8 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key, { networkstatus_t *v; - if (!(v = networkstatus_parse_vote_from_string(status, NULL, + if (!(v = networkstatus_parse_vote_from_string(status, strlen(status), + NULL, v3_ns->type))) { log_err(LD_BUG,"Generated a networkstatus %s we couldn't parse: " "<<%s>>", @@ -2398,7 +2399,8 @@ networkstatus_compute_consensus(smartlist_t *votes, { networkstatus_t *c; - if (!(c = networkstatus_parse_vote_from_string(result, NULL, + if (!(c = networkstatus_parse_vote_from_string(result, strlen(result), + NULL, NS_TYPE_CONSENSUS))) { log_err(LD_BUG, "Generated a networkstatus consensus we couldn't " "parse."); @@ -3121,7 +3123,8 @@ dirvote_add_vote(const char *vote_body, const char **msg_out, int *status_out) *msg_out = NULL; again: - vote = networkstatus_parse_vote_from_string(vote_body, &end_of_vote, + vote = networkstatus_parse_vote_from_string(vote_body, strlen(vote_body), + &end_of_vote, NS_TYPE_VOTE); if (!end_of_vote) end_of_vote = vote_body + strlen(vote_body); @@ -3379,7 +3382,9 @@ dirvote_compute_consensuses(void) flavor_name); continue; } - consensus = networkstatus_parse_vote_from_string(consensus_body, NULL, + consensus = networkstatus_parse_vote_from_string(consensus_body, + strlen(consensus_body), + NULL, NS_TYPE_CONSENSUS); if (!consensus) { log_warn(LD_DIR, "Couldn't parse %s consensus we generated!", @@ -3518,7 +3523,7 @@ dirvote_add_signatures_to_pending_consensus( * just in case we break detached signature processing at some point. */ { networkstatus_t *v = networkstatus_parse_vote_from_string( - pc->body, NULL, + pc->body, strlen(pc->body), NULL, NS_TYPE_CONSENSUS); tor_assert(v); networkstatus_vote_free(v); diff --git a/src/feature/nodelist/networkstatus.c b/src/feature/nodelist/networkstatus.c index 6492b828b1..4338fde599 100644 --- a/src/feature/nodelist/networkstatus.c +++ b/src/feature/nodelist/networkstatus.c @@ -1861,7 +1861,9 @@ networkstatus_set_current_consensus(const char *consensus, } /* Make sure it's parseable. */ - c = networkstatus_parse_vote_from_string(consensus, NULL, NS_TYPE_CONSENSUS); + c = networkstatus_parse_vote_from_string(consensus, + strlen(consensus), + NULL, NS_TYPE_CONSENSUS); if (!c) { log_warn(LD_DIR, "Unable to parse networkstatus consensus"); result = -2; diff --git a/src/feature/nodelist/routerparse.c b/src/feature/nodelist/routerparse.c index 9c51799d9b..9abdfb614c 100644 --- a/src/feature/nodelist/routerparse.c +++ b/src/feature/nodelist/routerparse.c @@ -1057,9 +1057,10 @@ router_get_networkstatus_v3_sha3_as_signed(uint8_t *digest_out, /** Set digests to all the digests of the consensus document in * s */ int -router_get_networkstatus_v3_hashes(const char *s, common_digests_t *digests) +router_get_networkstatus_v3_hashes(const char *s, size_t len, + common_digests_t *digests) { - return router_get_hashes_impl(s,strlen(s),digests, + return router_get_hashes_impl(s, len, digests, "network-status-version", "\ndirectory-signature", ' '); @@ -2489,18 +2490,19 @@ authority_cert_parse_from_string(const char *s, const char **end_of_string) return NULL; } -/** Helper: given a string s, return the start of the next router-status +/** Helper: given a string s ending at s_eos, return the + * start of the next router-status * object (starting with "r " at the start of a line). If none is found, * return the start of the directory footer, or the next directory signature. * If none is found, return the end of the string. */ static inline const char * -find_start_of_next_routerstatus(const char *s) +find_start_of_next_routerstatus(const char *s, const char *s_eos) { const char *eos, *footer, *sig; - if ((eos = strstr(s, "\nr "))) + if ((eos = tor_memstr(s, s_eos - s, "\nr "))) ++eos; else - eos = s + strlen(s); + eos = s_eos; footer = tor_memstr(s, eos-s, "\ndirectory-footer"); sig = tor_memstr(s, eos-s, "\ndirectory-signature"); @@ -2632,7 +2634,8 @@ summarize_protover_flags(protover_summary_flags_t *out, **/ STATIC routerstatus_t * routerstatus_parse_entry_from_string(memarea_t *area, - const char **s, smartlist_t *tokens, + const char **s, const char *s_eos, + smartlist_t *tokens, networkstatus_t *vote, vote_routerstatus_t *vote_rs, int consensus_method, @@ -2651,7 +2654,7 @@ routerstatus_parse_entry_from_string(memarea_t *area, flav = FLAV_NS; tor_assert(flav == FLAV_NS || flav == FLAV_MICRODESC); - eos = find_start_of_next_routerstatus(*s); + eos = find_start_of_next_routerstatus(*s, s_eos); if (tokenize_string(area,*s, eos, tokens, rtrstatus_token_table,0)) { log_warn(LD_DIR, "Error tokenizing router status"); @@ -3394,7 +3397,9 @@ extract_shared_random_srvs(networkstatus_t *ns, smartlist_t *tokens) /** Parse a v3 networkstatus vote, opinion, or consensus (depending on * ns_type), from s, and return the result. Return NULL on failure. */ networkstatus_t * -networkstatus_parse_vote_from_string(const char *s, const char **eos_out, +networkstatus_parse_vote_from_string(const char *s, + size_t s_len, + const char **eos_out, networkstatus_type_t ns_type) { smartlist_t *tokens = smartlist_new(); @@ -3410,21 +3415,22 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, memarea_t *area = NULL, *rs_area = NULL; consensus_flavor_t flav = FLAV_NS; char *last_kwd=NULL; + const char *eos = s + s_len; tor_assert(s); if (eos_out) *eos_out = NULL; - if (router_get_networkstatus_v3_hashes(s, &ns_digests) || + if (router_get_networkstatus_v3_hashes(s, s_len, &ns_digests) || router_get_networkstatus_v3_sha3_as_signed(sha3_as_signed, - s, strlen(s))<0) { + s, s_len)<0) { log_warn(LD_DIR, "Unable to compute digest of network-status"); goto err; } area = memarea_new(); - end_of_header = find_start_of_next_routerstatus(s); + end_of_header = find_start_of_next_routerstatus(s, eos); if (tokenize_string(area, s, end_of_header, tokens, (ns_type == NS_TYPE_CONSENSUS) ? networkstatus_consensus_token_table : @@ -3455,7 +3461,8 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, if (ns_type != NS_TYPE_CONSENSUS) { const char *end_of_cert = NULL; - if (!(cert = strstr(s, "\ndir-key-certificate-version"))) + if (!(cert = tor_memstr(s, end_of_header - s, + "\ndir-key-certificate-version"))) goto err; ++cert; ns->cert = authority_cert_parse_from_string(cert, &end_of_cert); @@ -3768,10 +3775,10 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, s = end_of_header; ns->routerstatus_list = smartlist_new(); - while (!strcmpstart(s, "r ")) { + while (eos-s >= 2 && fast_memeq(s, "r ", 2)) { if (ns->type != NS_TYPE_CONSENSUS) { vote_routerstatus_t *rs = tor_malloc_zero(sizeof(vote_routerstatus_t)); - if (routerstatus_parse_entry_from_string(rs_area, &s, rs_tokens, ns, + if (routerstatus_parse_entry_from_string(rs_area, &s, eos, rs_tokens, ns, rs, 0, 0)) { smartlist_add(ns->routerstatus_list, rs); } else { @@ -3779,7 +3786,8 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, } } else { routerstatus_t *rs; - if ((rs = routerstatus_parse_entry_from_string(rs_area, &s, rs_tokens, + if ((rs = routerstatus_parse_entry_from_string(rs_area, &s, eos, + rs_tokens, NULL, NULL, ns->consensus_method, flav))) { @@ -3824,10 +3832,10 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, /* Parse footer; check signature. */ footer_tokens = smartlist_new(); - if ((end_of_footer = strstr(s, "\nnetwork-status-version "))) + if ((end_of_footer = tor_memstr(s, eos-s, "\nnetwork-status-version "))) ++end_of_footer; else - end_of_footer = s + strlen(s); + end_of_footer = eos; if (tokenize_string(area,s, end_of_footer, footer_tokens, networkstatus_vote_footer_token_table, 0)) { log_warn(LD_DIR, "Error tokenizing network-status vote footer."); diff --git a/src/feature/nodelist/routerparse.h b/src/feature/nodelist/routerparse.h index be455984d1..390088c948 100644 --- a/src/feature/nodelist/routerparse.h +++ b/src/feature/nodelist/routerparse.h @@ -30,7 +30,7 @@ enum networkstatus_type_t; int router_get_router_hash(const char *s, size_t s_len, char *digest); int router_get_dir_hash(const char *s, char *digest); -int router_get_networkstatus_v3_hashes(const char *s, +int router_get_networkstatus_v3_hashes(const char *s, size_t len, common_digests_t *digests); int router_get_networkstatus_v3_signed_boundaries(const char *s, size_t len, const char **start_out, @@ -81,6 +81,7 @@ void dump_distinct_digest_count(int severity); int compare_vote_routerstatus_entries(const void **_a, const void **_b); int networkstatus_verify_bw_weights(networkstatus_t *ns, int); networkstatus_t *networkstatus_parse_vote_from_string(const char *s, + size_t len, const char **eos_out, enum networkstatus_type_t ns_type); ns_detached_signatures_t *networkstatus_parse_detached_signatures( @@ -139,7 +140,8 @@ STATIC void dump_desc_fifo_cleanup(void); struct memarea_t; STATIC routerstatus_t *routerstatus_parse_entry_from_string( struct memarea_t *area, - const char **s, smartlist_t *tokens, + const char **s, const char *eos, + smartlist_t *tokens, networkstatus_t *vote, vote_routerstatus_t *vote_rs, int consensus_method, diff --git a/src/test/fuzz/fuzz_consensus.c b/src/test/fuzz/fuzz_consensus.c index b170fd33d8..5a04683a11 100644 --- a/src/test/fuzz/fuzz_consensus.c +++ b/src/test/fuzz/fuzz_consensus.c @@ -59,13 +59,13 @@ int fuzz_main(const uint8_t *data, size_t sz) { networkstatus_t *ns; - char *str = tor_memdup_nulterm(data, sz); const char *eos = NULL; networkstatus_type_t tp = NS_TYPE_CONSENSUS; if (tor_memstr(data, MIN(sz, 1024), "tus vote")) tp = NS_TYPE_VOTE; const char *what = (tp == NS_TYPE_CONSENSUS) ? "consensus" : "vote"; - ns = networkstatus_parse_vote_from_string(str, + ns = networkstatus_parse_vote_from_string((const char *)data, + sz, &eos, tp); if (ns) { @@ -74,6 +74,6 @@ fuzz_main(const uint8_t *data, size_t sz) } else { log_debug(LD_GENERAL, "Parsing as %s failed", what); } - tor_free(str); + return 0; } diff --git a/src/test/fuzz/fuzz_vrs.c b/src/test/fuzz/fuzz_vrs.c index 8c96851b1f..5665ffeaca 100644 --- a/src/test/fuzz/fuzz_vrs.c +++ b/src/test/fuzz/fuzz_vrs.c @@ -52,24 +52,24 @@ fuzz_cleanup(void) int fuzz_main(const uint8_t *data, size_t sz) { - char *str = tor_memdup_nulterm(data, sz); const char *s; routerstatus_t *rs_ns = NULL, *rs_md = NULL, *rs_vote = NULL; vote_routerstatus_t *vrs = tor_malloc_zero(sizeof(*vrs)); smartlist_t *tokens = smartlist_new(); + const char *eos = (const char *)data + sz; - s = str; - rs_ns = routerstatus_parse_entry_from_string(area, &s, tokens, + s = (const char *)data; + rs_ns = routerstatus_parse_entry_from_string(area, &s, eos, tokens, NULL, NULL, 26, FLAV_NS); tor_assert(smartlist_len(tokens) == 0); - s = str; - rs_md = routerstatus_parse_entry_from_string(area, &s, tokens, + s = (const char *)data; + rs_md = routerstatus_parse_entry_from_string(area, &s, eos, tokens, NULL, NULL, 26, FLAV_MICRODESC); tor_assert(smartlist_len(tokens) == 0); - s = str; - rs_vote = routerstatus_parse_entry_from_string(area, &s, tokens, + s = (const char *)data; + rs_vote = routerstatus_parse_entry_from_string(area, &s, eos, tokens, dummy_vote, vrs, 26, FLAV_NS); tor_assert(smartlist_len(tokens) == 0); @@ -81,6 +81,6 @@ fuzz_main(const uint8_t *data, size_t sz) vote_routerstatus_free(vrs); memarea_clear(area); smartlist_free(tokens); - tor_free(str); + return 0; } diff --git a/src/test/test_dir.c b/src/test/test_dir.c index 723799ee8a..0fa5c31039 100644 --- a/src/test/test_dir.c +++ b/src/test/test_dir.c @@ -70,6 +70,23 @@ #define NS_MODULE dir +static networkstatus_t * +networkstatus_parse_vote_from_string_(const char *s, + const char **eos_out, + enum networkstatus_type_t ns_type) +{ + size_t len = strlen(s); + // memdup so that it won't be nul-terminated. + char *tmp = tor_memdup(s, len); + networkstatus_t *result = + networkstatus_parse_vote_from_string(tmp, len, eos_out, ns_type); + if (eos_out && *eos_out) { + *eos_out = s + (*eos_out - tmp); + } + tor_free(tmp); + return result; +} + static void test_dir_nicknames(void *arg) { @@ -2888,7 +2905,7 @@ test_a_networkstatus( sign_skey_leg1, FLAV_NS); tt_assert(consensus_text); - con = networkstatus_parse_vote_from_string(consensus_text, NULL, + con = networkstatus_parse_vote_from_string_(consensus_text, NULL, NS_TYPE_CONSENSUS); tt_assert(con); //log_notice(LD_GENERAL, "<<%s>>\n<<%s>>\n<<%s>>\n", @@ -2900,7 +2917,7 @@ test_a_networkstatus( sign_skey_leg1, FLAV_MICRODESC); tt_assert(consensus_text_md); - con_md = networkstatus_parse_vote_from_string(consensus_text_md, NULL, + con_md = networkstatus_parse_vote_from_string_(consensus_text_md, NULL, NS_TYPE_CONSENSUS); tt_assert(con_md); tt_int_op(con_md->flavor,OP_EQ, FLAV_MICRODESC); @@ -2999,13 +3016,13 @@ test_a_networkstatus( tt_assert(consensus_text3); tt_assert(consensus_text_md2); tt_assert(consensus_text_md3); - con2 = networkstatus_parse_vote_from_string(consensus_text2, NULL, + con2 = networkstatus_parse_vote_from_string_(consensus_text2, NULL, NS_TYPE_CONSENSUS); - con3 = networkstatus_parse_vote_from_string(consensus_text3, NULL, + con3 = networkstatus_parse_vote_from_string_(consensus_text3, NULL, NS_TYPE_CONSENSUS); - con_md2 = networkstatus_parse_vote_from_string(consensus_text_md2, NULL, + con_md2 = networkstatus_parse_vote_from_string_(consensus_text_md2, NULL, NS_TYPE_CONSENSUS); - con_md3 = networkstatus_parse_vote_from_string(consensus_text_md3, NULL, + con_md3 = networkstatus_parse_vote_from_string_(consensus_text_md3, NULL, NS_TYPE_CONSENSUS); tt_assert(con2); tt_assert(con3); @@ -6020,9 +6037,10 @@ test_dir_assumed_flags(void *arg) "192.168.0.1 9001 0\n" "m thisoneislongerbecauseitisa256bitmddigest33\n" "s Fast Guard Stable\n"; + const char *eos = str1 + strlen(str1); const char *cp = str1; - rs = routerstatus_parse_entry_from_string(area, &cp, tokens, NULL, NULL, + rs = routerstatus_parse_entry_from_string(area, &cp, eos, tokens, NULL, NULL, 24, FLAV_MICRODESC); tt_assert(rs); tt_assert(rs->is_flagged_running); diff --git a/src/test/test_dir_common.c b/src/test/test_dir_common.c index e65e2b0111..6e3bb9945d 100644 --- a/src/test/test_dir_common.c +++ b/src/test/test_dir_common.c @@ -264,7 +264,9 @@ dir_common_add_rs_and_parse(networkstatus_t *vote, networkstatus_t **vote_out, /* dump the vote and try to parse it. */ v_text = format_networkstatus_vote(sign_skey, vote); tt_assert(v_text); - *vote_out = networkstatus_parse_vote_from_string(v_text, NULL, NS_TYPE_VOTE); + *vote_out = networkstatus_parse_vote_from_string(v_text, + strlen(v_text), + NULL, NS_TYPE_VOTE); done: if (v_text) @@ -422,4 +424,3 @@ dir_common_construct_vote_3(networkstatus_t **vote, authority_cert_t *cert, return 0; } - diff --git a/src/test/test_routerlist.c b/src/test/test_routerlist.c index 89d1f4f90f..7fe4c15b18 100644 --- a/src/test/test_routerlist.c +++ b/src/test/test_routerlist.c @@ -270,7 +270,9 @@ test_router_pick_directory_server_impl(void *arg) construct_consensus(&consensus_text_md, now); tt_assert(consensus_text_md); - con_md = networkstatus_parse_vote_from_string(consensus_text_md, NULL, + con_md = networkstatus_parse_vote_from_string(consensus_text_md, + strlen(consensus_text_md), + NULL, NS_TYPE_CONSENSUS); tt_assert(con_md); tt_int_op(con_md->flavor,OP_EQ, FLAV_MICRODESC); From 7e3005af30b94fd1925b0be475d72875272b9044 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 7 Sep 2018 19:38:21 -0400 Subject: [PATCH 0026/2557] Replace "read consensus from disk" with "map consensus from disk". Implements 27244, and should save a bunch of RAM on clients. --- src/feature/dirauth/dirvote.c | 4 +- src/feature/dircache/directory.c | 18 +++--- src/feature/nodelist/networkstatus.c | 89 +++++++++++++++------------- src/feature/nodelist/networkstatus.h | 4 +- src/test/test_routerlist.c | 3 +- 5 files changed, 65 insertions(+), 53 deletions(-) diff --git a/src/feature/dirauth/dirvote.c b/src/feature/dirauth/dirvote.c index 42821760c5..771e0363aa 100644 --- a/src/feature/dirauth/dirvote.c +++ b/src/feature/dirauth/dirvote.c @@ -3648,7 +3648,9 @@ dirvote_publish_consensus(void) continue; } - if (networkstatus_set_current_consensus(pending->body, name, 0, NULL)) + if (networkstatus_set_current_consensus(pending->body, + strlen(pending->body), + name, 0, NULL)) log_warn(LD_DIR, "Error publishing %s consensus", name); else log_notice(LD_DIR, "Published %s consensus", name); diff --git a/src/feature/dircache/directory.c b/src/feature/dircache/directory.c index 8e5fc86836..1473299029 100644 --- a/src/feature/dircache/directory.c +++ b/src/feature/dircache/directory.c @@ -2606,17 +2606,17 @@ handle_response_fetch_consensus(dir_connection_t *conn, if (looks_like_a_consensus_diff(body, body_len)) { /* First find our previous consensus. Maybe it's in ram, maybe not. */ cached_dir_t *cd = dirserv_get_consensus(flavname); - const char *consensus_body; + const char *consensus_body = NULL; size_t consensus_body_len; - char *owned_consensus = NULL; + tor_mmap_t *mapped_consensus = NULL; if (cd) { consensus_body = cd->dir; consensus_body_len = cd->dir_len; } else { - owned_consensus = networkstatus_read_cached_consensus(flavname); - if (owned_consensus) { - consensus_body = owned_consensus; - consensus_body_len = strlen(consensus_body); + mapped_consensus = networkstatus_map_cached_consensus(flavname); + if (mapped_consensus) { + consensus_body = mapped_consensus->data; + consensus_body_len = mapped_consensus->size; } } if (!consensus_body) { @@ -2629,7 +2629,7 @@ handle_response_fetch_consensus(dir_connection_t *conn, new_consensus = consensus_diff_apply(consensus_body, consensus_body_len, body, body_len); - tor_free(owned_consensus); + tor_munmap_file(mapped_consensus); if (new_consensus == NULL) { log_warn(LD_DIR, "Could not apply consensus diff received from server " "'%s:%d'", conn->base_.address, conn->base_.port); @@ -2651,7 +2651,9 @@ handle_response_fetch_consensus(dir_connection_t *conn, sourcename = "downloaded"; } - if ((r=networkstatus_set_current_consensus(consensus, flavname, 0, + if ((r=networkstatus_set_current_consensus(consensus, + strlen(consensus), + flavname, 0, conn->identity_digest))<0) { log_fn(r<-1?LOG_WARN:LOG_INFO, LD_DIR, "Unable to load %s consensus directory %s from " diff --git a/src/feature/nodelist/networkstatus.c b/src/feature/nodelist/networkstatus.c index 4338fde599..847ca0cdfc 100644 --- a/src/feature/nodelist/networkstatus.c +++ b/src/feature/nodelist/networkstatus.c @@ -105,8 +105,6 @@ STATIC networkstatus_t *current_md_consensus = NULL; typedef struct consensus_waiting_for_certs_t { /** The consensus itself. */ networkstatus_t *consensus; - /** The encoded version of the consensus, nul-terminated. */ - char *body; /** When did we set the current value of consensus_waiting_for_certs? If * this is too recent, we shouldn't try to fetch a new consensus for a * little while, to give ourselves time to get certificates for this one. */ @@ -199,14 +197,11 @@ networkstatus_reset_download_failures(void) download_status_reset(&consensus_bootstrap_dl_status[i]); } -/** - * Read and and return the cached consensus of type flavorname. If - * unverified is false, get the one we haven't verified. Return NULL if - * the file isn't there. */ +/** Return the filename used to cache the consensus of a given flavor */ static char * -networkstatus_read_cached_consensus_impl(int flav, - const char *flavorname, - int unverified_consensus) +networkstatus_get_cache_fname(int flav, + const char *flavorname, + int unverified_consensus) { char buf[128]; const char *prefix; @@ -221,21 +216,35 @@ networkstatus_read_cached_consensus_impl(int flav, tor_snprintf(buf, sizeof(buf), "%s-%s-consensus", prefix, flavorname); } - char *filename = get_cachedir_fname(buf); - char *result = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL); + return get_cachedir_fname(buf); +} + +/** + * Read and and return the cached consensus of type flavorname. If + * unverified is false, get the one we haven't verified. Return NULL if + * the file isn't there. */ +static tor_mmap_t * +networkstatus_map_cached_consensus_impl(int flav, + const char *flavorname, + int unverified_consensus) +{ + char *filename = networkstatus_get_cache_fname(flav, + flavorname, + unverified_consensus); + tor_mmap_t *result = tor_mmap_file(filename); tor_free(filename); return result; } -/** Return a new string containing the current cached consensus of flavor - * flavorname. */ -char * -networkstatus_read_cached_consensus(const char *flavorname) - { +/** Map the file containing the current cached consensus of flavor + * flavorname */ +tor_mmap_t * +networkstatus_map_cached_consensus(const char *flavorname) +{ int flav = networkstatus_parse_flavor_name(flavorname); if (flav < 0) return NULL; - return networkstatus_read_cached_consensus_impl(flav, flavorname, 0); + return networkstatus_map_cached_consensus_impl(flav, flavorname, 0); } /** Read every cached v3 consensus networkstatus from the disk. */ @@ -248,24 +257,26 @@ router_reload_consensus_networkstatus(void) /* FFFF Suppress warnings if cached consensus is bad? */ for (flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) { const char *flavor = networkstatus_get_flavor_name(flav); - char *s = networkstatus_read_cached_consensus_impl(flav, flavor, 0); - if (s) { - if (networkstatus_set_current_consensus(s, flavor, flags, NULL) < -1) { + tor_mmap_t *m = networkstatus_map_cached_consensus_impl(flav, flavor, 0); + if (m) { + if (networkstatus_set_current_consensus(m->data, m->size, + flavor, flags, NULL) < -1) { log_warn(LD_FS, "Couldn't load consensus %s networkstatus from cache", flavor); } - tor_free(s); + tor_munmap_file(m); } - s = networkstatus_read_cached_consensus_impl(flav, flavor, 1); - if (s) { - if (networkstatus_set_current_consensus(s, flavor, + m = networkstatus_map_cached_consensus_impl(flav, flavor, 1); + if (m) { + if (networkstatus_set_current_consensus(m->data, m->size, + flavor, flags | NSSET_WAS_WAITING_FOR_CERTS, NULL)) { log_info(LD_FS, "Couldn't load unverified consensus %s networkstatus " "from cache", flavor); } - tor_free(s); + tor_munmap_file(m); } } @@ -1833,6 +1844,7 @@ warn_early_consensus(const networkstatus_t *c, const char *flavor, */ int networkstatus_set_current_consensus(const char *consensus, + size_t consensus_len, const char *flavor, unsigned flags, const char *source_dir) @@ -1862,7 +1874,7 @@ networkstatus_set_current_consensus(const char *consensus, /* Make sure it's parseable. */ c = networkstatus_parse_vote_from_string(consensus, - strlen(consensus), + consensus_len, NULL, NS_TYPE_CONSENSUS); if (!c) { log_warn(LD_DIR, "Unable to parse networkstatus consensus"); @@ -1951,14 +1963,12 @@ networkstatus_set_current_consensus(const char *consensus, c->valid_after > current_valid_after) { waiting = &consensus_waiting_for_certs[flav]; networkstatus_vote_free(waiting->consensus); - tor_free(waiting->body); waiting->consensus = c; free_consensus = 0; - waiting->body = tor_strdup(consensus); waiting->set_at = now; waiting->dl_failed = 0; if (!from_cache) { - write_str_to_file(unverified_fname, consensus, 0); + write_bytes_to_file(unverified_fname, consensus, consensus_len, 0); } if (dl_certs) authority_certs_fetch_missing(c, now, source_dir); @@ -2049,10 +2059,6 @@ networkstatus_set_current_consensus(const char *consensus, waiting->consensus->valid_after <= c->valid_after) { networkstatus_vote_free(waiting->consensus); waiting->consensus = NULL; - if (consensus != waiting->body) - tor_free(waiting->body); - else - waiting->body = NULL; waiting->set_at = 0; waiting->dl_failed = 0; if (unlink(unverified_fname) != 0) { @@ -2147,14 +2153,16 @@ networkstatus_note_certs_arrived(const char *source_dir) if (!waiting->consensus) continue; if (networkstatus_check_consensus_signature(waiting->consensus, 0)>=0) { - char *waiting_body = waiting->body; - if (!networkstatus_set_current_consensus( - waiting_body, - flavor_name, - NSSET_WAS_WAITING_FOR_CERTS, - source_dir)) { - tor_free(waiting_body); + tor_mmap_t *mapping = networkstatus_map_cached_consensus_impl( + i, flavor_name, 1); + if (mapping) { + networkstatus_set_current_consensus(mapping->data, + mapping->size, + flavor_name, + NSSET_WAS_WAITING_FOR_CERTS, + source_dir); } + tor_munmap_file(mapping); } } } @@ -2730,6 +2738,5 @@ networkstatus_free_all(void) networkstatus_vote_free(waiting->consensus); waiting->consensus = NULL; } - tor_free(waiting->body); } } diff --git a/src/feature/nodelist/networkstatus.h b/src/feature/nodelist/networkstatus.h index cc6badf0b2..997d7cfa39 100644 --- a/src/feature/nodelist/networkstatus.h +++ b/src/feature/nodelist/networkstatus.h @@ -16,7 +16,7 @@ void networkstatus_reset_warnings(void); void networkstatus_reset_download_failures(void); -char *networkstatus_read_cached_consensus(const char *flavorname); +tor_mmap_t *networkstatus_map_cached_consensus(const char *flavorname); int router_reload_consensus_networkstatus(void); void routerstatus_free_(routerstatus_t *rs); #define routerstatus_free(rs) \ @@ -108,6 +108,7 @@ int networkstatus_consensus_has_ipv6(const or_options_t* options); #define NSSET_ACCEPT_OBSOLETE 8 #define NSSET_REQUIRE_FLAVOR 16 int networkstatus_set_current_consensus(const char *consensus, + size_t consensus_len, const char *flavor, unsigned flags, const char *source_dir); @@ -159,4 +160,3 @@ extern networkstatus_t *current_md_consensus; #endif /* defined(NETWORKSTATUS_PRIVATE) */ #endif /* !defined(TOR_NETWORKSTATUS_H) */ - diff --git a/src/test/test_routerlist.c b/src/test/test_routerlist.c index 7fe4c15b18..bf76570b3f 100644 --- a/src/test/test_routerlist.c +++ b/src/test/test_routerlist.c @@ -659,7 +659,8 @@ test_skew_common(void *arg, time_t now, unsigned long *offset) MOCK(clock_skew_warning, mock_clock_skew_warning); /* Caller will call teardown_capture_of_logs() */ setup_capture_of_logs(LOG_WARN); - retval = networkstatus_set_current_consensus(consensus, "microdesc", 0, + retval = networkstatus_set_current_consensus(consensus, strlen(consensus), + "microdesc", 0, NULL); done: From 04bb70199be924804708bd4ace18b28b5acdbf19 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 11 Sep 2018 11:37:55 -0400 Subject: [PATCH 0027/2557] Followup: Make authority_cert_parse_from_string() take length too --- src/feature/nodelist/routerlist.c | 3 ++- src/feature/nodelist/routerparse.c | 19 +++++++++++-------- src/feature/nodelist/routerparse.h | 1 + src/feature/relay/router.c | 2 +- src/test/test_dir.c | 12 +++++++++--- src/test/test_dir_common.c | 12 +++++++++--- src/test/test_dir_handle_get.c | 16 ++++++++++++---- src/test/test_routerlist.c | 12 +++++++++--- src/test/test_shared_random.c | 12 +++++++++--- 9 files changed, 63 insertions(+), 26 deletions(-) diff --git a/src/feature/nodelist/routerlist.c b/src/feature/nodelist/routerlist.c index bcc5c1f074..c3c72e9c78 100644 --- a/src/feature/nodelist/routerlist.c +++ b/src/feature/nodelist/routerlist.c @@ -555,7 +555,8 @@ trusted_dirs_load_certs_from_string(const char *contents, int source, int added_trusted_cert = 0; for (s = contents; *s; s = eos) { - authority_cert_t *cert = authority_cert_parse_from_string(s, &eos); + authority_cert_t *cert = authority_cert_parse_from_string(s, strlen(s), + &eos); cert_list_t *cl; if (!cert) { failure_code = -1; diff --git a/src/feature/nodelist/routerparse.c b/src/feature/nodelist/routerparse.c index 9abdfb614c..8a5efc007e 100644 --- a/src/feature/nodelist/routerparse.c +++ b/src/feature/nodelist/routerparse.c @@ -2308,7 +2308,8 @@ extrainfo_parse_entry_from_string(const char *s, const char *end, /** Parse a key certificate from s; point end-of-string to * the first character after the certificate. */ authority_cert_t * -authority_cert_parse_from_string(const char *s, const char **end_of_string) +authority_cert_parse_from_string(const char *s, size_t maxlen, + const char **end_of_string) { /** Reject any certificate at least this big; it is probably an overflow, an * attack, a bug, or some other nonsense. */ @@ -2319,24 +2320,25 @@ authority_cert_parse_from_string(const char *s, const char **end_of_string) char digest[DIGEST_LEN]; directory_token_t *tok; char fp_declared[DIGEST_LEN]; - char *eos; + const char *eos; size_t len; int found; memarea_t *area = NULL; + const char *end_of_s = s + maxlen; const char *s_dup = s; - s = eat_whitespace(s); - eos = strstr(s, "\ndir-key-certification"); + s = eat_whitespace_eos(s, end_of_s); + eos = tor_memstr(s, end_of_s - s, "\ndir-key-certification"); if (! eos) { log_warn(LD_DIR, "No signature found on key certificate"); return NULL; } - eos = strstr(eos, "\n-----END SIGNATURE-----\n"); + eos = tor_memstr(eos, end_of_s - eos, "\n-----END SIGNATURE-----\n"); if (! eos) { log_warn(LD_DIR, "No end-of-signature found on key certificate"); return NULL; } - eos = strchr(eos+2, '\n'); + eos = memchr(eos+2, '\n', end_of_s - (eos+2)); tor_assert(eos); ++eos; len = eos - s; @@ -2353,7 +2355,7 @@ authority_cert_parse_from_string(const char *s, const char **end_of_string) log_warn(LD_DIR, "Error tokenizing key certificate"); goto err; } - if (router_get_hash_impl(s, strlen(s), digest, "dir-key-certificate-version", + if (router_get_hash_impl(s, eos-s, digest, "dir-key-certificate-version", "\ndir-key-certification", '\n', DIGEST_SHA1) < 0) goto err; tok = smartlist_get(tokens, 0); @@ -3465,7 +3467,8 @@ networkstatus_parse_vote_from_string(const char *s, "\ndir-key-certificate-version"))) goto err; ++cert; - ns->cert = authority_cert_parse_from_string(cert, &end_of_cert); + ns->cert = authority_cert_parse_from_string(cert, end_of_header - cert, + &end_of_cert); if (!ns->cert || !end_of_cert || end_of_cert > end_of_header) goto err; } diff --git a/src/feature/nodelist/routerparse.h b/src/feature/nodelist/routerparse.h index 390088c948..c60046e8e8 100644 --- a/src/feature/nodelist/routerparse.h +++ b/src/feature/nodelist/routerparse.h @@ -93,6 +93,7 @@ smartlist_t *microdescs_parse_from_string(const char *s, const char *eos, smartlist_t *invalid_digests_out); authority_cert_t *authority_cert_parse_from_string(const char *s, + size_t maxlen, const char **end_of_string); int rend_parse_v2_service_descriptor(rend_service_descriptor_t **parsed_out, char *desc_id_out, diff --git a/src/feature/relay/router.c b/src/feature/relay/router.c index 1f316ebf08..17f63e9983 100644 --- a/src/feature/relay/router.c +++ b/src/feature/relay/router.c @@ -717,7 +717,7 @@ load_authority_keyset(int legacy, crypto_pk_t **key_out, fname); goto done; } - parsed = authority_cert_parse_from_string(cert, &eos); + parsed = authority_cert_parse_from_string(cert, strlen(cert), &eos); if (!parsed) { log_warn(LD_DIR, "Unable to parse certificate in %s", fname); goto done; diff --git a/src/test/test_dir.c b/src/test/test_dir.c index 0fa5c31039..078a383dd8 100644 --- a/src/test/test_dir.c +++ b/src/test/test_dir.c @@ -2799,11 +2799,17 @@ test_a_networkstatus( MOCK(get_my_v3_authority_cert, get_my_v3_authority_cert_m); /* Parse certificates and keys. */ - cert1 = mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL); + cert1 = mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, + strlen(AUTHORITY_CERT_1), + NULL); tt_assert(cert1); - cert2 = authority_cert_parse_from_string(AUTHORITY_CERT_2, NULL); + cert2 = authority_cert_parse_from_string(AUTHORITY_CERT_2, + strlen(AUTHORITY_CERT_2), + NULL); tt_assert(cert2); - cert3 = authority_cert_parse_from_string(AUTHORITY_CERT_3, NULL); + cert3 = authority_cert_parse_from_string(AUTHORITY_CERT_3, + strlen(AUTHORITY_CERT_3), + NULL); tt_assert(cert3); sign_skey_1 = crypto_pk_new(); sign_skey_2 = crypto_pk_new(); diff --git a/src/test/test_dir_common.c b/src/test/test_dir_common.c index 6e3bb9945d..63bd082eea 100644 --- a/src/test/test_dir_common.c +++ b/src/test/test_dir_common.c @@ -40,14 +40,20 @@ dir_common_authority_pk_init(authority_cert_t **cert1, { /* Parse certificates and keys. */ authority_cert_t *cert; - cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL); + cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, + strlen(AUTHORITY_CERT_1), + NULL); tt_assert(cert); tt_assert(cert->identity_key); *cert1 = cert; tt_assert(*cert1); - *cert2 = authority_cert_parse_from_string(AUTHORITY_CERT_2, NULL); + *cert2 = authority_cert_parse_from_string(AUTHORITY_CERT_2, + strlen(AUTHORITY_CERT_2), + NULL); tt_assert(*cert2); - *cert3 = authority_cert_parse_from_string(AUTHORITY_CERT_3, NULL); + *cert3 = authority_cert_parse_from_string(AUTHORITY_CERT_3, + strlen(AUTHORITY_CERT_3), + NULL); tt_assert(*cert3); *sign_skey_1 = crypto_pk_new(); *sign_skey_2 = crypto_pk_new(); diff --git a/src/test/test_dir_handle_get.c b/src/test/test_dir_handle_get.c index 09799a0e5f..ecb7c1b5fc 100644 --- a/src/test/test_dir_handle_get.c +++ b/src/test/test_dir_handle_get.c @@ -1270,7 +1270,9 @@ test_dir_handle_get_server_keys_authority(void* data) size_t body_used = 0; (void) data; - mock_cert = authority_cert_parse_from_string(TEST_CERTIFICATE, NULL); + mock_cert = authority_cert_parse_from_string(TEST_CERTIFICATE, + strlen(TEST_CERTIFICATE), + NULL); MOCK(get_my_v3_authority_cert, get_my_v3_authority_cert_m); MOCK(connection_write_to_buf_impl_, connection_write_to_buf_mock); @@ -1420,7 +1422,9 @@ test_dir_handle_get_server_keys_sk(void* data) size_t body_used = 0; (void) data; - mock_cert = authority_cert_parse_from_string(TEST_CERTIFICATE, NULL); + mock_cert = authority_cert_parse_from_string(TEST_CERTIFICATE, + strlen(TEST_CERTIFICATE), + NULL); MOCK(get_my_v3_authority_cert, get_my_v3_authority_cert_m); MOCK(connection_write_to_buf_impl_, connection_write_to_buf_mock); @@ -2388,7 +2392,9 @@ test_dir_handle_get_status_vote_next_authority(void* data) routerlist_free_all(); dirvote_free_all(); - mock_cert = authority_cert_parse_from_string(TEST_CERTIFICATE, NULL); + mock_cert = authority_cert_parse_from_string(TEST_CERTIFICATE, + strlen(TEST_CERTIFICATE), + NULL); /* create a trusted ds */ ds = trusted_dir_server_new("ds", "127.0.0.1", 9059, 9060, NULL, digest, @@ -2466,7 +2472,9 @@ test_dir_handle_get_status_vote_current_authority(void* data) routerlist_free_all(); dirvote_free_all(); - mock_cert = authority_cert_parse_from_string(TEST_CERTIFICATE, NULL); + mock_cert = authority_cert_parse_from_string(TEST_CERTIFICATE, + strlen(TEST_CERTIFICATE), + NULL); /* create a trusted ds */ ds = trusted_dir_server_new("ds", "127.0.0.1", 9059, 9060, NULL, digest, diff --git a/src/test/test_routerlist.c b/src/test/test_routerlist.c index bf76570b3f..e7c577cf66 100644 --- a/src/test/test_routerlist.c +++ b/src/test/test_routerlist.c @@ -260,7 +260,9 @@ test_router_pick_directory_server_impl(void *arg) /* Init SR subsystem. */ MOCK(get_my_v3_authority_cert, get_my_v3_authority_cert_m); - mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL); + mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, + strlen(AUTHORITY_CERT_1), + NULL); sr_init(0); UNMOCK(get_my_v3_authority_cert); @@ -472,7 +474,9 @@ test_directory_guard_fetch_with_no_dirinfo(void *arg) /* Initialize the SRV subsystem */ MOCK(get_my_v3_authority_cert, get_my_v3_authority_cert_m); - mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL); + mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, + strlen(AUTHORITY_CERT_1), + NULL); sr_init(0); UNMOCK(get_my_v3_authority_cert); @@ -645,7 +649,9 @@ test_skew_common(void *arg, time_t now, unsigned long *offset) /* Initialize the SRV subsystem */ MOCK(get_my_v3_authority_cert, get_my_v3_authority_cert_m); - mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL); + mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, + strlen(AUTHORITY_CERT_1), + NULL); sr_init(0); UNMOCK(get_my_v3_authority_cert); diff --git a/src/test/test_shared_random.c b/src/test/test_shared_random.c index 70adf580ab..2043613641 100644 --- a/src/test/test_shared_random.c +++ b/src/test/test_shared_random.c @@ -64,7 +64,9 @@ init_authority_state(void) MOCK(get_my_v3_authority_cert, get_my_v3_authority_cert_m); or_options_t *options = get_options_mutable(); - mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL); + mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, + strlen(AUTHORITY_CERT_1), + NULL); tt_assert(mock_cert); options->AuthoritativeDir = 1; tt_int_op(load_ed_keys(options, time(NULL)), OP_GE, 0); @@ -420,7 +422,9 @@ test_sr_commit(void *arg) { /* Setup a minimal dirauth environment for this test */ or_options_t *options = get_options_mutable(); - auth_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL); + auth_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, + strlen(AUTHORITY_CERT_1), + NULL); tt_assert(auth_cert); options->AuthoritativeDir = 1; @@ -823,7 +827,9 @@ test_sr_setup_commits(void) { /* Setup a minimal dirauth environment for this test */ or_options_t *options = get_options_mutable(); - auth_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL); + auth_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, + strlen(AUTHORITY_CERT_1), + NULL); tt_assert(auth_cert); options->AuthoritativeDir = 1; From 81a5448c187458ea3edcc0a72044a4cfdf0273bf Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 11 Sep 2018 11:54:37 -0400 Subject: [PATCH 0028/2557] Changes file for feature27244 --- changes/feature27244 | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 changes/feature27244 diff --git a/changes/feature27244 b/changes/feature27244 new file mode 100644 index 0000000000..a4debbbe53 --- /dev/null +++ b/changes/feature27244 @@ -0,0 +1,5 @@ + o Minor features (memory usage): + - Tor clients no longer need to keep the full text of a consensus in + memory in order to parse it, or apply a diff to it. Instead, they + use mmap() to read the consensus files from disk. Closes ticket + 27244. From cb1891b412b40ad943306ad0894ad2fc726cf562 Mon Sep 17 00:00:00 2001 From: Colin Childs Date: Wed, 12 Sep 2018 13:39:49 -0500 Subject: [PATCH 0029/2557] Adding 1 new fallback --- scripts/maint/fallback.whitelist | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/maint/fallback.whitelist b/scripts/maint/fallback.whitelist index 375ac8f748..088545471b 100644 --- a/scripts/maint/fallback.whitelist +++ b/scripts/maint/fallback.whitelist @@ -1064,3 +1064,5 @@ # https://trac.torproject.org/projects/tor/ticket/27297 37.252.185.182:9030 orport=8080 id=113143469021882C3A4B82F084F8125B08EE471E ipv6=[2a00:63c1:a:182::2]:8080 +# Email sent directly to Phoul +139.99.130.178:80 orport=443 id=867B95CACD64653FEEC4D2CEFC5C49B4620307A7 From 7e862c3ec029d3cdfdd07483e31843c8f3b48a68 Mon Sep 17 00:00:00 2001 From: teor Date: Sat, 15 Sep 2018 01:23:02 +1000 Subject: [PATCH 0030/2557] check-changes: Check bugfix version formatting Check that bugfix versions in changes files look like Tor versions from the versions spec. Part of ticket 27761. --- changes/ticket27761 | 3 +++ scripts/maint/lintChanges.py | 6 ++++++ 2 files changed, 9 insertions(+) create mode 100644 changes/ticket27761 diff --git a/changes/ticket27761 b/changes/ticket27761 new file mode 100644 index 0000000000..ef4686bdca --- /dev/null +++ b/changes/ticket27761 @@ -0,0 +1,3 @@ + o Minor features (changelogs): + - Check that bugfix versions in changes files look like Tor versions + from the versions spec. Closes ticket 27761. diff --git a/scripts/maint/lintChanges.py b/scripts/maint/lintChanges.py index d5b8fcae5c..5c5d45da1d 100755 --- a/scripts/maint/lintChanges.py +++ b/scripts/maint/lintChanges.py @@ -87,6 +87,12 @@ def lintfile(fname): warn("Bugfix does not say 'Fixes bug X; bugfix on Y'") elif re.search('tor-([0-9]+)', contents): warn("Do not prefix versions with 'tor-'. ('0.1.2', not 'tor-0.1.2'.)") + else: + bugfix_match = re.search('bugfix on ([0-9]+\.[0-9]+\.[0-9]+)', contents) + if bugfix_match is None: + warn("Versions must have at least 3 digits. ('0.1.2', '0.3.4.8', or '0.3.5.1-alpha'.)") + elif bugfix_match.group(0) is None: + warn("Versions must have at least 3 digits. ('0.1.2', '0.3.4.8', or '0.3.5.1-alpha'.)") return have_warned != [] From 3eafa61f6324c91b4bb8c02a6e54db0f12ef42c2 Mon Sep 17 00:00:00 2001 From: teor Date: Tue, 18 Sep 2018 20:39:03 +1000 Subject: [PATCH 0031/2557] check-changes: Warn about bugfixes on future releases Warn when bugfix changes files say that the bug is in a future release. Closes ticket 27761. --- Makefile.am | 2 +- changes/ticket27761 | 3 ++- scripts/maint/lintChanges.py | 50 ++++++++++++++++++++++++++++++++++++ 3 files changed, 53 insertions(+), 2 deletions(-) diff --git a/Makefile.am b/Makefile.am index 36a5dd2e9e..9a65e083a5 100644 --- a/Makefile.am +++ b/Makefile.am @@ -408,7 +408,7 @@ endif check-changes: if USEPYTHON @if test -d "$(top_srcdir)/changes"; then \ - $(PYTHON) $(top_srcdir)/scripts/maint/lintChanges.py $(top_srcdir)/changes; \ + PACKAGE_VERSION=$(PACKAGE_VERSION) $(PYTHON) $(top_srcdir)/scripts/maint/lintChanges.py $(top_srcdir)/changes; \ fi endif diff --git a/changes/ticket27761 b/changes/ticket27761 index ef4686bdca..35106ee9c6 100644 --- a/changes/ticket27761 +++ b/changes/ticket27761 @@ -1,3 +1,4 @@ o Minor features (changelogs): - Check that bugfix versions in changes files look like Tor versions - from the versions spec. Closes ticket 27761. + from the versions spec. Warn when bugfixes claim to be on a future + release. Closes ticket 27761. diff --git a/scripts/maint/lintChanges.py b/scripts/maint/lintChanges.py index 5c5d45da1d..39fa08bb4a 100755 --- a/scripts/maint/lintChanges.py +++ b/scripts/maint/lintChanges.py @@ -35,6 +35,36 @@ NEEDS_SUBCATEGORIES = set([ "Major features", ]) +def split_tor_version(version): + ''' + Return the initial numeric components of the Tor version as a list of ints. + For versions earlier than 0.1.0, returns MAJOR, MINOR, and MICRO. + For versions 0.1.0 and later, returns MAJOR, MINOR, MICRO, and PATCHLEVEL if present. + + If the version is malformed, returns None. + ''' + version_match = re.search('([0-9]+)\.([0-9]+)\.([0-9]+)(\.([0-9]+))?', version) + if version_match is None: + return None + + version_groups = version_match.groups() + if version_groups is None: + return None + if len(version_groups) < 3: + return None + + if len(version_groups) != 5: + return None + version_components = version_groups[0:3] + version_components += version_groups[4:5] + + try: + version_list = [int(v) for v in version_components if v is not None] + except ValueError: + return None + + return version_list + def lintfile(fname): have_warned = [] @@ -93,6 +123,26 @@ def lintfile(fname): warn("Versions must have at least 3 digits. ('0.1.2', '0.3.4.8', or '0.3.5.1-alpha'.)") elif bugfix_match.group(0) is None: warn("Versions must have at least 3 digits. ('0.1.2', '0.3.4.8', or '0.3.5.1-alpha'.)") + else: + bugfix_match = re.search('bugfix on ([0-9a-z][-.0-9a-z]+[0-9a-z])', contents) + bugfix_group = bugfix_match.groups(0) if bugfix_match is not None else None + bugfix_version = bugfix_group[0] if bugfix_group is not None else None + package_version = os.environ.get('PACKAGE_VERSION', None) + if bugfix_version is None: + # This should be unreachable, unless the patterns are out of sync + warn("Malformed bugfix version.") + elif package_version is not None: + # If $PACKAGE_VERSION isn't set, skip this check + bugfix_split = split_tor_version(bugfix_version) + package_split = split_tor_version(package_version) + if bugfix_split is None: + # This should be unreachable, unless the patterns are out of sync + warn("Malformed bugfix version: '{}'.".format(bugfix_version)) + elif package_split is None: + # This should be unreachable, unless the patterns are out of sync, or the package versioning scheme has changed + warn("Malformed $PACKAGE_VERSION: '{}'.".format(package_version)) + elif bugfix_split > package_split: + warn("Bugfixes must be made on earlier versions (or this version). (Bugfix on version: '{}', current tor package version: '{}'.)".format(bugfix_version, package_version)) return have_warned != [] From 1663a1dd633213b750fe72b72b8c1bca7e15cd0a Mon Sep 17 00:00:00 2001 From: Colin Childs Date: Wed, 19 Sep 2018 15:39:39 -0500 Subject: [PATCH 0032/2557] Adjusting IP of fallback --- scripts/maint/fallback.whitelist | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/maint/fallback.whitelist b/scripts/maint/fallback.whitelist index 088545471b..81e6785333 100644 --- a/scripts/maint/fallback.whitelist +++ b/scripts/maint/fallback.whitelist @@ -731,7 +731,7 @@ 185.220.101.24:10024 orport=20024 id=FDA70EC93DB01E3CB418CB6943B0C68464B18B4C # niftyrat # Email sent directly to teor, verified using relay contact info -64.113.32.29:9030 orport=9001 id=30C19B81981F450C402306E2E7CFB6C3F79CB6B2 +198.232.165.2:9030 orport=9001 id=30C19B81981F450C402306E2E7CFB6C3F79CB6B2 # Emails sent directly to teor, verified using relay contact info 51.254.101.242:9002 orport=9001 id=4CC9CC9195EC38645B699A33307058624F660CCF From bc68e80e0a801d498ce215de4a689c09927181df Mon Sep 17 00:00:00 2001 From: Colin Childs Date: Thu, 20 Sep 2018 13:48:45 -0500 Subject: [PATCH 0033/2557] Adding fallback relay --- scripts/maint/fallback.whitelist | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/maint/fallback.whitelist b/scripts/maint/fallback.whitelist index 81e6785333..1fd231abf2 100644 --- a/scripts/maint/fallback.whitelist +++ b/scripts/maint/fallback.whitelist @@ -1066,3 +1066,7 @@ # Email sent directly to Phoul 139.99.130.178:80 orport=443 id=867B95CACD64653FEEC4D2CEFC5C49B4620307A7 + +# Email sent directly to Phoul +104.131.11.214:9030 orport=8080 id=32828476F4F84E15C42B4C360A5CD8DE4C3C2BE7 + From b0c41e3ec2bca832466ea7709827518159af0c08 Mon Sep 17 00:00:00 2001 From: Jay Bitron Date: Wed, 10 Oct 2018 14:12:53 -0700 Subject: [PATCH 0034/2557] Fix the missing unpack function in mmdb-convert.py --- src/config/mmdb-convert.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config/mmdb-convert.py b/src/config/mmdb-convert.py index 3a454a3fc1..706a8b03cc 100644 --- a/src/config/mmdb-convert.py +++ b/src/config/mmdb-convert.py @@ -77,7 +77,7 @@ def to_int32(s): def to_int28(s): "Parse a pair of big-endian 28-bit integers from bytestring s." - a, b = unpack("!LL", s + b'\x00') + a, b = struct.unpack("!LL", s + b'\x00') return (((a & 0xf0) << 20) + (a >> 8)), ((a & 0x0f) << 24) + (b >> 8) class Tree(object): From 8b5ad246e81a8346b79d37052320753f87e22a04 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Thu, 11 Oct 2018 20:28:11 +0300 Subject: [PATCH 0035/2557] Fix issues that shellcheck found in chutney-git-bisect.sh --- changes/ticket28006 | 3 +++ scripts/test/chutney-git-bisect.sh | 6 +++--- 2 files changed, 6 insertions(+), 3 deletions(-) create mode 100644 changes/ticket28006 diff --git a/changes/ticket28006 b/changes/ticket28006 new file mode 100644 index 0000000000..95a4b2cae4 --- /dev/null +++ b/changes/ticket28006 @@ -0,0 +1,3 @@ + o Code simplification and refactoring: + - Fix issues that shellcheck found in chutney-git-bisect.sh. + Resolves ticket 28006. diff --git a/scripts/test/chutney-git-bisect.sh b/scripts/test/chutney-git-bisect.sh index 8a3f2c70c8..dc1319a27a 100755 --- a/scripts/test/chutney-git-bisect.sh +++ b/scripts/test/chutney-git-bisect.sh @@ -20,7 +20,7 @@ if [ ! -z "$1" ]; then fi if [ ! -z "$2" ]; then - cd "$2" + cd "$2" || exit fi CHUTNEY_TEST_CMD="make test-network-all" @@ -54,9 +54,9 @@ while [ "$i" -le "$CHUTNEY_TRIES" ]; do echo "test '$CHUTNEY_TEST_CMD' succeeded after $i/$CHUTNEY_TRIES attempts, good" exit 0 fi - i=$[$i+1] + i=$((i+1)) done -i=$[$i-1] +i=$((i-1)) echo "test '$CHUTNEY_TEST_CMD' failed $i/$CHUTNEY_TRIES attempts, bad" exit 1 From 57bba19bbeb74d97f3e429259a19b5b6b1f8be53 Mon Sep 17 00:00:00 2001 From: Colin Childs Date: Fri, 12 Oct 2018 15:21:32 -0500 Subject: [PATCH 0036/2557] Remove aurora and chulak from fallback list --- scripts/maint/fallback.whitelist | 2 -- 1 file changed, 2 deletions(-) diff --git a/scripts/maint/fallback.whitelist b/scripts/maint/fallback.whitelist index 1fd231abf2..1b06299cca 100644 --- a/scripts/maint/fallback.whitelist +++ b/scripts/maint/fallback.whitelist @@ -172,8 +172,6 @@ # Email sent directly to teor, verified using relay contact info 94.242.246.24:23 orport=8080 id=EC116BCB80565A408CE67F8EC3FE3B0B02C3A065 ipv6=[2a01:608:ffff:ff07::1:24]:9004 -176.126.252.11:443 orport=9001 id=B0279A521375F3CB2AE210BDBFC645FDD2E1973A ipv6=[2a02:59e0:0:7::11]:9003 -176.126.252.12:21 orport=8080 id=379FB450010D17078B3766C2273303C358C3A442 ipv6=[2a02:59e0:0:7::12]:81 94.242.246.23:443 orport=9001 id=F65E0196C94DFFF48AFBF2F5F9E3E19AAE583FD0 ipv6=[2a01:608:ffff:ff07::1:23]:9003 85.248.227.164:444 orport=9002 id=B84F248233FEA90CAD439F292556A3139F6E1B82 ipv6=[2a00:1298:8011:212::164]:9004 85.248.227.163:443 orport=9001 id=C793AB88565DDD3C9E4C6F15CCB9D8C7EF964CE9 ipv6=[2a00:1298:8011:212::163]:9003 From 5033e950ccd0e0f37134f5c4124e1f79d4d784b3 Mon Sep 17 00:00:00 2001 From: Colin Childs Date: Fri, 12 Oct 2018 15:25:17 -0500 Subject: [PATCH 0037/2557] Adding Quake to fallback list --- scripts/maint/fallback.whitelist | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/maint/fallback.whitelist b/scripts/maint/fallback.whitelist index 1b06299cca..128cbc8613 100644 --- a/scripts/maint/fallback.whitelist +++ b/scripts/maint/fallback.whitelist @@ -1068,3 +1068,5 @@ # Email sent directly to Phoul 104.131.11.214:9030 orport=8080 id=32828476F4F84E15C42B4C360A5CD8DE4C3C2BE7 +# Email sent directly to Phoul / Teor +178.175.139.122:80 orport=443 id=490FB3FAAF8837407D94CA7E1DEF025DEF0F3516 ipv6=[2a00:1dc0:3002::3]:443 From 9b946c13a2e825311eff40713574ba1608137010 Mon Sep 17 00:00:00 2001 From: Colin Childs Date: Mon, 15 Oct 2018 11:43:02 -0500 Subject: [PATCH 0038/2557] Adding hviv104 to fallback list --- scripts/maint/fallback.whitelist | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/maint/fallback.whitelist b/scripts/maint/fallback.whitelist index 128cbc8613..3696da8e18 100644 --- a/scripts/maint/fallback.whitelist +++ b/scripts/maint/fallback.whitelist @@ -1070,3 +1070,6 @@ # Email sent directly to Phoul / Teor 178.175.139.122:80 orport=443 id=490FB3FAAF8837407D94CA7E1DEF025DEF0F3516 ipv6=[2a00:1dc0:3002::3]:443 + +# Email sent directly to Phoul +192.42.116.16:80 orport=443 id=81B75D534F91BFB7C57AB67DA10BCEF622582AE8 From 17f4388b0fa377f7bbdf6f9df23ff429a717d4f8 Mon Sep 17 00:00:00 2001 From: Colin Childs Date: Mon, 15 Oct 2018 13:47:31 -0500 Subject: [PATCH 0039/2557] Remove mullbinde9 from fallback list --- scripts/maint/fallback.whitelist | 3 --- 1 file changed, 3 deletions(-) diff --git a/scripts/maint/fallback.whitelist b/scripts/maint/fallback.whitelist index 3696da8e18..94db9272fd 100644 --- a/scripts/maint/fallback.whitelist +++ b/scripts/maint/fallback.whitelist @@ -747,9 +747,6 @@ # Email sent directly to teor, verified using relay contact info 51.254.136.195:80 orport=443 id=7BB70F8585DFC27E75D692970C0EEB0F22983A63 -# Email sent directly to teor, verified using relay contact info -163.172.13.165:9030 orport=9001 id=33DA0CAB7C27812EFF2E22C9705630A54D101FEB ipv6=[2001:bc8:38cb:201::8]:9001 - # Email sent directly to teor, verified using relay contact info 5.196.88.122:9030 orport=9001 id=0C2C599AFCB26F5CFC2C7592435924C1D63D9484 ipv6=[2001:41d0:a:fb7a::1]:9001 From 55412c4f3d3486d28fe337b919e7fddc2f93e1b4 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Thu, 11 Oct 2018 15:22:12 +0300 Subject: [PATCH 0040/2557] Add new source file to test target --- src/test/include.am | 1 + src/test/test.c | 1 + src/test/test.h | 1 + src/test/test_parsecommon.c | 44 +++++++++++++++++++++++++++++++++++++ 4 files changed, 47 insertions(+) create mode 100644 src/test/test_parsecommon.c diff --git a/src/test/include.am b/src/test/include.am index 1055cd0a81..dd2986c67c 100644 --- a/src/test/include.am +++ b/src/test/include.am @@ -182,6 +182,7 @@ src_test_test_SOURCES += \ src/test/test_x509.c \ src/test/test_helpers.c \ src/test/test_dns.c \ + src/test/test_parsecommon.c \ src/test/testing_common.c \ src/test/testing_rsakeys.c \ src/ext/tinytest.c diff --git a/src/test/test.c b/src/test/test.c index 70d91e3967..56eb153289 100644 --- a/src/test/test.c +++ b/src/test/test.c @@ -926,5 +926,6 @@ struct testgroup_t testgroups[] = { { "util/thread/", thread_tests }, { "util/handle/", handle_tests }, { "dns/", dns_tests }, + { "parsecommon/", parsecommon_tests }, END_OF_GROUPS }; diff --git a/src/test/test.h b/src/test/test.h index a46fedf3e0..281551aa69 100644 --- a/src/test/test.h +++ b/src/test/test.h @@ -266,6 +266,7 @@ extern struct testcase_t dns_tests[]; extern struct testcase_t handle_tests[]; extern struct testcase_t sr_tests[]; extern struct testcase_t x509_tests[]; +extern struct testcase_t parsecommon_tests[]; extern struct testcase_t slow_crypto_tests[]; extern struct testcase_t slow_util_tests[]; diff --git a/src/test/test_parsecommon.c b/src/test/test_parsecommon.c new file mode 100644 index 0000000000..f152450f5a --- /dev/null +++ b/src/test/test_parsecommon.c @@ -0,0 +1,44 @@ +/* Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "core/or/or.h" +#include "test/test.h" +#include "lib/memarea/memarea.h" +#include "lib/encoding/binascii.h" +#include "feature/dirparse/parsecommon.h" +#include "test/log_test_helpers.h" + +static void +test_parsecommon_tokenize_string_null(void *arg) +{ + + memarea_t *area = memarea_new(); + smartlist_t *tokens = smartlist_new(); + + (void)arg; + + const char *str_with_null = "a\0bccccccccc"; + + int retval = + tokenize_string(area, str_with_null, + str_with_null + 3, + tokens, NULL, 0); + + tt_int_op(retval, OP_EQ, -1); + + done: + memarea_drop_all(area); + smartlist_free(tokens); + return; +} + +#define PARSECOMMON_TEST(name) \ + { #name, test_parsecommon_ ## name, 0, NULL, NULL } + +struct testcase_t parsecommon_tests[] = { + PARSECOMMON_TEST(tokenize_string_null), + END_OF_TESTCASES +}; + From 7829e3a86875da16e3d7ac55be85145bd672fc12 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Fri, 12 Oct 2018 14:19:40 +0300 Subject: [PATCH 0041/2557] First testcase for get_next_token() --- src/test/test_parsecommon.c | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/src/test/test_parsecommon.c b/src/test/test_parsecommon.c index f152450f5a..ba778d9969 100644 --- a/src/test/test_parsecommon.c +++ b/src/test/test_parsecommon.c @@ -34,11 +34,38 @@ test_parsecommon_tokenize_string_null(void *arg) return; } +static void +test_parsecommon_get_next_token_success(void *arg) +{ + memarea_t *area = memarea_new(); + const char *str = "uptime 1024"; + const char *end = str + strlen(str); + const char **s = &str; + token_rule_t table = T01("uptime", K_UPTIME, GE(1), NO_OBJ); + (void)arg; + + directory_token_t *token = get_next_token(area, s, end, &table); + + tt_int_op(token->tp, OP_EQ, K_UPTIME); + tt_int_op(token->n_args, OP_EQ, 1); + tt_str_op(*(token->args), OP_EQ, "1024"); + tt_assert(!token->object_type); + tt_int_op(token->object_size, OP_EQ, 0); + tt_assert(!token->object_body); + + tt_ptr_op(*s, OP_EQ, end); + + done: + memarea_drop_all(area); + return; +} + #define PARSECOMMON_TEST(name) \ { #name, test_parsecommon_ ## name, 0, NULL, NULL } struct testcase_t parsecommon_tests[] = { PARSECOMMON_TEST(tokenize_string_null), + PARSECOMMON_TEST(get_next_token_success), END_OF_TESTCASES }; From 5c891dba770b752d16f7e7cea8f61f15abd09ef5 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Fri, 12 Oct 2018 15:48:38 +0300 Subject: [PATCH 0042/2557] Test argument concatenation in get_next_token() --- src/test/test_parsecommon.c | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/test/test_parsecommon.c b/src/test/test_parsecommon.c index ba778d9969..b63327ecd1 100644 --- a/src/test/test_parsecommon.c +++ b/src/test/test_parsecommon.c @@ -60,12 +60,33 @@ test_parsecommon_get_next_token_success(void *arg) return; } +static void +test_parsecommon_get_next_token_concat_args(void *arg) +{ + memarea_t *area = memarea_new(); + const char *str = "proto A=1 B=2"; + const char *end = str + strlen(str); + const char **s = &str; + token_rule_t rule = T01("proto", K_PROTO, CONCAT_ARGS, NO_OBJ); + (void)arg; + + directory_token_t *token = get_next_token(area, s, end, &rule); + + tt_int_op(token->tp, OP_EQ, K_PROTO); + tt_int_op(token->n_args, OP_EQ, 1); + tt_str_op(*(token->args), OP_EQ, "A=1 B=2"); + + done: + memarea_drop_all(area); +} + #define PARSECOMMON_TEST(name) \ { #name, test_parsecommon_ ## name, 0, NULL, NULL } struct testcase_t parsecommon_tests[] = { PARSECOMMON_TEST(tokenize_string_null), PARSECOMMON_TEST(get_next_token_success), + PARSECOMMON_TEST(get_next_token_concat_args), END_OF_TESTCASES }; From 569d8d8cd73987cd0ca126a6f98a75dd34b21137 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Fri, 12 Oct 2018 18:33:27 +0300 Subject: [PATCH 0043/2557] Test-case for public key parsing using get_next_token() --- src/test/test_parsecommon.c | 42 +++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/src/test/test_parsecommon.c b/src/test/test_parsecommon.c index b63327ecd1..519bbf79ea 100644 --- a/src/test/test_parsecommon.c +++ b/src/test/test_parsecommon.c @@ -80,6 +80,47 @@ test_parsecommon_get_next_token_concat_args(void *arg) memarea_drop_all(area); } +static void +test_parsecommon_get_next_token_parse_keys(void *arg) +{ + (void)arg; + + memarea_t *area = memarea_new(); + const char *base64_key = + "MIGJAoGBAMDdIya33BfNlHOkzoTKSTT8EjD64waMfUr372syVHiFjHhObwKwGA5u\n" + "sHaMIe9r+Ij/4C1dKyuXkcz3DOl6gWNhTD7dZ89I+Okoh1jWe30jxCiAcywC22p5\n" + "XLhrDkX1A63Z7XCH9ltwU2WMqWsVM98N2GR6MTujP7wtqdLExYN1AgMBAAE=\n"; + char *str; + tor_asprintf(&str, "onion-key\n" + "-----BEGIN RSA PUBLIC KEY-----\n" + "%s" + "-----END RSA PUBLIC KEY-----\n", base64_key); + const char *end = str + strlen(str); + const char **s = (const char **)&str; + const char decoded[128]; + + base64_decode((char *)decoded, sizeof(decoded), base64_key, + strlen(base64_key)); + + token_rule_t rule = T1("onion-key", R_IPO_ONION_KEY, NO_ARGS, NEED_KEY_1024); + + directory_token_t *token = get_next_token(area, s, end, &rule); + + tt_int_op(token->tp, OP_EQ, R_IPO_ONION_KEY); + tt_int_op(token->n_args, OP_EQ, 0); + tt_str_op(token->object_type, OP_EQ, "RSA PUBLIC KEY"); + tt_int_op(token->object_size, OP_EQ, 0); + tt_assert(!token->object_body); + tt_assert(token->key); + tt_assert(!token->error); + + // TODO: same with secret key + + + done: + memarea_drop_all(area); +} + #define PARSECOMMON_TEST(name) \ { #name, test_parsecommon_ ## name, 0, NULL, NULL } @@ -87,6 +128,7 @@ struct testcase_t parsecommon_tests[] = { PARSECOMMON_TEST(tokenize_string_null), PARSECOMMON_TEST(get_next_token_success), PARSECOMMON_TEST(get_next_token_concat_args), + PARSECOMMON_TEST(get_next_token_parse_keys), END_OF_TESTCASES }; From 7764d6dfc914d9a29c706cbcb2b399b09680c9b7 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Fri, 12 Oct 2018 19:19:40 +0300 Subject: [PATCH 0044/2557] Test RSA private key parsing with get_next_token() --- src/test/test_parsecommon.c | 39 ++++++++++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/src/test/test_parsecommon.c b/src/test/test_parsecommon.c index 519bbf79ea..ffd3902a8d 100644 --- a/src/test/test_parsecommon.c +++ b/src/test/test_parsecommon.c @@ -114,8 +114,45 @@ test_parsecommon_get_next_token_parse_keys(void *arg) tt_assert(token->key); tt_assert(!token->error); - // TODO: same with secret key + const char *base64_skey = + "MIICXAIBAAKBgQCwS810a2auH2PQchOBz9smNgjlDu31aq0IYlUohSYbhcv5AJ+d\n" + "DY0nfZWzS+mZPwzL3UiEnTt6PVv7AgoZ5V9ZJWJTKIURjJpkK0mstfJKHKIZhf84\n" + "pmFfRej9GQViB6NLtp1obOXJgJixSlMfw9doDI4NoAnEISCyH/tD77Qs2wIDAQAB\n" + "AoGAbDg8CKkdQOnX9c7xFpCnsE8fKqz9eddgHHNwXw1NFTwOt+2gDWKSMZmv2X5S\n" + "CVZg3owZxf5W0nT0D6Ny2+6nliak7foYAvkD0BsCiBhgftwC0zAo6k5rIbUKB3PJ\n" + "QLFXgpJhqWuXkODyt/hS/GTernR437WVSEGp1bnALqiFabECQQDaqHOxzoWY/nvH\n" + "KrfUi8EhqCnqERlRHwrW0MQZ1RPvF16OPPma+xa+ht/amfh3vYN5tZY82Zm43gGl\n" + "XWL5cZhNAkEAzmdSootYVnqLLLRMfHKXnO1XbaEcA/08MDNKGlSclBJixFenE8jX\n" + "iQsUbHwMJuGONvzWpRGPBP2f8xBd28ZtxwJARY+LZshtpfNniz/ixYJESaHG28je\n" + "xfjbKOW3TQSFV+2WTifFvHEeljQwKMoMyoMGvYRwLCGJjs9JtMLVxsdFjQJBAKwD\n" + "3BBvBQ39TuPQ1zWX4tb7zjMlY83HTFP3Sriq71tP/1QWoL2SUl56B2lp8E6vB/C3\n" + "wsMK4SCNprHRYAd7VZ0CQDKn6Zhd11P94PLs0msybFEh1VXr6CEW/BrxBgbL4ls6\n" + "dbX5XO0z4Ra8gYXgObgimhyMDYO98Idt5+Z3HIdyrSc=\n"; + const char decoded2[128]; + base64_decode((char *)decoded2, sizeof(decoded2), base64_skey, + strlen(base64_skey)); + + char *str2; + tor_asprintf(&str2, "client-key\n" + "-----BEGIN RSA PRIVATE KEY-----\n" + "%s" + "-----END RSA PRIVATE KEY-----\n", base64_skey); + const char *end2 = str2 + strlen(str2); + const char **s2 = (const char **)&str2; + + token_rule_t rule2 = T01("client-key", C_CLIENT_KEY, NO_ARGS, + NEED_SKEY_1024); + + directory_token_t *token2 = get_next_token(area, s2, end2, &rule2); + + tt_int_op(token2->tp, OP_EQ, C_CLIENT_KEY); + tt_int_op(token2->n_args, OP_EQ, 0); + tt_str_op(token2->object_type, OP_EQ, "RSA PRIVATE KEY"); + tt_int_op(token2->object_size, OP_EQ, 0); + tt_assert(!token2->object_body); + tt_assert(token2->key); + tt_assert(!token->error); done: memarea_drop_all(area); From 38a7033d3378a7b0cdc68258ebcd0e53371e8f24 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Mon, 15 Oct 2018 17:44:40 +0300 Subject: [PATCH 0045/2557] Fix memory management in test_parsecommon_get_next_token_parse_keys --- src/test/test_parsecommon.c | 41 +++++++++++++++---------------------- 1 file changed, 17 insertions(+), 24 deletions(-) diff --git a/src/test/test_parsecommon.c b/src/test/test_parsecommon.c index ffd3902a8d..6b5307619a 100644 --- a/src/test/test_parsecommon.c +++ b/src/test/test_parsecommon.c @@ -86,25 +86,22 @@ test_parsecommon_get_next_token_parse_keys(void *arg) (void)arg; memarea_t *area = memarea_new(); - const char *base64_key = + const char *str = + "onion-key\n" + "-----BEGIN RSA PUBLIC KEY-----\n" "MIGJAoGBAMDdIya33BfNlHOkzoTKSTT8EjD64waMfUr372syVHiFjHhObwKwGA5u\n" "sHaMIe9r+Ij/4C1dKyuXkcz3DOl6gWNhTD7dZ89I+Okoh1jWe30jxCiAcywC22p5\n" - "XLhrDkX1A63Z7XCH9ltwU2WMqWsVM98N2GR6MTujP7wtqdLExYN1AgMBAAE=\n"; - char *str; - tor_asprintf(&str, "onion-key\n" - "-----BEGIN RSA PUBLIC KEY-----\n" - "%s" - "-----END RSA PUBLIC KEY-----\n", base64_key); + "XLhrDkX1A63Z7XCH9ltwU2WMqWsVM98N2GR6MTujP7wtqdLExYN1AgMBAAE=\n" + "-----END RSA PUBLIC KEY-----\n"; + const char *end = str + strlen(str); const char **s = (const char **)&str; - const char decoded[128]; - - base64_decode((char *)decoded, sizeof(decoded), base64_key, - strlen(base64_key)); + directory_token_t *token = NULL; + directory_token_t *token2 = NULL; token_rule_t rule = T1("onion-key", R_IPO_ONION_KEY, NO_ARGS, NEED_KEY_1024); - directory_token_t *token = get_next_token(area, s, end, &rule); + token = get_next_token(area, s, end, &rule); tt_int_op(token->tp, OP_EQ, R_IPO_ONION_KEY); tt_int_op(token->n_args, OP_EQ, 0); @@ -114,7 +111,9 @@ test_parsecommon_get_next_token_parse_keys(void *arg) tt_assert(token->key); tt_assert(!token->error); - const char *base64_skey = + const char *str2 = + "client-key\n" + "-----BEGIN RSA PRIVATE KEY-----\n" "MIICXAIBAAKBgQCwS810a2auH2PQchOBz9smNgjlDu31aq0IYlUohSYbhcv5AJ+d\n" "DY0nfZWzS+mZPwzL3UiEnTt6PVv7AgoZ5V9ZJWJTKIURjJpkK0mstfJKHKIZhf84\n" "pmFfRej9GQViB6NLtp1obOXJgJixSlMfw9doDI4NoAnEISCyH/tD77Qs2wIDAQAB\n" @@ -127,24 +126,16 @@ test_parsecommon_get_next_token_parse_keys(void *arg) "xfjbKOW3TQSFV+2WTifFvHEeljQwKMoMyoMGvYRwLCGJjs9JtMLVxsdFjQJBAKwD\n" "3BBvBQ39TuPQ1zWX4tb7zjMlY83HTFP3Sriq71tP/1QWoL2SUl56B2lp8E6vB/C3\n" "wsMK4SCNprHRYAd7VZ0CQDKn6Zhd11P94PLs0msybFEh1VXr6CEW/BrxBgbL4ls6\n" - "dbX5XO0z4Ra8gYXgObgimhyMDYO98Idt5+Z3HIdyrSc=\n"; + "dbX5XO0z4Ra8gYXgObgimhyMDYO98Idt5+Z3HIdyrSc=\n" + "-----END RSA PRIVATE KEY-----\n"; - const char decoded2[128]; - base64_decode((char *)decoded2, sizeof(decoded2), base64_skey, - strlen(base64_skey)); - - char *str2; - tor_asprintf(&str2, "client-key\n" - "-----BEGIN RSA PRIVATE KEY-----\n" - "%s" - "-----END RSA PRIVATE KEY-----\n", base64_skey); const char *end2 = str2 + strlen(str2); const char **s2 = (const char **)&str2; token_rule_t rule2 = T01("client-key", C_CLIENT_KEY, NO_ARGS, NEED_SKEY_1024); - directory_token_t *token2 = get_next_token(area, s2, end2, &rule2); + token2 = get_next_token(area, s2, end2, &rule2); tt_int_op(token2->tp, OP_EQ, C_CLIENT_KEY); tt_int_op(token2->n_args, OP_EQ, 0); @@ -155,6 +146,8 @@ test_parsecommon_get_next_token_parse_keys(void *arg) tt_assert(!token->error); done: + if (token) token_clear(token); + if (token2) token_clear(token2); memarea_drop_all(area); } From 6c5ba2662af08c8094e024a2c04141e776d966a6 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Mon, 15 Oct 2018 18:51:08 +0300 Subject: [PATCH 0046/2557] Test object parsing in get_next_token() --- src/test/test_parsecommon.c | 57 +++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/src/test/test_parsecommon.c b/src/test/test_parsecommon.c index 6b5307619a..59e366bc4e 100644 --- a/src/test/test_parsecommon.c +++ b/src/test/test_parsecommon.c @@ -151,6 +151,62 @@ test_parsecommon_get_next_token_parse_keys(void *arg) memarea_drop_all(area); } +static void +test_parsecommon_get_next_token_object(void *arg) +{ + memarea_t *area = memarea_new(); + + const char *str = + "directory-signature 0232AF901C31A04EE9848595AF9BB7620D4C5B2E " + "CD1FD971855430880D3C31E0331C5C55800C2F79\n" + "-----BEGIN SIGNATURE-----\n" + "dLTbc1Lad/OWKBJhA/dERzDHumswTAzBFAWAz2vnQhLsebs1SOm0W/vceEsiEkiF\n" + "A+JJSzIyfywJc6Mnk7aKMEIFjOO/MaxuAp4zv+q+JonJkF0ExjMqvKR0D6pSFmfN\n" + "cnemnxGHxNuPDnKl0imbWKmWDsHtwgi4zWeTq3MekfMOXKi6gIh+bDFzCs9/Vquh\n" + "uNKJI1jW/A2DEKeaSAODEv9VoCsYSvbVVEuHCBWjeNAurd5aL26BrAolW6m7pkD6\n" + "I+cQ8dQG6Wa/Zt6gLXtBbOP2o/iDI7ahDP9diNkBI/rm4nfp9j4piTwsqpi7xz9J\n" + "Ua9DEZB9KbJHVX1rGShrLA==\n" + "-----END SIGNATURE-----\n"; + + const char *end = str + strlen(str); + const char **s = &str; + token_rule_t rule = T("directory-signature", K_DIRECTORY_SIGNATURE, + GE(2), NEED_OBJ); + (void)arg; + + directory_token_t *token = get_next_token(area, s, end, &rule); + + tt_int_op(token->tp, OP_EQ, K_DIRECTORY_SIGNATURE); + tt_int_op(token->n_args, OP_EQ, 2); + tt_str_op(token->args[0], OP_EQ, + "0232AF901C31A04EE9848595AF9BB7620D4C5B2E"); + tt_str_op(token->args[1], OP_EQ, + "CD1FD971855430880D3C31E0331C5C55800C2F79"); + + tt_assert(!token->error); + + char decoded[256]; + const char *signature = + "dLTbc1Lad/OWKBJhA/dERzDHumswTAzBFAWAz2vnQhLsebs1SOm0W/vceEsiEkiF\n" + "A+JJSzIyfywJc6Mnk7aKMEIFjOO/MaxuAp4zv+q+JonJkF0ExjMqvKR0D6pSFmfN\n" + "cnemnxGHxNuPDnKl0imbWKmWDsHtwgi4zWeTq3MekfMOXKi6gIh+bDFzCs9/Vquh\n" + "uNKJI1jW/A2DEKeaSAODEv9VoCsYSvbVVEuHCBWjeNAurd5aL26BrAolW6m7pkD6\n" + "I+cQ8dQG6Wa/Zt6gLXtBbOP2o/iDI7ahDP9diNkBI/rm4nfp9j4piTwsqpi7xz9J\n" + "Ua9DEZB9KbJHVX1rGShrLA==\n"; + tt_assert(signature); + size_t signature_len = strlen(signature); + base64_decode(decoded, sizeof(decoded), signature, signature_len); + + tt_str_op(token->object_type, OP_EQ, "SIGNATURE"); + tt_int_op(token->object_size, OP_EQ, 256); + tt_mem_op(token->object_body, OP_EQ, decoded, 256); + + tt_assert(!token->key); + + done: + memarea_drop_all(area); +} + #define PARSECOMMON_TEST(name) \ { #name, test_parsecommon_ ## name, 0, NULL, NULL } @@ -159,6 +215,7 @@ struct testcase_t parsecommon_tests[] = { PARSECOMMON_TEST(get_next_token_success), PARSECOMMON_TEST(get_next_token_concat_args), PARSECOMMON_TEST(get_next_token_parse_keys), + PARSECOMMON_TEST(get_next_token_object), END_OF_TESTCASES }; From 81731a290d2a806047901c1835afb557533a0350 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Mon, 15 Oct 2018 20:01:42 +0300 Subject: [PATCH 0047/2557] Unit-test some error conditions in get_next_token() --- src/test/test_parsecommon.c | 173 ++++++++++++++++++++++++++++++++++++ 1 file changed, 173 insertions(+) diff --git a/src/test/test_parsecommon.c b/src/test/test_parsecommon.c index 59e366bc4e..d8719747e2 100644 --- a/src/test/test_parsecommon.c +++ b/src/test/test_parsecommon.c @@ -207,6 +207,173 @@ test_parsecommon_get_next_token_object(void *arg) memarea_drop_all(area); } +static void +test_parsecommon_get_next_token_err_too_many_args(void *arg) +{ + memarea_t *area = memarea_new(); + const char *str = "uptime 1024 1024 1024"; + const char *end = str + strlen(str); + const char **s = &str; + token_rule_t table = T01("uptime", K_UPTIME, EQ(1), NO_OBJ); + (void)arg; + + directory_token_t *token = get_next_token(area, s, end, &table); + + tt_int_op(token->tp, OP_EQ, ERR_); + tt_str_op(token->error, OP_EQ, "Too many arguments to uptime"); + + done: + memarea_drop_all(area); + return; +} + +static void +test_parsecommon_get_next_token_err_too_few_args(void *arg) +{ + memarea_t *area = memarea_new(); + const char *str = "uptime"; + const char *end = str + strlen(str); + const char **s = &str; + token_rule_t table = T01("uptime", K_UPTIME, EQ(1), NO_OBJ); + (void)arg; + + directory_token_t *token = get_next_token(area, s, end, &table); + + tt_int_op(token->tp, OP_EQ, ERR_); + tt_str_op(token->error, OP_EQ, "Too few arguments to uptime"); + + done: + memarea_drop_all(area); + return; +} + +static void +test_parsecommon_get_next_token_err_obj_missing_endline(void *arg) +{ + memarea_t *area = memarea_new(); + + const char *str = + "directory-signature 0232AF901C31A04EE9848595AF9BB7620D4C5B2E " + "CD1FD971855430880D3C31E0331C5C55800C2F79\n" + "-----BEGIN SIGNATURE-----\n" + "dLTbc1Lad/OWKBJhA/dERzDHumswTAzBFAWAz2vnQhLsebs1SOm0W/vceEsiEkiF\n" + "A+JJSzIyfywJc6Mnk7aKMEIFjOO/MaxuAp4zv+q+JonJkF0ExjMqvKR0D6pSFmfN\n" + "cnemnxGHxNuPDnKl0imbWKmWDsHtwgi4zWeTq3MekfMOXKi6gIh+bDFzCs9/Vquh\n" + "uNKJI1jW/A2DEKeaSAODEv9VoCsYSvbVVEuHCBWjeNAurd5aL26BrAolW6m7pkD6\n" + "I+cQ8dQG6Wa/Zt6gLXtBbOP2o/iDI7ahDP9diNkBI/rm4nfp9j4piTwsqpi7xz9J\n" + "Ua9DEZB9KbJHVX1rGShrLA==\n"; + + const char *end = str + strlen(str); + const char **s = &str; + token_rule_t rule = T("directory-signature", K_DIRECTORY_SIGNATURE, + GE(2), NEED_OBJ); + (void)arg; + + directory_token_t *token = get_next_token(area, s, end, &rule); + + tt_int_op(token->tp, OP_EQ, ERR_); + tt_str_op(token->error, OP_EQ, "Malformed object: missing object end line"); + + done: + memarea_drop_all(area); + return; +} + +static void +test_parsecommon_get_next_token_err_bad_beginline(void *arg) +{ + memarea_t *area = memarea_new(); + + const char *str = + "directory-signature 0232AF901C31A04EE9848595AF9BB7620D4C5B2E " + "CD1FD971855430880D3C31E0331C5C55800C2F79\n" + "-----BEGIN SIGNATURE-Z---\n" + "dLTbc1Lad/OWKBJhA/dERzDHumswTAzBFAWAz2vnQhLsebs1SOm0W/vceEsiEkiF\n" + "A+JJSzIyfywJc6Mnk7aKMEIFjOO/MaxuAp4zv+q+JonJkF0ExjMqvKR0D6pSFmfN\n" + "cnemnxGHxNuPDnKl0imbWKmWDsHtwgi4zWeTq3MekfMOXKi6gIh+bDFzCs9/Vquh\n" + "uNKJI1jW/A2DEKeaSAODEv9VoCsYSvbVVEuHCBWjeNAurd5aL26BrAolW6m7pkD6\n" + "I+cQ8dQG6Wa/Zt6gLXtBbOP2o/iDI7ahDP9diNkBI/rm4nfp9j4piTwsqpi7xz9J\n" + "Ua9DEZB9KbJHVX1rGShrLA==\n" + "-----END SIGNATURE-----\n"; + + const char *end = str + strlen(str); + const char **s = &str; + token_rule_t rule = T("directory-signature", K_DIRECTORY_SIGNATURE, + GE(2), NEED_OBJ); + (void)arg; + + directory_token_t *token = get_next_token(area, s, end, &rule); + + tt_int_op(token->tp, OP_EQ, ERR_); + tt_str_op(token->error, OP_EQ, "Malformed object: bad begin line"); + + done: + memarea_drop_all(area); + return; +} + +static void +test_parsecommon_get_next_token_err_tag_mismatch(void *arg) +{ + memarea_t *area = memarea_new(); + + const char *str = + "directory-signature 0232AF901C31A04EE9848595AF9BB7620D4C5B2E " + "CD1FD971855430880D3C31E0331C5C55800C2F79\n" + "-----BEGIN SIGNATURE-----\n" + "dLTbc1Lad/OWKBJhA/dERzDHumswTAzBFAWAz2vnQhLsebs1SOm0W/vceEsiEkiF\n" + "A+JJSzIyfywJc6Mnk7aKMEIFjOO/MaxuAp4zv+q+JonJkF0ExjMqvKR0D6pSFmfN\n" + "cnemnxGHxNuPDnKl0imbWKmWDsHtwgi4zWeTq3MekfMOXKi6gIh+bDFzCs9/Vquh\n" + "uNKJI1jW/A2DEKeaSAODEv9VoCsYSvbVVEuHCBWjeNAurd5aL26BrAolW6m7pkD6\n" + "I+cQ8dQG6Wa/Zt6gLXtBbOP2o/iDI7ahDP9diNkBI/rm4nfp9j4piTwsqpi7xz9J\n" + "Ua9DEZB9KbJHVX1rGShrLA==\n" + "-----END SOMETHINGELSE-----\n"; + + const char *end = str + strlen(str); + const char **s = &str; + token_rule_t rule = T("directory-signature", K_DIRECTORY_SIGNATURE, + GE(2), NEED_OBJ); + (void)arg; + + directory_token_t *token = get_next_token(area, s, end, &rule); + + tt_int_op(token->tp, OP_EQ, ERR_); + tt_str_op(token->error, OP_EQ, + "Malformed object: mismatched end tag SIGNATURE"); + + done: + memarea_drop_all(area); + return; +} + +static void +test_parsecommon_get_next_token_err_bad_base64(void *arg) +{ + memarea_t *area = memarea_new(); + + const char *str = + "directory-signature 0232AF901C31A04EE9848595AF9BB7620D4C5B2E " + "CD1FD971855430880D3C31E0331C5C55800C2F79\n" + "-----BEGIN SIGNATURE-----\n" + "%%@%%%%%%%!!!'\n" + "-----END SIGNATURE-----\n"; + + const char *end = str + strlen(str); + const char **s = &str; + token_rule_t rule = T("directory-signature", K_DIRECTORY_SIGNATURE, + GE(2), NEED_OBJ); + (void)arg; + + directory_token_t *token = get_next_token(area, s, end, &rule); + + tt_int_op(token->tp, OP_EQ, ERR_); + tt_str_op(token->error, OP_EQ, "Malformed object: bad base64-encoded data"); + + done: + memarea_drop_all(area); + return; +} + #define PARSECOMMON_TEST(name) \ { #name, test_parsecommon_ ## name, 0, NULL, NULL } @@ -216,6 +383,12 @@ struct testcase_t parsecommon_tests[] = { PARSECOMMON_TEST(get_next_token_concat_args), PARSECOMMON_TEST(get_next_token_parse_keys), PARSECOMMON_TEST(get_next_token_object), + PARSECOMMON_TEST(get_next_token_err_too_many_args), + PARSECOMMON_TEST(get_next_token_err_too_few_args), + PARSECOMMON_TEST(get_next_token_err_obj_missing_endline), + PARSECOMMON_TEST(get_next_token_err_bad_beginline), + PARSECOMMON_TEST(get_next_token_err_tag_mismatch), + PARSECOMMON_TEST(get_next_token_err_bad_base64), END_OF_TESTCASES }; From 78c446af7ef9900c31e84496a4547550d4c191e2 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Tue, 16 Oct 2018 14:53:04 +0300 Subject: [PATCH 0048/2557] Unit-test multiple line parsing with tokenize_string() --- src/test/test_parsecommon.c | 46 +++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/src/test/test_parsecommon.c b/src/test/test_parsecommon.c index d8719747e2..0c754a8fee 100644 --- a/src/test/test_parsecommon.c +++ b/src/test/test_parsecommon.c @@ -34,6 +34,51 @@ test_parsecommon_tokenize_string_null(void *arg) return; } +static void +test_parsecommon_tokenize_string_multiple_lines(void *arg) +{ + memarea_t *area = memarea_new(); + smartlist_t *tokens = smartlist_new(); + + (void)arg; + + token_rule_t table[] = { + T01("uptime", K_UPTIME, GE(1), NO_OBJ), + T01("hibernating", K_HIBERNATING, GE(1), NO_OBJ), + T1( "published", K_PUBLISHED, CONCAT_ARGS, NO_OBJ), + END_OF_TABLE, + }; + + char *str = tor_strdup( + "hibernating 0\nuptime 1024\n" + "published 2018-10-15 10:00:00\n"); + + int retval = + tokenize_string(area, str, NULL, + tokens, table, 0); + + tt_int_op(smartlist_len(tokens), OP_EQ, 3); + directory_token_t *token = smartlist_get(tokens, 0); + + tt_int_op(token->tp, OP_EQ, K_HIBERNATING); + + token = smartlist_get(tokens, 1); + + tt_int_op(token->tp, OP_EQ, K_UPTIME); + + token = smartlist_get(tokens, 2); + + tt_int_op(token->tp, OP_EQ, K_PUBLISHED); + + tt_int_op(retval, OP_EQ, 0); + + done: + tor_free(str); + memarea_drop_all(area); + smartlist_free(tokens); + return; +} + static void test_parsecommon_get_next_token_success(void *arg) { @@ -379,6 +424,7 @@ test_parsecommon_get_next_token_err_bad_base64(void *arg) struct testcase_t parsecommon_tests[] = { PARSECOMMON_TEST(tokenize_string_null), + PARSECOMMON_TEST(tokenize_string_multiple_lines), PARSECOMMON_TEST(get_next_token_success), PARSECOMMON_TEST(get_next_token_concat_args), PARSECOMMON_TEST(get_next_token_parse_keys), From 7c8bf2f7c7adbae3b0f56631c2a2c036076b322d Mon Sep 17 00:00:00 2001 From: rl1987 Date: Tue, 16 Oct 2018 15:22:48 +0300 Subject: [PATCH 0049/2557] Add testcase for too few elements in tokenize_string() input --- src/test/test_parsecommon.c | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/src/test/test_parsecommon.c b/src/test/test_parsecommon.c index 0c754a8fee..e68d0fb884 100644 --- a/src/test/test_parsecommon.c +++ b/src/test/test_parsecommon.c @@ -79,6 +79,36 @@ test_parsecommon_tokenize_string_multiple_lines(void *arg) return; } +static void +test_parsecommon_tokenize_string_min_cnt(void *arg) +{ + memarea_t *area = memarea_new(); + smartlist_t *tokens = smartlist_new(); + + (void)arg; + + token_rule_t table[] = { + T01("uptime", K_UPTIME, EQ(2), NO_OBJ), + T01("hibernating", K_HIBERNATING, GE(1), NO_OBJ), + END_OF_TABLE, + }; + + // Missing "uptime" + char *str = tor_strdup("uptime 1024\nhibernating 0\n"); + + int retval = + tokenize_string(area, str, NULL, + tokens, table, 0); + + tt_int_op(retval, OP_EQ, -1); + + done: + tor_free(str); + memarea_drop_all(area); + smartlist_free(tokens); + return; +} + static void test_parsecommon_get_next_token_success(void *arg) { @@ -425,6 +455,7 @@ test_parsecommon_get_next_token_err_bad_base64(void *arg) struct testcase_t parsecommon_tests[] = { PARSECOMMON_TEST(tokenize_string_null), PARSECOMMON_TEST(tokenize_string_multiple_lines), + PARSECOMMON_TEST(tokenize_string_min_cnt), PARSECOMMON_TEST(get_next_token_success), PARSECOMMON_TEST(get_next_token_concat_args), PARSECOMMON_TEST(get_next_token_parse_keys), From 1a4edceee9049d19a97b08dc28e87fa346d02536 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Tue, 16 Oct 2018 15:31:47 +0300 Subject: [PATCH 0050/2557] Add testcase for too many elements in tokenize_string() input --- src/test/test_parsecommon.c | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/src/test/test_parsecommon.c b/src/test/test_parsecommon.c index e68d0fb884..182f6fba5d 100644 --- a/src/test/test_parsecommon.c +++ b/src/test/test_parsecommon.c @@ -109,6 +109,37 @@ test_parsecommon_tokenize_string_min_cnt(void *arg) return; } +static void +test_parsecommon_tokenize_string_max_cnt(void *arg) +{ + memarea_t *area = memarea_new(); + smartlist_t *tokens = smartlist_new(); + + (void)arg; + + token_rule_t table[] = { + T01("uptime", K_UPTIME, EQ(1), NO_OBJ), + T01("hibernating", K_HIBERNATING, GE(1), NO_OBJ), + END_OF_TABLE, + }; + + // "uptime" expected once, but occurs twice in input. + char *str = tor_strdup( + "uptime 1024\nuptime 2048\nhibernating 0\n"); + + int retval = + tokenize_string(area, str, NULL, + tokens, table, 0); + + tt_int_op(retval, OP_EQ, -1); + + done: + tor_free(str); + memarea_drop_all(area); + smartlist_free(tokens); + return; +} + static void test_parsecommon_get_next_token_success(void *arg) { @@ -456,6 +487,7 @@ struct testcase_t parsecommon_tests[] = { PARSECOMMON_TEST(tokenize_string_null), PARSECOMMON_TEST(tokenize_string_multiple_lines), PARSECOMMON_TEST(tokenize_string_min_cnt), + PARSECOMMON_TEST(tokenize_string_max_cnt), PARSECOMMON_TEST(get_next_token_success), PARSECOMMON_TEST(get_next_token_concat_args), PARSECOMMON_TEST(get_next_token_parse_keys), From f10d664fd14b318fba23c7290c800590d0a474b4 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Tue, 16 Oct 2018 15:54:11 +0300 Subject: [PATCH 0051/2557] Test AT_END checking in tokenize_string() --- src/test/test_parsecommon.c | 62 +++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/src/test/test_parsecommon.c b/src/test/test_parsecommon.c index 182f6fba5d..13e4ac8c7f 100644 --- a/src/test/test_parsecommon.c +++ b/src/test/test_parsecommon.c @@ -140,6 +140,66 @@ test_parsecommon_tokenize_string_max_cnt(void *arg) return; } +static void +test_parsecommon_tokenize_string_at_start(void *arg) +{ + memarea_t *area = memarea_new(); + smartlist_t *tokens = smartlist_new(); + + (void)arg; + + token_rule_t table[] = { + T1_START("client-name", C_CLIENT_NAME, CONCAT_ARGS, NO_OBJ), + T01("uptime", K_UPTIME, EQ(1), NO_OBJ), + END_OF_TABLE, + }; + + // "client-name" is not the first line. + char *str = tor_strdup( + "uptime 1024\nclient-name Alice\n"); + + int retval = + tokenize_string(area, str, NULL, tokens, table, 0); + + tt_int_op(retval, OP_EQ, -1); + + done: + tor_free(str); + memarea_drop_all(area); + smartlist_free(tokens); + return; +} + +static void +test_parsecommon_tokenize_string_at_end(void *arg) +{ + memarea_t *area = memarea_new(); + smartlist_t *tokens = smartlist_new(); + + (void)arg; + + token_rule_t table[] = { + T1_END("client-name", C_CLIENT_NAME, CONCAT_ARGS, NO_OBJ), + T01("uptime", K_UPTIME, EQ(1), NO_OBJ), + END_OF_TABLE, + }; + + // "client-name" is not the last line. + char *str = tor_strdup( + "client-name Alice\nuptime 1024\n"); + + int retval = + tokenize_string(area, str, NULL, tokens, table, 0); + + tt_int_op(retval, OP_EQ, -1); + + done: + tor_free(str); + memarea_drop_all(area); + smartlist_free(tokens); + return; +} + static void test_parsecommon_get_next_token_success(void *arg) { @@ -488,6 +548,8 @@ struct testcase_t parsecommon_tests[] = { PARSECOMMON_TEST(tokenize_string_multiple_lines), PARSECOMMON_TEST(tokenize_string_min_cnt), PARSECOMMON_TEST(tokenize_string_max_cnt), + PARSECOMMON_TEST(tokenize_string_at_start), + PARSECOMMON_TEST(tokenize_string_at_end), PARSECOMMON_TEST(get_next_token_success), PARSECOMMON_TEST(get_next_token_concat_args), PARSECOMMON_TEST(get_next_token_parse_keys), From 7fd82a4570c4c3ea887c02190a54e424eb5f4c5f Mon Sep 17 00:00:00 2001 From: rl1987 Date: Tue, 16 Oct 2018 17:39:52 +0300 Subject: [PATCH 0052/2557] One testcase for annotation handling in tokenize_string() --- src/test/test_parsecommon.c | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/src/test/test_parsecommon.c b/src/test/test_parsecommon.c index 13e4ac8c7f..7ce4b71b00 100644 --- a/src/test/test_parsecommon.c +++ b/src/test/test_parsecommon.c @@ -200,6 +200,33 @@ test_parsecommon_tokenize_string_at_end(void *arg) return; } +static void +test_parsecommon_tokenize_string_no_annotations(void *arg) +{ + memarea_t *area = memarea_new(); + smartlist_t *tokens = smartlist_new(); + + (void)arg; + + token_rule_t table[] = { + A01("@last-listed", A_LAST_LISTED, CONCAT_ARGS, NO_OBJ), + END_OF_TABLE, + }; + + char *str = tor_strdup("@last-listed 2018-09-21 15:30:03\n"); + + int retval = + tokenize_string(area, str, NULL, tokens, table, 0); + + tt_int_op(retval, OP_EQ, -1); + + done: + tor_free(str); + memarea_drop_all(area); + smartlist_free(tokens); + return; +} + static void test_parsecommon_get_next_token_success(void *arg) { @@ -550,6 +577,7 @@ struct testcase_t parsecommon_tests[] = { PARSECOMMON_TEST(tokenize_string_max_cnt), PARSECOMMON_TEST(tokenize_string_at_start), PARSECOMMON_TEST(tokenize_string_at_end), + PARSECOMMON_TEST(tokenize_string_no_annotations), PARSECOMMON_TEST(get_next_token_success), PARSECOMMON_TEST(get_next_token_concat_args), PARSECOMMON_TEST(get_next_token_parse_keys), From 65864be9bc59fb3743d2cebf497d0a4268596f95 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Tue, 16 Oct 2018 17:53:17 +0300 Subject: [PATCH 0053/2557] Add changes file --- changes/ticket27625 | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 changes/ticket27625 diff --git a/changes/ticket27625 b/changes/ticket27625 new file mode 100644 index 0000000000..33d40adf34 --- /dev/null +++ b/changes/ticket27625 @@ -0,0 +1,4 @@ + o Testing: + - Write some unit tests for tokenize_string() and + get_next_token() functions. Resolves ticket 27625. + From 2f0744b3e6f579f25db1ed6e048d0418ac2ab570 Mon Sep 17 00:00:00 2001 From: cypherpunks Date: Wed, 17 Oct 2018 00:16:21 +0000 Subject: [PATCH 0054/2557] rust/tor_util: drop unsafe block in cstr! This is unnecessary just to get an empty string, there's Default::default(). Fix on 8fff331bb095dc6f5e2fe2ecfc9ab08ea9e2fe97. --- changes/ticket28077 | 3 +++ src/rust/tor_util/strings.rs | 6 +----- 2 files changed, 4 insertions(+), 5 deletions(-) create mode 100644 changes/ticket28077 diff --git a/changes/ticket28077 b/changes/ticket28077 new file mode 100644 index 0000000000..2b5afb1678 --- /dev/null +++ b/changes/ticket28077 @@ -0,0 +1,3 @@ + o Code simplification and refactoring: + - Remove unnecessarily unsafe code from the rust macro cstr!. Closes + ticket 28077. diff --git a/src/rust/tor_util/strings.rs b/src/rust/tor_util/strings.rs index d64275e06b..71a908a58c 100644 --- a/src/rust/tor_util/strings.rs +++ b/src/rust/tor_util/strings.rs @@ -105,11 +105,7 @@ macro_rules! cstr { ($($bytes:expr),*) => ( ::std::ffi::CStr::from_bytes_with_nul( concat!($($bytes),*, "\0").as_bytes() - ).unwrap_or( - unsafe{ - ::std::ffi::CStr::from_bytes_with_nul_unchecked(b"\0") - } - ) + ).unwrap_or_default() ) } From fd2e0ac1c39088a315ccce648e52536a43ba3c27 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 17 Oct 2018 14:46:38 -0400 Subject: [PATCH 0055/2557] Bump to 0.3.6.0-alpha-dev. --- configure.ac | 2 +- contrib/win32build/tor-mingw.nsi.in | 2 +- src/win32/orconfig.h | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/configure.ac b/configure.ac index d6e0c84904..292244f56f 100644 --- a/configure.ac +++ b/configure.ac @@ -4,7 +4,7 @@ dnl Copyright (c) 2007-2018, The Tor Project, Inc. dnl See LICENSE for licensing information AC_PREREQ([2.63]) -AC_INIT([tor],[0.3.5.3-alpha]) +AC_INIT([tor],[0.3.6.0-alpha-dev]) AC_CONFIG_SRCDIR([src/app/main/tor_main.c]) AC_CONFIG_MACRO_DIR([m4]) diff --git a/contrib/win32build/tor-mingw.nsi.in b/contrib/win32build/tor-mingw.nsi.in index 8a82b013e4..8b9b709698 100644 --- a/contrib/win32build/tor-mingw.nsi.in +++ b/contrib/win32build/tor-mingw.nsi.in @@ -8,7 +8,7 @@ !include "LogicLib.nsh" !include "FileFunc.nsh" !insertmacro GetParameters -!define VERSION "0.3.5.3-alpha" +!define VERSION "0.3.6.0-alpha-dev" !define INSTALLER "tor-${VERSION}-win32.exe" !define WEBSITE "https://www.torproject.org/" !define LICENSE "LICENSE" diff --git a/src/win32/orconfig.h b/src/win32/orconfig.h index 7a4a1d72c0..de3bf09282 100644 --- a/src/win32/orconfig.h +++ b/src/win32/orconfig.h @@ -218,7 +218,7 @@ #define USING_TWOS_COMPLEMENT /* Version number of package */ -#define VERSION "0.3.5.3-alpha" +#define VERSION "0.3.6.0-alpha-dev" From f07ab5b95c3d38cf3fb85f9c1ea1f08aed003971 Mon Sep 17 00:00:00 2001 From: cypherpunks Date: Sun, 30 Sep 2018 14:05:48 +0000 Subject: [PATCH 0056/2557] evloop: fix docs alert_sockets_t was moved in 544ab27a949406628809869111b7288017a5bcb1. --- src/lib/evloop/workqueue.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/evloop/workqueue.c b/src/lib/evloop/workqueue.c index 931f65e710..5e59b48fe6 100644 --- a/src/lib/evloop/workqueue.c +++ b/src/lib/evloop/workqueue.c @@ -15,7 +15,7 @@ * * The main thread informs the worker threads of pending work by using a * condition variable. The workers inform the main process of completed work - * by using an alert_sockets_t object, as implemented in compat_threads.c. + * by using an alert_sockets_t object, as implemented in net/alertsock.c. * * The main thread can also queue an "update" that will be handled by all the * workers. This is useful for updating state that all the workers share. From a56451af4269235df45416a37a5e37be6e63c827 Mon Sep 17 00:00:00 2001 From: cypherpunks Date: Sun, 30 Sep 2018 14:37:11 +0000 Subject: [PATCH 0057/2557] evloop: fix docs for threadpool_register_reply_event Commit 6a5f62f68f2c73dbbbbddb4fa3759586f4c2b0dc ultimately didn't include the base argument, and the callback is named cb. --- src/lib/evloop/workqueue.c | 4 ++-- src/lib/evloop/workqueue.h | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/lib/evloop/workqueue.c b/src/lib/evloop/workqueue.c index 5e59b48fe6..5471f87b04 100644 --- a/src/lib/evloop/workqueue.c +++ b/src/lib/evloop/workqueue.c @@ -622,8 +622,8 @@ reply_event_cb(evutil_socket_t sock, short events, void *arg) tp->reply_cb(tp); } -/** Register the threadpool tp's reply queue with the libevent - * mainloop of base. If tp is provided, it is run after +/** Register the threadpool tp's reply queue with Tor's global + * libevent mainloop. If cb is provided, it is run after * each time there is work to process from the reply queue. Return 0 on * success, -1 on failure. */ diff --git a/src/lib/evloop/workqueue.h b/src/lib/evloop/workqueue.h index da292d1f05..10d5d47464 100644 --- a/src/lib/evloop/workqueue.h +++ b/src/lib/evloop/workqueue.h @@ -63,7 +63,6 @@ replyqueue_t *threadpool_get_replyqueue(threadpool_t *tp); replyqueue_t *replyqueue_new(uint32_t alertsocks_flags); void replyqueue_process(replyqueue_t *queue); -struct event_base; int threadpool_register_reply_event(threadpool_t *tp, void (*cb)(threadpool_t *tp)); From 3a8f32067d1cb4d5aee320fbd3d1c02541f4e112 Mon Sep 17 00:00:00 2001 From: David Goulet Date: Tue, 18 Sep 2018 13:50:12 -0400 Subject: [PATCH 0058/2557] hs-v3: Consolidate descriptor cookie computation code Both client and service had their own code for this. Consolidate into one place so we avoid duplication. Closes #27549 Signed-off-by: David Goulet --- changes/ticket27549 | 3 ++ src/feature/hs/hs_descriptor.c | 89 ++++++++++++++++++++++------------ 2 files changed, 61 insertions(+), 31 deletions(-) create mode 100644 changes/ticket27549 diff --git a/changes/ticket27549 b/changes/ticket27549 new file mode 100644 index 0000000000..51d0f24757 --- /dev/null +++ b/changes/ticket27549 @@ -0,0 +1,3 @@ + o Code simplification and refactoring (hidden service v3): + - Consolidate the authorized client descriptor cookie computation code + from client and service into one function. Closes ticket 27549. diff --git a/src/feature/hs/hs_descriptor.c b/src/feature/hs/hs_descriptor.c index d0cdffdf10..1232182ea5 100644 --- a/src/feature/hs/hs_descriptor.c +++ b/src/feature/hs/hs_descriptor.c @@ -1400,6 +1400,49 @@ encrypted_data_length_is_valid(size_t len) return 0; } +/* Build the KEYS component for the authorized client computation. The format + * of the construction is: + * + * SECRET_SEED = x25519(sk, pk) + * KEYS = KDF(subcredential | SECRET_SEED, 40) + * + * The keys_out parameter will points to the buffer containing the KEYS. The + * caller should wipe and free its content once done with it. This function + * can't fail. */ +static void +build_descriptor_cookie_keys(const uint8_t *subcredential, + size_t subcredential_len, + const curve25519_secret_key_t *sk, + const curve25519_public_key_t *pk, + uint8_t **keys_out) +{ + uint8_t secret_seed[CURVE25519_OUTPUT_LEN]; + uint8_t *keystream; + size_t keystream_len = HS_DESC_CLIENT_ID_LEN + HS_DESC_COOKIE_KEY_LEN; + crypto_xof_t *xof; + + tor_assert(subcredential); + tor_assert(sk); + tor_assert(pk); + tor_assert(keys_out); + + keystream = tor_malloc_zero(keystream_len); + + /* Calculate x25519(sk, pk) to get the secret seed. */ + curve25519_handshake(secret_seed, sk, pk); + + /* Calculate KEYS = KDF(subcredential | SECRET_SEED, 40) */ + xof = crypto_xof_new(); + crypto_xof_add_bytes(xof, subcredential, subcredential_len); + crypto_xof_add_bytes(xof, secret_seed, sizeof(secret_seed)); + crypto_xof_squeeze_bytes(xof, keystream, keystream_len); + crypto_xof_free(xof); + + memwipe(secret_seed, 0, sizeof(secret_seed)); + + *keys_out = keystream; +} + /* Decrypt the descriptor cookie given the descriptor, the auth client, * and the client secret key. On sucess, return 0 and a newly allocated * descriptor cookie descriptor_cookie_out. On error or if the client id @@ -1412,12 +1455,10 @@ decrypt_descriptor_cookie(const hs_descriptor_t *desc, uint8_t **descriptor_cookie_out) { int ret = -1; - uint8_t secret_seed[CURVE25519_OUTPUT_LEN]; - uint8_t keystream[HS_DESC_CLIENT_ID_LEN + HS_DESC_COOKIE_KEY_LEN]; - uint8_t *cookie_key = NULL; + uint8_t *keystream = NULL; uint8_t *descriptor_cookie = NULL; + const uint8_t *cookie_key = NULL; crypto_cipher_t *cipher = NULL; - crypto_xof_t *xof = NULL; tor_assert(desc); tor_assert(client); @@ -1429,16 +1470,11 @@ decrypt_descriptor_cookie(const hs_descriptor_t *desc, sizeof(*client_auth_sk))); tor_assert(!tor_mem_is_zero((char *) desc->subcredential, DIGEST256_LEN)); - /* Calculate x25519(client_x, hs_Y) */ - curve25519_handshake(secret_seed, client_auth_sk, - &desc->superencrypted_data.auth_ephemeral_pubkey); - - /* Calculate KEYS = KDF(subcredential | SECRET_SEED, 40) */ - xof = crypto_xof_new(); - crypto_xof_add_bytes(xof, desc->subcredential, DIGEST256_LEN); - crypto_xof_add_bytes(xof, secret_seed, sizeof(secret_seed)); - crypto_xof_squeeze_bytes(xof, keystream, sizeof(keystream)); - crypto_xof_free(xof); + /* Get the KEYS component to derive the CLIENT-ID and COOKIE-KEY. */ + build_descriptor_cookie_keys(desc->subcredential, DIGEST256_LEN, + client_auth_sk, + &desc->superencrypted_data.auth_ephemeral_pubkey, + &keystream); /* If the client id of auth client is not the same as the calculcated * client id, it means that this auth client is invaild according to the @@ -1464,8 +1500,8 @@ decrypt_descriptor_cookie(const hs_descriptor_t *desc, if (cipher) { crypto_cipher_free(cipher); } - memwipe(secret_seed, 0, sizeof(secret_seed)); memwipe(keystream, 0, sizeof(keystream)); + tor_free(keystream); return ret; } @@ -2864,11 +2900,9 @@ hs_desc_build_authorized_client(const uint8_t *subcredential, const uint8_t *descriptor_cookie, hs_desc_authorized_client_t *client_out) { - uint8_t secret_seed[CURVE25519_OUTPUT_LEN]; - uint8_t keystream[HS_DESC_CLIENT_ID_LEN + HS_DESC_COOKIE_KEY_LEN]; - uint8_t *cookie_key; + uint8_t *keystream = NULL; + const uint8_t *cookie_key; crypto_cipher_t *cipher; - crypto_xof_t *xof; tor_assert(client_auth_pk); tor_assert(auth_ephemeral_sk); @@ -2884,18 +2918,11 @@ hs_desc_build_authorized_client(const uint8_t *subcredential, tor_assert(!tor_mem_is_zero((char *) subcredential, DIGEST256_LEN)); - /* Calculate x25519(hs_y, client_X) */ - curve25519_handshake(secret_seed, - auth_ephemeral_sk, - client_auth_pk); - - /* Calculate KEYS = KDF(subcredential | SECRET_SEED, 40) */ - xof = crypto_xof_new(); - crypto_xof_add_bytes(xof, subcredential, DIGEST256_LEN); - crypto_xof_add_bytes(xof, secret_seed, sizeof(secret_seed)); - crypto_xof_squeeze_bytes(xof, keystream, sizeof(keystream)); - crypto_xof_free(xof); + /* Get the KEYS part so we can derive the CLIENT-ID and COOKIE-KEY. */ + build_descriptor_cookie_keys(subcredential, DIGEST256_LEN, + auth_ephemeral_sk, client_auth_pk, &keystream); + /* Extract the CLIENT-ID and COOKIE-KEY from the KEYS. */ memcpy(client_out->client_id, keystream, HS_DESC_CLIENT_ID_LEN); cookie_key = keystream + HS_DESC_CLIENT_ID_LEN; @@ -2910,8 +2937,8 @@ hs_desc_build_authorized_client(const uint8_t *subcredential, (const char *) descriptor_cookie, HS_DESC_DESCRIPTOR_COOKIE_LEN); - memwipe(secret_seed, 0, sizeof(secret_seed)); memwipe(keystream, 0, sizeof(keystream)); + tor_free(keystream); crypto_cipher_free(cipher); } From 91fa12aedc5f30e0f3703bed252e5b30ce374c5c Mon Sep 17 00:00:00 2001 From: rl1987 Date: Mon, 13 Aug 2018 18:31:27 +0300 Subject: [PATCH 0059/2557] Fallback to local DNS when no other nameservers are known --- src/feature/relay/dns.c | 55 ++++++++++++++++++++++++++++------------- 1 file changed, 38 insertions(+), 17 deletions(-) diff --git a/src/feature/relay/dns.c b/src/feature/relay/dns.c index bc507d47f6..f056629b8d 100644 --- a/src/feature/relay/dns.c +++ b/src/feature/relay/dns.c @@ -1388,16 +1388,23 @@ configure_nameservers(int force) evdns_set_log_fn(evdns_log_cb); if (conf_fname) { log_debug(LD_FS, "stat()ing %s", conf_fname); - if (stat(sandbox_intern_string(conf_fname), &st)) { + int missing_resolv_conf = 0; + int stat_res = stat(sandbox_intern_string(conf_fname), &st); + + if (stat_res) { log_warn(LD_EXIT, "Unable to stat resolver configuration in '%s': %s", conf_fname, strerror(errno)); - goto err; - } - if (!force && resolv_conf_fname && !strcmp(conf_fname,resolv_conf_fname) + missing_resolv_conf = 1; + } else if (!force && resolv_conf_fname && + !strcmp(conf_fname,resolv_conf_fname) && st.st_mtime == resolv_conf_mtime) { log_info(LD_EXIT, "No change to '%s'", conf_fname); return 0; } + + if (stat_res == 0 && st.st_size == 0) + missing_resolv_conf = 1; + if (nameservers_configured) { evdns_base_search_clear(the_evdns_base); evdns_base_clear_nameservers_and_suspend(the_evdns_base); @@ -1410,20 +1417,34 @@ configure_nameservers(int force) sandbox_intern_string("/etc/hosts")); } #endif /* defined(DNS_OPTION_HOSTSFILE) && defined(USE_LIBSECCOMP) */ - log_info(LD_EXIT, "Parsing resolver configuration in '%s'", conf_fname); - if ((r = evdns_base_resolv_conf_parse(the_evdns_base, flags, - sandbox_intern_string(conf_fname)))) { - log_warn(LD_EXIT, "Unable to parse '%s', or no nameservers in '%s' (%d)", - conf_fname, conf_fname, r); - goto err; + + if (!missing_resolv_conf) { + log_info(LD_EXIT, "Parsing resolver configuration in '%s'", conf_fname); + if ((r = evdns_base_resolv_conf_parse(the_evdns_base, flags, + sandbox_intern_string(conf_fname)))) { + log_warn(LD_EXIT, "Unable to parse '%s', or no nameservers " + "in '%s' (%d)", conf_fname, conf_fname, r); + + if (r != 6) // "r = 6" means "no DNS servers were in resolv.conf" - + goto err; // in which case we expect libevent to add 127.0.0.1 as + // fallback. + } + if (evdns_base_count_nameservers(the_evdns_base) == 0) { + log_warn(LD_EXIT, "Unable to find any nameservers in '%s'.", + conf_fname); + } + + tor_free(resolv_conf_fname); + resolv_conf_fname = tor_strdup(conf_fname); + resolv_conf_mtime = st.st_mtime; + } else { + log_warn(LD_EXIT, "Could not read your DNS config from '%s' - " + "please investigate your DNS configuration. " + "This is possibly a problem. Meanwhile, falling" + " back to local DNS at 127.0.0.1.", conf_fname); + evdns_base_nameserver_ip_add(the_evdns_base, "127.0.0.1"); } - if (evdns_base_count_nameservers(the_evdns_base) == 0) { - log_warn(LD_EXIT, "Unable to find any nameservers in '%s'.", conf_fname); - goto err; - } - tor_free(resolv_conf_fname); - resolv_conf_fname = tor_strdup(conf_fname); - resolv_conf_mtime = st.st_mtime; + if (nameservers_configured) evdns_base_resume(the_evdns_base); } From 98fd3b410463a25be0864cfaa8ef2e56420125fe Mon Sep 17 00:00:00 2001 From: rl1987 Date: Mon, 13 Aug 2018 18:48:32 +0300 Subject: [PATCH 0060/2557] Mention DNS fallback in manpage --- doc/tor.1.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/tor.1.txt b/doc/tor.1.txt index 406372433f..8729b8ed50 100644 --- a/doc/tor.1.txt +++ b/doc/tor.1.txt @@ -2211,7 +2211,8 @@ is non-zero): __filename__. The file format is the same as the standard Unix "**resolv.conf**" file (7). This option, like all other ServerDNS options, only affects name lookups that your server does on behalf of clients. - (Defaults to use the system DNS configuration.) + (Defaults to use the system DNS configuration or a localhost DNS service + in case no nameservers are found in a given configuration.) [[ServerDNSAllowBrokenConfig]] **ServerDNSAllowBrokenConfig** **0**|**1**:: If this option is false, Tor exits immediately if there are problems From 1a1b088f8cab73ca258c35e83d21df900f8e2e32 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Mon, 13 Aug 2018 19:05:40 +0300 Subject: [PATCH 0061/2557] Add changes file --- changes/bug21900 | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 changes/bug21900 diff --git a/changes/bug21900 b/changes/bug21900 new file mode 100644 index 0000000000..686cb6c584 --- /dev/null +++ b/changes/bug21900 @@ -0,0 +1,4 @@ + o Minor bugfixes (DNS): + - Gracefully handle empty or absent resolve.conf file by falling + back to using localhost DNS service and hoping it works. Fixes + bug 21900; bugfix on 0.2.1.10-alpha. From d827902cb1107593473552974be5fcbb1fbb4abe Mon Sep 17 00:00:00 2001 From: rl1987 Date: Sat, 20 Oct 2018 20:15:06 +0300 Subject: [PATCH 0062/2557] Unit test for DNS fallback in configure_nameservers --- src/feature/relay/dns.c | 30 ++++++++++++++++++++ src/feature/relay/dns.h | 3 ++ src/test/test_dns.c | 61 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 94 insertions(+) diff --git a/src/feature/relay/dns.c b/src/feature/relay/dns.c index f056629b8d..1b4914b49a 100644 --- a/src/feature/relay/dns.c +++ b/src/feature/relay/dns.c @@ -1357,6 +1357,36 @@ evdns_err_is_transient(int err) } } +/** + * Return number of configured nameservers in the_evdns_base. + */ +size_t +number_of_configured_nameservers(void) +{ + return evdns_base_count_nameservers(the_evdns_base); +} + +/** + * Return address of configured nameserver in the_evdns_base + * at index idx. + */ +tor_addr_t * +configured_nameserver_address(const size_t idx) +{ + struct sockaddr_storage sa; + ev_socklen_t sa_len = sizeof(sa); + + if (evdns_base_get_nameserver_addr(the_evdns_base, (int)idx, + (struct sockaddr *)&sa, + sa_len) > 0) { + tor_addr_t *tor_addr = tor_malloc(sizeof(tor_addr_t)); + tor_addr_from_sockaddr(tor_addr, (const struct sockaddr *)&sa, NULL); + return tor_addr; + } + + return NULL; +} + /** Configure eventdns nameservers if force is true, or if the configuration * has changed since the last time we called this function, or if we failed on * our last attempt. On Unix, this reads from /etc/resolv.conf or diff --git a/src/feature/relay/dns.h b/src/feature/relay/dns.h index 1dd6f903d1..3a17ea7b36 100644 --- a/src/feature/relay/dns.h +++ b/src/feature/relay/dns.h @@ -45,6 +45,9 @@ size_t dns_cache_handle_oom(time_t now, size_t min_remove_bytes); #ifdef DNS_PRIVATE #include "feature/relay/dns_structs.h" +size_t number_of_configured_nameservers(void); +tor_addr_t *configured_nameserver_address(const size_t idx); + MOCK_DECL(STATIC int,dns_resolve_impl,(edge_connection_t *exitconn, int is_resolve,or_circuit_t *oncirc, char **hostname_out, int *made_connection_pending_out, cached_resolve_t **resolve_out)); diff --git a/src/test/test_dns.c b/src/test/test_dns.c index 8369f844f6..ea59a49b5b 100644 --- a/src/test/test_dns.c +++ b/src/test/test_dns.c @@ -13,9 +13,69 @@ #include "core/or/edge_connection_st.h" #include "core/or/or_circuit_st.h" +#include "app/config/or_options_st.h" +#include "app/config/config.h" + +#include +#include #define NS_MODULE dns +#define NS_SUBMODULE configure_nameservers_fallback + +static or_options_t options = { + .ORPort_set = 1, +}; + +static const or_options_t * +mock_get_options(void) +{ + return &options; +} + +static void +NS(test_main)(void *arg) +{ + (void)arg; + tor_addr_t *nameserver_addr = NULL; + + MOCK(get_options, mock_get_options); + + options.ServerDNSResolvConfFile = (char *)"no_such_file!!!"; + + dns_init(); // calls configure_nameservers() + + tt_int_op(number_of_configured_nameservers(), OP_EQ, 1); + + nameserver_addr = configured_nameserver_address(0); + + tt_assert(tor_addr_family(nameserver_addr) == AF_INET); + tt_assert(tor_addr_eq_ipv4h(nameserver_addr, 0x7f000001)); + +#ifndef _WIN32 + tor_free(nameserver_addr); + + options.ServerDNSResolvConfFile = (char *)"/dev/null"; + + dns_init(); + + tt_int_op(number_of_configured_nameservers(), OP_EQ, 1); + + nameserver_addr = configured_nameserver_address(0); + + tt_assert(tor_addr_family(nameserver_addr) == AF_INET); + tt_assert(tor_addr_eq_ipv4h(nameserver_addr, 0x7f000001)); +#endif + + UNMOCK(get_options); + + done: + tor_free(nameserver_addr); + return; +} + +#undef NS_SUBMODULE + #define NS_SUBMODULE clip_ttl static void @@ -736,6 +796,7 @@ NS(test_main)(void *arg) #undef NS_SUBMODULE struct testcase_t dns_tests[] = { + TEST_CASE(configure_nameservers_fallback), TEST_CASE(clip_ttl), TEST_CASE(resolve), TEST_CASE_ASPECT(resolve_impl, addr_is_ip_no_need_to_resolve), From 92f0e04f8d08574be60da7e6ebad90a2883a488a Mon Sep 17 00:00:00 2001 From: rl1987 Date: Sat, 20 Oct 2018 20:18:54 +0300 Subject: [PATCH 0063/2557] Check if libevent comes with evdns_base_get_nameserver_addr() --- configure.ac | 2 ++ 1 file changed, 2 insertions(+) diff --git a/configure.ac b/configure.ac index 292244f56f..af3d64b7f2 100644 --- a/configure.ac +++ b/configure.ac @@ -812,6 +812,8 @@ fi dnl Now check for particular libevent functions. AC_CHECK_FUNCS([evutil_secure_rng_set_urandom_device_file \ evutil_secure_rng_add_bytes \ + evdns_base_get_nameserver_addr \ + ]) LIBS="$save_LIBS" From 98cef6807eb70e7c459f6f80a06f894fac63100a Mon Sep 17 00:00:00 2001 From: rl1987 Date: Sat, 20 Oct 2018 20:32:26 +0300 Subject: [PATCH 0064/2557] Exclude test and a supporting function when evdns_base_get_nameserver_addr() is not available --- src/feature/relay/dns.c | 2 ++ src/feature/relay/dns.h | 2 ++ src/test/test_dns.c | 5 +++++ 3 files changed, 9 insertions(+) diff --git a/src/feature/relay/dns.c b/src/feature/relay/dns.c index 1b4914b49a..dfd84652ad 100644 --- a/src/feature/relay/dns.c +++ b/src/feature/relay/dns.c @@ -1366,6 +1366,7 @@ number_of_configured_nameservers(void) return evdns_base_count_nameservers(the_evdns_base); } +#ifdef HAVE_EVDNS_BASE_GET_NAMESERVER_ADDR /** * Return address of configured nameserver in the_evdns_base * at index idx. @@ -1386,6 +1387,7 @@ configured_nameserver_address(const size_t idx) return NULL; } +#endif /** Configure eventdns nameservers if force is true, or if the configuration * has changed since the last time we called this function, or if we failed on diff --git a/src/feature/relay/dns.h b/src/feature/relay/dns.h index 3a17ea7b36..5758ea4363 100644 --- a/src/feature/relay/dns.h +++ b/src/feature/relay/dns.h @@ -46,7 +46,9 @@ size_t dns_cache_handle_oom(time_t now, size_t min_remove_bytes); #include "feature/relay/dns_structs.h" size_t number_of_configured_nameservers(void); +#ifdef HAVE_EVDNS_BASE_GET_NAMESERVER_ADDR tor_addr_t *configured_nameserver_address(const size_t idx); +#endif MOCK_DECL(STATIC int,dns_resolve_impl,(edge_connection_t *exitconn, int is_resolve,or_circuit_t *oncirc, char **hostname_out, diff --git a/src/test/test_dns.c b/src/test/test_dns.c index ea59a49b5b..ea0fcf8c5e 100644 --- a/src/test/test_dns.c +++ b/src/test/test_dns.c @@ -1,6 +1,7 @@ /* Copyright (c) 2015-2018, The Tor Project, Inc. */ /* See LICENSE for licensing information */ +#include "orconfig.h" #include "core/or/or.h" #include "test/test.h" @@ -21,6 +22,7 @@ #define NS_MODULE dns +#ifdef HAVE_EVDNS_BASE_GET_NAMESERVER_ADDR #define NS_SUBMODULE configure_nameservers_fallback static or_options_t options = { @@ -75,6 +77,7 @@ NS(test_main)(void *arg) } #undef NS_SUBMODULE +#endif #define NS_SUBMODULE clip_ttl @@ -796,7 +799,9 @@ NS(test_main)(void *arg) #undef NS_SUBMODULE struct testcase_t dns_tests[] = { +#ifdef HAVE_EVDNS_BASE_GET_NAMESERVER_ADDR TEST_CASE(configure_nameservers_fallback), +#endif TEST_CASE(clip_ttl), TEST_CASE(resolve), TEST_CASE_ASPECT(resolve_impl, addr_is_ip_no_need_to_resolve), From 7aa9fc1637aeb876c0eebceec3dcdcbcba603302 Mon Sep 17 00:00:00 2001 From: Roger Dingledine Date: Sun, 21 Oct 2018 23:46:09 -0400 Subject: [PATCH 0065/2557] clean up a tor2web comment --- src/feature/rend/rendmid.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/feature/rend/rendmid.c b/src/feature/rend/rendmid.c index 8ca19a2522..67a8b5f8f4 100644 --- a/src/feature/rend/rendmid.c +++ b/src/feature/rend/rendmid.c @@ -236,8 +236,8 @@ rend_mid_establish_rendezvous(or_circuit_t *circ, const uint8_t *request, goto err; } - /* Check if we are configured to accept established rendezvous cells from - * client or in other words Tor2Web clients. */ + /* Check if we are configured to defend ourselves from clients that + * attempt to establish rendezvous points directly to us. */ if (channel_is_client(circ->p_chan) && dos_should_refuse_single_hop_client()) { /* Note it down for the heartbeat log purposes. */ From 3704c4a01219cca7273fa5fa0bc6751d98ad05ac Mon Sep 17 00:00:00 2001 From: cypherpunks Date: Wed, 29 Aug 2018 13:32:52 +0000 Subject: [PATCH 0066/2557] string: add BOM helper --- src/lib/string/util_string.c | 13 +++++++++++++ src/lib/string/util_string.h | 1 + 2 files changed, 14 insertions(+) diff --git a/src/lib/string/util_string.c b/src/lib/string/util_string.c index b2b85d151d..e76e73046f 100644 --- a/src/lib/string/util_string.c +++ b/src/lib/string/util_string.c @@ -541,3 +541,16 @@ string_is_utf8(const char *str, size_t len) } return true; } + +/** As string_is_utf8(), but returns false if the string begins with a UTF-8 + * byte order mark (BOM). + */ +int +string_is_utf8_no_bom(const char *str, size_t len) +{ + if (len >= 3 && (!strcmpstart(str, "\uFEFF") || + !strcmpstart(str, "\uFFFE"))) { + return false; + } + return string_is_utf8(str, len); +} diff --git a/src/lib/string/util_string.h b/src/lib/string/util_string.h index 746ece0d33..99467a27c3 100644 --- a/src/lib/string/util_string.h +++ b/src/lib/string/util_string.h @@ -53,5 +53,6 @@ const char *find_str_at_start_of_line(const char *haystack, int string_is_C_identifier(const char *string); int string_is_utf8(const char *str, size_t len); +int string_is_utf8_no_bom(const char *str, size_t len); #endif /* !defined(TOR_UTIL_STRING_H) */ From f874ab26401ca269074963697ddcad879b3b4e3a Mon Sep 17 00:00:00 2001 From: cypherpunks Date: Wed, 29 Aug 2018 08:49:10 +0000 Subject: [PATCH 0067/2557] dircache: make dirauths reject non UTF-8 descriptors and extrainfo Ticket #27367. --- changes/feature27367 | 4 ++++ src/feature/dirauth/process_descs.c | 12 +++++++++--- src/feature/dirauth/process_descs.h | 3 ++- src/feature/dircache/dircache.c | 4 ++-- 4 files changed, 17 insertions(+), 6 deletions(-) create mode 100644 changes/feature27367 diff --git a/changes/feature27367 b/changes/feature27367 new file mode 100644 index 0000000000..99c0839621 --- /dev/null +++ b/changes/feature27367 @@ -0,0 +1,4 @@ + o Minor features (parsing): + - Directory authorities now validate that router descriptors and ExtraInfo + documents are in a valid subset of UTF-8, and reject them if not. + Closes ticket 27367. diff --git a/src/feature/dirauth/process_descs.c b/src/feature/dirauth/process_descs.c index c379f25bdd..dca87b3eaf 100644 --- a/src/feature/dirauth/process_descs.c +++ b/src/feature/dirauth/process_descs.c @@ -519,7 +519,8 @@ WRA_MORE_SEVERE(was_router_added_t a, was_router_added_t b) /** As for dirserv_add_descriptor(), but accepts multiple documents, and * returns the most severe error that occurred for any one of them. */ was_router_added_t -dirserv_add_multiple_descriptors(const char *desc, uint8_t purpose, +dirserv_add_multiple_descriptors(const char *desc, size_t desclen, + uint8_t purpose, const char *source, const char **msg) { @@ -536,6 +537,11 @@ dirserv_add_multiple_descriptors(const char *desc, uint8_t purpose, r=ROUTER_ADDED_SUCCESSFULLY; /*Least severe return value. */ + if (!string_is_utf8_no_bom(desc, desclen)) { + *msg = "descriptor(s) or extrainfo(s) not valid UTF-8 or had BOM."; + return ROUTER_AUTHDIR_REJECTS; + } + format_iso_time(time_buf, now); if (tor_snprintf(annotation_buf, sizeof(annotation_buf), "@uploaded-at %s\n" @@ -552,7 +558,7 @@ dirserv_add_multiple_descriptors(const char *desc, uint8_t purpose, s = desc; list = smartlist_new(); - if (!router_parse_list_from_string(&s, NULL, list, SAVED_NOWHERE, 0, 0, + if (!router_parse_list_from_string(&s, s+desclen, list, SAVED_NOWHERE, 0, 0, annotation_buf, NULL)) { SMARTLIST_FOREACH(list, routerinfo_t *, ri, { msg_out = NULL; @@ -568,7 +574,7 @@ dirserv_add_multiple_descriptors(const char *desc, uint8_t purpose, smartlist_clear(list); s = desc; - if (!router_parse_list_from_string(&s, NULL, list, SAVED_NOWHERE, 1, 0, + if (!router_parse_list_from_string(&s, s+desclen, list, SAVED_NOWHERE, 1, 0, NULL, NULL)) { SMARTLIST_FOREACH(list, extrainfo_t *, ei, { msg_out = NULL; diff --git a/src/feature/dirauth/process_descs.h b/src/feature/dirauth/process_descs.h index ad9d5c3d4c..5a0914acd8 100644 --- a/src/feature/dirauth/process_descs.h +++ b/src/feature/dirauth/process_descs.h @@ -17,7 +17,8 @@ void dirserv_free_fingerprint_list(void); int dirserv_add_own_fingerprint(crypto_pk_t *pk); enum was_router_added_t dirserv_add_multiple_descriptors( - const char *desc, uint8_t purpose, + const char *desc, size_t desclen, + uint8_t purpose, const char *source, const char **msg); enum was_router_added_t dirserv_add_descriptor(routerinfo_t *ri, diff --git a/src/feature/dircache/dircache.c b/src/feature/dircache/dircache.c index 872a88018f..930a8b87ef 100644 --- a/src/feature/dircache/dircache.c +++ b/src/feature/dircache/dircache.c @@ -1608,8 +1608,8 @@ directory_handle_command_post,(dir_connection_t *conn, const char *headers, const char *msg = "[None]"; uint8_t purpose = authdir_mode_bridge(options) ? ROUTER_PURPOSE_BRIDGE : ROUTER_PURPOSE_GENERAL; - was_router_added_t r = dirserv_add_multiple_descriptors(body, purpose, - conn->base_.address, &msg); + was_router_added_t r = dirserv_add_multiple_descriptors(body, body_len, + purpose, conn->base_.address, &msg); tor_assert(msg); if (r == ROUTER_ADDED_SUCCESSFULLY) { From c0bd800d267285d7372094b77121c63a4ba720d2 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 24 Oct 2018 09:09:40 -0400 Subject: [PATCH 0068/2557] Re-alphabetize the list of tests in tests.[ch] --- src/test/test.c | 20 ++++++++++---------- src/test/test.h | 34 +++++++++++++++++----------------- 2 files changed, 27 insertions(+), 27 deletions(-) diff --git a/src/test/test.c b/src/test/test.c index 56eb153289..17b736d305 100644 --- a/src/test/test.c +++ b/src/test/test.c @@ -847,8 +847,8 @@ struct testgroup_t testgroups[] = { { "circuitbuild/", circuitbuild_tests }, { "circuitlist/", circuitlist_tests }, { "circuitmux/", circuitmux_tests }, - { "circuituse/", circuituse_tests }, { "circuitstats/", circuitstats_tests }, + { "circuituse/", circuituse_tests }, { "compat/libevent/", compat_libevent_tests }, { "config/", config_tests }, { "connection/", connection_tests }, @@ -865,34 +865,36 @@ struct testgroup_t testgroups[] = { #endif { "crypto/pem/", pem_tests }, { "dir/", dir_tests }, - { "dir_handle_get/", dir_handle_get_tests }, { "dir/md/", microdesc_tests }, { "dir/voting-schedule/", voting_schedule_tests }, + { "dir_handle_get/", dir_handle_get_tests }, + { "dns/", dns_tests }, { "dos/", dos_tests }, { "entryconn/", entryconn_tests }, { "entrynodes/", entrynodes_tests }, - { "guardfraction/", guardfraction_tests }, { "extorport/", extorport_tests }, { "geoip/", geoip_tests }, - { "legacy_hs/", hs_tests }, + { "guardfraction/", guardfraction_tests }, { "hs_cache/", hs_cache }, { "hs_cell/", hs_cell_tests }, + { "hs_client/", hs_client_tests }, { "hs_common/", hs_common_tests }, { "hs_config/", hs_config_tests }, { "hs_control/", hs_control_tests }, { "hs_descriptor/", hs_descriptor }, + { "hs_intropoint/", hs_intropoint_tests }, { "hs_ntor/", hs_ntor_tests }, { "hs_service/", hs_service_tests }, - { "hs_client/", hs_client_tests }, - { "hs_intropoint/", hs_intropoint_tests }, { "introduce/", introduce_tests }, { "keypin/", keypin_tests }, + { "legacy_hs/", hs_tests }, { "link-handshake/", link_handshake_tests }, { "mainloop/", mainloop_tests }, { "nodelist/", nodelist_tests }, { "oom/", oom_tests }, { "oos/", oos_tests }, { "options/", options_tests }, + { "parsecommon/", parsecommon_tests }, { "periodic-event/" , periodic_event_tests }, { "policy/" , policy_tests }, { "procmon/", procmon_tests }, @@ -910,8 +912,8 @@ struct testgroup_t testgroups[] = { { "routerlist/", routerlist_tests }, { "routerset/" , routerset_tests }, { "scheduler/", scheduler_tests }, - { "socks/", socks_tests }, { "shared-random/", sr_tests }, + { "socks/", socks_tests }, { "status/" , status_tests }, { "storagedir/", storagedir_tests }, { "tortls/", tortls_tests }, @@ -921,11 +923,9 @@ struct testgroup_t testgroups[] = { { "tortls/x509/", x509_tests }, { "util/", util_tests }, { "util/format/", util_format_tests }, + { "util/handle/", handle_tests }, { "util/logging/", logging_tests }, { "util/process/", util_process_tests }, { "util/thread/", thread_tests }, - { "util/handle/", handle_tests }, - { "dns/", dns_tests }, - { "parsecommon/", parsecommon_tests }, END_OF_GROUPS }; diff --git a/src/test/test.h b/src/test/test.h index 281551aa69..092356f0fb 100644 --- a/src/test/test.h +++ b/src/test/test.h @@ -177,11 +177,11 @@ extern const struct testcase_setup_t ed25519_test_setup; extern struct testcase_t accounting_tests[]; extern struct testcase_t addr_tests[]; -extern struct testcase_t address_tests[]; extern struct testcase_t address_set_tests[]; +extern struct testcase_t address_tests[]; extern struct testcase_t bridges_tests[]; -extern struct testcase_t bwmgt_tests[]; extern struct testcase_t buffer_tests[]; +extern struct testcase_t bwmgt_tests[]; extern struct testcase_t cell_format_tests[]; extern struct testcase_t cell_queue_tests[]; extern struct testcase_t channel_tests[]; @@ -191,8 +191,8 @@ extern struct testcase_t checkdir_tests[]; extern struct testcase_t circuitbuild_tests[]; extern struct testcase_t circuitlist_tests[]; extern struct testcase_t circuitmux_tests[]; -extern struct testcase_t circuituse_tests[]; extern struct testcase_t circuitstats_tests[]; +extern struct testcase_t circuituse_tests[]; extern struct testcase_t compat_libevent_tests[]; extern struct testcase_t config_tests[]; extern struct testcase_t connection_tests[]; @@ -200,30 +200,32 @@ extern struct testcase_t conscache_tests[]; extern struct testcase_t consdiff_tests[]; extern struct testcase_t consdiffmgr_tests[]; extern struct testcase_t container_tests[]; -extern struct testcase_t controller_tests[]; extern struct testcase_t controller_event_tests[]; -extern struct testcase_t crypto_tests[]; +extern struct testcase_t controller_tests[]; extern struct testcase_t crypto_ope_tests[]; extern struct testcase_t crypto_openssl_tests[]; -extern struct testcase_t dir_tests[]; +extern struct testcase_t crypto_tests[]; extern struct testcase_t dir_handle_get_tests[]; +extern struct testcase_t dir_tests[]; +extern struct testcase_t dns_tests[]; extern struct testcase_t dos_tests[]; extern struct testcase_t entryconn_tests[]; extern struct testcase_t entrynodes_tests[]; -extern struct testcase_t guardfraction_tests[]; extern struct testcase_t extorport_tests[]; extern struct testcase_t geoip_tests[]; -extern struct testcase_t hs_tests[]; +extern struct testcase_t guardfraction_tests[]; +extern struct testcase_t handle_tests[]; extern struct testcase_t hs_cache[]; extern struct testcase_t hs_cell_tests[]; +extern struct testcase_t hs_client_tests[]; extern struct testcase_t hs_common_tests[]; extern struct testcase_t hs_config_tests[]; extern struct testcase_t hs_control_tests[]; extern struct testcase_t hs_descriptor[]; +extern struct testcase_t hs_intropoint_tests[]; extern struct testcase_t hs_ntor_tests[]; extern struct testcase_t hs_service_tests[]; -extern struct testcase_t hs_client_tests[]; -extern struct testcase_t hs_intropoint_tests[]; +extern struct testcase_t hs_tests[]; extern struct testcase_t introduce_tests[]; extern struct testcase_t keypin_tests[]; extern struct testcase_t link_handshake_tests[]; @@ -234,6 +236,7 @@ extern struct testcase_t nodelist_tests[]; extern struct testcase_t oom_tests[]; extern struct testcase_t oos_tests[]; extern struct testcase_t options_tests[]; +extern struct testcase_t parsecommon_tests[]; extern struct testcase_t pem_tests[]; extern struct testcase_t periodic_event_tests[]; extern struct testcase_t policy_tests[]; @@ -252,21 +255,18 @@ extern struct testcase_t routerkeys_tests[]; extern struct testcase_t routerlist_tests[]; extern struct testcase_t routerset_tests[]; extern struct testcase_t scheduler_tests[]; -extern struct testcase_t storagedir_tests[]; extern struct testcase_t socks_tests[]; +extern struct testcase_t sr_tests[]; extern struct testcase_t status_tests[]; +extern struct testcase_t storagedir_tests[]; extern struct testcase_t thread_tests[]; -extern struct testcase_t tortls_tests[]; extern struct testcase_t tortls_openssl_tests[]; -extern struct testcase_t util_tests[]; +extern struct testcase_t tortls_tests[]; extern struct testcase_t util_format_tests[]; extern struct testcase_t util_process_tests[]; +extern struct testcase_t util_tests[]; extern struct testcase_t voting_schedule_tests[]; -extern struct testcase_t dns_tests[]; -extern struct testcase_t handle_tests[]; -extern struct testcase_t sr_tests[]; extern struct testcase_t x509_tests[]; -extern struct testcase_t parsecommon_tests[]; extern struct testcase_t slow_crypto_tests[]; extern struct testcase_t slow_util_tests[]; From 594140574e7366efac693d440a636a1e1cce82ff Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 24 Oct 2018 11:06:34 -0400 Subject: [PATCH 0069/2557] Fix remaining cases of using consensus without a len parameter. (Thanks to cyberpunks for noting two of them!) --- src/feature/dircache/consdiffmgr.c | 28 ++++++++++++++++++++++++---- src/feature/dircache/consdiffmgr.h | 6 ++++++ src/feature/dircache/dirserv.c | 5 ++++- src/feature/dircache/dirserv.h | 1 + src/feature/nodelist/networkstatus.c | 5 +++-- src/test/test_consdiffmgr.c | 2 ++ src/test/test_dir_handle_get.c | 2 ++ 7 files changed, 42 insertions(+), 7 deletions(-) diff --git a/src/feature/dircache/consdiffmgr.c b/src/feature/dircache/consdiffmgr.c index bf3a0ef3cf..2cddf68b69 100644 --- a/src/feature/dircache/consdiffmgr.c +++ b/src/feature/dircache/consdiffmgr.c @@ -189,6 +189,7 @@ static consdiff_cfg_t consdiff_cfg = { static int consdiffmgr_ensure_space_for_files(int n); static int consensus_queue_compression_work(const char *consensus, + size_t consensus_len, const networkstatus_t *as_parsed); static int consensus_diff_queue_diff_work(consensus_cache_entry_t *diff_from, consensus_cache_entry_t *diff_to); @@ -509,8 +510,25 @@ get_max_age_to_cache(void) MAX_MAX_AGE_TO_CACHE); } +#ifdef TOR_UNIT_TESTS +/** As consdiffmgr_add_consensus, but requires a nul-terminated input. For + * testing. */ +int +consdiffmgr_add_consensus_nulterm(const char *consensus, + const networkstatus_t *as_parsed) +{ + size_t len = strlen(consensus); + /* make a non-nul-terminated copy so that we can have a better chance + * of catching errors. */ + char *ctmp = tor_memdup(consensus, len); + int r = consdiffmgr_add_consensus(ctmp, len, as_parsed); + tor_free(ctmp); + return r; +} +#endif + /** - * Given a string containing a networkstatus consensus, and the results of + * Given a buffer containing a networkstatus consensus, and the results of * having parsed that consensus, add that consensus to the cache if it is not * already present and not too old. Create new consensus diffs from or to * that consensus as appropriate. @@ -519,6 +537,7 @@ get_max_age_to_cache(void) */ int consdiffmgr_add_consensus(const char *consensus, + size_t consensus_len, const networkstatus_t *as_parsed) { if (BUG(consensus == NULL) || BUG(as_parsed == NULL)) @@ -544,7 +563,7 @@ consdiffmgr_add_consensus(const char *consensus, } /* We don't have it. Add it to the cache. */ - return consensus_queue_compression_work(consensus, as_parsed); + return consensus_queue_compression_work(consensus, consensus_len, as_parsed); } /** @@ -1826,14 +1845,15 @@ static int background_compression = 0; */ static int consensus_queue_compression_work(const char *consensus, + size_t consensus_len, const networkstatus_t *as_parsed) { tor_assert(consensus); tor_assert(as_parsed); consensus_compress_worker_job_t *job = tor_malloc_zero(sizeof(*job)); - job->consensus = tor_strdup(consensus); - job->consensus_len = strlen(consensus); + job->consensus = tor_memdup_nulterm(consensus, consensus_len); + job->consensus_len = strlen(job->consensus); job->flavor = as_parsed->flavor; char va_str[ISO_TIME_LEN+1]; diff --git a/src/feature/dircache/consdiffmgr.h b/src/feature/dircache/consdiffmgr.h index d6f273cc4e..011c8799d6 100644 --- a/src/feature/dircache/consdiffmgr.h +++ b/src/feature/dircache/consdiffmgr.h @@ -22,6 +22,7 @@ typedef struct consdiff_cfg_t { struct consensus_cache_entry_t; // from conscache.h int consdiffmgr_add_consensus(const char *consensus, + size_t consensus_len, const networkstatus_t *as_parsed); consdiff_status_t consdiffmgr_find_consensus( @@ -73,4 +74,9 @@ STATIC int uncompress_or_set_ptr(const char **out, size_t *outlen, consensus_cache_entry_t *ent); #endif /* defined(CONSDIFFMGR_PRIVATE) */ +#ifdef TOR_UNIT_TESTS +int consdiffmgr_add_consensus_nulterm(const char *consensus, + const networkstatus_t *as_parsed); +#endif + #endif /* !defined(TOR_CONSDIFFMGR_H) */ diff --git a/src/feature/dircache/dirserv.c b/src/feature/dircache/dirserv.c index b85db8324f..433d3f4ce4 100644 --- a/src/feature/dircache/dirserv.c +++ b/src/feature/dircache/dirserv.c @@ -1272,6 +1272,7 @@ free_cached_dir_(void *_d) * validation is performed. */ void dirserv_set_cached_consensus_networkstatus(const char *networkstatus, + size_t networkstatus_len, const char *flavor_name, const common_digests_t *digests, const uint8_t *sha3_as_signed, @@ -1282,7 +1283,9 @@ dirserv_set_cached_consensus_networkstatus(const char *networkstatus, if (!cached_consensuses) cached_consensuses = strmap_new(); - new_networkstatus = new_cached_dir(tor_strdup(networkstatus), published); + new_networkstatus = + new_cached_dir(tor_memdup_nulterm(networkstatus, networkstatus_len), + published); memcpy(&new_networkstatus->digests, digests, sizeof(common_digests_t)); memcpy(&new_networkstatus->digest_sha3_as_signed, sha3_as_signed, DIGEST256_LEN); diff --git a/src/feature/dircache/dirserv.h b/src/feature/dircache/dirserv.h index 9be4bf9db2..c430987aa7 100644 --- a/src/feature/dircache/dirserv.h +++ b/src/feature/dircache/dirserv.h @@ -148,6 +148,7 @@ int directory_too_idle_to_fetch_descriptors(const or_options_t *options, cached_dir_t *dirserv_get_consensus(const char *flavor_name); void dirserv_set_cached_consensus_networkstatus(const char *consensus, + size_t consensus_len, const char *flavor_name, const common_digests_t *digests, const uint8_t *sha3_as_signed, diff --git a/src/feature/nodelist/networkstatus.c b/src/feature/nodelist/networkstatus.c index 847ca0cdfc..97a8779bea 100644 --- a/src/feature/nodelist/networkstatus.c +++ b/src/feature/nodelist/networkstatus.c @@ -2107,17 +2107,18 @@ networkstatus_set_current_consensus(const char *consensus, if (we_want_to_fetch_flavor(options, flav)) { dirserv_set_cached_consensus_networkstatus(consensus, + consensus_len, flavor, &c->digests, c->digest_sha3_as_signed, c->valid_after); if (dir_server_mode(get_options())) { - consdiffmgr_add_consensus(consensus, c); + consdiffmgr_add_consensus(consensus, consensus_len, c); } } if (!from_cache) { - write_str_to_file(consensus_fname, consensus, 0); + write_bytes_to_file(consensus_fname, consensus, consensus_len, 0); } warn_early_consensus(c, flavor, now); diff --git a/src/test/test_consdiffmgr.c b/src/test/test_consdiffmgr.c index 4b49fdb6aa..966314be13 100644 --- a/src/test/test_consdiffmgr.c +++ b/src/test/test_consdiffmgr.c @@ -21,6 +21,8 @@ #include "test/test.h" #include "test/log_test_helpers.h" +#define consdiffmgr_add_consensus consdiffmgr_add_consensus_nulterm + static char * consensus_diff_apply_(const char *c, const char *d) { diff --git a/src/test/test_dir_handle_get.c b/src/test/test_dir_handle_get.c index ecb7c1b5fc..b591396913 100644 --- a/src/test/test_dir_handle_get.c +++ b/src/test/test_dir_handle_get.c @@ -67,6 +67,8 @@ ENABLE_GCC_WARNING(overlength-strings) #define NOT_ENOUGH_CONSENSUS_SIGNATURES "HTTP/1.0 404 " \ "Consensus not signed by sufficient number of requested authorities\r\n\r\n" +#define consdiffmgr_add_consensus consdiffmgr_add_consensus_nulterm + static dir_connection_t * new_dir_conn(void) { From 52a82bc53c82c82f754c1267aaa68b48737ba07c Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 25 Oct 2018 09:24:15 -0400 Subject: [PATCH 0070/2557] Add a couple more checks to test_parsecommon.c These checks should make coverity stop giving us a "dereference before null check" warning here. --- src/test/test_parsecommon.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/test/test_parsecommon.c b/src/test/test_parsecommon.c index 7ce4b71b00..6da125dd0a 100644 --- a/src/test/test_parsecommon.c +++ b/src/test/test_parsecommon.c @@ -295,6 +295,7 @@ test_parsecommon_get_next_token_parse_keys(void *arg) token_rule_t rule = T1("onion-key", R_IPO_ONION_KEY, NO_ARGS, NEED_KEY_1024); token = get_next_token(area, s, end, &rule); + tt_assert(token); tt_int_op(token->tp, OP_EQ, R_IPO_ONION_KEY); tt_int_op(token->n_args, OP_EQ, 0); @@ -329,6 +330,7 @@ test_parsecommon_get_next_token_parse_keys(void *arg) NEED_SKEY_1024); token2 = get_next_token(area, s2, end2, &rule2); + tt_assert(token2); tt_int_op(token2->tp, OP_EQ, C_CLIENT_KEY); tt_int_op(token2->n_args, OP_EQ, 0); @@ -590,4 +592,3 @@ struct testcase_t parsecommon_tests[] = { PARSECOMMON_TEST(get_next_token_err_bad_base64), END_OF_TESTCASES }; - From b7edfcbf6bb3a27d914ad883ae75413a4d25739a Mon Sep 17 00:00:00 2001 From: rl1987 Date: Fri, 26 Oct 2018 10:26:47 +0300 Subject: [PATCH 0071/2557] In configured_nameserver_address, check if tor_addr_from_sockaddr succeeded --- src/feature/relay/dns.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/feature/relay/dns.c b/src/feature/relay/dns.c index dfd84652ad..371c2f5069 100644 --- a/src/feature/relay/dns.c +++ b/src/feature/relay/dns.c @@ -1381,8 +1381,11 @@ configured_nameserver_address(const size_t idx) (struct sockaddr *)&sa, sa_len) > 0) { tor_addr_t *tor_addr = tor_malloc(sizeof(tor_addr_t)); - tor_addr_from_sockaddr(tor_addr, (const struct sockaddr *)&sa, NULL); - return tor_addr; + if (tor_addr_from_sockaddr(tor_addr, + (const struct sockaddr *)&sa, + NULL) == 0) { + return tor_addr; + } } return NULL; From b59eedc25992f8c460d4c99968a1fbaa28896610 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Fri, 21 Sep 2018 12:39:11 +0300 Subject: [PATCH 0072/2557] Add trunnel spec and impl for NETINFO cells --- src/trunnel/include.am | 6 +- src/trunnel/netinfo.c | 721 ++++++++++++++++++++++++++++++++++++ src/trunnel/netinfo.h | 226 +++++++++++ src/trunnel/netinfo.trunnel | 21 ++ 4 files changed, 972 insertions(+), 2 deletions(-) create mode 100644 src/trunnel/netinfo.c create mode 100644 src/trunnel/netinfo.h create mode 100644 src/trunnel/netinfo.trunnel diff --git a/src/trunnel/include.am b/src/trunnel/include.am index 03c1753e96..b5db0609a8 100644 --- a/src/trunnel/include.am +++ b/src/trunnel/include.am @@ -23,7 +23,8 @@ TRUNNELSOURCES = \ src/trunnel/hs/cell_introduce1.c \ src/trunnel/hs/cell_rendezvous.c \ src/trunnel/channelpadding_negotiation.c \ - src/trunnel/socks5.c + src/trunnel/socks5.c \ + src/trunnel/netinfo.c TRUNNELHEADERS = \ src/ext/trunnel/trunnel.h \ @@ -37,7 +38,8 @@ TRUNNELHEADERS = \ src/trunnel/hs/cell_introduce1.h \ src/trunnel/hs/cell_rendezvous.h \ src/trunnel/channelpadding_negotiation.h \ - src/trunnel/socks5.h + src/trunnel/socks5.h \ + src/trunnel/netinfo.h src_trunnel_libor_trunnel_a_SOURCES = $(TRUNNELSOURCES) src_trunnel_libor_trunnel_a_CPPFLAGS = \ diff --git a/src/trunnel/netinfo.c b/src/trunnel/netinfo.c new file mode 100644 index 0000000000..170e9bf034 --- /dev/null +++ b/src/trunnel/netinfo.c @@ -0,0 +1,721 @@ +/* netinfo.c -- generated by Trunnel v1.5.2. + * https://gitweb.torproject.org/trunnel.git + * You probably shouldn't edit this file. + */ +#include +#include "trunnel-impl.h" + +#include "netinfo.h" + +#define TRUNNEL_SET_ERROR_CODE(obj) \ + do { \ + (obj)->trunnel_error_code_ = 1; \ + } while (0) + +#if defined(__COVERITY__) || defined(__clang_analyzer__) +/* If we're running a static analysis tool, we don't want it to complain + * that some of our remaining-bytes checks are dead-code. */ +int netinfo_deadcode_dummy__ = 0; +#define OR_DEADCODE_DUMMY || netinfo_deadcode_dummy__ +#else +#define OR_DEADCODE_DUMMY +#endif + +#define CHECK_REMAINING(nbytes, label) \ + do { \ + if (remaining < (nbytes) OR_DEADCODE_DUMMY) { \ + goto label; \ + } \ + } while (0) + +netinfo_addr_t * +netinfo_addr_new(void) +{ + netinfo_addr_t *val = trunnel_calloc(1, sizeof(netinfo_addr_t)); + if (NULL == val) + return NULL; + val->addr_type = NETINFO_ADDR_TYPE_IPV4; + val->len = 4; + return val; +} + +/** Release all storage held inside 'obj', but do not free 'obj'. + */ +static void +netinfo_addr_clear(netinfo_addr_t *obj) +{ + (void) obj; +} + +void +netinfo_addr_free(netinfo_addr_t *obj) +{ + if (obj == NULL) + return; + netinfo_addr_clear(obj); + trunnel_memwipe(obj, sizeof(netinfo_addr_t)); + trunnel_free_(obj); +} + +uint8_t +netinfo_addr_get_addr_type(const netinfo_addr_t *inp) +{ + return inp->addr_type; +} +int +netinfo_addr_set_addr_type(netinfo_addr_t *inp, uint8_t val) +{ + if (! ((val == NETINFO_ADDR_TYPE_IPV4 || val == NETINFO_ADDR_TYPE_IPV6))) { + TRUNNEL_SET_ERROR_CODE(inp); + return -1; + } + inp->addr_type = val; + return 0; +} +uint8_t +netinfo_addr_get_len(const netinfo_addr_t *inp) +{ + return inp->len; +} +int +netinfo_addr_set_len(netinfo_addr_t *inp, uint8_t val) +{ + if (! ((val == 4 || val == 16))) { + TRUNNEL_SET_ERROR_CODE(inp); + return -1; + } + inp->len = val; + return 0; +} +uint32_t +netinfo_addr_get_addr_ipv4(const netinfo_addr_t *inp) +{ + return inp->addr_ipv4; +} +int +netinfo_addr_set_addr_ipv4(netinfo_addr_t *inp, uint32_t val) +{ + inp->addr_ipv4 = val; + return 0; +} +size_t +netinfo_addr_getlen_addr_ipv6(const netinfo_addr_t *inp) +{ + (void)inp; return 16; +} + +uint8_t +netinfo_addr_get_addr_ipv6(netinfo_addr_t *inp, size_t idx) +{ + trunnel_assert(idx < 16); + return inp->addr_ipv6[idx]; +} + +uint8_t +netinfo_addr_getconst_addr_ipv6(const netinfo_addr_t *inp, size_t idx) +{ + return netinfo_addr_get_addr_ipv6((netinfo_addr_t*)inp, idx); +} +int +netinfo_addr_set_addr_ipv6(netinfo_addr_t *inp, size_t idx, uint8_t elt) +{ + trunnel_assert(idx < 16); + inp->addr_ipv6[idx] = elt; + return 0; +} + +uint8_t * +netinfo_addr_getarray_addr_ipv6(netinfo_addr_t *inp) +{ + return inp->addr_ipv6; +} +const uint8_t * +netinfo_addr_getconstarray_addr_ipv6(const netinfo_addr_t *inp) +{ + return (const uint8_t *)netinfo_addr_getarray_addr_ipv6((netinfo_addr_t*)inp); +} +const char * +netinfo_addr_check(const netinfo_addr_t *obj) +{ + if (obj == NULL) + return "Object was NULL"; + if (obj->trunnel_error_code_) + return "A set function failed on this object"; + if (! (obj->addr_type == NETINFO_ADDR_TYPE_IPV4 || obj->addr_type == NETINFO_ADDR_TYPE_IPV6)) + return "Integer out of bounds"; + if (! (obj->len == 4 || obj->len == 16)) + return "Integer out of bounds"; + switch (obj->addr_type) { + + case NETINFO_ADDR_TYPE_IPV4: + break; + + case NETINFO_ADDR_TYPE_IPV6: + break; + + default: + return "Bad tag for union"; + break; + } + return NULL; +} + +ssize_t +netinfo_addr_encoded_len(const netinfo_addr_t *obj) +{ + ssize_t result = 0; + + if (NULL != netinfo_addr_check(obj)) + return -1; + + + /* Length of u8 addr_type IN [NETINFO_ADDR_TYPE_IPV4, NETINFO_ADDR_TYPE_IPV6] */ + result += 1; + + /* Length of u8 len IN [4, 16] */ + result += 1; + switch (obj->addr_type) { + + case NETINFO_ADDR_TYPE_IPV4: + + /* Length of u32 addr_ipv4 */ + result += 4; + break; + + case NETINFO_ADDR_TYPE_IPV6: + + /* Length of u8 addr_ipv6[16] */ + result += 16; + break; + + default: + trunnel_assert(0); + break; + } + return result; +} +int +netinfo_addr_clear_errors(netinfo_addr_t *obj) +{ + int r = obj->trunnel_error_code_; + obj->trunnel_error_code_ = 0; + return r; +} +ssize_t +netinfo_addr_encode(uint8_t *output, const size_t avail, const netinfo_addr_t *obj) +{ + ssize_t result = 0; + size_t written = 0; + uint8_t *ptr = output; + const char *msg; +#ifdef TRUNNEL_CHECK_ENCODED_LEN + const ssize_t encoded_len = netinfo_addr_encoded_len(obj); +#endif + + if (NULL != (msg = netinfo_addr_check(obj))) + goto check_failed; + +#ifdef TRUNNEL_CHECK_ENCODED_LEN + trunnel_assert(encoded_len >= 0); +#endif + + /* Encode u8 addr_type IN [NETINFO_ADDR_TYPE_IPV4, NETINFO_ADDR_TYPE_IPV6] */ + trunnel_assert(written <= avail); + if (avail - written < 1) + goto truncated; + trunnel_set_uint8(ptr, (obj->addr_type)); + written += 1; ptr += 1; + + /* Encode u8 len IN [4, 16] */ + trunnel_assert(written <= avail); + if (avail - written < 1) + goto truncated; + trunnel_set_uint8(ptr, (obj->len)); + written += 1; ptr += 1; + + /* Encode union addr[addr_type] */ + trunnel_assert(written <= avail); + switch (obj->addr_type) { + + case NETINFO_ADDR_TYPE_IPV4: + + /* Encode u32 addr_ipv4 */ + trunnel_assert(written <= avail); + if (avail - written < 4) + goto truncated; + trunnel_set_uint32(ptr, trunnel_htonl(obj->addr_ipv4)); + written += 4; ptr += 4; + break; + + case NETINFO_ADDR_TYPE_IPV6: + + /* Encode u8 addr_ipv6[16] */ + trunnel_assert(written <= avail); + if (avail - written < 16) + goto truncated; + memcpy(ptr, obj->addr_ipv6, 16); + written += 16; ptr += 16; + break; + + default: + trunnel_assert(0); + break; + } + + + trunnel_assert(ptr == output + written); +#ifdef TRUNNEL_CHECK_ENCODED_LEN + { + trunnel_assert(encoded_len >= 0); + trunnel_assert((size_t)encoded_len == written); + } + +#endif + + return written; + + truncated: + result = -2; + goto fail; + check_failed: + (void)msg; + result = -1; + goto fail; + fail: + trunnel_assert(result < 0); + return result; +} + +/** As netinfo_addr_parse(), but do not allocate the output object. + */ +static ssize_t +netinfo_addr_parse_into(netinfo_addr_t *obj, const uint8_t *input, const size_t len_in) +{ + const uint8_t *ptr = input; + size_t remaining = len_in; + ssize_t result = 0; + (void)result; + + /* Parse u8 addr_type IN [NETINFO_ADDR_TYPE_IPV4, NETINFO_ADDR_TYPE_IPV6] */ + CHECK_REMAINING(1, truncated); + obj->addr_type = (trunnel_get_uint8(ptr)); + remaining -= 1; ptr += 1; + if (! (obj->addr_type == NETINFO_ADDR_TYPE_IPV4 || obj->addr_type == NETINFO_ADDR_TYPE_IPV6)) + goto fail; + + /* Parse u8 len IN [4, 16] */ + CHECK_REMAINING(1, truncated); + obj->len = (trunnel_get_uint8(ptr)); + remaining -= 1; ptr += 1; + if (! (obj->len == 4 || obj->len == 16)) + goto fail; + + /* Parse union addr[addr_type] */ + switch (obj->addr_type) { + + case NETINFO_ADDR_TYPE_IPV4: + + /* Parse u32 addr_ipv4 */ + CHECK_REMAINING(4, truncated); + obj->addr_ipv4 = trunnel_ntohl(trunnel_get_uint32(ptr)); + remaining -= 4; ptr += 4; + break; + + case NETINFO_ADDR_TYPE_IPV6: + + /* Parse u8 addr_ipv6[16] */ + CHECK_REMAINING(16, truncated); + memcpy(obj->addr_ipv6, ptr, 16); + remaining -= 16; ptr += 16; + break; + + default: + goto fail; + break; + } + trunnel_assert(ptr + remaining == input + len_in); + return len_in - remaining; + + truncated: + return -2; + fail: + result = -1; + return result; +} + +ssize_t +netinfo_addr_parse(netinfo_addr_t **output, const uint8_t *input, const size_t len_in) +{ + ssize_t result; + *output = netinfo_addr_new(); + if (NULL == *output) + return -1; + result = netinfo_addr_parse_into(*output, input, len_in); + if (result < 0) { + netinfo_addr_free(*output); + *output = NULL; + } + return result; +} +netinfo_cell_t * +netinfo_cell_new(void) +{ + netinfo_cell_t *val = trunnel_calloc(1, sizeof(netinfo_cell_t)); + if (NULL == val) + return NULL; + return val; +} + +/** Release all storage held inside 'obj', but do not free 'obj'. + */ +static void +netinfo_cell_clear(netinfo_cell_t *obj) +{ + (void) obj; + netinfo_addr_free(obj->other_addr); + obj->other_addr = NULL; + { + + unsigned idx; + for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->my_addrs); ++idx) { + netinfo_addr_free(TRUNNEL_DYNARRAY_GET(&obj->my_addrs, idx)); + } + } + TRUNNEL_DYNARRAY_WIPE(&obj->my_addrs); + TRUNNEL_DYNARRAY_CLEAR(&obj->my_addrs); +} + +void +netinfo_cell_free(netinfo_cell_t *obj) +{ + if (obj == NULL) + return; + netinfo_cell_clear(obj); + trunnel_memwipe(obj, sizeof(netinfo_cell_t)); + trunnel_free_(obj); +} + +uint32_t +netinfo_cell_get_timestamp(const netinfo_cell_t *inp) +{ + return inp->timestamp; +} +int +netinfo_cell_set_timestamp(netinfo_cell_t *inp, uint32_t val) +{ + inp->timestamp = val; + return 0; +} +struct netinfo_addr_st * +netinfo_cell_get_other_addr(netinfo_cell_t *inp) +{ + return inp->other_addr; +} +const struct netinfo_addr_st * +netinfo_cell_getconst_other_addr(const netinfo_cell_t *inp) +{ + return netinfo_cell_get_other_addr((netinfo_cell_t*) inp); +} +int +netinfo_cell_set_other_addr(netinfo_cell_t *inp, struct netinfo_addr_st *val) +{ + if (inp->other_addr && inp->other_addr != val) + netinfo_addr_free(inp->other_addr); + return netinfo_cell_set0_other_addr(inp, val); +} +int +netinfo_cell_set0_other_addr(netinfo_cell_t *inp, struct netinfo_addr_st *val) +{ + inp->other_addr = val; + return 0; +} +uint8_t +netinfo_cell_get_n_my_addrs(const netinfo_cell_t *inp) +{ + return inp->n_my_addrs; +} +int +netinfo_cell_set_n_my_addrs(netinfo_cell_t *inp, uint8_t val) +{ + inp->n_my_addrs = val; + return 0; +} +size_t +netinfo_cell_getlen_my_addrs(const netinfo_cell_t *inp) +{ + return TRUNNEL_DYNARRAY_LEN(&inp->my_addrs); +} + +struct netinfo_addr_st * +netinfo_cell_get_my_addrs(netinfo_cell_t *inp, size_t idx) +{ + return TRUNNEL_DYNARRAY_GET(&inp->my_addrs, idx); +} + + const struct netinfo_addr_st * +netinfo_cell_getconst_my_addrs(const netinfo_cell_t *inp, size_t idx) +{ + return netinfo_cell_get_my_addrs((netinfo_cell_t*)inp, idx); +} +int +netinfo_cell_set_my_addrs(netinfo_cell_t *inp, size_t idx, struct netinfo_addr_st * elt) +{ + netinfo_addr_t *oldval = TRUNNEL_DYNARRAY_GET(&inp->my_addrs, idx); + if (oldval && oldval != elt) + netinfo_addr_free(oldval); + return netinfo_cell_set0_my_addrs(inp, idx, elt); +} +int +netinfo_cell_set0_my_addrs(netinfo_cell_t *inp, size_t idx, struct netinfo_addr_st * elt) +{ + TRUNNEL_DYNARRAY_SET(&inp->my_addrs, idx, elt); + return 0; +} +int +netinfo_cell_add_my_addrs(netinfo_cell_t *inp, struct netinfo_addr_st * elt) +{ +#if SIZE_MAX >= UINT8_MAX + if (inp->my_addrs.n_ == UINT8_MAX) + goto trunnel_alloc_failed; +#endif + TRUNNEL_DYNARRAY_ADD(struct netinfo_addr_st *, &inp->my_addrs, elt, {}); + return 0; + trunnel_alloc_failed: + TRUNNEL_SET_ERROR_CODE(inp); + return -1; +} + +struct netinfo_addr_st * * +netinfo_cell_getarray_my_addrs(netinfo_cell_t *inp) +{ + return inp->my_addrs.elts_; +} +const struct netinfo_addr_st * const * +netinfo_cell_getconstarray_my_addrs(const netinfo_cell_t *inp) +{ + return (const struct netinfo_addr_st * const *)netinfo_cell_getarray_my_addrs((netinfo_cell_t*)inp); +} +int +netinfo_cell_setlen_my_addrs(netinfo_cell_t *inp, size_t newlen) +{ + struct netinfo_addr_st * *newptr; +#if UINT8_MAX < SIZE_MAX + if (newlen > UINT8_MAX) + goto trunnel_alloc_failed; +#endif + newptr = trunnel_dynarray_setlen(&inp->my_addrs.allocated_, + &inp->my_addrs.n_, inp->my_addrs.elts_, newlen, + sizeof(inp->my_addrs.elts_[0]), (trunnel_free_fn_t) netinfo_addr_free, + &inp->trunnel_error_code_); + if (newlen != 0 && newptr == NULL) + goto trunnel_alloc_failed; + inp->my_addrs.elts_ = newptr; + return 0; + trunnel_alloc_failed: + TRUNNEL_SET_ERROR_CODE(inp); + return -1; +} +const char * +netinfo_cell_check(const netinfo_cell_t *obj) +{ + if (obj == NULL) + return "Object was NULL"; + if (obj->trunnel_error_code_) + return "A set function failed on this object"; + { + const char *msg; + if (NULL != (msg = netinfo_addr_check(obj->other_addr))) + return msg; + } + { + const char *msg; + + unsigned idx; + for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->my_addrs); ++idx) { + if (NULL != (msg = netinfo_addr_check(TRUNNEL_DYNARRAY_GET(&obj->my_addrs, idx)))) + return msg; + } + } + if (TRUNNEL_DYNARRAY_LEN(&obj->my_addrs) != obj->n_my_addrs) + return "Length mismatch for my_addrs"; + return NULL; +} + +ssize_t +netinfo_cell_encoded_len(const netinfo_cell_t *obj) +{ + ssize_t result = 0; + + if (NULL != netinfo_cell_check(obj)) + return -1; + + + /* Length of u32 timestamp */ + result += 4; + + /* Length of struct netinfo_addr other_addr */ + result += netinfo_addr_encoded_len(obj->other_addr); + + /* Length of u8 n_my_addrs */ + result += 1; + + /* Length of struct netinfo_addr my_addrs[n_my_addrs] */ + { + + unsigned idx; + for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->my_addrs); ++idx) { + result += netinfo_addr_encoded_len(TRUNNEL_DYNARRAY_GET(&obj->my_addrs, idx)); + } + } + return result; +} +int +netinfo_cell_clear_errors(netinfo_cell_t *obj) +{ + int r = obj->trunnel_error_code_; + obj->trunnel_error_code_ = 0; + return r; +} +ssize_t +netinfo_cell_encode(uint8_t *output, const size_t avail, const netinfo_cell_t *obj) +{ + ssize_t result = 0; + size_t written = 0; + uint8_t *ptr = output; + const char *msg; +#ifdef TRUNNEL_CHECK_ENCODED_LEN + const ssize_t encoded_len = netinfo_cell_encoded_len(obj); +#endif + + if (NULL != (msg = netinfo_cell_check(obj))) + goto check_failed; + +#ifdef TRUNNEL_CHECK_ENCODED_LEN + trunnel_assert(encoded_len >= 0); +#endif + + /* Encode u32 timestamp */ + trunnel_assert(written <= avail); + if (avail - written < 4) + goto truncated; + trunnel_set_uint32(ptr, trunnel_htonl(obj->timestamp)); + written += 4; ptr += 4; + + /* Encode struct netinfo_addr other_addr */ + trunnel_assert(written <= avail); + result = netinfo_addr_encode(ptr, avail - written, obj->other_addr); + if (result < 0) + goto fail; /* XXXXXXX !*/ + written += result; ptr += result; + + /* Encode u8 n_my_addrs */ + trunnel_assert(written <= avail); + if (avail - written < 1) + goto truncated; + trunnel_set_uint8(ptr, (obj->n_my_addrs)); + written += 1; ptr += 1; + + /* Encode struct netinfo_addr my_addrs[n_my_addrs] */ + { + + unsigned idx; + for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->my_addrs); ++idx) { + trunnel_assert(written <= avail); + result = netinfo_addr_encode(ptr, avail - written, TRUNNEL_DYNARRAY_GET(&obj->my_addrs, idx)); + if (result < 0) + goto fail; /* XXXXXXX !*/ + written += result; ptr += result; + } + } + + + trunnel_assert(ptr == output + written); +#ifdef TRUNNEL_CHECK_ENCODED_LEN + { + trunnel_assert(encoded_len >= 0); + trunnel_assert((size_t)encoded_len == written); + } + +#endif + + return written; + + truncated: + result = -2; + goto fail; + check_failed: + (void)msg; + result = -1; + goto fail; + fail: + trunnel_assert(result < 0); + return result; +} + +/** As netinfo_cell_parse(), but do not allocate the output object. + */ +static ssize_t +netinfo_cell_parse_into(netinfo_cell_t *obj, const uint8_t *input, const size_t len_in) +{ + const uint8_t *ptr = input; + size_t remaining = len_in; + ssize_t result = 0; + (void)result; + + /* Parse u32 timestamp */ + CHECK_REMAINING(4, truncated); + obj->timestamp = trunnel_ntohl(trunnel_get_uint32(ptr)); + remaining -= 4; ptr += 4; + + /* Parse struct netinfo_addr other_addr */ + result = netinfo_addr_parse(&obj->other_addr, ptr, remaining); + if (result < 0) + goto relay_fail; + trunnel_assert((size_t)result <= remaining); + remaining -= result; ptr += result; + + /* Parse u8 n_my_addrs */ + CHECK_REMAINING(1, truncated); + obj->n_my_addrs = (trunnel_get_uint8(ptr)); + remaining -= 1; ptr += 1; + + /* Parse struct netinfo_addr my_addrs[n_my_addrs] */ + TRUNNEL_DYNARRAY_EXPAND(netinfo_addr_t *, &obj->my_addrs, obj->n_my_addrs, {}); + { + netinfo_addr_t * elt; + unsigned idx; + for (idx = 0; idx < obj->n_my_addrs; ++idx) { + result = netinfo_addr_parse(&elt, ptr, remaining); + if (result < 0) + goto relay_fail; + trunnel_assert((size_t)result <= remaining); + remaining -= result; ptr += result; + TRUNNEL_DYNARRAY_ADD(netinfo_addr_t *, &obj->my_addrs, elt, {netinfo_addr_free(elt);}); + } + } + trunnel_assert(ptr + remaining == input + len_in); + return len_in - remaining; + + truncated: + return -2; + relay_fail: + trunnel_assert(result < 0); + return result; + trunnel_alloc_failed: + return -1; +} + +ssize_t +netinfo_cell_parse(netinfo_cell_t **output, const uint8_t *input, const size_t len_in) +{ + ssize_t result; + *output = netinfo_cell_new(); + if (NULL == *output) + return -1; + result = netinfo_cell_parse_into(*output, input, len_in); + if (result < 0) { + netinfo_cell_free(*output); + *output = NULL; + } + return result; +} diff --git a/src/trunnel/netinfo.h b/src/trunnel/netinfo.h new file mode 100644 index 0000000000..ac46e603ba --- /dev/null +++ b/src/trunnel/netinfo.h @@ -0,0 +1,226 @@ +/* netinfo.h -- generated by Trunnel v1.5.2. + * https://gitweb.torproject.org/trunnel.git + * You probably shouldn't edit this file. + */ +#ifndef TRUNNEL_NETINFO_H +#define TRUNNEL_NETINFO_H + +#include +#include "trunnel.h" + +#define NETINFO_ADDR_TYPE_IPV4 4 +#define NETINFO_ADDR_TYPE_IPV6 6 +#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_NETINFO_ADDR) +struct netinfo_addr_st { + uint8_t addr_type; + uint8_t len; + uint32_t addr_ipv4; + uint8_t addr_ipv6[16]; + uint8_t trunnel_error_code_; +}; +#endif +typedef struct netinfo_addr_st netinfo_addr_t; +#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_NETINFO_CELL) +struct netinfo_cell_st { + uint32_t timestamp; + struct netinfo_addr_st *other_addr; + uint8_t n_my_addrs; + TRUNNEL_DYNARRAY_HEAD(, struct netinfo_addr_st *) my_addrs; + uint8_t trunnel_error_code_; +}; +#endif +typedef struct netinfo_cell_st netinfo_cell_t; +/** Return a newly allocated netinfo_addr with all elements set to + * zero. + */ +netinfo_addr_t *netinfo_addr_new(void); +/** Release all storage held by the netinfo_addr in 'victim'. (Do + * nothing if 'victim' is NULL.) + */ +void netinfo_addr_free(netinfo_addr_t *victim); +/** Try to parse a netinfo_addr from the buffer in 'input', using up + * to 'len_in' bytes from the input buffer. On success, return the + * number of bytes consumed and set *output to the newly allocated + * netinfo_addr_t. On failure, return -2 if the input appears + * truncated, and -1 if the input is otherwise invalid. + */ +ssize_t netinfo_addr_parse(netinfo_addr_t **output, const uint8_t *input, const size_t len_in); +/** Return the number of bytes we expect to need to encode the + * netinfo_addr in 'obj'. On failure, return a negative value. Note + * that this value may be an overestimate, and can even be an + * underestimate for certain unencodeable objects. + */ +ssize_t netinfo_addr_encoded_len(const netinfo_addr_t *obj); +/** Try to encode the netinfo_addr from 'input' into the buffer at + * 'output', using up to 'avail' bytes of the output buffer. On + * success, return the number of bytes used. On failure, return -2 if + * the buffer was not long enough, and -1 if the input was invalid. + */ +ssize_t netinfo_addr_encode(uint8_t *output, size_t avail, const netinfo_addr_t *input); +/** Check whether the internal state of the netinfo_addr in 'obj' is + * consistent. Return NULL if it is, and a short message if it is not. + */ +const char *netinfo_addr_check(const netinfo_addr_t *obj); +/** Clear any errors that were set on the object 'obj' by its setter + * functions. Return true iff errors were cleared. + */ +int netinfo_addr_clear_errors(netinfo_addr_t *obj); +/** Return the value of the addr_type field of the netinfo_addr_t in + * 'inp' + */ +uint8_t netinfo_addr_get_addr_type(const netinfo_addr_t *inp); +/** Set the value of the addr_type field of the netinfo_addr_t in + * 'inp' to 'val'. Return 0 on success; return -1 and set the error + * code on 'inp' on failure. + */ +int netinfo_addr_set_addr_type(netinfo_addr_t *inp, uint8_t val); +/** Return the value of the len field of the netinfo_addr_t in 'inp' + */ +uint8_t netinfo_addr_get_len(const netinfo_addr_t *inp); +/** Set the value of the len field of the netinfo_addr_t in 'inp' to + * 'val'. Return 0 on success; return -1 and set the error code on + * 'inp' on failure. + */ +int netinfo_addr_set_len(netinfo_addr_t *inp, uint8_t val); +/** Return the value of the addr_ipv4 field of the netinfo_addr_t in + * 'inp' + */ +uint32_t netinfo_addr_get_addr_ipv4(const netinfo_addr_t *inp); +/** Set the value of the addr_ipv4 field of the netinfo_addr_t in + * 'inp' to 'val'. Return 0 on success; return -1 and set the error + * code on 'inp' on failure. + */ +int netinfo_addr_set_addr_ipv4(netinfo_addr_t *inp, uint32_t val); +/** Return the (constant) length of the array holding the addr_ipv6 + * field of the netinfo_addr_t in 'inp'. + */ +size_t netinfo_addr_getlen_addr_ipv6(const netinfo_addr_t *inp); +/** Return the element at position 'idx' of the fixed array field + * addr_ipv6 of the netinfo_addr_t in 'inp'. + */ +uint8_t netinfo_addr_get_addr_ipv6(netinfo_addr_t *inp, size_t idx); +/** As netinfo_addr_get_addr_ipv6, but take and return a const pointer + */ +uint8_t netinfo_addr_getconst_addr_ipv6(const netinfo_addr_t *inp, size_t idx); +/** Change the element at position 'idx' of the fixed array field + * addr_ipv6 of the netinfo_addr_t in 'inp', so that it will hold the + * value 'elt'. + */ +int netinfo_addr_set_addr_ipv6(netinfo_addr_t *inp, size_t idx, uint8_t elt); +/** Return a pointer to the 16-element array field addr_ipv6 of 'inp'. + */ +uint8_t * netinfo_addr_getarray_addr_ipv6(netinfo_addr_t *inp); +/** As netinfo_addr_get_addr_ipv6, but take and return a const pointer + */ +const uint8_t * netinfo_addr_getconstarray_addr_ipv6(const netinfo_addr_t *inp); +/** Return a newly allocated netinfo_cell with all elements set to + * zero. + */ +netinfo_cell_t *netinfo_cell_new(void); +/** Release all storage held by the netinfo_cell in 'victim'. (Do + * nothing if 'victim' is NULL.) + */ +void netinfo_cell_free(netinfo_cell_t *victim); +/** Try to parse a netinfo_cell from the buffer in 'input', using up + * to 'len_in' bytes from the input buffer. On success, return the + * number of bytes consumed and set *output to the newly allocated + * netinfo_cell_t. On failure, return -2 if the input appears + * truncated, and -1 if the input is otherwise invalid. + */ +ssize_t netinfo_cell_parse(netinfo_cell_t **output, const uint8_t *input, const size_t len_in); +/** Return the number of bytes we expect to need to encode the + * netinfo_cell in 'obj'. On failure, return a negative value. Note + * that this value may be an overestimate, and can even be an + * underestimate for certain unencodeable objects. + */ +ssize_t netinfo_cell_encoded_len(const netinfo_cell_t *obj); +/** Try to encode the netinfo_cell from 'input' into the buffer at + * 'output', using up to 'avail' bytes of the output buffer. On + * success, return the number of bytes used. On failure, return -2 if + * the buffer was not long enough, and -1 if the input was invalid. + */ +ssize_t netinfo_cell_encode(uint8_t *output, size_t avail, const netinfo_cell_t *input); +/** Check whether the internal state of the netinfo_cell in 'obj' is + * consistent. Return NULL if it is, and a short message if it is not. + */ +const char *netinfo_cell_check(const netinfo_cell_t *obj); +/** Clear any errors that were set on the object 'obj' by its setter + * functions. Return true iff errors were cleared. + */ +int netinfo_cell_clear_errors(netinfo_cell_t *obj); +/** Return the value of the timestamp field of the netinfo_cell_t in + * 'inp' + */ +uint32_t netinfo_cell_get_timestamp(const netinfo_cell_t *inp); +/** Set the value of the timestamp field of the netinfo_cell_t in + * 'inp' to 'val'. Return 0 on success; return -1 and set the error + * code on 'inp' on failure. + */ +int netinfo_cell_set_timestamp(netinfo_cell_t *inp, uint32_t val); +/** Return the value of the other_addr field of the netinfo_cell_t in + * 'inp' + */ +struct netinfo_addr_st * netinfo_cell_get_other_addr(netinfo_cell_t *inp); +/** As netinfo_cell_get_other_addr, but take and return a const + * pointer + */ +const struct netinfo_addr_st * netinfo_cell_getconst_other_addr(const netinfo_cell_t *inp); +/** Set the value of the other_addr field of the netinfo_cell_t in + * 'inp' to 'val'. Free the old value if any. Steals the referenceto + * 'val'.Return 0 on success; return -1 and set the error code on + * 'inp' on failure. + */ +int netinfo_cell_set_other_addr(netinfo_cell_t *inp, struct netinfo_addr_st *val); +/** As netinfo_cell_set_other_addr, but does not free the previous + * value. + */ +int netinfo_cell_set0_other_addr(netinfo_cell_t *inp, struct netinfo_addr_st *val); +/** Return the value of the n_my_addrs field of the netinfo_cell_t in + * 'inp' + */ +uint8_t netinfo_cell_get_n_my_addrs(const netinfo_cell_t *inp); +/** Set the value of the n_my_addrs field of the netinfo_cell_t in + * 'inp' to 'val'. Return 0 on success; return -1 and set the error + * code on 'inp' on failure. + */ +int netinfo_cell_set_n_my_addrs(netinfo_cell_t *inp, uint8_t val); +/** Return the length of the dynamic array holding the my_addrs field + * of the netinfo_cell_t in 'inp'. + */ +size_t netinfo_cell_getlen_my_addrs(const netinfo_cell_t *inp); +/** Return the element at position 'idx' of the dynamic array field + * my_addrs of the netinfo_cell_t in 'inp'. + */ +struct netinfo_addr_st * netinfo_cell_get_my_addrs(netinfo_cell_t *inp, size_t idx); +/** As netinfo_cell_get_my_addrs, but take and return a const pointer + */ + const struct netinfo_addr_st * netinfo_cell_getconst_my_addrs(const netinfo_cell_t *inp, size_t idx); +/** Change the element at position 'idx' of the dynamic array field + * my_addrs of the netinfo_cell_t in 'inp', so that it will hold the + * value 'elt'. Free the previous value, if any. + */ +int netinfo_cell_set_my_addrs(netinfo_cell_t *inp, size_t idx, struct netinfo_addr_st * elt); +/** As netinfo_cell_set_my_addrs, but does not free the previous + * value. + */ +int netinfo_cell_set0_my_addrs(netinfo_cell_t *inp, size_t idx, struct netinfo_addr_st * elt); +/** Append a new element 'elt' to the dynamic array field my_addrs of + * the netinfo_cell_t in 'inp'. + */ +int netinfo_cell_add_my_addrs(netinfo_cell_t *inp, struct netinfo_addr_st * elt); +/** Return a pointer to the variable-length array field my_addrs of + * 'inp'. + */ +struct netinfo_addr_st * * netinfo_cell_getarray_my_addrs(netinfo_cell_t *inp); +/** As netinfo_cell_get_my_addrs, but take and return a const pointer + */ +const struct netinfo_addr_st * const * netinfo_cell_getconstarray_my_addrs(const netinfo_cell_t *inp); +/** Change the length of the variable-length array field my_addrs of + * 'inp' to 'newlen'.Fill extra elements with NULL; free removed + * elements. Return 0 on success; return -1 and set the error code on + * 'inp' on failure. + */ +int netinfo_cell_setlen_my_addrs(netinfo_cell_t *inp, size_t newlen); + + +#endif diff --git a/src/trunnel/netinfo.trunnel b/src/trunnel/netinfo.trunnel new file mode 100644 index 0000000000..9c80fcc533 --- /dev/null +++ b/src/trunnel/netinfo.trunnel @@ -0,0 +1,21 @@ +const NETINFO_ADDR_TYPE_IPV4 = 4; +const NETINFO_ADDR_TYPE_IPV6 = 6; + +struct netinfo_addr { + u8 addr_type IN [NETINFO_ADDR_TYPE_IPV4, NETINFO_ADDR_TYPE_IPV6]; + u8 len IN [4, 16]; + union addr[addr_type] { + NETINFO_ADDR_TYPE_IPV4: u32 ipv4; + NETINFO_ADDR_TYPE_IPV6: u8 ipv6[16]; + default: fail; + }; + +} + +struct netinfo_cell { + u32 timestamp; + struct netinfo_addr other_addr; + u8 n_my_addrs; + struct netinfo_addr my_addrs[n_my_addrs]; +} + From d3e6112bb2536f0437cabc3e675177a95d3e374c Mon Sep 17 00:00:00 2001 From: rl1987 Date: Fri, 21 Sep 2018 14:58:19 +0300 Subject: [PATCH 0073/2557] Use trunnel for NETINFO cell parsing --- src/core/or/channeltls.c | 94 ++++++++++++++++++++++++++++++---------- 1 file changed, 70 insertions(+), 24 deletions(-) diff --git a/src/core/or/channeltls.c b/src/core/or/channeltls.c index 883acacf37..392e78506e 100644 --- a/src/core/or/channeltls.c +++ b/src/core/or/channeltls.c @@ -59,6 +59,7 @@ #include "feature/nodelist/torcert.h" #include "feature/nodelist/networkstatus.h" #include "trunnel/channelpadding_negotiation.h" +#include "trunnel/netinfo.h" #include "core/or/channelpadding.h" #include "core/or/cell_st.h" @@ -1636,6 +1637,35 @@ channel_tls_process_padding_negotiate_cell(cell_t *cell, channel_tls_t *chan) channelpadding_negotiate_free(negotiation); } +/** + * Convert netinfo_addr into corresponding tor_addr. + * Return 0 on success; on failure, return -1 and log a warning. + */ +static int +tor_addr_from_netinfo_addr(tor_addr_t *tor_addr, + const netinfo_addr_t *netinfo_addr) { + tor_assert(tor_addr); + tor_assert(netinfo_addr); + + uint8_t type = netinfo_addr_get_addr_type(netinfo_addr); + uint8_t len = netinfo_addr_get_len(netinfo_addr); + + if (type == NETINFO_ADDR_TYPE_IPV4 && len == 4) { + uint32_t ipv4 = netinfo_addr_get_addr_ipv4(netinfo_addr); + tor_addr_from_ipv4h(tor_addr, ipv4); + } else if (type == NETINFO_ADDR_TYPE_IPV6 && len == 16) { + const uint8_t *ipv6_bytes = netinfo_addr_getconstarray_addr_ipv6( + netinfo_addr); + tor_addr_from_ipv6_bytes(tor_addr, (const char *)ipv6_bytes); + } else { + log_fn(LOG_PROTOCOL_WARN, LD_OR, "Cannot read address from NETINFO " + "- wrong type/length."); + return -1; + } + + return 0; +} + /** * Process a 'netinfo' cell. * @@ -1648,8 +1678,6 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan) time_t timestamp; uint8_t my_addr_type; uint8_t my_addr_len; - const uint8_t *my_addr_ptr; - const uint8_t *cp, *end; uint8_t n_other_addrs; time_t now = time(NULL); const routerinfo_t *me = router_get_my_routerinfo(); @@ -1720,34 +1748,48 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan) } /* Decode the cell. */ - timestamp = ntohl(get_uint32(cell->payload)); + netinfo_cell_t *netinfo_cell = NULL; + + ssize_t parsed = netinfo_cell_parse(&netinfo_cell, cell->payload, + CELL_PAYLOAD_SIZE); + + if (parsed < 0) { + log_fn(LOG_PROTOCOL_WARN, LD_OR, + "Failed to parse NETINFO cell - closing connection."); + connection_or_close_for_error(chan->conn, 0); + return; + } + + timestamp = netinfo_cell_get_timestamp(netinfo_cell); + + const netinfo_addr_t *my_addr = + netinfo_cell_getconst_other_addr(netinfo_cell); + + my_addr_type = netinfo_addr_get_addr_type(my_addr); + my_addr_len = netinfo_addr_get_len(my_addr); + if (labs(now - chan->conn->handshake_state->sent_versions_at) < 180) { apparent_skew = now - timestamp; } - - my_addr_type = (uint8_t) cell->payload[4]; - my_addr_len = (uint8_t) cell->payload[5]; - my_addr_ptr = (uint8_t*) cell->payload + 6; - end = cell->payload + CELL_PAYLOAD_SIZE; - cp = cell->payload + 6 + my_addr_len; - /* We used to check: * if (my_addr_len >= CELL_PAYLOAD_SIZE - 6) { * * This is actually never going to happen, since my_addr_len is at most 255, * and CELL_PAYLOAD_LEN - 6 is 503. So we know that cp is < end. */ - if (my_addr_type == RESOLVED_TYPE_IPV4 && my_addr_len == 4) { - tor_addr_from_ipv4n(&my_apparent_addr, get_uint32(my_addr_ptr)); + if (tor_addr_from_netinfo_addr(&my_apparent_addr, my_addr) == -1) { + connection_or_close_for_error(chan->conn, 0); + netinfo_cell_free(netinfo_cell); + return; + } + if (my_addr_type == NETINFO_ADDR_TYPE_IPV4 && my_addr_len == 4) { if (!get_options()->BridgeRelay && me && - get_uint32(my_addr_ptr) == htonl(me->addr)) { + tor_addr_eq_ipv4h(&my_apparent_addr, me->addr)) { TLS_CHAN_TO_BASE(chan)->is_canonical_to_peer = 1; } - - } else if (my_addr_type == RESOLVED_TYPE_IPV6 && my_addr_len == 16) { - tor_addr_from_ipv6_bytes(&my_apparent_addr, (const char *) my_addr_ptr); - + } else if (my_addr_type == NETINFO_ADDR_TYPE_IPV6 && + my_addr_len == 16) { if (!get_options()->BridgeRelay && me && !tor_addr_is_null(&me->ipv6_addr) && tor_addr_eq(&my_apparent_addr, &me->ipv6_addr)) { @@ -1755,17 +1797,21 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan) } } - n_other_addrs = (uint8_t) *cp++; - while (n_other_addrs && cp < end-2) { + n_other_addrs = netinfo_cell_get_n_my_addrs(netinfo_cell); + for (uint8_t i = 0; i < n_other_addrs; i++) { /* Consider all the other addresses; if any matches, this connection is * "canonical." */ + + const netinfo_addr_t *netinfo_addr = + netinfo_cell_getconst_my_addrs(netinfo_cell, i); + tor_addr_t addr; - const uint8_t *next = - decode_address_from_payload(&addr, cp, (int)(end-cp)); - if (next == NULL) { + + if (tor_addr_from_netinfo_addr(&addr, netinfo_addr) == -1) { log_fn(LOG_PROTOCOL_WARN, LD_OR, "Bad address in netinfo cell; closing connection."); connection_or_close_for_error(chan->conn, 0); + netinfo_cell_free(netinfo_cell); return; } /* A relay can connect from anywhere and be canonical, so @@ -1779,10 +1825,10 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan) connection_or_set_canonical(chan->conn, 1); break; } - cp = next; - --n_other_addrs; } + netinfo_cell_free(netinfo_cell); + if (me && !TLS_CHAN_TO_BASE(chan)->is_canonical_to_peer && channel_is_canonical(TLS_CHAN_TO_BASE(chan))) { const char *descr = From 5cc86e364f4efbaa39b599e007765fc1770c8010 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Fri, 21 Sep 2018 17:56:39 +0300 Subject: [PATCH 0074/2557] Generate NETINFO cell using trunnel --- src/core/or/connection_or.c | 82 +++++++++++++++++++++++++++---------- 1 file changed, 61 insertions(+), 21 deletions(-) diff --git a/src/core/or/connection_or.c b/src/core/or/connection_or.c index 65f4e28c92..274cf85904 100644 --- a/src/core/or/connection_or.c +++ b/src/core/or/connection_or.c @@ -46,6 +46,7 @@ #include "lib/geoip/geoip.h" #include "core/mainloop/mainloop.h" #include "trunnel/link_handshake.h" +#include "trunnel/netinfo.h" #include "feature/nodelist/microdesc.h" #include "feature/nodelist/networkstatus.h" #include "feature/nodelist/nodelist.h" @@ -2428,6 +2429,31 @@ connection_or_send_versions(or_connection_t *conn, int v3_plus) return 0; } +static netinfo_addr_t * +netinfo_addr_from_tor_addr(const tor_addr_t *tor_addr) +{ + sa_family_t addr_family = tor_addr_family(tor_addr); + + if (BUG(addr_family != AF_INET && addr_family != AF_INET6)) + return NULL; + + netinfo_addr_t *netinfo_addr = netinfo_addr_new(); + + if (addr_family == AF_INET) { + netinfo_addr_set_addr_type(netinfo_addr, NETINFO_ADDR_TYPE_IPV4); + netinfo_addr_set_len(netinfo_addr, 4); + netinfo_addr_set_addr_ipv4(netinfo_addr, tor_addr_to_ipv4h(tor_addr)); + } else if (addr_family == AF_INET6) { + netinfo_addr_set_addr_type(netinfo_addr, NETINFO_ADDR_TYPE_IPV6); + netinfo_addr_set_len(netinfo_addr, 16); + uint8_t *ipv6_buf = netinfo_addr_getarray_addr_ipv6(netinfo_addr); + const uint8_t *in6_addr = tor_addr_to_in6_addr8(tor_addr); + memcpy(ipv6_buf, in6_addr, 16); + } + + return netinfo_addr; +} + /** Send a NETINFO cell on conn, telling the other server what we know * about their address, our address, and the current time. */ MOCK_IMPL(int, @@ -2436,8 +2462,7 @@ connection_or_send_netinfo,(or_connection_t *conn)) cell_t cell; time_t now = time(NULL); const routerinfo_t *me; - int len; - uint8_t *out; + int r = -1; tor_assert(conn->handshake_state); @@ -2450,20 +2475,21 @@ connection_or_send_netinfo,(or_connection_t *conn)) memset(&cell, 0, sizeof(cell_t)); cell.command = CELL_NETINFO; + netinfo_cell_t *netinfo_cell = netinfo_cell_new(); + /* Timestamp, if we're a relay. */ if (public_server_mode(get_options()) || ! conn->is_outgoing) - set_uint32(cell.payload, htonl((uint32_t)now)); + netinfo_cell_set_timestamp(netinfo_cell, (uint32_t)now); /* Their address. */ - out = cell.payload + 4; + const tor_addr_t *remote_tor_addr = + !tor_addr_is_null(&conn->real_addr) ? &conn->real_addr : &conn->base_.addr; /* We use &conn->real_addr below, unless it hasn't yet been set. If it * hasn't yet been set, we know that base_.addr hasn't been tampered with * yet either. */ - len = append_address_to_payload(out, !tor_addr_is_null(&conn->real_addr) - ? &conn->real_addr : &conn->base_.addr); - if (len<0) - return -1; - out += len; + netinfo_addr_t *their_addr = netinfo_addr_from_tor_addr(remote_tor_addr); + + netinfo_cell_set_other_addr(netinfo_cell, their_addr); /* My address -- only include it if I'm a public relay, or if I'm a * bridge and this is an incoming connection. If I'm a bridge and this @@ -2471,28 +2497,42 @@ connection_or_send_netinfo,(or_connection_t *conn)) if ((public_server_mode(get_options()) || !conn->is_outgoing) && (me = router_get_my_routerinfo())) { tor_addr_t my_addr; - *out++ = 1 + !tor_addr_is_null(&me->ipv6_addr); - tor_addr_from_ipv4h(&my_addr, me->addr); - len = append_address_to_payload(out, &my_addr); - if (len < 0) - return -1; - out += len; + + uint8_t n_my_addrs = 1 + !tor_addr_is_null(&me->ipv6_addr); + netinfo_cell_set_n_my_addrs(netinfo_cell, n_my_addrs); + + netinfo_cell_add_my_addrs(netinfo_cell, + netinfo_addr_from_tor_addr(&my_addr)); if (!tor_addr_is_null(&me->ipv6_addr)) { - len = append_address_to_payload(out, &me->ipv6_addr); - if (len < 0) - return -1; + netinfo_cell_add_my_addrs(netinfo_cell, + netinfo_addr_from_tor_addr(&me->ipv6_addr)); } - } else { - *out = 0; + } + + const char *errmsg = NULL; + if ((errmsg = netinfo_cell_check(netinfo_cell))) { + log_warn(LD_OR, "Failed to validate NETINFO cell with error: %s", + errmsg); + goto cleanup; + } + + if (netinfo_cell_encode(cell.payload, CELL_PAYLOAD_SIZE, + netinfo_cell) < 0) { + log_warn(LD_OR, "Failed generating NETINFO cell"); + goto cleanup; } conn->handshake_state->digest_sent_data = 0; conn->handshake_state->sent_netinfo = 1; connection_or_write_cell_to_buf(&cell, conn); - return 0; + r = 0; + cleanup: + netinfo_cell_free(netinfo_cell); + + return r; } /** Helper used to add an encoded certs to a cert cell */ From b64e1e602bac5e108e40a284616e3fe7f7597b17 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Sat, 22 Sep 2018 20:01:15 +0300 Subject: [PATCH 0075/2557] Add changes file --- changes/ticket27325 | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 changes/ticket27325 diff --git a/changes/ticket27325 b/changes/ticket27325 new file mode 100644 index 0000000000..2cf2bb7d69 --- /dev/null +++ b/changes/ticket27325 @@ -0,0 +1,4 @@ + o Code simplification and refactoring: + - Reimplement NETINFO cell parsing and generation to rely on + trunnel-generated wire format handling code. Closes ticket + 27325. From bdf6540edf5b9d3d45214393412b8fe7e42da5ca Mon Sep 17 00:00:00 2001 From: rl1987 Date: Fri, 26 Oct 2018 11:30:26 +0300 Subject: [PATCH 0076/2557] Add a comment about address type field to netinfo.trunnel --- src/trunnel/netinfo.trunnel | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/trunnel/netinfo.trunnel b/src/trunnel/netinfo.trunnel index 9c80fcc533..21afcd136e 100644 --- a/src/trunnel/netinfo.trunnel +++ b/src/trunnel/netinfo.trunnel @@ -1,3 +1,6 @@ +// Warning: make sure these values are consistent with RESOLVED_TYPE_* +// constants in Tor code and numbers in Section 6.4 of tor-spec.txt. + const NETINFO_ADDR_TYPE_IPV4 = 4; const NETINFO_ADDR_TYPE_IPV6 = 6; From 5c2212c7346c5368a9d5e35ac78b143999682297 Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Fri, 26 Oct 2018 14:55:17 +0300 Subject: [PATCH 0077/2557] HSv3: Correctly memwipe client auth keystream. Wipe the whole thing, not just the size of the pointer. --- src/feature/hs/hs_descriptor.c | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/src/feature/hs/hs_descriptor.c b/src/feature/hs/hs_descriptor.c index 0e3c761bf6..1b2008c804 100644 --- a/src/feature/hs/hs_descriptor.c +++ b/src/feature/hs/hs_descriptor.c @@ -1406,10 +1406,10 @@ encrypted_data_length_is_valid(size_t len) * SECRET_SEED = x25519(sk, pk) * KEYS = KDF(subcredential | SECRET_SEED, 40) * - * The keys_out parameter will points to the buffer containing the KEYS. The - * caller should wipe and free its content once done with it. This function - * can't fail. */ -static void + * Set the keys_out argument to point to the buffer containing the KEYS, + * and return the buffer's length. The caller should wipe and free its content + * once done with it. This function can't fail. */ +static size_t build_descriptor_cookie_keys(const uint8_t *subcredential, size_t subcredential_len, const curve25519_secret_key_t *sk, @@ -1441,6 +1441,7 @@ build_descriptor_cookie_keys(const uint8_t *subcredential, memwipe(secret_seed, 0, sizeof(secret_seed)); *keys_out = keystream; + return keystream_len; } /* Decrypt the descriptor cookie given the descriptor, the auth client, @@ -1456,6 +1457,7 @@ decrypt_descriptor_cookie(const hs_descriptor_t *desc, { int ret = -1; uint8_t *keystream = NULL; + size_t keystream_length = 0; uint8_t *descriptor_cookie = NULL; const uint8_t *cookie_key = NULL; crypto_cipher_t *cipher = NULL; @@ -1471,10 +1473,12 @@ decrypt_descriptor_cookie(const hs_descriptor_t *desc, tor_assert(!tor_mem_is_zero((char *) desc->subcredential, DIGEST256_LEN)); /* Get the KEYS component to derive the CLIENT-ID and COOKIE-KEY. */ - build_descriptor_cookie_keys(desc->subcredential, DIGEST256_LEN, - client_auth_sk, - &desc->superencrypted_data.auth_ephemeral_pubkey, - &keystream); + keystream_length = + build_descriptor_cookie_keys(desc->subcredential, DIGEST256_LEN, + client_auth_sk, + &desc->superencrypted_data.auth_ephemeral_pubkey, + &keystream); + tor_assert(keystream_length > 0); /* If the client id of auth client is not the same as the calculcated * client id, it means that this auth client is invaild according to the @@ -1500,7 +1504,7 @@ decrypt_descriptor_cookie(const hs_descriptor_t *desc, if (cipher) { crypto_cipher_free(cipher); } - memwipe(keystream, 0, sizeof(keystream)); + memwipe(keystream, 0, keystream_length); tor_free(keystream); return ret; } @@ -2915,6 +2919,7 @@ hs_desc_build_authorized_client(const uint8_t *subcredential, hs_desc_authorized_client_t *client_out) { uint8_t *keystream = NULL; + size_t keystream_length = 0; const uint8_t *cookie_key; crypto_cipher_t *cipher; @@ -2933,8 +2938,11 @@ hs_desc_build_authorized_client(const uint8_t *subcredential, DIGEST256_LEN)); /* Get the KEYS part so we can derive the CLIENT-ID and COOKIE-KEY. */ - build_descriptor_cookie_keys(subcredential, DIGEST256_LEN, - auth_ephemeral_sk, client_auth_pk, &keystream); + keystream_length = + build_descriptor_cookie_keys(subcredential, DIGEST256_LEN, + auth_ephemeral_sk, client_auth_pk, + &keystream); + tor_assert(keystream_length > 0); /* Extract the CLIENT-ID and COOKIE-KEY from the KEYS. */ memcpy(client_out->client_id, keystream, HS_DESC_CLIENT_ID_LEN); @@ -2951,7 +2959,7 @@ hs_desc_build_authorized_client(const uint8_t *subcredential, (const char *) descriptor_cookie, HS_DESC_DESCRIPTOR_COOKIE_LEN); - memwipe(keystream, 0, sizeof(keystream)); + memwipe(keystream, 0, keystream_length); tor_free(keystream); crypto_cipher_free(cipher); From fe89d9df6935e3a61f8014ef9033b1ca2cfff5a1 Mon Sep 17 00:00:00 2001 From: teor Date: Mon, 29 Oct 2018 13:47:43 +1000 Subject: [PATCH 0078/2557] doc: Spell make test-network-all correctly in ReleasingTor.md Closes ticket 28821. --- doc/HACKING/ReleasingTor.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/HACKING/ReleasingTor.md b/doc/HACKING/ReleasingTor.md index 55a40fc89b..29e1ae1668 100644 --- a/doc/HACKING/ReleasingTor.md +++ b/doc/HACKING/ReleasingTor.md @@ -39,7 +39,7 @@ new Tor release: Does 'make distcheck' complain? How about 'make test-stem' and 'make test-network' and - `make test-network-full`? + `make test-network-all`? - Are all those tests still happy with --enable-expensive-hardening ? From b0c456e578229420ed833ba919c181f7dd5f5e2b Mon Sep 17 00:00:00 2001 From: teor Date: Mon, 29 Oct 2018 13:49:47 +1000 Subject: [PATCH 0079/2557] doc: Use `` for commands and "" for quotes in ReleasingTor.md --- doc/HACKING/ReleasingTor.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/HACKING/ReleasingTor.md b/doc/HACKING/ReleasingTor.md index 29e1ae1668..b5444afa96 100644 --- a/doc/HACKING/ReleasingTor.md +++ b/doc/HACKING/ReleasingTor.md @@ -36,9 +36,9 @@ new Tor release: What about clang scan-build? - Does 'make distcheck' complain? + Does `make distcheck` complain? - How about 'make test-stem' and 'make test-network' and + How about `make test-stem` and `make test-network` and `make test-network-all`? - Are all those tests still happy with --enable-expensive-hardening ? @@ -79,7 +79,7 @@ new Tor release: Present and imperative tense: not past. - 'Relays', not 'servers' or 'nodes' or 'Tor relays'. + "Relays", not "servers" or "nodes" or "Tor relays". "Stop FOOing", not "Fix a bug where we would FOO". @@ -100,7 +100,7 @@ new Tor release: For stable releases that backport things from later, we try to compose their releases, we try to make sure that we keep the changelog entries - identical to their original versions, with a 'backport from 0.x.y.z' + identical to their original versions, with a "backport from 0.x.y.z" note added to each section. So in this case, once you have the items from the changes files copied together, don't use them to build a new changelog: instead, look up the corrected versions that were merged From 635312fc2a8fe96aa282ed242279629f814b9333 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Mon, 29 Oct 2018 10:45:14 +0200 Subject: [PATCH 0080/2557] Silence shellcheck SC2086 in run_trunnel.sh --- scripts/codegen/run_trunnel.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/codegen/run_trunnel.sh b/scripts/codegen/run_trunnel.sh index 428804342a..9df402cf59 100755 --- a/scripts/codegen/run_trunnel.sh +++ b/scripts/codegen/run_trunnel.sh @@ -10,7 +10,7 @@ OPTIONS="--require-version=1.5.1" # Get all .trunnel files recursively from that directory so we can support # multiple sub-directories. for file in `find ./src/trunnel/ -name '*.trunnel'`; do - python -m trunnel ${OPTIONS} $file + python -m trunnel ${OPTIONS} "$file" done python -m trunnel ${OPTIONS} --write-c-files --target-dir=./src/ext/trunnel/ From a0dd6bfdb03f53a52778e65161b9f3c68234bb82 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Mon, 29 Oct 2018 10:54:31 +0200 Subject: [PATCH 0081/2557] run_trunnel.sh: Use 'find -exec' instead of a 'for' loop This fixes shellcheck warnings SC2044 and SC2006. --- scripts/codegen/run_trunnel.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/scripts/codegen/run_trunnel.sh b/scripts/codegen/run_trunnel.sh index 9df402cf59..645b3c2158 100755 --- a/scripts/codegen/run_trunnel.sh +++ b/scripts/codegen/run_trunnel.sh @@ -9,9 +9,7 @@ OPTIONS="--require-version=1.5.1" # Get all .trunnel files recursively from that directory so we can support # multiple sub-directories. -for file in `find ./src/trunnel/ -name '*.trunnel'`; do - python -m trunnel ${OPTIONS} "$file" -done +find ./src/trunnel/ -name '*.trunnel' -exec python -m trunnel ${OPTIONS} {} \; python -m trunnel ${OPTIONS} --write-c-files --target-dir=./src/ext/trunnel/ From 4af27e016814f4817174ed87e7b660d65c6eaf9c Mon Sep 17 00:00:00 2001 From: rl1987 Date: Mon, 29 Oct 2018 11:05:55 +0200 Subject: [PATCH 0082/2557] Add changes file --- changes/ticket28010 | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 changes/ticket28010 diff --git a/changes/ticket28010 b/changes/ticket28010 new file mode 100644 index 0000000000..4fca17d022 --- /dev/null +++ b/changes/ticket28010 @@ -0,0 +1,3 @@ + o Code simplification and refactoring: + - Fix shellcheck warnings in run_trunnel.sh. Resolves issue + 28010. From eab81b12e92ea3ae53d14337a6fafc9d6c32b39a Mon Sep 17 00:00:00 2001 From: rl1987 Date: Mon, 29 Oct 2018 11:31:03 +0200 Subject: [PATCH 0083/2557] Fix shellcheck warning SC2086 in run_calltool.sh --- scripts/maint/run_calltool.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/maint/run_calltool.sh b/scripts/maint/run_calltool.sh index efb8706fea..f36810f5b6 100755 --- a/scripts/maint/run_calltool.sh +++ b/scripts/maint/run_calltool.sh @@ -15,7 +15,7 @@ SUBITEMS="fn_graph fn_invgraph fn_scc fn_scc_weaklinks module_graph module_invgr for calculation in $SUBITEMS; do echo "======== $calculation" - python -m calltool $calculation > callgraph/$calculation + python -m calltool "$calculation" > callgraph/"$calculation" done echo < callgraph/README From 6aef0ce94e9bfe5d3bd06f0a3349f9e8930218f5 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Mon, 29 Oct 2018 11:37:16 +0200 Subject: [PATCH 0084/2557] Fix shellcheck issue SC2217 in run_calltool.sh --- scripts/maint/run_calltool.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/maint/run_calltool.sh b/scripts/maint/run_calltool.sh index f36810f5b6..b0268322f4 100755 --- a/scripts/maint/run_calltool.sh +++ b/scripts/maint/run_calltool.sh @@ -18,7 +18,7 @@ for calculation in $SUBITEMS; do python -m calltool "$calculation" > callgraph/"$calculation" done -echo < callgraph/README +cat < callgraph/README This directory holds output from calltool, as run on Tor. For more information about each of these files, see the NOTES and README files in the calltool distribution. From 5a3cb495ce5bf010440fc0288c1ca00fff6ec8e5 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Mon, 29 Oct 2018 11:41:36 +0200 Subject: [PATCH 0085/2557] Add changes file --- changes/ticket28011 | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 changes/ticket28011 diff --git a/changes/ticket28011 b/changes/ticket28011 new file mode 100644 index 0000000000..5efc3c917b --- /dev/null +++ b/changes/ticket28011 @@ -0,0 +1,3 @@ + o Code simplification and refactoring: + - Fix shellcheck warnings in run_calltool.sh. Resolves + ticket 28011. From 91748cd17c1c1c2d5590d3b5533b74fcc1cd2140 Mon Sep 17 00:00:00 2001 From: David Goulet Date: Mon, 29 Oct 2018 08:58:19 -0400 Subject: [PATCH 0086/2557] doc: Add Maintaining.md documentation Closes #28225 Signed-off-by: David Goulet --- doc/HACKING/Maintaining.md | 113 +++++++++++++++++++++++++++++++++++++ 1 file changed, 113 insertions(+) create mode 100644 doc/HACKING/Maintaining.md diff --git a/doc/HACKING/Maintaining.md b/doc/HACKING/Maintaining.md new file mode 100644 index 0000000000..22d62b5471 --- /dev/null +++ b/doc/HACKING/Maintaining.md @@ -0,0 +1,113 @@ +# Maintaining Tor + +This document details the duties and processes on maintaining the Tor code +base. + +The first section describes who is the current Tor maintainer and what are the +responsabilities. Tor has one main single maintainer but does have many +committers and subsystem maintainers. + +The second third section describes how the **alpha and master** branches are +maintained and by whom. + +Finally, the last section describes how the **stable** branches are maintained +and by whom. + +This document does not cover how Tor is released, please see +[ReleasingTor.md](ReleasingTor.md) for that information. + +## Tor Maintainer + +The current maintainer is Nick Mathewson . + +The maintainer takes final decisions in terms of engineering, architecture and +protocol design. Releasing Tor falls under their responsability. + +## Alpha and Master Branches + +The Tor repository always has at all time a **master** branch which contains +the upstream ongoing development. + +It may also contains a branch for a released feature freezed version which is +called the **alpha** branch. The git tag and version number is always +postfixed with `-alpha[-dev]`. For example: `tor-0.3.5.0-alpha-dev` or +`tor-0.3.5.3-alpha`. + +Tor is separated into subsystems and some of those are maintained by other +developers than the main maintainer. Those people have commit access to the +code base but only commit (in most cases) into the subsystem they maintain. + +Upstream merges are restricted to the alpha and master branches. Subsystem +maintainers should never push a patch into a stable branch which is the +responsability of the [stable branch maintainer](#stable-branches). + +### Who + +In alphabetical order, the following people have upstream commit access and +maintain the following subsystems: + +- David Goulet + * Onion Service (including Shared Random). + ***keywords:*** *[tor-hs]* + * Channels, Circuitmux, Connection, Scheduler. + ***keywords:*** *[tor-chan, tor-cmux, tor-sched, tor-conn]* + * Cell Logic (Handling/Parsing). + ***keywords:*** *[tor-cell]* + * Threading backend. + ***keywords:*** *[tor-thread]* + +- George Kadianakis + * Onion Service (including Shared Random). + ***keywords:*** *[tor-hs]* + * Guard. + ***keywords:*** *[tor-guard]* + * Pluggable Transport (excluding Bridge networking). + ***keywords:*** *[tor-pt]* + +### Tasks + +These are the tasks of a subsystem maintainer: + +1. Regurlarly go over `merge_ready` tickets relevant to the related subsystem + and for the current alpha or development (master branch) Milestone. + +2. A subsystem maintainer is expected to contribute to any design changes + (including proposals) or large patch set about the subsystem. + +3. Leave their ego at the door. Mistakes will be made but they have to be + taking care of seriously. Learn and move on quickly. + +### Merging Policy + +These are few important items to follow when merging code upstream: + +1. To merge code upstream, the patch must have passed our CI (currently + github.com/torproject), have a corresponding ticket and reviewed by + **at least** one person that is not the original coder. + + Example A: If Alice writes a patch then Bob, a Tor network team member, + reviews it and flags it `merge_ready`. Then, the maintainter is required + to look at the patch and makes a decision. + + Example B: If the maintainer writes a patch then Bob, a Tor network + team member, reviews it and flags it `merge_ready`, then the maintainer + can merge the code upstream. + +2. Maintainer makes sure the commit message should describe what was fixed + and, if it applies, how was it fixed. It should also always refer to + the ticket number. + +3. Trivial patches such as comment change, documentation, syntax issues or + typos can be merged without a ticket or reviewers. + +4. Tor uses the "merge forward" method that is if a patch applies to the + alpha branch, it has to be merged there first and then merged forward + into master. + +5. Maintainer should always consult with the network team about any doubts, + mis-understandings or unknowns of a patch. Final word will always go to the + main Tor maintainer. + +## Stable Branches + +(Currently being drafted and reviewed by the network team.) From 742cd1564993faefded2d33b6839428a1fe4412a Mon Sep 17 00:00:00 2001 From: rl1987 Date: Sat, 1 Sep 2018 20:56:33 +0300 Subject: [PATCH 0087/2557] Move a check for trailing colon to tor_inet_pton() That way, string_is_valid_ipv6_address() can benefit from it --- src/lib/net/address.c | 5 +---- src/lib/net/inaddr.c | 8 +++++++- src/test/test_util.c | 13 +++++++++++++ 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/src/lib/net/address.c b/src/lib/net/address.c index 336693b464..17f4b1cf7a 100644 --- a/src/lib/net/address.c +++ b/src/lib/net/address.c @@ -1198,10 +1198,7 @@ tor_addr_parse(tor_addr_t *addr, const char *src) len -= 2; } - /* Reject if src has needless trailing ':'. */ - if (len > 2 && src[len - 1] == ':' && src[len - 2] != ':') { - result = -1; - } else if (tor_inet_pton(AF_INET6, src, &in6_tmp) > 0) { + if (tor_inet_pton(AF_INET6, src, &in6_tmp) > 0) { result = AF_INET6; tor_addr_from_in6(addr, &in6_tmp); } else if (!brackets_detected && diff --git a/src/lib/net/inaddr.c b/src/lib/net/inaddr.c index dcd8fcdd65..0960d323c5 100644 --- a/src/lib/net/inaddr.c +++ b/src/lib/net/inaddr.c @@ -168,6 +168,13 @@ tor_inet_pton(int af, const char *src, void *dst) if (af == AF_INET) { return tor_inet_aton(src, dst); } else if (af == AF_INET6) { + ssize_t len = strlen(src); + + /* Reject if src has needless trailing ':'. */ + if (len > 2 && src[len - 1] == ':' && src[len - 2] != ':') { + return 0; + } + struct in6_addr *out = dst; uint16_t words[8]; int gapPos = -1, i, setWords=0; @@ -207,7 +214,6 @@ tor_inet_pton(int af, const char *src, void *dst) return 0; if (TOR_ISXDIGIT(*src)) { char *next; - ssize_t len; long r = strtol(src, &next, 16); if (next == NULL || next == src) { /* The 'next == src' error case can happen on versions of openbsd diff --git a/src/test/test_util.c b/src/test/test_util.c index 6cbd504e34..0921bae109 100644 --- a/src/test/test_util.c +++ b/src/test/test_util.c @@ -5749,6 +5749,18 @@ test_util_ipv4_validation(void *arg) return; } +static void +test_util_ipv6_validation(void *arg) +{ + (void)arg; + + tt_assert(string_is_valid_ipv6_address("2a00:1450:401b:800::200e")); + tt_assert(!string_is_valid_ipv6_address("11:22::33:44:")); + + done: + return; +} + static void test_util_writepid(void *arg) { @@ -6439,6 +6451,7 @@ struct testcase_t util_tests[] = { UTIL_TEST(hostname_validation, 0), UTIL_TEST(dest_validation_edgecase, 0), UTIL_TEST(ipv4_validation, 0), + UTIL_TEST(ipv6_validation, 0), UTIL_TEST(writepid, 0), UTIL_TEST(get_avail_disk_space, 0), UTIL_TEST(touch_file, 0), From 1425549ca61cab8aa9476a25be0a31f726672bcc Mon Sep 17 00:00:00 2001 From: rl1987 Date: Sat, 1 Sep 2018 21:22:47 +0300 Subject: [PATCH 0088/2557] Code cleanups for tor_addr_parse() --- src/lib/net/address.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/lib/net/address.c b/src/lib/net/address.c index 17f4b1cf7a..a87d4a36a3 100644 --- a/src/lib/net/address.c +++ b/src/lib/net/address.c @@ -1189,13 +1189,13 @@ tor_addr_parse(tor_addr_t *addr, const char *src) struct in6_addr in6_tmp; int brackets_detected = 0; + tor_assert(addr && src); + size_t len = strlen(src); - tor_assert(addr && src); - if (src[0] == '[' && src[1] && src[len - 1] == ']') { + if (len && src[0] == '[' && src[len - 1] == ']') { brackets_detected = 1; src = tmp = tor_strndup(src+1, strlen(src)-2); - len -= 2; } if (tor_inet_pton(AF_INET6, src, &in6_tmp) > 0) { From 067b16eae2b7d37c7ec1595226bc7bf26aac1ff5 Mon Sep 17 00:00:00 2001 From: Neel Chauhan Date: Wed, 5 Sep 2018 10:12:35 -0400 Subject: [PATCH 0089/2557] Check IPv6 subnets as well as IPv4 subnets where possible when choosing client paths --- changes/bug24393 | 6 ++++++ src/feature/nodelist/nodelist.c | 18 ++++++++++++++++-- 2 files changed, 22 insertions(+), 2 deletions(-) create mode 100644 changes/bug24393 diff --git a/changes/bug24393 b/changes/bug24393 new file mode 100644 index 0000000000..e190192319 --- /dev/null +++ b/changes/bug24393 @@ -0,0 +1,6 @@ + o Minor features (ipv6): + - When using addrs_in_same_network_family(), check IPv6 subnets as well as + IPv4 ones where possible when a client chooses circuit paths. Previously, + we used this function only for IPv4 subnets. Closes ticket 24393. Patch + by Neel Chauhan. + diff --git a/src/feature/nodelist/nodelist.c b/src/feature/nodelist/nodelist.c index a98a5c8655..a1a1b0ea37 100644 --- a/src/feature/nodelist/nodelist.c +++ b/src/feature/nodelist/nodelist.c @@ -1867,6 +1867,9 @@ int addrs_in_same_network_family(const tor_addr_t *a1, const tor_addr_t *a2) { + if (tor_addr_is_null(a1) || tor_addr_is_null(a2)) + return 0; + switch (tor_addr_family(a1)) { case AF_INET: return 0 == tor_addr_compare_masked(a1, a2, 16, CMP_SEMANTIC); @@ -1917,7 +1920,13 @@ nodes_in_same_family(const node_t *node1, const node_t *node2) tor_addr_t a1, a2; node_get_addr(node1, &a1); node_get_addr(node2, &a2); - if (addrs_in_same_network_family(&a1, &a2)) + + tor_addr_port_t ap6_1, ap6_2; + node_get_pref_ipv6_orport(node1, &ap6_1); + node_get_pref_ipv6_orport(node2, &ap6_2); + + if (addrs_in_same_network_family(&a1, &a2) || + addrs_in_same_network_family(&ap6_1.addr, &ap6_2.addr)) return 1; } @@ -1974,12 +1983,17 @@ nodelist_add_node_and_family(smartlist_t *sl, const node_t *node) /* First, add any nodes with similar network addresses. */ if (options->EnforceDistinctSubnets) { tor_addr_t node_addr; + tor_addr_port_t node_ap6; node_get_addr(node, &node_addr); + node_get_pref_ipv6_orport(node, &node_ap6); SMARTLIST_FOREACH_BEGIN(all_nodes, const node_t *, node2) { tor_addr_t a; + tor_addr_port_t ap6; node_get_addr(node2, &a); - if (addrs_in_same_network_family(&a, &node_addr)) + node_get_pref_ipv6_orport(node2, &ap6); + if (addrs_in_same_network_family(&a, &node_addr) || + addrs_in_same_network_family(&ap6.addr, &node_ap6.addr)) smartlist_add(sl, (void*)node2); } SMARTLIST_FOREACH_END(node2); } From a182301152afe9cd066516ae02f588840b2efc43 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 31 Oct 2018 08:30:48 -0400 Subject: [PATCH 0090/2557] Fix memory leak (#28257, CID 1440805). --- src/feature/relay/dns.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/feature/relay/dns.c b/src/feature/relay/dns.c index 371c2f5069..701719af95 100644 --- a/src/feature/relay/dns.c +++ b/src/feature/relay/dns.c @@ -1386,6 +1386,7 @@ configured_nameserver_address(const size_t idx) NULL) == 0) { return tor_addr; } + tor_free(tor_addr); } return NULL; From e9adc200aab8bf2068dc7d7fb0cf2e2d43149182 Mon Sep 17 00:00:00 2001 From: Neel Chauhan Date: Wed, 12 Sep 2018 15:50:11 -0400 Subject: [PATCH 0091/2557] Add test for nodes_in_same_family() --- src/test/test_address.c | 75 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/src/test/test_address.c b/src/test/test_address.c index e99220f838..69de072cfa 100644 --- a/src/test/test_address.c +++ b/src/test/test_address.c @@ -24,6 +24,8 @@ #endif /* defined(HAVE_IFCONF_TO_SMARTLIST) */ #include "core/or/or.h" +#include "feature/nodelist/routerinfo_st.h" +#include "feature/nodelist/node_st.h" #include "feature/nodelist/nodelist.h" #include "lib/net/address.h" #include "test/test.h" @@ -1170,6 +1172,78 @@ test_address_tor_addr_in_same_network_family(void *ignored) return; } +static node_t * +helper_create_mock_node(char id_char) +{ + node_t *node = tor_malloc_zero(sizeof(node_t)); + routerinfo_t *ri = tor_malloc_zero(sizeof(routerinfo_t)); + tor_addr_make_null(&ri->ipv6_addr, AF_INET6); + node->ri = ri; + memset(node->identity, id_char, sizeof(node->identity)); + return node; +} + +static void +helper_free_mock_node(node_t *node) +{ + tor_free(node->ri); + tor_free(node); +} + +#define NODE_SET_IPV4(node, ipv4_addr, ipv4_port) { \ + tor_addr_t addr; \ + tor_addr_parse(&addr, ipv4_addr); \ + node->ri->addr = tor_addr_to_ipv4h(&addr); \ + node->ri->or_port = ipv4_port; \ + } + +#define NODE_CLEAR_IPV4(node) { \ + node->ri->addr = 0; \ + node->ri->or_port = 0; \ + } + +#define NODE_SET_IPV6(node, ipv6_addr_str, ipv6_port) { \ + tor_addr_parse(&node->ri->ipv6_addr, ipv6_addr_str); \ + node->ri->ipv6_orport = ipv6_port; \ + } + +static void +test_address_tor_node_in_same_network_family(void *ignored) +{ + (void)ignored; + node_t *node_a = helper_create_mock_node('a'); + node_t *node_b = helper_create_mock_node('b'); + + NODE_SET_IPV4(node_a, "8.8.8.8", 1); + NODE_SET_IPV4(node_b, "8.8.4.4", 1); + + tt_int_op(nodes_in_same_family(node_a, node_b), OP_EQ, 1); + + NODE_SET_IPV4(node_a, "8.8.8.8", 1); + NODE_SET_IPV4(node_b, "1.1.1.1", 1); + + tt_int_op(nodes_in_same_family(node_a, node_b), OP_EQ, 0); + + NODE_CLEAR_IPV4(node_a); + NODE_SET_IPV6(node_a, "2001:470:20::2", 1); + + tt_int_op(nodes_in_same_family(node_a, node_b), OP_EQ, 0); + + NODE_CLEAR_IPV4(node_b); + NODE_SET_IPV6(node_b, "2606:4700:4700::1111", 1); + + tt_int_op(nodes_in_same_family(node_a, node_b), OP_EQ, 0); + + NODE_SET_IPV6(node_a, "2606:4700:4700::1001", 1); + tt_int_op(nodes_in_same_family(node_a, node_b), OP_EQ, 1); + + helper_free_mock_node(node_a); + helper_free_mock_node(node_b); + + done: + return; +} + #define ADDRESS_TEST(name, flags) \ { #name, test_address_ ## name, flags, NULL, NULL } @@ -1202,6 +1276,7 @@ struct testcase_t address_tests[] = { ADDRESS_TEST(tor_addr_to_mapped_ipv4h, 0), ADDRESS_TEST(tor_addr_eq_ipv4h, 0), ADDRESS_TEST(tor_addr_in_same_network_family, 0), + ADDRESS_TEST(tor_node_in_same_network_family, 0), END_OF_TESTCASES }; From f60607ee96bf85286c7ef58f7fcf5ab00f0c3ad0 Mon Sep 17 00:00:00 2001 From: Fernando Fernandez Mancera Date: Mon, 29 Oct 2018 17:07:41 +0100 Subject: [PATCH 0092/2557] Improve log message in hs_service.c Signed-off-by: Fernando Fernandez Mancera --- src/feature/hs/hs_service.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/feature/hs/hs_service.c b/src/feature/hs/hs_service.c index 7d56c9e2ad..c288e28e80 100644 --- a/src/feature/hs/hs_service.c +++ b/src/feature/hs/hs_service.c @@ -2931,8 +2931,8 @@ set_descriptor_revision_counter(hs_service_descriptor_t *hs_desc, time_t now, /* The OPE module returns CRYPTO_OPE_ERROR in case of errors. */ tor_assert_nonfatal(rev_counter < CRYPTO_OPE_ERROR); - log_info(LD_REND, "Encrypted revision counter %d to %ld", - (int) seconds_since_start_of_srv, (long int) rev_counter); + log_info(LD_REND, "Encrypted revision counter %d to %" PRIu64, + (int) seconds_since_start_of_srv, rev_counter); hs_desc->desc->plaintext_data.revision_counter = rev_counter; } From a0402c6f33206468f57c381c0022e547520d14c2 Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Thu, 1 Nov 2018 12:37:17 +0200 Subject: [PATCH 0093/2557] Add changes file for #27707. --- changes/bug27707 | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 changes/bug27707 diff --git a/changes/bug27707 b/changes/bug27707 new file mode 100644 index 0000000000..e114222741 --- /dev/null +++ b/changes/bug27707 @@ -0,0 +1,3 @@ + o Minor features (log messages): + - Improve log message in HSv3 service that could print out negative + revision counters. Closes ticket 27707. Patch by "ffmancera". \ No newline at end of file From da716fdfbb08952b971882eba1dabca2fef9c7f3 Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Thu, 1 Nov 2018 12:55:04 +0200 Subject: [PATCH 0094/2557] Add tests for the string_is_utf8_no_bom() function. --- src/test/test_util.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/test/test_util.c b/src/test/test_util.c index 7bc1b7921a..0678251137 100644 --- a/src/test/test_util.c +++ b/src/test/test_util.c @@ -4024,6 +4024,13 @@ test_util_string_is_utf8(void *ptr) tt_int_op(1, OP_EQ, string_is_utf8("ascii\x7f\n", 7)); tt_int_op(1, OP_EQ, string_is_utf8("Risqu\u00e9=1", 9)); + /* Test the utf8_no_bom function */ + tt_int_op(0, OP_EQ, string_is_utf8_no_bom("\uFEFF", 3)); + tt_int_op(0, OP_EQ, string_is_utf8_no_bom("\uFFFE", 3)); + tt_int_op(0, OP_EQ, string_is_utf8_no_bom("\uFEFFlove", 7)); + tt_int_op(1, OP_EQ, string_is_utf8_no_bom("loveandrespect", + strlen("loveandrespect"))); + // Validate exactly 'len' bytes. tt_int_op(0, OP_EQ, string_is_utf8("\0\x80", 2)); tt_int_op(0, OP_EQ, string_is_utf8("Risqu\u00e9=1", 6)); From 0ce1f2d46646fd73abee56888650288055f16a53 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 1 Nov 2018 08:18:29 -0400 Subject: [PATCH 0095/2557] Declare the subsystem structure. --- src/include.am | 1 + src/lib/subsys/.may_include | 1 + src/lib/subsys/include.am | 3 ++ src/lib/subsys/subsys.h | 63 +++++++++++++++++++++++++++++++++++++ 4 files changed, 68 insertions(+) create mode 100644 src/lib/subsys/.may_include create mode 100644 src/lib/subsys/include.am create mode 100644 src/lib/subsys/subsys.h diff --git a/src/include.am b/src/include.am index d2f83da814..247b0db8da 100644 --- a/src/include.am +++ b/src/include.am @@ -25,6 +25,7 @@ include src/lib/osinfo/include.am include src/lib/process/include.am include src/lib/sandbox/include.am include src/lib/string/include.am +include src/lib/subsys/include.am include src/lib/smartlist_core/include.am include src/lib/term/include.am include src/lib/testsupport/include.am diff --git a/src/lib/subsys/.may_include b/src/lib/subsys/.may_include new file mode 100644 index 0000000000..2b06e8519c --- /dev/null +++ b/src/lib/subsys/.may_include @@ -0,0 +1 @@ +orconfig.h diff --git a/src/lib/subsys/include.am b/src/lib/subsys/include.am new file mode 100644 index 0000000000..4741126b14 --- /dev/null +++ b/src/lib/subsys/include.am @@ -0,0 +1,3 @@ + +noinst_HEADERS += \ + src/lib/subsys/subsys.h diff --git a/src/lib/subsys/subsys.h b/src/lib/subsys/subsys.h new file mode 100644 index 0000000000..7e4fe53636 --- /dev/null +++ b/src/lib/subsys/subsys.h @@ -0,0 +1,63 @@ +/* Copyright (c) 2003-2004, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef TOR_SUBSYS_T +#define TOR_SUBSYS_T + +#include + +struct dispatch_connector_t; + +/** + * A subsystem is a part of Tor that is initialized, shut down, configured, + * and connected to other parts of Tor. + * + * Subsystems + **/ +typedef struct subsys_fns_t { + /** + * The name of this subsystem. It should be a programmer-readable + * identifier. + **/ + const char *name; + + /** + * Whether this subsystem is supported -- that is, whether it is compiled + * into Tor. For most subsystems, this should be true. + **/ + bool supported; + + /** + * The 'initialization level' for the subsystem. It should run from -100 + * through +100. The subsystems are initialized from lowest level to + * highest, and shut down from highest level to lowest. + **/ + int level; + + /** + * Initialize any global components of this subsystem. + * + * This function MAY rely on any lower-level subsystem being initialized. + * + * This function MUST NOT rely on any runtime configuration information; + * it is only for global state or pre-configuration state. + **/ + int (*initialize)(void); + + /** + * Connect a subsystem to the message dispatch system. + **/ + int (*add_pubsub)(struct dispatch_connector_t *); + + /** + * Free all resources held by this subsystem. + * + * This function is not allowed to fail. + **/ + void (*shutdown)(void); + +} subsys_fns_t; + +#endif From 1b75de85b3cbc7706078a9899e483d18579a6fd1 Mon Sep 17 00:00:00 2001 From: "Alex Xu (Hello71)" Date: Fri, 19 Oct 2018 09:53:23 -0400 Subject: [PATCH 0096/2557] Don't overwrite the Content-Type when compressing --- changes/ticket28100 | 3 +++ src/feature/dircache/dircache.c | 14 ++++---------- 2 files changed, 7 insertions(+), 10 deletions(-) create mode 100644 changes/ticket28100 diff --git a/changes/ticket28100 b/changes/ticket28100 new file mode 100644 index 0000000000..b8e3271013 --- /dev/null +++ b/changes/ticket28100 @@ -0,0 +1,3 @@ + o Minor features (HTTP standards compliance): + - Don't send Content-Type: application/octet-stream for transparently + compressed documents, which confused browsers. Closes ticket 28100. diff --git a/src/feature/dircache/dircache.c b/src/feature/dircache/dircache.c index 872a88018f..dff4b85caa 100644 --- a/src/feature/dircache/dircache.c +++ b/src/feature/dircache/dircache.c @@ -166,22 +166,16 @@ write_http_response_header_impl(dir_connection_t *conn, ssize_t length, buf_free(buf); } -/** As write_http_response_header_impl, but sets encoding and content-typed - * based on whether the response will be compressed or not. */ +/** As write_http_response_header_impl, but translates method into + * encoding */ static void write_http_response_headers(dir_connection_t *conn, ssize_t length, compress_method_t method, const char *extra_headers, long cache_lifetime) { - const char *methodname = compression_method_get_name(method); - const char *doctype; - if (method == NO_METHOD) - doctype = "text/plain"; - else - doctype = "application/octet-stream"; write_http_response_header_impl(conn, length, - doctype, - methodname, + "text/plain", + compression_method_get_name(method), extra_headers, cache_lifetime); } From 3e2423d19b78ab71cb1ac8205406c7575296ecd3 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 2 Nov 2018 13:29:24 -0400 Subject: [PATCH 0097/2557] Update address tests to avoid offending coverity. --- src/test/test_address.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/test/test_address.c b/src/test/test_address.c index 69de072cfa..a823fd9cd5 100644 --- a/src/test/test_address.c +++ b/src/test/test_address.c @@ -1186,6 +1186,8 @@ helper_create_mock_node(char id_char) static void helper_free_mock_node(node_t *node) { + if (!node) + return; tor_free(node->ri); tor_free(node); } @@ -1237,11 +1239,9 @@ test_address_tor_node_in_same_network_family(void *ignored) NODE_SET_IPV6(node_a, "2606:4700:4700::1001", 1); tt_int_op(nodes_in_same_family(node_a, node_b), OP_EQ, 1); + done: helper_free_mock_node(node_a); helper_free_mock_node(node_b); - - done: - return; } #define ADDRESS_TEST(name, flags) \ @@ -1279,4 +1279,3 @@ struct testcase_t address_tests[] = { ADDRESS_TEST(tor_node_in_same_network_family, 0), END_OF_TESTCASES }; - From 18a4eaf5c142bae55780716464d43c2f8a9e2e49 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 2 Nov 2018 13:30:55 -0400 Subject: [PATCH 0098/2557] Avoid mmap leak if we get a consensus diff we can't use. Fixes CID 1440819; bug not in any released Tor. --- src/feature/dirclient/dirclient.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/feature/dirclient/dirclient.c b/src/feature/dirclient/dirclient.c index 705bf75e5c..cd88fa5ebf 100644 --- a/src/feature/dirclient/dirclient.c +++ b/src/feature/dirclient/dirclient.c @@ -2221,6 +2221,7 @@ handle_response_fetch_consensus(dir_connection_t *conn, if (!consensus_body) { log_warn(LD_DIR, "Received a consensus diff, but we can't find " "any %s-flavored consensus in our current cache.",flavname); + tor_munmap_file(mapped_consensus); networkstatus_consensus_download_failed(0, flavname); // XXXX if this happens too much, see below return -1; From 674ef53a7e953a724b4f3bfe2f1e06ba2897bba2 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 2 Nov 2018 13:32:43 -0400 Subject: [PATCH 0099/2557] Add a warning if we can't write networkstatus-bridges Fixes CID 1440818. --- src/feature/nodelist/networkstatus.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/feature/nodelist/networkstatus.c b/src/feature/nodelist/networkstatus.c index ec1a69b9e2..f1def9afb1 100644 --- a/src/feature/nodelist/networkstatus.c +++ b/src/feature/nodelist/networkstatus.c @@ -2403,7 +2403,9 @@ networkstatus_dump_bridge_status_to_file(time_t now) published, thresholds, fingerprint_line ? fingerprint_line : "", status); fname = get_datadir_fname("networkstatus-bridges"); - write_str_to_file(fname,published_thresholds_and_status,0); + if (write_str_to_file(fname,published_thresholds_and_status,0)<0) { + log_warn(LD_DIRSERV, "Unable to write networkstatus-bridges file."); + } tor_free(thresholds); tor_free(published_thresholds_and_status); tor_free(fname); From 5b48af9c4cc591d4db00cc1060de4d519a80023b Mon Sep 17 00:00:00 2001 From: rl1987 Date: Sun, 4 Nov 2018 19:34:46 +0200 Subject: [PATCH 0100/2557] Fix all instances of shellcheck warning SC2006 --- scripts/maint/updateRustDependencies.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/maint/updateRustDependencies.sh b/scripts/maint/updateRustDependencies.sh index a5a92579d3..d2680856b8 100755 --- a/scripts/maint/updateRustDependencies.sh +++ b/scripts/maint/updateRustDependencies.sh @@ -20,11 +20,11 @@ set -e -HERE=`dirname $(realpath $0)` -TOPLEVEL=`dirname $(dirname $HERE)` +HERE=$(dirname $(realpath $0)) +TOPLEVEL=$(dirname $(dirname $HERE)) TOML="$TOPLEVEL/src/rust/Cargo.toml" VENDORED="$TOPLEVEL/src/ext/rust/crates" -CARGO=`which cargo` +CARGO=$(which cargo) if ! test -f "$TOML" ; then printf "Error: Couldn't find workspace Cargo.toml in expected location: %s\n" "$TOML" @@ -38,7 +38,7 @@ if test -z "$CARGO" ; then printf "Error: cargo must be installed and in your \$PATH\n" fi -if test -z `cargo --list | grep vendor` ; then +if test -z $(cargo --list | grep vendor) ; then printf "Error: cargo-vendor not installed\n" fi From 45b28167d7e2b1d5afb26db6f76ca2329a9bbc04 Mon Sep 17 00:00:00 2001 From: Neel Chauhan Date: Wed, 17 Oct 2018 21:43:59 -0400 Subject: [PATCH 0101/2557] In count_acceptable_nodes(), count direct and indirect nodes with node_has_preferred_descriptor() --- changes/bug25885 | 7 +++++++ src/core/or/circuitbuild.c | 21 ++++++++++++--------- src/core/or/circuitbuild.h | 3 ++- src/test/test_circuitbuild.c | 4 ++-- 4 files changed, 23 insertions(+), 12 deletions(-) create mode 100644 changes/bug25885 diff --git a/changes/bug25885 b/changes/bug25885 new file mode 100644 index 0000000000..1b89acfe06 --- /dev/null +++ b/changes/bug25885 @@ -0,0 +1,7 @@ + o Minor bugfixes (guards): + - In count_acceptable_nodes(), check if we have at least one bridge + or guard node, and two non-guard nodes for a circuit. Previously, + we have added up the sum of all nodes with a descriptor, but that + could cause us to build circuits that fail if we had either too + many bridges, or not enough guard nodes. Fixes bug 25885; bugfix + on 0.3.6.1-alpha. Patch by Neel Chauhan. diff --git a/src/core/or/circuitbuild.c b/src/core/or/circuitbuild.c index a69457571e..4f9f09bc8f 100644 --- a/src/core/or/circuitbuild.c +++ b/src/core/or/circuitbuild.c @@ -1658,22 +1658,25 @@ route_len_for_purpose(uint8_t purpose, extend_info_t *exit_ei) STATIC int new_route_len(uint8_t purpose, extend_info_t *exit_ei, smartlist_t *nodes) { - int num_acceptable_routers; int routelen; tor_assert(nodes); routelen = route_len_for_purpose(purpose, exit_ei); - num_acceptable_routers = count_acceptable_nodes(nodes); + int num_acceptable_direct = count_acceptable_nodes(nodes, 1); + int num_acceptable_indirect = count_acceptable_nodes(nodes, 0); - log_debug(LD_CIRC,"Chosen route length %d (%d/%d routers suitable).", - routelen, num_acceptable_routers, smartlist_len(nodes)); + log_debug(LD_CIRC,"Chosen route length %d (%d direct and %d indirect " + "routers suitable).", routelen, num_acceptable_direct, + num_acceptable_indirect); - if (num_acceptable_routers < routelen) { + if (num_acceptable_direct < 1 || num_acceptable_indirect < routelen - 1) { log_info(LD_CIRC, - "Not enough acceptable routers (%d/%d). Discarding this circuit.", - num_acceptable_routers, routelen); + "Not enough acceptable routers (%d/%d direct and %d/%d " + "indirect routers suitable). Discarding this circuit.", + num_acceptable_direct, routelen, + num_acceptable_indirect, routelen); return -1; } @@ -2315,7 +2318,7 @@ circuit_extend_to_new_exit(origin_circuit_t *circ, extend_info_t *exit_ei) * particular router. See bug #25885.) */ MOCK_IMPL(STATIC int, -count_acceptable_nodes, (smartlist_t *nodes)) +count_acceptable_nodes, (smartlist_t *nodes, int direct)) { int num=0; @@ -2329,7 +2332,7 @@ count_acceptable_nodes, (smartlist_t *nodes)) if (! node->is_valid) // log_debug(LD_CIRC,"Nope, the directory says %d is not valid.",i); continue; - if (! node_has_any_descriptor(node)) + if (! node_has_preferred_descriptor(node, direct)) continue; /* The node has a descriptor, so we can just check the ntor key directly */ if (!node_has_curve25519_onion_key(node)) diff --git a/src/core/or/circuitbuild.h b/src/core/or/circuitbuild.h index cee71b297b..93f903f060 100644 --- a/src/core/or/circuitbuild.h +++ b/src/core/or/circuitbuild.h @@ -84,7 +84,8 @@ void circuit_upgrade_circuits_from_guard_wait(void); STATIC circid_t get_unique_circ_id_by_chan(channel_t *chan); STATIC int new_route_len(uint8_t purpose, extend_info_t *exit_ei, smartlist_t *nodes); -MOCK_DECL(STATIC int, count_acceptable_nodes, (smartlist_t *nodes)); +MOCK_DECL(STATIC int, count_acceptable_nodes, (smartlist_t *nodes, + int direct)); STATIC int onion_extend_cpath(origin_circuit_t *circ); diff --git a/src/test/test_circuitbuild.c b/src/test/test_circuitbuild.c index 02eadecd98..dd47ad7689 100644 --- a/src/test/test_circuitbuild.c +++ b/src/test/test_circuitbuild.c @@ -21,11 +21,11 @@ static smartlist_t dummy_nodes; static extend_info_t dummy_ei; static int -mock_count_acceptable_nodes(smartlist_t *nodes) +mock_count_acceptable_nodes(smartlist_t *nodes, int direct) { (void)nodes; - return DEFAULT_ROUTE_LEN + 1; + return direct ? 1 : DEFAULT_ROUTE_LEN + 1; } /* Test route lengths when the caller of new_route_len() doesn't From 7bb76b24cf755799b7950ef078ac5ccf4d6e3a8a Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 1 Nov 2018 11:51:33 -0400 Subject: [PATCH 0102/2557] Code to manage the list of subsystems. --- src/app/main/main.c | 6 ++ src/app/main/subsysmgr.c | 130 ++++++++++++++++++++++++++++++++++ src/app/main/subsysmgr.h | 20 ++++++ src/app/main/subsystem_list.c | 20 ++++++ src/core/include.am | 3 + src/lib/subsys/subsys.h | 5 ++ 6 files changed, 184 insertions(+) create mode 100644 src/app/main/subsysmgr.c create mode 100644 src/app/main/subsysmgr.h create mode 100644 src/app/main/subsystem_list.c diff --git a/src/app/main/main.c b/src/app/main/main.c index ae87add67d..444d6ea7ec 100644 --- a/src/app/main/main.c +++ b/src/app/main/main.c @@ -15,6 +15,7 @@ #include "app/config/statefile.h" #include "app/main/main.h" #include "app/main/ntmain.h" +#include "app/main/subsysmgr.h" #include "core/mainloop/connection.h" #include "core/mainloop/cpuworker.h" #include "core/mainloop/mainloop.h" @@ -813,6 +814,9 @@ tor_free_all(int postfork) release_lockfile(); } tor_libevent_free_all(); + + subsystems_shutdown(); + /* Stuff in util.c and address.c*/ if (!postfork) { escaped(NULL); @@ -1426,6 +1430,8 @@ tor_run_main(const tor_main_configuration_t *tor_cfg) event_set_mem_functions(tor_malloc_, tor_realloc_, tor_free_); #endif + subsystems_init(); + init_protocol_warning_severity_level(); update_approx_time(time(NULL)); diff --git a/src/app/main/subsysmgr.c b/src/app/main/subsysmgr.c new file mode 100644 index 0000000000..7974f2d238 --- /dev/null +++ b/src/app/main/subsysmgr.c @@ -0,0 +1,130 @@ +/* Copyright (c) 2003-2004, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "orconfig.h" +#include "app/main/subsysmgr.h" +#include "lib/err/torerr.h" + +#include +#include +#include + +/** + * True iff we have checked tor_subsystems for consistency. + **/ +static bool subsystem_array_validated = false; + +/** + * True if a given subsystem is initialized. Expand this array if there + * are more than this number of subsystems. (We'd rather not + * dynamically allocate in this module.) + **/ +static bool sys_initialized[128]; + +/** + * Exit with a raw assertion if the subsystems list is inconsistent; + * initialize the subsystem_initialized array. + **/ +static void +check_and_setup(void) +{ + if (subsystem_array_validated) + return; + + raw_assert(ARRAY_LENGTH(sys_initialized) >= n_tor_subsystems); + memset(sys_initialized, 0, sizeof(sys_initialized)); + + int last_level = MIN_SUBSYS_LEVEL; + + for (unsigned i = 0; i < n_tor_subsystems; ++i) { + const subsys_fns_t *sys = tor_subsystems[i]; + if (sys->level < MIN_SUBSYS_LEVEL || sys->level > MAX_SUBSYS_LEVEL) { + fprintf(stderr, "BUG: Subsystem %s (at %u) has an invalid level %d. " + "It is supposed to be between %d and %d (inclusive).\n", + sys->name, i, sys->level, MIN_SUBSYS_LEVEL, MAX_SUBSYS_LEVEL); + raw_assert_unreached_msg("There is a bug in subsystem_list.c"); + } + if (sys->level < last_level) { + fprintf(stderr, "BUG: Subsystem %s (at #%u) is in the wrong position. " + "Its level is %d; but the previous subsystem's level was %d.\n", + sys->name, i, sys->level, last_level); + raw_assert_unreached_msg("There is a bug in subsystem_list.c"); + } + last_level = sys->level; + } + + subsystem_array_validated = true; +} + +/** + * Initialize all the subsystems; exit on failure. + **/ +int +subsystems_init(void) +{ + return subsystems_init_upto(MAX_SUBSYS_LEVEL); +} + +/** + * Initialize all the subsystems whose level is less than or equal to + * target_level; exit on failure. + **/ +int +subsystems_init_upto(int target_level) +{ + check_and_setup(); + + for (unsigned i = 0; i < n_tor_subsystems; ++i) { + const subsys_fns_t *sys = tor_subsystems[i]; + if (!sys->supported) + continue; + if (sys->level > target_level) + break; + if (sys_initialized[i]) + continue; + int r = 0; + if (sys->initialize) + r = sys->initialize(); + if (r < 0) { + fprintf(stderr, "BUG: subsystem %s (at %u) initialization failed.\n", + sys->name, i); + raw_assert_unreached_msg("A subsystem couldn't be initialized."); + } + sys_initialized[i] = true; + } + + return 0; +} + +/** + * Shut down all the subsystems. + **/ +void +subsystems_shutdown(void) +{ + subsystems_shutdown_downto(MIN_SUBSYS_LEVEL - 1); +} + +/** + * Shut down all the subsystems whose level is above target_level. + **/ +void +subsystems_shutdown_downto(int target_level) +{ + check_and_setup(); + + for (int i = (int)n_tor_subsystems - 1; i >= 0; --i) { + const subsys_fns_t *sys = tor_subsystems[i]; + if (!sys->supported) + continue; + if (sys->level <= target_level) + break; + if (! sys_initialized[i]) + continue; + if (sys->shutdown) + sys->shutdown(); + sys_initialized[i] = false; + } +} diff --git a/src/app/main/subsysmgr.h b/src/app/main/subsysmgr.h new file mode 100644 index 0000000000..c9b892eee4 --- /dev/null +++ b/src/app/main/subsysmgr.h @@ -0,0 +1,20 @@ +/* Copyright (c) 2003-2004, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef TOR_SUBSYSMGR_T +#define TOR_SUBSYSMGR_T + +#include "lib/subsys/subsys.h" + +extern const struct subsys_fns_t *tor_subsystems[]; +extern const unsigned n_tor_subsystems; + +int subsystems_init(void); +int subsystems_init_upto(int level); + +void subsystems_shutdown(void); +void subsystems_shutdown_downto(int level); + +#endif diff --git a/src/app/main/subsystem_list.c b/src/app/main/subsystem_list.c new file mode 100644 index 0000000000..fc1249e1c6 --- /dev/null +++ b/src/app/main/subsystem_list.c @@ -0,0 +1,20 @@ +/* Copyright (c) 2003-2004, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "orconfig.h" +#include "app/main/subsysmgr.h" +#include "lib/cc/compat_compiler.h" +#include "lib/cc/torint.h" + +#include + +/** + * Global list of the subsystems in Tor, in the order of their initialization. + **/ +const subsys_fns_t *tor_subsystems[] = { + NULL // placeholder. +}; + +const unsigned n_tor_subsystems = ARRAY_LENGTH(tor_subsystems); diff --git a/src/core/include.am b/src/core/include.am index 1b8ef2ac58..d3fce54285 100644 --- a/src/core/include.am +++ b/src/core/include.am @@ -11,6 +11,8 @@ LIBTOR_APP_A_SOURCES = \ src/app/config/confparse.c \ src/app/config/statefile.c \ src/app/main/main.c \ + src/app/main/subsystem_list.c \ + src/app/main/subsysmgr.c \ src/core/crypto/hs_ntor.c \ src/core/crypto/onion_crypto.c \ src/core/crypto/onion_fast.c \ @@ -191,6 +193,7 @@ noinst_HEADERS += \ src/app/config/statefile.h \ src/app/main/main.h \ src/app/main/ntmain.h \ + src/app/main/subsysmgr.h \ src/core/crypto/hs_ntor.h \ src/core/crypto/onion_crypto.h \ src/core/crypto/onion_fast.h \ diff --git a/src/lib/subsys/subsys.h b/src/lib/subsys/subsys.h index 7e4fe53636..012b218da7 100644 --- a/src/lib/subsys/subsys.h +++ b/src/lib/subsys/subsys.h @@ -43,6 +43,8 @@ typedef struct subsys_fns_t { * * This function MUST NOT rely on any runtime configuration information; * it is only for global state or pre-configuration state. + * + * This function MUST NOT have any parts that can fail. **/ int (*initialize)(void); @@ -60,4 +62,7 @@ typedef struct subsys_fns_t { } subsys_fns_t; +#define MIN_SUBSYS_LEVEL -100 +#define MAX_SUBSYS_LEVEL 100 + #endif From 6e7ff8cba0efaf803e3ef5b5aba4123633fe0658 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 1 Nov 2018 12:33:22 -0400 Subject: [PATCH 0103/2557] Move the code that knows our tor version into a lowest-level lib --- .gitignore | 2 + Makefile.am | 2 + src/app/config/config.c | 41 +------------------ src/app/config/config.h | 2 - src/app/config/statefile.c | 1 + src/app/main/main.c | 1 + src/feature/control/control.c | 1 + src/feature/dirauth/shared_random_state.c | 1 + src/feature/relay/router.c | 1 + src/include.am | 1 + src/lib/log/.may_include | 3 +- src/lib/log/include.am | 8 ---- src/lib/log/log.c | 2 +- src/lib/version/.may_include | 3 ++ src/lib/{log => version}/git_revision.c | 2 +- src/lib/{log => version}/git_revision.h | 0 src/lib/version/include.am | 25 ++++++++++++ src/lib/version/torversion.h | 12 ++++++ src/lib/version/version.c | 50 +++++++++++++++++++++++ src/rust/build.rs | 1 + src/test/fuzz/fuzzing_common.c | 1 + src/test/testing_common.c | 1 + 22 files changed, 107 insertions(+), 54 deletions(-) create mode 100644 src/lib/version/.may_include rename src/lib/{log => version}/git_revision.c (94%) rename src/lib/{log => version}/git_revision.h (100%) create mode 100644 src/lib/version/include.am create mode 100644 src/lib/version/torversion.h create mode 100644 src/lib/version/version.c diff --git a/.gitignore b/.gitignore index cedff8fb37..ee2de376a6 100644 --- a/.gitignore +++ b/.gitignore @@ -210,6 +210,8 @@ uptime-*.json /src/lib/libtor-tls.a /src/lib/libtor-tls-testing.a /src/lib/libtor-trace.a +/src/lib/libtor-version.a +/src/lib/libtor-version-testing.a /src/lib/libtor-wallclock.a /src/lib/libtor-wallclock-testing.a diff --git a/Makefile.am b/Makefile.am index e5c1be31b5..cb76edfa2f 100644 --- a/Makefile.am +++ b/Makefile.am @@ -62,6 +62,7 @@ TOR_UTIL_LIBS = \ src/lib/libtor-malloc.a \ src/lib/libtor-wallclock.a \ src/lib/libtor-err.a \ + src/lib/libtor-version.a \ src/lib/libtor-intmath.a \ src/lib/libtor-ctime.a @@ -91,6 +92,7 @@ TOR_UTIL_TESTING_LIBS = \ src/lib/libtor-malloc-testing.a \ src/lib/libtor-wallclock-testing.a \ src/lib/libtor-err-testing.a \ + src/lib/libtor-version-testing.a \ src/lib/libtor-intmath.a \ src/lib/libtor-ctime-testing.a endif diff --git a/src/app/config/config.c b/src/app/config/config.c index 6e7e131055..7b49387bcf 100644 --- a/src/app/config/config.c +++ b/src/app/config/config.c @@ -112,9 +112,9 @@ #include "lib/crypt_ops/crypto_rand.h" #include "lib/crypt_ops/crypto_util.h" #include "lib/encoding/confline.h" -#include "lib/log/git_revision.h" #include "lib/net/resolve.h" #include "lib/sandbox/sandbox.h" +#include "lib/version/torversion.h" #ifdef ENABLE_NSS #include "lib/crypt_ops/crypto_nss_mgt.h" @@ -972,42 +972,6 @@ set_options(or_options_t *new_val, char **msg) return 0; } -/** The version of this Tor process, as parsed. */ -static char *the_tor_version = NULL; -/** A shorter version of this Tor process's version, for export in our router - * descriptor. (Does not include the git version, if any.) */ -static char *the_short_tor_version = NULL; - -/** Return the current Tor version. */ -const char * -get_version(void) -{ - if (the_tor_version == NULL) { - if (strlen(tor_git_revision)) { - tor_asprintf(&the_tor_version, "%s (git-%s)", get_short_version(), - tor_git_revision); - } else { - the_tor_version = tor_strdup(get_short_version()); - } - } - return the_tor_version; -} - -/** Return the current Tor version, without any git tag. */ -const char * -get_short_version(void) -{ - - if (the_short_tor_version == NULL) { -#ifdef TOR_BUILD_TAG - tor_asprintf(&the_short_tor_version, "%s (%s)", VERSION, TOR_BUILD_TAG); -#else - the_short_tor_version = tor_strdup(VERSION); -#endif - } - return the_short_tor_version; -} - /** Release additional memory allocated in options */ STATIC void @@ -1067,9 +1031,6 @@ config_free_all(void) tor_free(torrc_defaults_fname); tor_free(global_dirfrontpagecontents); - tor_free(the_short_tor_version); - tor_free(the_tor_version); - cleanup_protocol_warning_severity_level(); have_parsed_cmdline = 0; diff --git a/src/app/config/config.h b/src/app/config/config.h index a169cfd451..4c497b83a6 100644 --- a/src/app/config/config.h +++ b/src/app/config/config.h @@ -41,8 +41,6 @@ const char *escaped_safe_str_client(const char *address); const char *escaped_safe_str(const char *address); void init_protocol_warning_severity_level(void); int get_protocol_warning_severity_level(void); -const char *get_version(void); -const char *get_short_version(void); /** An error from options_trial_assign() or options_init_from_string(). */ typedef enum setopt_err_t { diff --git a/src/app/config/statefile.c b/src/app/config/statefile.c index 8a8b7ced01..4ba7be1519 100644 --- a/src/app/config/statefile.c +++ b/src/app/config/statefile.c @@ -45,6 +45,7 @@ #include "app/config/statefile.h" #include "lib/encoding/confline.h" #include "lib/net/resolve.h" +#include "lib/version/torversion.h" #include "app/config/or_state_st.h" diff --git a/src/app/main/main.c b/src/app/main/main.c index 444d6ea7ec..031f570097 100644 --- a/src/app/main/main.c +++ b/src/app/main/main.c @@ -84,6 +84,7 @@ #include "lib/encoding/confline.h" #include "lib/evloop/timers.h" #include "lib/crypt_ops/crypto_init.h" +#include "lib/version/torversion.h" #include diff --git a/src/feature/control/control.c b/src/feature/control/control.c index 3fa47747eb..b31b448e96 100644 --- a/src/feature/control/control.c +++ b/src/feature/control/control.c @@ -92,6 +92,7 @@ #include "lib/crypt_ops/crypto_util.h" #include "lib/encoding/confline.h" #include "lib/evloop/compat_libevent.h" +#include "lib/version/torversion.h" #include "feature/dircache/cached_dir_st.h" #include "feature/control/control_connection_st.h" diff --git a/src/feature/dirauth/shared_random_state.c b/src/feature/dirauth/shared_random_state.c index 38c7fd76d0..1ce06744d4 100644 --- a/src/feature/dirauth/shared_random_state.c +++ b/src/feature/dirauth/shared_random_state.c @@ -22,6 +22,7 @@ #include "feature/dirauth/shared_random_state.h" #include "feature/dircommon/voting_schedule.h" #include "lib/encoding/confline.h" +#include "lib/version/torversion.h" #include "app/config/or_state_st.h" diff --git a/src/feature/relay/router.c b/src/feature/relay/router.c index 3a819f592c..9d61ced11c 100644 --- a/src/feature/relay/router.c +++ b/src/feature/relay/router.c @@ -49,6 +49,7 @@ #include "lib/encoding/confline.h" #include "lib/osinfo/uname.h" #include "lib/tls/tortls.h" +#include "lib/version/torversion.h" #include "feature/dirauth/authmode.h" diff --git a/src/include.am b/src/include.am index 247b0db8da..8279499936 100644 --- a/src/include.am +++ b/src/include.am @@ -33,6 +33,7 @@ include src/lib/thread/include.am include src/lib/time/include.am include src/lib/tls/include.am include src/lib/trace/include.am +include src/lib/version/include.am include src/lib/wallclock/include.am include src/trunnel/include.am diff --git a/src/lib/log/.may_include b/src/lib/log/.may_include index 852173aab3..7ca1863a52 100644 --- a/src/lib/log/.may_include +++ b/src/lib/log/.may_include @@ -10,6 +10,5 @@ lib/log/*.h lib/malloc/*.h lib/string/*.h lib/testsupport/*.h +lib/version/*.h lib/wallclock/*.h - -micro-revision.i \ No newline at end of file diff --git a/src/lib/log/include.am b/src/lib/log/include.am index 4a6c9b3686..c6f404e269 100644 --- a/src/lib/log/include.am +++ b/src/lib/log/include.am @@ -7,7 +7,6 @@ endif src_lib_libtor_log_a_SOURCES = \ src/lib/log/escape.c \ - src/lib/log/git_revision.c \ src/lib/log/ratelim.c \ src/lib/log/log.c \ src/lib/log/util_bug.c @@ -21,15 +20,8 @@ src_lib_libtor_log_testing_a_SOURCES = \ src_lib_libtor_log_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_log_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) -# Declare that these object files depend on micro-revision.i. Without this -# rule, we could try to build them before micro-revision.i was created. -src/lib/log/git_revision.$(OBJEXT) \ - src/lib/log/src_lib_libtor_log_testing_a-git_revision.$(OBJEXT): \ - micro-revision.i - noinst_HEADERS += \ src/lib/log/escape.h \ - src/lib/log/git_revision.h \ src/lib/log/ratelim.h \ src/lib/log/log.h \ src/lib/log/util_bug.h \ diff --git a/src/lib/log/log.c b/src/lib/log/log.c index d60ce6308a..bc7b36dcb9 100644 --- a/src/lib/log/log.c +++ b/src/lib/log/log.c @@ -32,7 +32,7 @@ #define LOG_PRIVATE #include "lib/log/log.h" -#include "lib/log/git_revision.h" +#include "lib/version/git_revision.h" #include "lib/log/ratelim.h" #include "lib/lock/compat_mutex.h" #include "lib/smartlist_core/smartlist_core.h" diff --git a/src/lib/version/.may_include b/src/lib/version/.may_include new file mode 100644 index 0000000000..d159ceb41f --- /dev/null +++ b/src/lib/version/.may_include @@ -0,0 +1,3 @@ +orconfig.h +micro-revision.i +lib/version/*.h \ No newline at end of file diff --git a/src/lib/log/git_revision.c b/src/lib/version/git_revision.c similarity index 94% rename from src/lib/log/git_revision.c rename to src/lib/version/git_revision.c index 9d29ecd2a2..e5b2ff534e 100644 --- a/src/lib/log/git_revision.c +++ b/src/lib/version/git_revision.c @@ -4,7 +4,7 @@ /* See LICENSE for licensing information */ #include "orconfig.h" -#include "lib/log/git_revision.h" +#include "lib/version/git_revision.h" /** String describing which Tor Git repository version the source was * built from. This string is generated by a bit of shell kludging in diff --git a/src/lib/log/git_revision.h b/src/lib/version/git_revision.h similarity index 100% rename from src/lib/log/git_revision.h rename to src/lib/version/git_revision.h diff --git a/src/lib/version/include.am b/src/lib/version/include.am new file mode 100644 index 0000000000..6944eb05e3 --- /dev/null +++ b/src/lib/version/include.am @@ -0,0 +1,25 @@ + +noinst_LIBRARIES += src/lib/libtor-version.a + +if UNITTESTS_ENABLED +noinst_LIBRARIES += src/lib/libtor-version-testing.a +endif + +src_lib_libtor_version_a_SOURCES = \ + src/lib/version/git_revision.c \ + src/lib/version/version.c + +src_lib_libtor_version_testing_a_SOURCES = \ + $(src_lib_libtor_version_a_SOURCES) +src_lib_libtor_version_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) +src_lib_libtor_version_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) + +# Declare that these object files depend on micro-revision.i. Without this +# rule, we could try to build them before micro-revision.i was created. +src/lib/version/git_revision.$(OBJEXT) \ + src/lib/version/src_lib_libtor_version_testing_a-git_revision.$(OBJEXT): \ + micro-revision.i + +noinst_HEADERS += \ + src/lib/version/git_revision.h \ + src/lib/version/torversion.h diff --git a/src/lib/version/torversion.h b/src/lib/version/torversion.h new file mode 100644 index 0000000000..761d6f25ab --- /dev/null +++ b/src/lib/version/torversion.h @@ -0,0 +1,12 @@ +/* Copyright 2001-2004 Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef TOR_VERSION_H +#define TOR_VERSION_H + +const char *get_version(void); +const char *get_short_version(void); + +#endif /* !defined(TOR_VERSION_H) */ diff --git a/src/lib/version/version.c b/src/lib/version/version.c new file mode 100644 index 0000000000..29ada39c9d --- /dev/null +++ b/src/lib/version/version.c @@ -0,0 +1,50 @@ +/* Copyright 2001-2004 Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "orconfig.h" +#include "lib/version/torversion.h" +#include "lib/version/git_revision.h" + +#include +#include + +/** A shorter version of this Tor process's version, for export in our router + * descriptor. (Does not include the git version, if any.) */ +static const char the_short_tor_version[] = + VERSION +#ifdef TOR_BUILD_TAG + " ("TOR_BUILD_TAG")" +#endif + ""; + +#define MAX_VERSION_LEN 128 + +/** The version of this Tor process, possibly including git version */ +static char the_tor_version[MAX_VERSION_LEN] = ""; + +/** Return the current Tor version. */ +const char * +get_version(void) +{ + if (the_tor_version[0] == 0) { + if (strlen(tor_git_revision)) { + snprintf(the_tor_version, sizeof(the_tor_version), + "%s (git-%s)", the_short_tor_version, tor_git_revision); + } else { + snprintf(the_tor_version, sizeof(the_tor_version), + "%s", the_short_tor_version); + } + the_tor_version[sizeof(the_tor_version)-1] = 0; + } + + return the_tor_version; +} + +/** Return the current Tor version, without any git tag. */ +const char * +get_short_version(void) +{ + return the_short_tor_version; +} diff --git a/src/rust/build.rs b/src/rust/build.rs index 123d5c0682..bf566c56bf 100644 --- a/src/rust/build.rs +++ b/src/rust/build.rs @@ -162,6 +162,7 @@ pub fn main() { cfg.component("tor-malloc"); cfg.component("tor-wallclock"); cfg.component("tor-err-testing"); + cfg.component("tor-version-testing"); cfg.component("tor-intmath-testing"); cfg.component("tor-ctime-testing"); cfg.component("curve25519_donna"); diff --git a/src/test/fuzz/fuzzing_common.c b/src/test/fuzz/fuzzing_common.c index 1401e4c28d..879f9e74dc 100644 --- a/src/test/fuzz/fuzzing_common.c +++ b/src/test/fuzz/fuzzing_common.c @@ -9,6 +9,7 @@ #include "lib/compress/compress.h" #include "lib/crypt_ops/crypto_ed25519.h" #include "lib/crypt_ops/crypto_init.h" +#include "lib/version/torversion.h" static or_options_t *mock_options = NULL; static const or_options_t * diff --git a/src/test/testing_common.c b/src/test/testing_common.c index c52683afca..8d648ee175 100644 --- a/src/test/testing_common.c +++ b/src/test/testing_common.c @@ -25,6 +25,7 @@ #include "lib/compress/compress.h" #include "lib/evloop/compat_libevent.h" #include "lib/crypt_ops/crypto_init.h" +#include "lib/version/torversion.h" #include #ifdef HAVE_FCNTL_H From 175153a3290b3987faacac9d5390e87e1ad4a457 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 1 Nov 2018 12:40:55 -0400 Subject: [PATCH 0104/2557] Make initialization for the "err" library into a subsystem. --- src/app/main/main.c | 10 --------- src/app/main/subsystem_list.c | 4 +++- src/lib/err/.may_include | 2 ++ src/lib/err/include.am | 8 ++++--- src/lib/err/torerr.c | 10 +++++++++ src/lib/err/torerr.h | 1 + src/lib/err/torerr_sys.c | 39 +++++++++++++++++++++++++++++++++++ src/lib/err/torerr_sys.h | 14 +++++++++++++ 8 files changed, 74 insertions(+), 14 deletions(-) create mode 100644 src/lib/err/torerr_sys.c create mode 100644 src/lib/err/torerr_sys.h diff --git a/src/app/main/main.c b/src/app/main/main.c index 031f570097..e3d7610c82 100644 --- a/src/app/main/main.c +++ b/src/app/main/main.c @@ -69,7 +69,6 @@ #include "lib/container/buffers.h" #include "lib/crypt_ops/crypto_rand.h" #include "lib/crypt_ops/crypto_s2k.h" -#include "lib/err/backtrace.h" #include "lib/geoip/geoip.h" #include "lib/process/waitpid.h" @@ -822,7 +821,6 @@ tor_free_all(int postfork) if (!postfork) { escaped(NULL); esc_router_info(NULL); - clean_up_backtrace_handler(); logs_free_all(); /* free log strings. do this last so logs keep working. */ } } @@ -1419,14 +1417,6 @@ tor_run_main(const tor_main_configuration_t *tor_cfg) #endif /* !defined(_WIN64) */ #endif /* defined(_WIN32) */ - { - int bt_err = configure_backtrace_handler(get_version()); - if (bt_err < 0) { - log_warn(LD_BUG, "Unable to install backtrace handler: %s", - strerror(-bt_err)); - } - } - #ifdef EVENT_SET_MEM_FUNCTIONS_IMPLEMENTED event_set_mem_functions(tor_malloc_, tor_realloc_, tor_free_); #endif diff --git a/src/app/main/subsystem_list.c b/src/app/main/subsystem_list.c index fc1249e1c6..244dbadbd9 100644 --- a/src/app/main/subsystem_list.c +++ b/src/app/main/subsystem_list.c @@ -8,13 +8,15 @@ #include "lib/cc/compat_compiler.h" #include "lib/cc/torint.h" +#include "lib/err/torerr_sys.h" + #include /** * Global list of the subsystems in Tor, in the order of their initialization. **/ const subsys_fns_t *tor_subsystems[] = { - NULL // placeholder. + &sys_torerr, }; const unsigned n_tor_subsystems = ARRAY_LENGTH(tor_subsystems); diff --git a/src/lib/err/.may_include b/src/lib/err/.may_include index 48cc0ef088..daa1b6e4ca 100644 --- a/src/lib/err/.may_include +++ b/src/lib/err/.may_include @@ -1,3 +1,5 @@ orconfig.h lib/cc/*.h lib/err/*.h +lib/subsys/*.h +lib/version/*.h \ No newline at end of file diff --git a/src/lib/err/include.am b/src/lib/err/include.am index f2a409c51e..43adcd2694 100644 --- a/src/lib/err/include.am +++ b/src/lib/err/include.am @@ -6,8 +6,9 @@ noinst_LIBRARIES += src/lib/libtor-err-testing.a endif src_lib_libtor_err_a_SOURCES = \ - src/lib/err/backtrace.c \ - src/lib/err/torerr.c + src/lib/err/backtrace.c \ + src/lib/err/torerr.c \ + src/lib/err/torerr_sys.c src_lib_libtor_err_testing_a_SOURCES = \ $(src_lib_libtor_err_a_SOURCES) @@ -16,4 +17,5 @@ src_lib_libtor_err_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) noinst_HEADERS += \ src/lib/err/backtrace.h \ - src/lib/err/torerr.h + src/lib/err/torerr.h \ + src/lib/err/torerr_sys.h diff --git a/src/lib/err/torerr.c b/src/lib/err/torerr.c index f9e139f967..e9de86837f 100644 --- a/src/lib/err/torerr.c +++ b/src/lib/err/torerr.c @@ -122,6 +122,16 @@ tor_log_set_sigsafe_err_fds(const int *fds, int n) n_sigsafe_log_fds = n; } +/** + * Reset the list of emergency error fds to its default. + */ +void +tor_log_reset_sigsafe_err_fds(void) +{ + int fds[] = { STDERR_FILENO }; + tor_log_set_sigsafe_err_fds(fds, 1); +} + /** * Set the granularity (in ms) to use when reporting fatal errors outside * the logging system. diff --git a/src/lib/err/torerr.h b/src/lib/err/torerr.h index d4bba6916f..b415ef73ef 100644 --- a/src/lib/err/torerr.h +++ b/src/lib/err/torerr.h @@ -39,6 +39,7 @@ void tor_raw_assertion_failed_msg_(const char *file, int line, void tor_log_err_sigsafe(const char *m, ...); int tor_log_get_sigsafe_err_fds(const int **out); void tor_log_set_sigsafe_err_fds(const int *fds, int n); +void tor_log_reset_sigsafe_err_fds(void); void tor_log_sigsafe_err_set_granularity(int ms); int format_hex_number_sigsafe(unsigned long x, char *buf, int max_len); diff --git a/src/lib/err/torerr_sys.c b/src/lib/err/torerr_sys.c new file mode 100644 index 0000000000..54666f4106 --- /dev/null +++ b/src/lib/err/torerr_sys.c @@ -0,0 +1,39 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file torerr_sys.c + * \brief Subsystem object for the error handling subsystem. + **/ + +#include "orconfig.h" +#include "lib/err/backtrace.h" +#include "lib/err/torerr.h" +#include "lib/err/torerr_sys.h" +#include "lib/subsys/subsys.h" +#include "lib/version/torversion.h" + +#include + +static int +torerr_subsys_init(void) +{ + configure_backtrace_handler(get_version()); + tor_log_reset_sigsafe_err_fds(); + + return 0; +} +static void +torerr_subsys_shutdown(void) +{ + tor_log_reset_sigsafe_err_fds(); + clean_up_backtrace_handler(); +} + +const subsys_fns_t sys_torerr = { + .name = "err", + .level = -100, + .supported = true, + .initialize = torerr_subsys_init, + .shutdown = torerr_subsys_shutdown +}; diff --git a/src/lib/err/torerr_sys.h b/src/lib/err/torerr_sys.h new file mode 100644 index 0000000000..b56270d538 --- /dev/null +++ b/src/lib/err/torerr_sys.h @@ -0,0 +1,14 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file torerr_sys.h + * \brief Declare subsystem object for torerr.c + **/ + +#ifndef TOR_TORERR_SYS_H +#define TOR_TORERR_SYS_H + +extern const struct subsys_fns_t sys_torerr; + +#endif /* !defined(TOR_TORERR_SYS_H) */ From 178c1821b2115972ce3c3f194d1fcbd0d75ca364 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 1 Nov 2018 12:55:10 -0400 Subject: [PATCH 0105/2557] Make the windows process parameter initialization a subsystem Also, move it from "main" into lib/process --- src/app/main/main.c | 29 --------------- src/app/main/subsystem_list.c | 2 + src/lib/process/.may_include | 1 + src/lib/process/include.am | 6 ++- src/lib/process/winprocess_sys.c | 64 ++++++++++++++++++++++++++++++++ src/lib/process/winprocess_sys.h | 14 +++++++ 6 files changed, 85 insertions(+), 31 deletions(-) create mode 100644 src/lib/process/winprocess_sys.c create mode 100644 src/lib/process/winprocess_sys.h diff --git a/src/app/main/main.c b/src/app/main/main.c index e3d7610c82..1e4cd37feb 100644 --- a/src/app/main/main.c +++ b/src/app/main/main.c @@ -1388,35 +1388,6 @@ tor_run_main(const tor_main_configuration_t *tor_cfg) { int result = 0; -#ifdef _WIN32 -#ifndef HeapEnableTerminationOnCorruption -#define HeapEnableTerminationOnCorruption 1 -#endif - /* On heap corruption, just give up; don't try to play along. */ - HeapSetInformation(NULL, HeapEnableTerminationOnCorruption, NULL, 0); - - /* SetProcessDEPPolicy is only supported on 32-bit Windows. - * (On 64-bit Windows it always fails, and some compilers don't like the - * PSETDEP cast.) - * 32-bit Windows defines _WIN32. - * 64-bit Windows defines _WIN32 and _WIN64. */ -#ifndef _WIN64 - /* Call SetProcessDEPPolicy to permanently enable DEP. - The function will not resolve on earlier versions of Windows, - and failure is not dangerous. */ - HMODULE hMod = GetModuleHandleA("Kernel32.dll"); - if (hMod) { - typedef BOOL (WINAPI *PSETDEP)(DWORD); - PSETDEP setdeppolicy = (PSETDEP)GetProcAddress(hMod, - "SetProcessDEPPolicy"); - if (setdeppolicy) { - /* PROCESS_DEP_ENABLE | PROCESS_DEP_DISABLE_ATL_THUNK_EMULATION */ - setdeppolicy(3); - } - } -#endif /* !defined(_WIN64) */ -#endif /* defined(_WIN32) */ - #ifdef EVENT_SET_MEM_FUNCTIONS_IMPLEMENTED event_set_mem_functions(tor_malloc_, tor_realloc_, tor_free_); #endif diff --git a/src/app/main/subsystem_list.c b/src/app/main/subsystem_list.c index 244dbadbd9..0f7d5d2ccc 100644 --- a/src/app/main/subsystem_list.c +++ b/src/app/main/subsystem_list.c @@ -9,6 +9,7 @@ #include "lib/cc/torint.h" #include "lib/err/torerr_sys.h" +#include "lib/process/winprocess_sys.h" #include @@ -16,6 +17,7 @@ * Global list of the subsystems in Tor, in the order of their initialization. **/ const subsys_fns_t *tor_subsystems[] = { + &sys_winprocess, &sys_torerr, }; diff --git a/src/lib/process/.may_include b/src/lib/process/.may_include index 05414d2a96..a2d57a52f3 100644 --- a/src/lib/process/.may_include +++ b/src/lib/process/.may_include @@ -11,6 +11,7 @@ lib/malloc/*.h lib/net/*.h lib/process/*.h lib/string/*.h +lib/subsys/*.h lib/testsupport/*.h lib/thread/*.h diff --git a/src/lib/process/include.am b/src/lib/process/include.am index c6cc3a6699..2aa30cc3c1 100644 --- a/src/lib/process/include.am +++ b/src/lib/process/include.am @@ -12,7 +12,8 @@ src_lib_libtor_process_a_SOURCES = \ src/lib/process/restrict.c \ src/lib/process/setuid.c \ src/lib/process/subprocess.c \ - src/lib/process/waitpid.c + src/lib/process/waitpid.c \ + src/lib/process/winprocess_sys.c src_lib_libtor_process_testing_a_SOURCES = \ $(src_lib_libtor_process_a_SOURCES) @@ -26,4 +27,5 @@ noinst_HEADERS += \ src/lib/process/restrict.h \ src/lib/process/setuid.h \ src/lib/process/subprocess.h \ - src/lib/process/waitpid.h + src/lib/process/waitpid.h \ + src/lib/process/winprocess_sys.h diff --git a/src/lib/process/winprocess_sys.c b/src/lib/process/winprocess_sys.c new file mode 100644 index 0000000000..e00f94c915 --- /dev/null +++ b/src/lib/process/winprocess_sys.c @@ -0,0 +1,64 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file winprocess_sys.c + * \brief Subsystem object for windows process setup. + **/ + +#include "orconfig.h" +#include "lib/subsys/subsys.h" +#include "lib/process/winprocess_sys.h" + +#include +#include + +#ifdef _WIN32 +#include + +#define WINPROCESS_SYS_ENABLED true + +static int +init_windows_process_params(void) +{ +#ifndef HeapEnableTerminationOnCorruption +#define HeapEnableTerminationOnCorruption 1 +#endif + + /* On heap corruption, just give up; don't try to play along. */ + HeapSetInformation(NULL, HeapEnableTerminationOnCorruption, NULL, 0); + + /* SetProcessDEPPolicy is only supported on 32-bit Windows. + * (On 64-bit Windows it always fails, and some compilers don't like the + * PSETDEP cast.) + * 32-bit Windows defines _WIN32. + * 64-bit Windows defines _WIN32 and _WIN64. */ +#ifndef _WIN64 + /* Call SetProcessDEPPolicy to permanently enable DEP. + The function will not resolve on earlier versions of Windows, + and failure is not dangerous. */ + HMODULE hMod = GetModuleHandleA("Kernel32.dll"); + if (hMod) { + typedef BOOL (WINAPI *PSETDEP)(DWORD); + PSETDEP setdeppolicy = (PSETDEP)GetProcAddress(hMod, + "SetProcessDEPPolicy"); + if (setdeppolicy) { + /* PROCESS_DEP_ENABLE | PROCESS_DEP_DISABLE_ATL_THUNK_EMULATION */ + setdeppolicy(3); + } + } +#endif /* !defined(_WIN64) */ + + return 0; +} +#else /* !defined(_WIN32) */ +#define WINPROCESS_SYS_ENABLED false +#define init_windows_process_params NULL +#endif /* defined(_WIN32) */ + +const subsys_fns_t sys_winprocess = { + .name = "winprocess", + .level = -100, + .supported = WINPROCESS_SYS_ENABLED, + .initialize = init_windows_process_params, +}; diff --git a/src/lib/process/winprocess_sys.h b/src/lib/process/winprocess_sys.h new file mode 100644 index 0000000000..cb096e0c92 --- /dev/null +++ b/src/lib/process/winprocess_sys.h @@ -0,0 +1,14 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file winprocess_sys.h + * \brief Declare subsystem object for winprocess.c + **/ + +#ifndef TOR_WINPROCESS_SYS_H +#define TOR_WINPROCESS_SYS_H + +extern const struct subsys_fns_t sys_winprocess; + +#endif /* !defined(TOR_WINPROCESS_SYS_H) */ From b8c50eabfee1bd9f5ed03f8ec569cc53b980f1d1 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 1 Nov 2018 13:14:43 -0400 Subject: [PATCH 0106/2557] Add a subsystem for our threading support --- src/app/main/main.c | 1 - src/app/main/subsystem_list.c | 2 ++ src/lib/thread/.may_include | 1 + src/lib/thread/compat_threads.c | 16 ++++++++++++++++ src/lib/thread/include.am | 5 +++-- src/lib/thread/thread_sys.h | 14 ++++++++++++++ 6 files changed, 36 insertions(+), 3 deletions(-) create mode 100644 src/lib/thread/thread_sys.h diff --git a/src/app/main/main.c b/src/app/main/main.c index 1e4cd37feb..21a2832781 100644 --- a/src/app/main/main.c +++ b/src/app/main/main.c @@ -1397,7 +1397,6 @@ tor_run_main(const tor_main_configuration_t *tor_cfg) init_protocol_warning_severity_level(); update_approx_time(time(NULL)); - tor_threads_init(); tor_compress_init(); init_logging(0); monotime_init(); diff --git a/src/app/main/subsystem_list.c b/src/app/main/subsystem_list.c index 0f7d5d2ccc..c3b731ca39 100644 --- a/src/app/main/subsystem_list.c +++ b/src/app/main/subsystem_list.c @@ -10,6 +10,7 @@ #include "lib/err/torerr_sys.h" #include "lib/process/winprocess_sys.h" +#include "lib/thread/thread_sys.h" #include @@ -19,6 +20,7 @@ const subsys_fns_t *tor_subsystems[] = { &sys_winprocess, &sys_torerr, + &sys_threads, }; const unsigned n_tor_subsystems = ARRAY_LENGTH(tor_subsystems); diff --git a/src/lib/thread/.may_include b/src/lib/thread/.may_include index 93ad0cd734..c26a426923 100644 --- a/src/lib/thread/.may_include +++ b/src/lib/thread/.may_include @@ -2,5 +2,6 @@ orconfig.h lib/cc/*.h lib/lock/*.h lib/log/*.h +lib/subsys/*.h lib/testsupport/*.h lib/thread/*.h diff --git a/src/lib/thread/compat_threads.c b/src/lib/thread/compat_threads.c index 7f1970af45..3d41faa8ce 100644 --- a/src/lib/thread/compat_threads.c +++ b/src/lib/thread/compat_threads.c @@ -14,9 +14,11 @@ #include "orconfig.h" #include #include "lib/thread/threads.h" +#include "lib/thread/thread_sys.h" #include "lib/log/log.h" #include "lib/log/util_bug.h" +#include "lib/subsys/subsys.h" #include @@ -109,3 +111,17 @@ atomic_counter_exchange(atomic_counter_t *counter, size_t newval) return oldval; } #endif /* !defined(HAVE_WORKING_STDATOMIC) */ + +static int +sys_threads_initialize(void) +{ + tor_threads_init(); + return 0; +} + +const subsys_fns_t sys_threads = { + .name = "threads", + .supported = true, + .level = -95, + .initialize = sys_threads_initialize, +}; diff --git a/src/lib/thread/include.am b/src/lib/thread/include.am index 9ec23d166e..695795a2c8 100644 --- a/src/lib/thread/include.am +++ b/src/lib/thread/include.am @@ -23,5 +23,6 @@ src_lib_libtor_thread_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_thread_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) noinst_HEADERS += \ - src/lib/thread/threads.h \ - src/lib/thread/numcpus.h + src/lib/thread/numcpus.h \ + src/lib/thread/thread_sys.h \ + src/lib/thread/threads.h diff --git a/src/lib/thread/thread_sys.h b/src/lib/thread/thread_sys.h new file mode 100644 index 0000000000..984abe88e8 --- /dev/null +++ b/src/lib/thread/thread_sys.h @@ -0,0 +1,14 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file threads_sys.h + * \brief Declare subsystem object for threads library + **/ + +#ifndef TOR_THREADS_SYS_H +#define TOR_THREADS_SYS_H + +extern const struct subsys_fns_t sys_threads; + +#endif /* !defined(TOR_THREADS_SYS_H) */ From d3e4afcc9b835e0f862207ef16d7e706ceea9ce1 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 1 Nov 2018 13:26:33 -0400 Subject: [PATCH 0107/2557] Turn the logging code into a subsystem --- src/app/main/main.c | 3 --- src/app/main/subsystem_list.c | 2 ++ src/lib/log/.may_include | 1 + src/lib/log/include.am | 2 ++ src/lib/log/log.c | 1 + src/lib/log/log_sys.c | 35 +++++++++++++++++++++++++++++++++++ src/lib/log/log_sys.h | 14 ++++++++++++++ 7 files changed, 55 insertions(+), 3 deletions(-) create mode 100644 src/lib/log/log_sys.c create mode 100644 src/lib/log/log_sys.h diff --git a/src/app/main/main.c b/src/app/main/main.c index 21a2832781..f44f3475dd 100644 --- a/src/app/main/main.c +++ b/src/app/main/main.c @@ -819,9 +819,7 @@ tor_free_all(int postfork) /* Stuff in util.c and address.c*/ if (!postfork) { - escaped(NULL); esc_router_info(NULL); - logs_free_all(); /* free log strings. do this last so logs keep working. */ } } @@ -1398,7 +1396,6 @@ tor_run_main(const tor_main_configuration_t *tor_cfg) update_approx_time(time(NULL)); tor_compress_init(); - init_logging(0); monotime_init(); int argc = tor_cfg->argc + tor_cfg->argc_owned; diff --git a/src/app/main/subsystem_list.c b/src/app/main/subsystem_list.c index c3b731ca39..4a2994ec49 100644 --- a/src/app/main/subsystem_list.c +++ b/src/app/main/subsystem_list.c @@ -11,6 +11,7 @@ #include "lib/err/torerr_sys.h" #include "lib/process/winprocess_sys.h" #include "lib/thread/thread_sys.h" +#include "lib/log/log_sys.h" #include @@ -21,6 +22,7 @@ const subsys_fns_t *tor_subsystems[] = { &sys_winprocess, &sys_torerr, &sys_threads, + &sys_logging, }; const unsigned n_tor_subsystems = ARRAY_LENGTH(tor_subsystems); diff --git a/src/lib/log/.may_include b/src/lib/log/.may_include index 7ca1863a52..11c87f0a0d 100644 --- a/src/lib/log/.may_include +++ b/src/lib/log/.may_include @@ -9,6 +9,7 @@ lib/lock/*.h lib/log/*.h lib/malloc/*.h lib/string/*.h +lib/subsys/*.h lib/testsupport/*.h lib/version/*.h lib/wallclock/*.h diff --git a/src/lib/log/include.am b/src/lib/log/include.am index c6f404e269..9d3dbe3104 100644 --- a/src/lib/log/include.am +++ b/src/lib/log/include.am @@ -9,6 +9,7 @@ src_lib_libtor_log_a_SOURCES = \ src/lib/log/escape.c \ src/lib/log/ratelim.c \ src/lib/log/log.c \ + src/lib/log/log_sys.c \ src/lib/log/util_bug.c if WIN32 @@ -24,5 +25,6 @@ noinst_HEADERS += \ src/lib/log/escape.h \ src/lib/log/ratelim.h \ src/lib/log/log.h \ + src/lib/log/log_sys.h \ src/lib/log/util_bug.h \ src/lib/log/win32err.h diff --git a/src/lib/log/log.c b/src/lib/log/log.c index bc7b36dcb9..46107fe848 100644 --- a/src/lib/log/log.c +++ b/src/lib/log/log.c @@ -32,6 +32,7 @@ #define LOG_PRIVATE #include "lib/log/log.h" +#include "lib/log/log_sys.h" #include "lib/version/git_revision.h" #include "lib/log/ratelim.h" #include "lib/lock/compat_mutex.h" diff --git a/src/lib/log/log_sys.c b/src/lib/log/log_sys.c new file mode 100644 index 0000000000..94ec97fdc1 --- /dev/null +++ b/src/lib/log/log_sys.c @@ -0,0 +1,35 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file log_sys.c + * \brief Setup and tear down the logging module. + **/ + +#include "orconfig.h" +#include "lib/subsys/subsys.h" +#include "lib/log/escape.h" +#include "lib/log/log.h" +#include "lib/log/log_sys.h" + +static int +init_logging_subsys(void) +{ + init_logging(0); + return 0; +} + +static void +shutdown_logging_subsys(void) +{ + logs_free_all(); + escaped(NULL); +} + +const subsys_fns_t sys_logging = { + .name = "log", + .supported = true, + .level = -90, + .initialize = init_logging_subsys, + .shutdown = shutdown_logging_subsys, +}; diff --git a/src/lib/log/log_sys.h b/src/lib/log/log_sys.h new file mode 100644 index 0000000000..f7afbb279d --- /dev/null +++ b/src/lib/log/log_sys.h @@ -0,0 +1,14 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file log_sys.h + * \brief Declare subsystem object for the logging module. + **/ + +#ifndef TOR_LOG_SYS_H +#define TOR_LOG_SYS_H + +extern const struct subsys_fns_t sys_logging; + +#endif /* !defined(TOR_LOG_SYS_H) */ From a0ee54549fec3ae710ab5e3623d707bd08adcafe Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 1 Nov 2018 13:34:07 -0400 Subject: [PATCH 0108/2557] Turn the wallclock module into a subsystem. (This may be slightly gratuitous.) --- src/app/main/main.c | 1 - src/app/main/subsystem_list.c | 4 +++- src/lib/wallclock/.may_include | 1 + src/lib/wallclock/approx_time.c | 16 ++++++++++++++++ src/lib/wallclock/include.am | 3 ++- src/lib/wallclock/wallclock_sys.h | 14 ++++++++++++++ 6 files changed, 36 insertions(+), 3 deletions(-) create mode 100644 src/lib/wallclock/wallclock_sys.h diff --git a/src/app/main/main.c b/src/app/main/main.c index f44f3475dd..5740efb0b6 100644 --- a/src/app/main/main.c +++ b/src/app/main/main.c @@ -1394,7 +1394,6 @@ tor_run_main(const tor_main_configuration_t *tor_cfg) init_protocol_warning_severity_level(); - update_approx_time(time(NULL)); tor_compress_init(); monotime_init(); diff --git a/src/app/main/subsystem_list.c b/src/app/main/subsystem_list.c index 4a2994ec49..3d03a9a4df 100644 --- a/src/app/main/subsystem_list.c +++ b/src/app/main/subsystem_list.c @@ -9,9 +9,10 @@ #include "lib/cc/torint.h" #include "lib/err/torerr_sys.h" +#include "lib/log/log_sys.h" #include "lib/process/winprocess_sys.h" #include "lib/thread/thread_sys.h" -#include "lib/log/log_sys.h" +#include "lib/wallclock/wallclock_sys.h" #include @@ -21,6 +22,7 @@ const subsys_fns_t *tor_subsystems[] = { &sys_winprocess, &sys_torerr, + &sys_wallclock, &sys_threads, &sys_logging, }; diff --git a/src/lib/wallclock/.may_include b/src/lib/wallclock/.may_include index dc010da063..ce7a26472b 100644 --- a/src/lib/wallclock/.may_include +++ b/src/lib/wallclock/.may_include @@ -3,4 +3,5 @@ lib/cc/*.h lib/err/*.h lib/wallclock/*.h lib/string/*.h +lib/subsys/*.h lib/testsupport/*.h diff --git a/src/lib/wallclock/approx_time.c b/src/lib/wallclock/approx_time.c index bb9a292369..c7a7ae9bd7 100644 --- a/src/lib/wallclock/approx_time.c +++ b/src/lib/wallclock/approx_time.c @@ -9,7 +9,9 @@ **/ #include "orconfig.h" +#include "lib/subsys/subsys.h" #include "lib/wallclock/approx_time.h" +#include "lib/wallclock/wallclock_sys.h" /* ===== * Cached time @@ -41,3 +43,17 @@ update_approx_time(time_t now) cached_approx_time = now; } #endif /* !defined(TIME_IS_FAST) */ + +static int +init_wallclock_subsys(void) +{ + update_approx_time(time(NULL)); + return 0; +} + +const subsys_fns_t sys_wallclock = { + .name = "wallclock", + .supported = true, + .level = -99, + .initialize = init_wallclock_subsys, +}; diff --git a/src/lib/wallclock/include.am b/src/lib/wallclock/include.am index 1961639bd7..2351252e0c 100644 --- a/src/lib/wallclock/include.am +++ b/src/lib/wallclock/include.am @@ -19,4 +19,5 @@ noinst_HEADERS += \ src/lib/wallclock/approx_time.h \ src/lib/wallclock/timeval.h \ src/lib/wallclock/time_to_tm.h \ - src/lib/wallclock/tor_gettimeofday.h + src/lib/wallclock/tor_gettimeofday.h \ + src/lib/wallclock/wallclock_sys.h diff --git a/src/lib/wallclock/wallclock_sys.h b/src/lib/wallclock/wallclock_sys.h new file mode 100644 index 0000000000..e009578a83 --- /dev/null +++ b/src/lib/wallclock/wallclock_sys.h @@ -0,0 +1,14 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file wallclock_sys.h + * \brief Declare subsystem object for the wallclock module. + **/ + +#ifndef TOR_WALLCLOCK_SYS_H +#define TOR_WALLCLOCK_SYS_H + +extern const struct subsys_fns_t sys_wallclock; + +#endif /* !defined(TOR_WALLCLOCK_SYS_H) */ From 05b54f6a6a24ebdb47de4f7e41cf94f2f6be93bd Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 1 Nov 2018 14:13:36 -0400 Subject: [PATCH 0109/2557] Use subsystems manager for subsystems used in tests. --- src/lib/subsys/subsys.h | 4 ++++ src/test/bench.c | 7 ++++--- src/test/fuzz/fuzzing_common.c | 9 ++++----- src/test/testing_common.c | 8 +++----- 4 files changed, 15 insertions(+), 13 deletions(-) diff --git a/src/lib/subsys/subsys.h b/src/lib/subsys/subsys.h index 012b218da7..25451bc450 100644 --- a/src/lib/subsys/subsys.h +++ b/src/lib/subsys/subsys.h @@ -65,4 +65,8 @@ typedef struct subsys_fns_t { #define MIN_SUBSYS_LEVEL -100 #define MAX_SUBSYS_LEVEL 100 +/* All tor "libraries" (in src/libs) should have a subsystem level equal to or + * less than this value. */ +#define SUBSYS_LEVEL_LIBS -10 + #endif diff --git a/src/test/bench.c b/src/test/bench.c index 9da1b46a1b..ff8707d41c 100644 --- a/src/test/bench.c +++ b/src/test/bench.c @@ -24,6 +24,7 @@ #include "core/or/circuitlist.h" #include "app/config/config.h" +#include "app/main/subsysmgr.h" #include "lib/crypt_ops/crypto_curve25519.h" #include "lib/crypt_ops/crypto_dh.h" #include "core/crypto/onion_ntor.h" @@ -690,9 +691,10 @@ main(int argc, const char **argv) char *errmsg; or_options_t *options; - tor_threads_init(); + subsystems_init_upto(SUBSYS_LEVEL_LIBS); + flush_log_messages_from_startup(); + tor_compress_init(); - init_logging(1); if (argc == 4 && !strcmp(argv[1], "diff")) { const int N = 200; @@ -739,7 +741,6 @@ main(int argc, const char **argv) init_protocol_warning_severity_level(); options = options_new(); - init_logging(1); options->command = CMD_RUN_UNITTESTS; options->DataDirectory = tor_strdup(""); options->KeyDirectory = tor_strdup(""); diff --git a/src/test/fuzz/fuzzing_common.c b/src/test/fuzz/fuzzing_common.c index 879f9e74dc..21aa07cfe2 100644 --- a/src/test/fuzz/fuzzing_common.c +++ b/src/test/fuzz/fuzzing_common.c @@ -3,6 +3,7 @@ #define CRYPTO_ED25519_PRIVATE #include "orconfig.h" #include "core/or/or.h" +#include "app/main/subsysmgr.h" #include "lib/err/backtrace.h" #include "app/config/config.h" #include "test/fuzz/fuzzing.h" @@ -95,12 +96,10 @@ disable_signature_checking(void) static void global_init(void) { - tor_threads_init(); - tor_compress_init(); + subsystems_init_upto(SUBSYS_LEVEL_LIBS); + flush_log_messages_from_startup(); - /* Initialise logging first */ - init_logging(1); - configure_backtrace_handler(get_version()); + tor_compress_init(); if (crypto_global_init(0, NULL, NULL) < 0) abort(); diff --git a/src/test/testing_common.c b/src/test/testing_common.c index 8d648ee175..eef393d3a8 100644 --- a/src/test/testing_common.c +++ b/src/test/testing_common.c @@ -26,6 +26,7 @@ #include "lib/evloop/compat_libevent.h" #include "lib/crypt_ops/crypto_init.h" #include "lib/version/torversion.h" +#include "app/main/subsysmgr.h" #include #ifdef HAVE_FCNTL_H @@ -251,12 +252,9 @@ main(int c, const char **v) int loglevel = LOG_ERR; int accel_crypto = 0; - /* We must initialise logs before we call tor_assert() */ - init_logging(1); + subsystems_init_upto(SUBSYS_LEVEL_LIBS); - update_approx_time(time(NULL)); options = options_new(); - tor_threads_init(); tor_compress_init(); network_init(); @@ -268,7 +266,6 @@ main(int c, const char **v) tor_libevent_initialize(&cfg); control_initialize_event_queue(); - configure_backtrace_handler(get_version()); for (i_out = i = 1; i < c; ++i) { if (!strcmp(v[i], "--warn")) { @@ -295,6 +292,7 @@ main(int c, const char **v) s.masks[LOG_WARN-LOG_ERR] |= LD_BUG; add_stream_log(&s, "", fileno(stdout)); } + flush_log_messages_from_startup(); init_protocol_warning_severity_level(); options->command = CMD_RUN_UNITTESTS; From cfe5b35edb38cef6312ef0b4ae44fb0e20342706 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 2 Nov 2018 11:11:21 -0400 Subject: [PATCH 0110/2557] Move networking startup/cleanup logic into a subsystem. --- src/app/main/main.c | 19 --------------- src/app/main/subsystem_list.c | 2 ++ src/lib/net/.may_include | 1 + src/lib/net/include.am | 2 ++ src/lib/net/network_sys.c | 44 +++++++++++++++++++++++++++++++++++ src/lib/net/network_sys.h | 14 +++++++++++ src/test/testing_common.c | 2 -- 7 files changed, 63 insertions(+), 21 deletions(-) create mode 100644 src/lib/net/network_sys.c create mode 100644 src/lib/net/network_sys.h diff --git a/src/app/main/main.c b/src/app/main/main.c index 5740efb0b6..3e80725b9a 100644 --- a/src/app/main/main.c +++ b/src/app/main/main.c @@ -427,18 +427,6 @@ dumpstats(int severity) rend_service_dump_stats(severity); } -/** Called by exit() as we shut down the process. - */ -static void -exit_function(void) -{ - /* NOTE: If we ever daemonize, this gets called immediately. That's - * okay for now, because we only use this on Windows. */ -#ifdef _WIN32 - WSACleanup(); -#endif -} - #ifdef _WIN32 #define UNIX_ONLY 0 #else @@ -632,12 +620,6 @@ tor_init(int argc, char *argv[]) rust_log_welcome_string(); #endif /* defined(HAVE_RUST) */ - if (network_init()<0) { - log_err(LD_BUG,"Error initializing network; exiting."); - return -1; - } - atexit(exit_function); - int init_rv = options_init_from_torrc(argc,argv); if (init_rv < 0) { log_err(LD_CONFIG,"Reading config failed--see warnings above."); @@ -784,7 +766,6 @@ tor_free_all(int postfork) routerparse_free_all(); ext_orport_free_all(); control_free_all(); - tor_free_getaddrinfo_cache(); protover_free_all(); bridges_free_all(); consdiffmgr_free_all(); diff --git a/src/app/main/subsystem_list.c b/src/app/main/subsystem_list.c index 3d03a9a4df..cb186c14d9 100644 --- a/src/app/main/subsystem_list.c +++ b/src/app/main/subsystem_list.c @@ -10,6 +10,7 @@ #include "lib/err/torerr_sys.h" #include "lib/log/log_sys.h" +#include "lib/net/network_sys.h" #include "lib/process/winprocess_sys.h" #include "lib/thread/thread_sys.h" #include "lib/wallclock/wallclock_sys.h" @@ -25,6 +26,7 @@ const subsys_fns_t *tor_subsystems[] = { &sys_wallclock, &sys_threads, &sys_logging, + &sys_network, }; const unsigned n_tor_subsystems = ARRAY_LENGTH(tor_subsystems); diff --git a/src/lib/net/.may_include b/src/lib/net/.may_include index 13b209bbed..f93f0e1552 100644 --- a/src/lib/net/.may_include +++ b/src/lib/net/.may_include @@ -11,5 +11,6 @@ lib/lock/*.h lib/log/*.h lib/net/*.h lib/string/*.h +lib/subsys/*.h lib/testsupport/*.h lib/malloc/*.h \ No newline at end of file diff --git a/src/lib/net/include.am b/src/lib/net/include.am index ff0967e786..8a88f0f2ae 100644 --- a/src/lib/net/include.am +++ b/src/lib/net/include.am @@ -11,6 +11,7 @@ src_lib_libtor_net_a_SOURCES = \ src/lib/net/buffers_net.c \ src/lib/net/gethostname.c \ src/lib/net/inaddr.c \ + src/lib/net/network_sys.c \ src/lib/net/resolve.c \ src/lib/net/socket.c \ src/lib/net/socketpair.c @@ -28,6 +29,7 @@ noinst_HEADERS += \ src/lib/net/inaddr.h \ src/lib/net/inaddr_st.h \ src/lib/net/nettypes.h \ + src/lib/net/network_sys.h \ src/lib/net/resolve.h \ src/lib/net/socket.h \ src/lib/net/socketpair.h \ diff --git a/src/lib/net/network_sys.c b/src/lib/net/network_sys.c new file mode 100644 index 0000000000..c9d33a94d3 --- /dev/null +++ b/src/lib/net/network_sys.c @@ -0,0 +1,44 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file network_sys.c + * \brief Subsystem object for networking setup. + **/ + +#include "orconfig.h" +#include "lib/subsys/subsys.h" +#include "lib/net/network_sys.h" +#include "lib/net/resolve.h" +#include "lib/net/socket.h" + +#ifdef _WIN32 +#include +#include +#endif + +static int +init_network_sys(void) +{ + if (network_init() < 0) + return -1; + + return 0; +} + +static void +shutdown_network_sys(void) +{ +#ifdef _WIN32 + WSACleanup(); +#endif + tor_free_getaddrinfo_cache(); +} + +const subsys_fns_t sys_network = { + .name = "network", + .level = -90, + .supported = true, + .initialize = init_network_sys, + .shutdown = shutdown_network_sys, +}; diff --git a/src/lib/net/network_sys.h b/src/lib/net/network_sys.h new file mode 100644 index 0000000000..62b778bb66 --- /dev/null +++ b/src/lib/net/network_sys.h @@ -0,0 +1,14 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file log_network.h + * \brief Declare subsystem object for the network module. + **/ + +#ifndef TOR_NETWORK_SYS_H +#define TOR_NETWORK_SYS_H + +extern const struct subsys_fns_t sys_network; + +#endif /* !defined(TOR_NETWORK_SYS_H) */ diff --git a/src/test/testing_common.c b/src/test/testing_common.c index eef393d3a8..818bb58c9a 100644 --- a/src/test/testing_common.c +++ b/src/test/testing_common.c @@ -257,8 +257,6 @@ main(int c, const char **v) options = options_new(); tor_compress_init(); - network_init(); - monotime_init(); struct tor_libevent_cfg cfg; From 50436ccea4bd200e45196ccce7acff28f293a4de Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 2 Nov 2018 11:21:06 -0400 Subject: [PATCH 0111/2557] Add crypto module as a subsystem. --- src/app/main/main.c | 7 ------- src/app/main/subsystem_list.c | 2 ++ src/lib/crypt_ops/.may_include | 1 + src/lib/crypt_ops/crypto_init.c | 26 ++++++++++++++++++++++++++ src/lib/crypt_ops/crypto_sys.h | 14 ++++++++++++++ src/lib/crypt_ops/include.am | 1 + src/test/testing_common.c | 2 -- 7 files changed, 44 insertions(+), 9 deletions(-) create mode 100644 src/lib/crypt_ops/crypto_sys.h diff --git a/src/app/main/main.c b/src/app/main/main.c index 3e80725b9a..74c3c41e5b 100644 --- a/src/app/main/main.c +++ b/src/app/main/main.c @@ -535,12 +535,6 @@ tor_init(int argc, char *argv[]) tor_snprintf(progname, sizeof(progname), "Tor %s", get_version()); log_set_application_name(progname); - /* Set up the crypto nice and early */ - if (crypto_early_init() < 0) { - log_err(LD_GENERAL, "Unable to initialize the crypto subsystem!"); - return -1; - } - /* Initialize the history structures. */ rep_hist_init(); /* Initialize the service cache. */ @@ -859,7 +853,6 @@ tor_cleanup(void) later, if it makes shutdown unacceptably slow. But for now, leave it here: it's helped us catch bugs in the past. */ - crypto_global_cleanup(); } /** Read/create keys as needed, and echo our fingerprint to stdout. */ diff --git a/src/app/main/subsystem_list.c b/src/app/main/subsystem_list.c index cb186c14d9..dd64568226 100644 --- a/src/app/main/subsystem_list.c +++ b/src/app/main/subsystem_list.c @@ -8,6 +8,7 @@ #include "lib/cc/compat_compiler.h" #include "lib/cc/torint.h" +#include "lib/crypt_ops/crypto_sys.h" #include "lib/err/torerr_sys.h" #include "lib/log/log_sys.h" #include "lib/net/network_sys.h" @@ -27,6 +28,7 @@ const subsys_fns_t *tor_subsystems[] = { &sys_threads, &sys_logging, &sys_network, + &sys_crypto, }; const unsigned n_tor_subsystems = ARRAY_LENGTH(tor_subsystems); diff --git a/src/lib/crypt_ops/.may_include b/src/lib/crypt_ops/.may_include index a0fa4ec05c..352fde858c 100644 --- a/src/lib/crypt_ops/.may_include +++ b/src/lib/crypt_ops/.may_include @@ -12,6 +12,7 @@ lib/malloc/*.h lib/intmath/*.h lib/sandbox/*.h lib/string/*.h +lib/subsys/*.h lib/testsupport/testsupport.h lib/thread/*.h lib/log/*.h diff --git a/src/lib/crypt_ops/crypto_init.c b/src/lib/crypt_ops/crypto_init.c index 9d6e2da0d0..cc7865ef72 100644 --- a/src/lib/crypt_ops/crypto_init.c +++ b/src/lib/crypt_ops/crypto_init.c @@ -20,6 +20,9 @@ #include "lib/crypt_ops/crypto_openssl_mgt.h" #include "lib/crypt_ops/crypto_nss_mgt.h" #include "lib/crypt_ops/crypto_rand.h" +#include "lib/crypt_ops/crypto_sys.h" + +#include "lib/subsys/subsys.h" #include "siphash.h" @@ -202,3 +205,26 @@ tor_is_using_nss(void) return 0; #endif } + +static int +init_crypto_sys(void) +{ + if (crypto_early_init() < 0) + return -1; + crypto_dh_init(); + return 0; +} + +static void +shutdown_crypto_sys(void) +{ + crypto_global_cleanup(); +} + +const struct subsys_fns_t sys_crypto = { + .name = "crypto", + .supported = true, + .level = -60, + .initialize = init_crypto_sys, + .shutdown = shutdown_crypto_sys, +}; diff --git a/src/lib/crypt_ops/crypto_sys.h b/src/lib/crypt_ops/crypto_sys.h new file mode 100644 index 0000000000..31644d088b --- /dev/null +++ b/src/lib/crypt_ops/crypto_sys.h @@ -0,0 +1,14 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file log_crypto.h + * \brief Declare subsystem object for the crypto module. + **/ + +#ifndef TOR_CRYPTO_SYS_H +#define TOR_CRYPTO_SYS_H + +extern const struct subsys_fns_t sys_crypto; + +#endif /* !defined(TOR_CRYPTO_SYS_H) */ diff --git a/src/lib/crypt_ops/include.am b/src/lib/crypt_ops/include.am index 1022096fdc..d0ccc13bff 100644 --- a/src/lib/crypt_ops/include.am +++ b/src/lib/crypt_ops/include.am @@ -66,5 +66,6 @@ noinst_HEADERS += \ src/lib/crypt_ops/crypto_rand.h \ src/lib/crypt_ops/crypto_rsa.h \ src/lib/crypt_ops/crypto_s2k.h \ + src/lib/crypt_ops/crypto_sys.h \ src/lib/crypt_ops/crypto_util.h \ src/lib/crypt_ops/digestset.h diff --git a/src/test/testing_common.c b/src/test/testing_common.c index 818bb58c9a..d4c5632334 100644 --- a/src/test/testing_common.c +++ b/src/test/testing_common.c @@ -331,8 +331,6 @@ main(int c, const char **v) free_pregenerated_keys(); - crypto_global_cleanup(); - if (have_failed) return 1; else From cad61f0f6de48c6eab6e811a081f154b03de57b8 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 2 Nov 2018 18:00:56 -0400 Subject: [PATCH 0112/2557] Move prefork, postfork, and thread-exit hooks into subsys So far, crypto is the only module that uses them, but others are likely to do so in the future. --- src/app/config/config.c | 5 +-- src/app/main/subsysmgr.c | 57 +++++++++++++++++++++++++++++++++ src/app/main/subsysmgr.h | 4 +++ src/lib/crypt_ops/crypto_init.c | 3 ++ src/lib/subsys/subsys.h | 16 +++++++++ src/test/testing_common.c | 4 +-- 6 files changed, 85 insertions(+), 4 deletions(-) diff --git a/src/app/config/config.c b/src/app/config/config.c index 7b49387bcf..76df7ec67e 100644 --- a/src/app/config/config.c +++ b/src/app/config/config.c @@ -64,6 +64,7 @@ #include "app/config/confparse.h" #include "app/config/statefile.h" #include "app/main/main.h" +#include "app/main/subsysmgr.h" #include "core/mainloop/connection.h" #include "core/mainloop/cpuworker.h" #include "core/mainloop/mainloop.h" @@ -1393,10 +1394,10 @@ options_act_reversible(const or_options_t *old_options, char **msg) * processes. */ if (running_tor && options->RunAsDaemon) { if (! start_daemon_has_been_called()) - crypto_prefork(); + subsystems_prefork(); /* No need to roll back, since you can't change the value. */ if (start_daemon()) - crypto_postfork(); + subsystems_postfork(); } #ifdef HAVE_SYSTEMD diff --git a/src/app/main/subsysmgr.c b/src/app/main/subsysmgr.c index 7974f2d238..05803ee946 100644 --- a/src/app/main/subsysmgr.c +++ b/src/app/main/subsysmgr.c @@ -128,3 +128,60 @@ subsystems_shutdown_downto(int target_level) sys_initialized[i] = false; } } + +/** + * Run pre-fork code on all subsystems that declare any + **/ +void +subsystems_prefork(void) +{ + check_and_setup(); + + for (int i = (int)n_tor_subsystems - 1; i >= 0; --i) { + const subsys_fns_t *sys = tor_subsystems[i]; + if (!sys->supported) + continue; + if (! sys_initialized[i]) + continue; + if (sys->prefork) + sys->prefork(); + } +} + +/** + * Run post-fork code on all subsystems that declare any + **/ +void +subsystems_postfork(void) +{ + check_and_setup(); + + for (unsigned i = 0; i < n_tor_subsystems; ++i) { + const subsys_fns_t *sys = tor_subsystems[i]; + if (!sys->supported) + continue; + if (! sys_initialized[i]) + continue; + if (sys->postfork) + sys->postfork(); + } +} + +/** + * Run thread-clanup code on all subsystems that declare any + **/ +void +subsystems_thread_cleanup(void) +{ + check_and_setup(); + + for (int i = (int)n_tor_subsystems - 1; i >= 0; --i) { + const subsys_fns_t *sys = tor_subsystems[i]; + if (!sys->supported) + continue; + if (! sys_initialized[i]) + continue; + if (sys->thread_cleanup) + sys->thread_cleanup(); + } +} diff --git a/src/app/main/subsysmgr.h b/src/app/main/subsysmgr.h index c9b892eee4..4b3cad62ad 100644 --- a/src/app/main/subsysmgr.h +++ b/src/app/main/subsysmgr.h @@ -17,4 +17,8 @@ int subsystems_init_upto(int level); void subsystems_shutdown(void); void subsystems_shutdown_downto(int level); +void subsystems_prefork(void); +void subsystems_postfork(void); +void subsystems_thread_cleanup(void); + #endif diff --git a/src/lib/crypt_ops/crypto_init.c b/src/lib/crypt_ops/crypto_init.c index cc7865ef72..a03f5eff7c 100644 --- a/src/lib/crypt_ops/crypto_init.c +++ b/src/lib/crypt_ops/crypto_init.c @@ -227,4 +227,7 @@ const struct subsys_fns_t sys_crypto = { .level = -60, .initialize = init_crypto_sys, .shutdown = shutdown_crypto_sys, + .prefork = crypto_prefork, + .postfork = crypto_postfork, + .thread_cleanup = crypto_thread_cleanup, }; diff --git a/src/lib/subsys/subsys.h b/src/lib/subsys/subsys.h index 25451bc450..b06d67e624 100644 --- a/src/lib/subsys/subsys.h +++ b/src/lib/subsys/subsys.h @@ -53,6 +53,22 @@ typedef struct subsys_fns_t { **/ int (*add_pubsub)(struct dispatch_connector_t *); + /** + * Perform any necessary pre-fork cleanup. This function may not fail. + */ + void (*prefork)(void); + + /** + * Perform any necessary post-fork setup. This function may not fail. + */ + void (*postfork)(void); + + /** + * Free any thread-local resources held by this subsystem. Called before + * the thread exits. + */ + void (*thread_cleanup)(void); + /** * Free all resources held by this subsystem. * diff --git a/src/test/testing_common.c b/src/test/testing_common.c index d4c5632334..1362f29711 100644 --- a/src/test/testing_common.c +++ b/src/test/testing_common.c @@ -232,12 +232,12 @@ void tinytest_prefork(void) { free_pregenerated_keys(); - crypto_prefork(); + subsystems_prefork(); } void tinytest_postfork(void) { - crypto_postfork(); + subsystems_postfork(); init_pregenerated_keys(); } From 207253df8d7c040840c7f4305534e6979bfc7bf7 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 2 Nov 2018 18:09:44 -0400 Subject: [PATCH 0113/2557] Move monotonic time setup into a subsystem --- src/app/main/main.c | 2 -- src/app/main/subsystem_list.c | 2 ++ src/lib/time/.may_include | 1 + src/lib/time/include.am | 2 ++ src/lib/time/time_sys.c | 26 ++++++++++++++++++++++++++ src/lib/time/time_sys.h | 14 ++++++++++++++ src/test/testing_common.c | 2 -- 7 files changed, 45 insertions(+), 4 deletions(-) create mode 100644 src/lib/time/time_sys.c create mode 100644 src/lib/time/time_sys.h diff --git a/src/app/main/main.c b/src/app/main/main.c index 74c3c41e5b..bb2e9f5cdb 100644 --- a/src/app/main/main.c +++ b/src/app/main/main.c @@ -1248,7 +1248,6 @@ static int run_tor_main_loop(void) { handle_signals(); - monotime_init(); timers_initialize(); initialize_mainloop_events(); @@ -1369,7 +1368,6 @@ tor_run_main(const tor_main_configuration_t *tor_cfg) init_protocol_warning_severity_level(); tor_compress_init(); - monotime_init(); int argc = tor_cfg->argc + tor_cfg->argc_owned; char **argv = tor_calloc(argc, sizeof(char*)); diff --git a/src/app/main/subsystem_list.c b/src/app/main/subsystem_list.c index dd64568226..a9189b9941 100644 --- a/src/app/main/subsystem_list.c +++ b/src/app/main/subsystem_list.c @@ -14,6 +14,7 @@ #include "lib/net/network_sys.h" #include "lib/process/winprocess_sys.h" #include "lib/thread/thread_sys.h" +#include "lib/time/time_sys.h" #include "lib/wallclock/wallclock_sys.h" #include @@ -27,6 +28,7 @@ const subsys_fns_t *tor_subsystems[] = { &sys_wallclock, &sys_threads, &sys_logging, + &sys_time, &sys_network, &sys_crypto, }; diff --git a/src/lib/time/.may_include b/src/lib/time/.may_include index 2c7e37a836..40a18805ac 100644 --- a/src/lib/time/.may_include +++ b/src/lib/time/.may_include @@ -4,6 +4,7 @@ lib/cc/*.h lib/err/*.h lib/intmath/*.h lib/log/*.h +lib/subsys/*.h lib/time/*.h lib/wallclock/*.h diff --git a/src/lib/time/include.am b/src/lib/time/include.am index a3f93a3744..dae16f49ac 100644 --- a/src/lib/time/include.am +++ b/src/lib/time/include.am @@ -7,6 +7,7 @@ endif src_lib_libtor_time_a_SOURCES = \ src/lib/time/compat_time.c \ + src/lib/time/time_sys.c \ src/lib/time/tvdiff.c src_lib_libtor_time_testing_a_SOURCES = \ @@ -16,4 +17,5 @@ src_lib_libtor_time_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) noinst_HEADERS += \ src/lib/time/compat_time.h \ + src/lib/time/time_sys.h \ src/lib/time/tvdiff.h diff --git a/src/lib/time/time_sys.c b/src/lib/time/time_sys.c new file mode 100644 index 0000000000..2303874f29 --- /dev/null +++ b/src/lib/time/time_sys.c @@ -0,0 +1,26 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file time_sys.c + * \brief Subsystem object for monotime setup. + **/ + +#include "orconfig.h" +#include "lib/subsys/subsys.h" +#include "lib/time/time_sys.h" +#include "lib/time/compat_time.h" + +static int +init_time_sys(void) +{ + monotime_init(); + return 0; +} + +const subsys_fns_t sys_time = { + .name = "time", + .level = -90, + .supported = true, + .initialize = init_time_sys, +}; diff --git a/src/lib/time/time_sys.h b/src/lib/time/time_sys.h new file mode 100644 index 0000000000..0f1aebc268 --- /dev/null +++ b/src/lib/time/time_sys.h @@ -0,0 +1,14 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file log_time.h + * \brief Declare subsystem object for the time module. + **/ + +#ifndef TOR_TIME_SYS_H +#define TOR_TIME_SYS_H + +extern const struct subsys_fns_t sys_time; + +#endif /* !defined(TOR_TIME_SYS_H) */ diff --git a/src/test/testing_common.c b/src/test/testing_common.c index 1362f29711..333dbc436f 100644 --- a/src/test/testing_common.c +++ b/src/test/testing_common.c @@ -257,8 +257,6 @@ main(int c, const char **v) options = options_new(); tor_compress_init(); - monotime_init(); - struct tor_libevent_cfg cfg; memset(&cfg, 0, sizeof(cfg)); tor_libevent_initialize(&cfg); From 019a044e5e6586fb42a171cb98aea15a72bd5a13 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 2 Nov 2018 18:34:56 -0400 Subject: [PATCH 0114/2557] Turn "compress" into a subsystem. --- src/app/main/main.c | 2 -- src/app/main/subsystem_list.c | 2 ++ src/lib/compress/.may_include | 1 + src/lib/compress/compress.c | 15 ++++++++++++++- src/lib/compress/compress.h | 2 +- src/lib/compress/compress_sys.h | 14 ++++++++++++++ src/lib/compress/include.am | 1 + src/test/testing_common.c | 1 - 8 files changed, 33 insertions(+), 5 deletions(-) create mode 100644 src/lib/compress/compress_sys.h diff --git a/src/app/main/main.c b/src/app/main/main.c index bb2e9f5cdb..6240609ee6 100644 --- a/src/app/main/main.c +++ b/src/app/main/main.c @@ -1367,8 +1367,6 @@ tor_run_main(const tor_main_configuration_t *tor_cfg) init_protocol_warning_severity_level(); - tor_compress_init(); - int argc = tor_cfg->argc + tor_cfg->argc_owned; char **argv = tor_calloc(argc, sizeof(char*)); memcpy(argv, tor_cfg->argv, tor_cfg->argc*sizeof(char*)); diff --git a/src/app/main/subsystem_list.c b/src/app/main/subsystem_list.c index a9189b9941..e47b05da15 100644 --- a/src/app/main/subsystem_list.c +++ b/src/app/main/subsystem_list.c @@ -8,6 +8,7 @@ #include "lib/cc/compat_compiler.h" #include "lib/cc/torint.h" +#include "lib/compress/compress_sys.h" #include "lib/crypt_ops/crypto_sys.h" #include "lib/err/torerr_sys.h" #include "lib/log/log_sys.h" @@ -30,6 +31,7 @@ const subsys_fns_t *tor_subsystems[] = { &sys_logging, &sys_time, &sys_network, + &sys_compress, &sys_crypto, }; diff --git a/src/lib/compress/.may_include b/src/lib/compress/.may_include index 68fe9f1c54..4870259ec9 100644 --- a/src/lib/compress/.may_include +++ b/src/lib/compress/.may_include @@ -8,5 +8,6 @@ lib/intmath/*.h lib/log/*.h lib/malloc/*.h lib/string/*.h +lib/subsys/*.h lib/testsupport/*.h lib/thread/*.h diff --git a/src/lib/compress/compress.c b/src/lib/compress/compress.c index 2ad9b15b2e..0d134fd1be 100644 --- a/src/lib/compress/compress.c +++ b/src/lib/compress/compress.c @@ -29,10 +29,12 @@ #include "lib/compress/compress.h" #include "lib/compress/compress_lzma.h" #include "lib/compress/compress_none.h" +#include "lib/compress/compress_sys.h" #include "lib/compress/compress_zlib.h" #include "lib/compress/compress_zstd.h" #include "lib/intmath/cmp.h" #include "lib/malloc/malloc.h" +#include "lib/subsys/subsys.h" #include "lib/thread/threads.h" /** Total number of bytes allocated for compression state overhead. */ @@ -660,7 +662,7 @@ tor_compress_state_size(const tor_compress_state_t *state) } /** Initialize all compression modules. */ -void +int tor_compress_init(void) { atomic_counter_init(&total_compress_allocation); @@ -668,6 +670,8 @@ tor_compress_init(void) tor_zlib_init(); tor_lzma_init(); tor_zstd_init(); + + return 0; } /** Warn if we had any problems while setting up our compression libraries. @@ -677,5 +681,14 @@ tor_compress_init(void) void tor_compress_log_init_warnings(void) { + // XXXX can we move this into tor_compress_init() after all? log.c queues + // XXXX log messages at startup. tor_zstd_warn_if_version_mismatched(); } + +const subsys_fns_t sys_compress = { + .name = "compress", + .supported = true, + .level = -70, + .initialize = tor_compress_init, +}; diff --git a/src/lib/compress/compress.h b/src/lib/compress/compress.h index 4466e27c4d..4dd6506238 100644 --- a/src/lib/compress/compress.h +++ b/src/lib/compress/compress.h @@ -89,7 +89,7 @@ void tor_compress_free_(tor_compress_state_t *state); size_t tor_compress_state_size(const tor_compress_state_t *state); -void tor_compress_init(void); +int tor_compress_init(void); void tor_compress_log_init_warnings(void); struct buf_t; diff --git a/src/lib/compress/compress_sys.h b/src/lib/compress/compress_sys.h new file mode 100644 index 0000000000..a162140cfb --- /dev/null +++ b/src/lib/compress/compress_sys.h @@ -0,0 +1,14 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file compress_sys.h + * \brief Declare subsystem object for the compress module + **/ + +#ifndef TOR_COMPRESS_SYS_H +#define TOR_COMPRESS_SYS_H + +extern const struct subsys_fns_t sys_compress; + +#endif /* !defined(TOR_COMPRESS_SYS_H) */ diff --git a/src/lib/compress/include.am b/src/lib/compress/include.am index 75c9032bd2..b952779578 100644 --- a/src/lib/compress/include.am +++ b/src/lib/compress/include.am @@ -22,5 +22,6 @@ noinst_HEADERS += \ src/lib/compress/compress.h \ src/lib/compress/compress_lzma.h \ src/lib/compress/compress_none.h \ + src/lib/compress/compress_sys.h \ src/lib/compress/compress_zlib.h \ src/lib/compress/compress_zstd.h diff --git a/src/test/testing_common.c b/src/test/testing_common.c index 333dbc436f..6d2db28f15 100644 --- a/src/test/testing_common.c +++ b/src/test/testing_common.c @@ -255,7 +255,6 @@ main(int c, const char **v) subsystems_init_upto(SUBSYS_LEVEL_LIBS); options = options_new(); - tor_compress_init(); struct tor_libevent_cfg cfg; memset(&cfg, 0, sizeof(cfg)); From 32b23a4c40880591ecadab59f932f4a4c1e7560a Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 2 Nov 2018 18:46:35 -0400 Subject: [PATCH 0115/2557] Make tortls use the subsystems interface This one only needs a shutdown right now. --- src/app/main/main.c | 1 - src/app/main/subsystem_list.c | 2 ++ src/lib/tls/.may_include | 1 + src/lib/tls/include.am | 1 + src/lib/tls/tortls.c | 8 ++++++++ src/lib/tls/tortls_sys.h | 14 ++++++++++++++ 6 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 src/lib/tls/tortls_sys.h diff --git a/src/app/main/main.c b/src/app/main/main.c index 6240609ee6..4dedae9c0c 100644 --- a/src/app/main/main.c +++ b/src/app/main/main.c @@ -776,7 +776,6 @@ tor_free_all(int postfork) policies_free_all(); } if (!postfork) { - tor_tls_free_all(); #ifndef _WIN32 tor_getpwnam(NULL); #endif diff --git a/src/app/main/subsystem_list.c b/src/app/main/subsystem_list.c index e47b05da15..62c87005c6 100644 --- a/src/app/main/subsystem_list.c +++ b/src/app/main/subsystem_list.c @@ -16,6 +16,7 @@ #include "lib/process/winprocess_sys.h" #include "lib/thread/thread_sys.h" #include "lib/time/time_sys.h" +#include "lib/tls/tortls_sys.h" #include "lib/wallclock/wallclock_sys.h" #include @@ -33,6 +34,7 @@ const subsys_fns_t *tor_subsystems[] = { &sys_network, &sys_compress, &sys_crypto, + &sys_tortls, }; const unsigned n_tor_subsystems = ARRAY_LENGTH(tor_subsystems); diff --git a/src/lib/tls/.may_include b/src/lib/tls/.may_include index 2840e590b8..79301bc318 100644 --- a/src/lib/tls/.may_include +++ b/src/lib/tls/.may_include @@ -11,6 +11,7 @@ lib/log/*.h lib/malloc/*.h lib/net/*.h lib/string/*.h +lib/subsys/*.h lib/testsupport/testsupport.h lib/tls/*.h diff --git a/src/lib/tls/include.am b/src/lib/tls/include.am index a664b29fb2..1817739eef 100644 --- a/src/lib/tls/include.am +++ b/src/lib/tls/include.am @@ -36,5 +36,6 @@ noinst_HEADERS += \ src/lib/tls/tortls.h \ src/lib/tls/tortls_internal.h \ src/lib/tls/tortls_st.h \ + src/lib/tls/tortls_sys.h \ src/lib/tls/x509.h \ src/lib/tls/x509_internal.h diff --git a/src/lib/tls/tortls.c b/src/lib/tls/tortls.c index 56f70bc371..fdeea9e0d4 100644 --- a/src/lib/tls/tortls.c +++ b/src/lib/tls/tortls.c @@ -7,6 +7,7 @@ #define TOR_X509_PRIVATE #include "lib/tls/x509.h" #include "lib/tls/x509_internal.h" +#include "lib/tls/tortls_sys.h" #include "lib/tls/tortls.h" #include "lib/tls/tortls_st.h" #include "lib/tls/tortls_internal.h" @@ -15,6 +16,7 @@ #include "lib/crypt_ops/crypto_rsa.h" #include "lib/crypt_ops/crypto_rand.h" #include "lib/net/socket.h" +#include "lib/subsys/subsys.h" #ifdef _WIN32 #include @@ -440,3 +442,9 @@ tor_tls_verify(int severity, tor_tls_t *tls, crypto_pk_t **identity) return rv; } + +const subsys_fns_t sys_tortls = { + .name = "tortls", + .level = -50, + .shutdown = tor_tls_free_all +}; diff --git a/src/lib/tls/tortls_sys.h b/src/lib/tls/tortls_sys.h new file mode 100644 index 0000000000..fd909f6019 --- /dev/null +++ b/src/lib/tls/tortls_sys.h @@ -0,0 +1,14 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file tortls_sys.h + * \brief Declare subsystem object for the tortls module + **/ + +#ifndef TOR_TORTLS_SYS_H +#define TOR_TORTLS_SYS_H + +extern const struct subsys_fns_t sys_tortls; + +#endif /* !defined(TOR_TORTLS_SYS_H) */ From adecda753996611e9a5b82c5fa87ea78ec683806 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 5 Nov 2018 09:42:16 -0500 Subject: [PATCH 0116/2557] changes file for subsystems api (28330) --- changes/subsystems | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 changes/subsystems diff --git a/changes/subsystems b/changes/subsystems new file mode 100644 index 0000000000..a51fb8e2b1 --- /dev/null +++ b/changes/subsystems @@ -0,0 +1,6 @@ + o Major features (refactoring): + - Tor now uses an explicit list of its own subsystems when initializing + and shutting down. Previously, these systems were managed implicitly + though various places throughout the codebase. (There still some + subsystems using the old system.) + Closes ticket 28330. From 1a6060fa4263d20d3c1ccf29163165126a413957 Mon Sep 17 00:00:00 2001 From: Taylor R Campbell Date: Tue, 23 Oct 2018 23:53:39 +0000 Subject: [PATCH 0117/2557] New macro CTASSERT(condition) to assert condition at compile-time. To get it, use: #include "lib/cc/ctassert.h" --- src/lib/cc/ctassert.h | 75 +++++++++++++++++++++++++++++++++++++++++++ src/lib/cc/include.am | 1 + 2 files changed, 76 insertions(+) create mode 100644 src/lib/cc/ctassert.h diff --git a/src/lib/cc/ctassert.h b/src/lib/cc/ctassert.h new file mode 100644 index 0000000000..7307bca53a --- /dev/null +++ b/src/lib/cc/ctassert.h @@ -0,0 +1,75 @@ +/*- + * Copyright (c) 2018 Taylor R. Campbell + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +/** + * \file ctassert.h + * + * \brief Compile-time assertions: CTASSERT(expression). + */ + +#ifndef TOR_CTASSERT_H +#define TOR_CTASSERT_H + +#include "lib/cc/compat_compiler.h" + +/** + * CTASSERT(expression) + * + * Trigger a compiler error if expression is false. + */ +#if __STDC_VERSION__ >= 201112L + +/* If C11 is available, just use _Static_assert. */ +#define CTASSERT(x) _Static_assert(x, #x) + +#else + +/* + * If C11 is not available, expand __COUNTER__, or __INCLUDE_LEVEL__ + * and __LINE__, or just __LINE__, with an intermediate preprocessor + * macro CTASSERT_EXPN, and then use CTASSERT_DECL to paste the + * expansions together into a unique name. + * + * We use this name as a typedef of an array type with a positive + * length if the assertion is true, and a negative length of the + * assertion is false, which is invalid and hence triggers a compiler + * error. + */ +#if defined(__COUNTER__) +#define CTASSERT(x) CTASSERT_EXPN(x, c, __COUNTER__) +#elif defined(__INCLUDE_LEVEL__) +#define CTASSERT(x) CTASSERT_EXPN(x, __INCLUDE_LEVEL__, __LINE__) +#else +#define CTASSERT(x) CTASSERT_EXPN(x, l, __LINE__) /* hope it's unique enough */ +#endif + +#define CTASSERT_EXPN(x, a, b) CTASSERT_DECL(x, a, b) +#define CTASSERT_DECL(x, a, b) \ + typedef char tor_ctassert_##a##_##b[(x) ? 1 : -1] ATTR_UNUSED + +#endif + +#endif /* !defined(TOR_CTASSERT_H) */ diff --git a/src/lib/cc/include.am b/src/lib/cc/include.am index 2ae90f97dd..52cf8a9f72 100644 --- a/src/lib/cc/include.am +++ b/src/lib/cc/include.am @@ -1,4 +1,5 @@ noinst_HEADERS += \ src/lib/cc/compat_compiler.h \ + src/lib/cc/ctassert.h \ src/lib/cc/torint.h From e69a4ad6b321dbdb63236687ac8924c301c60f9d Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 6 Nov 2018 15:39:58 -0500 Subject: [PATCH 0118/2557] Add a user of CTASSERT(). --- src/lib/net/address.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/lib/net/address.c b/src/lib/net/address.c index c97a17037a..a351b9df28 100644 --- a/src/lib/net/address.c +++ b/src/lib/net/address.c @@ -40,6 +40,7 @@ #include "lib/net/address.h" #include "lib/net/socket.h" +#include "lib/cc/ctassert.h" #include "lib/container/smartlist.h" #include "lib/ctime/di_ops.h" #include "lib/log/log.h" @@ -98,6 +99,7 @@ #if AF_UNSPEC != 0 #error We rely on AF_UNSPEC being 0. Let us know about your platform, please! #endif +CTASSERT(AF_UNSPEC == 0); /** Convert the tor_addr_t in a, with port in port, into a * sockaddr object in *sa_out of object size len. If not enough From 3c9dd9ef2d86463e535952528bc5151ce499a74f Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 6 Nov 2018 15:41:14 -0500 Subject: [PATCH 0119/2557] Add parentheses to the ctassert macro expansions --- src/lib/cc/ctassert.h | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/lib/cc/ctassert.h b/src/lib/cc/ctassert.h index 7307bca53a..5a1b137cba 100644 --- a/src/lib/cc/ctassert.h +++ b/src/lib/cc/ctassert.h @@ -43,7 +43,7 @@ #if __STDC_VERSION__ >= 201112L /* If C11 is available, just use _Static_assert. */ -#define CTASSERT(x) _Static_assert(x, #x) +#define CTASSERT(x) _Static_assert((x), #x) #else @@ -59,11 +59,12 @@ * error. */ #if defined(__COUNTER__) -#define CTASSERT(x) CTASSERT_EXPN(x, c, __COUNTER__) +#define CTASSERT(x) CTASSERT_EXPN((x), c, __COUNTER__) #elif defined(__INCLUDE_LEVEL__) -#define CTASSERT(x) CTASSERT_EXPN(x, __INCLUDE_LEVEL__, __LINE__) +#define CTASSERT(x) CTASSERT_EXPN((x), __INCLUDE_LEVEL__, __LINE__) #else -#define CTASSERT(x) CTASSERT_EXPN(x, l, __LINE__) /* hope it's unique enough */ +/* hope it's unique enough */ +#define CTASSERT(x) CTASSERT_EXPN((x), l, __LINE__) #endif #define CTASSERT_EXPN(x, a, b) CTASSERT_DECL(x, a, b) From 6b706bcf199b82c6b1a87c839e8d13bf9d8e60b7 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 6 Nov 2018 15:41:32 -0500 Subject: [PATCH 0120/2557] Remove a tab. --- src/lib/cc/ctassert.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/cc/ctassert.h b/src/lib/cc/ctassert.h index 5a1b137cba..674c0c2e10 100644 --- a/src/lib/cc/ctassert.h +++ b/src/lib/cc/ctassert.h @@ -38,7 +38,7 @@ /** * CTASSERT(expression) * - * Trigger a compiler error if expression is false. + * Trigger a compiler error if expression is false. */ #if __STDC_VERSION__ >= 201112L From d9508d8ede7ecbdfa35ce8a12c6e1d66d1bbb4b7 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 6 Nov 2018 15:42:18 -0500 Subject: [PATCH 0121/2557] Change copyright statement (with permission) --- src/lib/cc/ctassert.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/cc/ctassert.h b/src/lib/cc/ctassert.h index 674c0c2e10..c6de3bf1ec 100644 --- a/src/lib/cc/ctassert.h +++ b/src/lib/cc/ctassert.h @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2018 Taylor R. Campbell + * Copyright (c) 2018 The Tor Project, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without From 770653ff456c9ae9ecf29008f7abf367a776e557 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 6 Nov 2018 16:59:39 -0500 Subject: [PATCH 0122/2557] Allow lib/cc to include its own files. --- src/lib/cc/.may_include | 1 + 1 file changed, 1 insertion(+) diff --git a/src/lib/cc/.may_include b/src/lib/cc/.may_include index 2b06e8519c..fa1478ce46 100644 --- a/src/lib/cc/.may_include +++ b/src/lib/cc/.may_include @@ -1 +1,2 @@ orconfig.h +lib/cc/*.h \ No newline at end of file From a7a060a637ac714abdb3f944df1bbbb30a7c9b14 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 7 Nov 2018 10:37:02 -0500 Subject: [PATCH 0123/2557] Switch ctassert.h to 3bsd (with permission) --- src/lib/cc/ctassert.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/lib/cc/ctassert.h b/src/lib/cc/ctassert.h index c6de3bf1ec..ea3a3e119e 100644 --- a/src/lib/cc/ctassert.h +++ b/src/lib/cc/ctassert.h @@ -10,6 +10,9 @@ * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. + * 3. Neither the names of the copyright owners nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE From 6d93820499a8bfb19128759893b18c1437f99c6b Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 19 Oct 2018 15:04:45 -0400 Subject: [PATCH 0124/2557] Memoize summarize_protover_flags() Our tests showed that this function is responsible for a huge number of our malloc/free() calls. It's a prime candidate for being memoized. Closes ticket 27225. --- changes/ticket27225 | 5 +++ src/app/main/main.c | 2 ++ src/core/or/versions.c | 82 ++++++++++++++++++++++++++++++++++-------- src/core/or/versions.h | 2 ++ 4 files changed, 76 insertions(+), 15 deletions(-) create mode 100644 changes/ticket27225 diff --git a/changes/ticket27225 b/changes/ticket27225 new file mode 100644 index 0000000000..4c05a269d6 --- /dev/null +++ b/changes/ticket27225 @@ -0,0 +1,5 @@ + o Minor features (performance): + - Avoid parsing the same protocol-versions string over and over + in summarize_protover_flags(). This should save us a huge number + of malloc calls on startup, and may reduce memory fragmentation with + some allocators. Closes ticket 27225. diff --git a/src/app/main/main.c b/src/app/main/main.c index ae87add67d..04bbfadcb7 100644 --- a/src/app/main/main.c +++ b/src/app/main/main.c @@ -33,6 +33,7 @@ #include "core/or/relay.h" #include "core/or/scheduler.h" #include "core/or/status.h" +#include "core/or/versions.h" #include "feature/api/tor_api.h" #include "feature/api/tor_api_internal.h" #include "feature/client/addressmap.h" @@ -791,6 +792,7 @@ tor_free_all(int postfork) dos_free_all(); circuitmux_ewma_free_all(); accounting_free_all(); + protover_summary_cache_free_all(); if (!postfork) { config_free_all(); diff --git a/src/core/or/versions.c b/src/core/or/versions.c index 06274996a7..6f8eea7a67 100644 --- a/src/core/or/versions.c +++ b/src/core/or/versions.c @@ -377,6 +377,62 @@ sort_version_list(smartlist_t *versions, int remove_duplicates) smartlist_uniq(versions, compare_tor_version_str_ptr_, tor_free_); } +/** If there are more than this many entries, we're probably under + * some kind of weird DoS. */ +static const int MAX_PROTOVER_SUMMARY_MAP_LEN = 1024; + +/** + * Map from protover string to protover_summary_flags_t. + */ +static strmap_t *protover_summary_map = NULL; + +/** + * Helper. Given a non-NULL protover string protocols, set out + * to its summary, and memoize the result in protover_summary_map. + */ +static void +memoize_protover_summary(protover_summary_flags_t *out, + const char *protocols) +{ + if (!protover_summary_map) + protover_summary_map = strmap_new(); + + if (strmap_size(protover_summary_map) >= MAX_PROTOVER_SUMMARY_MAP_LEN) { + protover_summary_cache_free_all(); + } + + const protover_summary_flags_t *cached = + strmap_get(protover_summary_map, protocols); + + if (cached != NULL) { + /* We found a cached entry; no need to parse this one. */ + memcpy(out, cached, sizeof(protover_summary_flags_t)); + tor_assert(out->protocols_known); + return; + } + + memset(out, 0, sizeof(*out)); + out->protocols_known = 1; + out->supports_extend2_cells = + protocol_list_supports_protocol(protocols, PRT_RELAY, 2); + out->supports_ed25519_link_handshake_compat = + protocol_list_supports_protocol(protocols, PRT_LINKAUTH, 3); + out->supports_ed25519_link_handshake_any = + protocol_list_supports_protocol_or_later(protocols, PRT_LINKAUTH, 3); + out->supports_ed25519_hs_intro = + protocol_list_supports_protocol(protocols, PRT_HSINTRO, 4); + out->supports_v3_hsdir = + protocol_list_supports_protocol(protocols, PRT_HSDIR, + PROTOVER_HSDIR_V3); + out->supports_v3_rendezvous_point = + protocol_list_supports_protocol(protocols, PRT_HSREND, + PROTOVER_HS_RENDEZVOUS_POINT_V3); + + protover_summary_flags_t *new_cached = tor_memdup(out, sizeof(*out)); + cached = strmap_set(protover_summary_map, protocols, new_cached); + tor_assert(!cached); +} + /** Summarize the protocols listed in protocols into out, * falling back or correcting them based on version as appropriate. */ @@ -388,21 +444,7 @@ summarize_protover_flags(protover_summary_flags_t *out, tor_assert(out); memset(out, 0, sizeof(*out)); if (protocols) { - out->protocols_known = 1; - out->supports_extend2_cells = - protocol_list_supports_protocol(protocols, PRT_RELAY, 2); - out->supports_ed25519_link_handshake_compat = - protocol_list_supports_protocol(protocols, PRT_LINKAUTH, 3); - out->supports_ed25519_link_handshake_any = - protocol_list_supports_protocol_or_later(protocols, PRT_LINKAUTH, 3); - out->supports_ed25519_hs_intro = - protocol_list_supports_protocol(protocols, PRT_HSINTRO, 4); - out->supports_v3_hsdir = - protocol_list_supports_protocol(protocols, PRT_HSDIR, - PROTOVER_HSDIR_V3); - out->supports_v3_rendezvous_point = - protocol_list_supports_protocol(protocols, PRT_HSREND, - PROTOVER_HS_RENDEZVOUS_POINT_V3); + memoize_protover_summary(out, protocols); } if (version && !strcmpstart(version, "Tor ")) { if (!out->protocols_known) { @@ -420,3 +462,13 @@ summarize_protover_flags(protover_summary_flags_t *out, } } } + +/** + * Free all space held in the protover_summary_map. + */ +void +protover_summary_cache_free_all(void) +{ + strmap_free(protover_summary_map, tor_free_); + protover_summary_map = NULL; +} diff --git a/src/core/or/versions.h b/src/core/or/versions.h index 0c773f3f4c..4fc50a0018 100644 --- a/src/core/or/versions.h +++ b/src/core/or/versions.h @@ -41,4 +41,6 @@ void summarize_protover_flags(protover_summary_flags_t *out, const char *protocols, const char *version); +void protover_summary_cache_free_all(void); + #endif /* !defined(TOR_VERSIONS_H) */ From 9e48d9a920c1e1f8c6fc551363b28905f5580f8e Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 7 Nov 2018 10:54:03 -0500 Subject: [PATCH 0125/2557] Change version on master to 0.4.0.0-alpha-dev. --- configure.ac | 2 +- contrib/win32build/tor-mingw.nsi.in | 2 +- src/win32/orconfig.h | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/configure.ac b/configure.ac index b80aa821ad..4524c6b467 100644 --- a/configure.ac +++ b/configure.ac @@ -4,7 +4,7 @@ dnl Copyright (c) 2007-2018, The Tor Project, Inc. dnl See LICENSE for licensing information AC_PREREQ([2.63]) -AC_INIT([tor],[0.3.6.0-alpha-dev]) +AC_INIT([tor],[0.4.0.0-alpha-dev]) AC_CONFIG_SRCDIR([src/app/main/tor_main.c]) AC_CONFIG_MACRO_DIR([m4]) diff --git a/contrib/win32build/tor-mingw.nsi.in b/contrib/win32build/tor-mingw.nsi.in index 8b9b709698..af01a2b499 100644 --- a/contrib/win32build/tor-mingw.nsi.in +++ b/contrib/win32build/tor-mingw.nsi.in @@ -8,7 +8,7 @@ !include "LogicLib.nsh" !include "FileFunc.nsh" !insertmacro GetParameters -!define VERSION "0.3.6.0-alpha-dev" +!define VERSION "0.4.0.0-alpha-dev" !define INSTALLER "tor-${VERSION}-win32.exe" !define WEBSITE "https://www.torproject.org/" !define LICENSE "LICENSE" diff --git a/src/win32/orconfig.h b/src/win32/orconfig.h index de3bf09282..cfc3bc9e9e 100644 --- a/src/win32/orconfig.h +++ b/src/win32/orconfig.h @@ -218,7 +218,7 @@ #define USING_TWOS_COMPLEMENT /* Version number of package */ -#define VERSION "0.3.6.0-alpha-dev" +#define VERSION "0.4.0.0-alpha-dev" From 6e828ced56c04aa38fa6e2d595528bf0d046ca8c Mon Sep 17 00:00:00 2001 From: Roger Dingledine Date: Wed, 7 Nov 2018 11:02:26 -0500 Subject: [PATCH 0126/2557] simplify now that it uses tor's copyright and license --- src/lib/cc/ctassert.h | 30 ++---------------------------- 1 file changed, 2 insertions(+), 28 deletions(-) diff --git a/src/lib/cc/ctassert.h b/src/lib/cc/ctassert.h index ea3a3e119e..e42976360f 100644 --- a/src/lib/cc/ctassert.h +++ b/src/lib/cc/ctassert.h @@ -1,31 +1,5 @@ -/*- - * Copyright (c) 2018 The Tor Project, Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. Neither the names of the copyright owners nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ +/* Copyright (c) 2018 The Tor Project, Inc. */ +/* See LICENSE for licensing information */ /** * \file ctassert.h From 4fe4bcf8a10967a668895e962099f50635ba9e4b Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 9 Nov 2018 10:55:18 -0500 Subject: [PATCH 0127/2557] Explain that configuration should happen elsewhere, but not init. --- src/lib/subsys/subsys.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/lib/subsys/subsys.h b/src/lib/subsys/subsys.h index b06d67e624..462314567e 100644 --- a/src/lib/subsys/subsys.h +++ b/src/lib/subsys/subsys.h @@ -44,6 +44,9 @@ typedef struct subsys_fns_t { * This function MUST NOT rely on any runtime configuration information; * it is only for global state or pre-configuration state. * + * (If you need to do any setup that depends on configuration, you'll need + * to declare a configuration callback. (Not yet designed)) + * * This function MUST NOT have any parts that can fail. **/ int (*initialize)(void); From 61695e3d622dfcc196b8b829842f2b12fecebeab Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 9 Nov 2018 10:58:20 -0500 Subject: [PATCH 0128/2557] Document that subsystem callbacks are optional. --- src/lib/subsys/subsys.h | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/lib/subsys/subsys.h b/src/lib/subsys/subsys.h index 462314567e..2452ec6e2f 100644 --- a/src/lib/subsys/subsys.h +++ b/src/lib/subsys/subsys.h @@ -14,7 +14,11 @@ struct dispatch_connector_t; * A subsystem is a part of Tor that is initialized, shut down, configured, * and connected to other parts of Tor. * - * Subsystems + * All callbacks are optional -- if a callback is set to NULL, the subsystem + * manager will treat it as a no-op. + * + * You should use c99 named-field initializers with this structure: we + * will be adding more fields, often in the middle of the structure. **/ typedef struct subsys_fns_t { /** From e80595f562e199049a41fdf1f3e12baced7e74d5 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 9 Nov 2018 11:00:31 -0500 Subject: [PATCH 0129/2557] fixup! Make initialization for the "err" library into a subsystem. Check for failure to install backtrace handler. --- src/lib/err/torerr_sys.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/lib/err/torerr_sys.c b/src/lib/err/torerr_sys.c index 54666f4106..2f9e33e233 100644 --- a/src/lib/err/torerr_sys.c +++ b/src/lib/err/torerr_sys.c @@ -18,7 +18,8 @@ static int torerr_subsys_init(void) { - configure_backtrace_handler(get_version()); + if (configure_backtrace_handler(get_version()) < 0) + return -1; tor_log_reset_sigsafe_err_fds(); return 0; From c6336727cac937b4b5ca38c9b49ed3a66ce0b579 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 9 Nov 2018 11:12:12 -0500 Subject: [PATCH 0130/2557] Rename subsystem callback functions to make them consistent --- src/lib/compress/compress.c | 8 +++++++- src/lib/crypt_ops/crypto_init.c | 32 +++++++++++++++++++++++++------- src/lib/err/torerr_sys.c | 8 ++++---- src/lib/log/log_sys.c | 8 ++++---- src/lib/net/network_sys.c | 8 ++++---- src/lib/process/winprocess_sys.c | 6 +++--- src/lib/thread/compat_threads.c | 4 ++-- src/lib/time/time_sys.c | 4 ++-- src/lib/tls/tortls.c | 8 +++++++- src/lib/wallclock/approx_time.c | 4 ++-- 10 files changed, 60 insertions(+), 30 deletions(-) diff --git a/src/lib/compress/compress.c b/src/lib/compress/compress.c index 0d134fd1be..6cb9bd492b 100644 --- a/src/lib/compress/compress.c +++ b/src/lib/compress/compress.c @@ -686,9 +686,15 @@ tor_compress_log_init_warnings(void) tor_zstd_warn_if_version_mismatched(); } +static int +subsys_compress_initialize(void) +{ + return tor_compress_init(); +} + const subsys_fns_t sys_compress = { .name = "compress", .supported = true, .level = -70, - .initialize = tor_compress_init, + .initialize = subsys_compress_initialize, }; diff --git a/src/lib/crypt_ops/crypto_init.c b/src/lib/crypt_ops/crypto_init.c index a03f5eff7c..4c4cc3e43b 100644 --- a/src/lib/crypt_ops/crypto_init.c +++ b/src/lib/crypt_ops/crypto_init.c @@ -207,7 +207,7 @@ tor_is_using_nss(void) } static int -init_crypto_sys(void) +subsys_crypto_initialize(void) { if (crypto_early_init() < 0) return -1; @@ -216,18 +216,36 @@ init_crypto_sys(void) } static void -shutdown_crypto_sys(void) +subsys_crypto_shutdown(void) { crypto_global_cleanup(); } +static void +subsys_crypto_prefork(void) +{ + crypto_prefork(); +} + +static void +subsys_crypto_postfork(void) +{ + crypto_postfork(); +} + +static void +subsys_crypto_thread_cleanup(void) +{ + crypto_thread_cleanup(); +} + const struct subsys_fns_t sys_crypto = { .name = "crypto", .supported = true, .level = -60, - .initialize = init_crypto_sys, - .shutdown = shutdown_crypto_sys, - .prefork = crypto_prefork, - .postfork = crypto_postfork, - .thread_cleanup = crypto_thread_cleanup, + .initialize = subsys_crypto_initialize, + .shutdown = subsys_crypto_shutdown, + .prefork = subsys_crypto_prefork, + .postfork = subsys_crypto_postfork, + .thread_cleanup = subsys_crypto_thread_cleanup, }; diff --git a/src/lib/err/torerr_sys.c b/src/lib/err/torerr_sys.c index 2f9e33e233..96bb1308a4 100644 --- a/src/lib/err/torerr_sys.c +++ b/src/lib/err/torerr_sys.c @@ -16,7 +16,7 @@ #include static int -torerr_subsys_init(void) +subsys_torerr_initialize(void) { if (configure_backtrace_handler(get_version()) < 0) return -1; @@ -25,7 +25,7 @@ torerr_subsys_init(void) return 0; } static void -torerr_subsys_shutdown(void) +subsys_torerr_shutdown(void) { tor_log_reset_sigsafe_err_fds(); clean_up_backtrace_handler(); @@ -35,6 +35,6 @@ const subsys_fns_t sys_torerr = { .name = "err", .level = -100, .supported = true, - .initialize = torerr_subsys_init, - .shutdown = torerr_subsys_shutdown + .initialize = subsys_torerr_initialize, + .shutdown = subsys_torerr_shutdown }; diff --git a/src/lib/log/log_sys.c b/src/lib/log/log_sys.c index 94ec97fdc1..e20f3156ca 100644 --- a/src/lib/log/log_sys.c +++ b/src/lib/log/log_sys.c @@ -13,14 +13,14 @@ #include "lib/log/log_sys.h" static int -init_logging_subsys(void) +subsys_logging_initialize(void) { init_logging(0); return 0; } static void -shutdown_logging_subsys(void) +subsys_logging_shutdown(void) { logs_free_all(); escaped(NULL); @@ -30,6 +30,6 @@ const subsys_fns_t sys_logging = { .name = "log", .supported = true, .level = -90, - .initialize = init_logging_subsys, - .shutdown = shutdown_logging_subsys, + .initialize = subsys_logging_initialize, + .shutdown = subsys_logging_shutdown, }; diff --git a/src/lib/net/network_sys.c b/src/lib/net/network_sys.c index c9d33a94d3..ac49288ee6 100644 --- a/src/lib/net/network_sys.c +++ b/src/lib/net/network_sys.c @@ -18,7 +18,7 @@ #endif static int -init_network_sys(void) +subsys_network_initialize(void) { if (network_init() < 0) return -1; @@ -27,7 +27,7 @@ init_network_sys(void) } static void -shutdown_network_sys(void) +subsys_network_shutdown(void) { #ifdef _WIN32 WSACleanup(); @@ -39,6 +39,6 @@ const subsys_fns_t sys_network = { .name = "network", .level = -90, .supported = true, - .initialize = init_network_sys, - .shutdown = shutdown_network_sys, + .initialize = subsys_network_initialize, + .shutdown = subsys_network_shutdown, }; diff --git a/src/lib/process/winprocess_sys.c b/src/lib/process/winprocess_sys.c index e00f94c915..ef66f8bfb1 100644 --- a/src/lib/process/winprocess_sys.c +++ b/src/lib/process/winprocess_sys.c @@ -19,7 +19,7 @@ #define WINPROCESS_SYS_ENABLED true static int -init_windows_process_params(void) +subsys_winprocess_initialize(void) { #ifndef HeapEnableTerminationOnCorruption #define HeapEnableTerminationOnCorruption 1 @@ -53,12 +53,12 @@ init_windows_process_params(void) } #else /* !defined(_WIN32) */ #define WINPROCESS_SYS_ENABLED false -#define init_windows_process_params NULL +#define subsys_winprocess_initialize NULL #endif /* defined(_WIN32) */ const subsys_fns_t sys_winprocess = { .name = "winprocess", .level = -100, .supported = WINPROCESS_SYS_ENABLED, - .initialize = init_windows_process_params, + .initialize = subsys_winprocess_initialize, }; diff --git a/src/lib/thread/compat_threads.c b/src/lib/thread/compat_threads.c index 3d41faa8ce..0b466da212 100644 --- a/src/lib/thread/compat_threads.c +++ b/src/lib/thread/compat_threads.c @@ -113,7 +113,7 @@ atomic_counter_exchange(atomic_counter_t *counter, size_t newval) #endif /* !defined(HAVE_WORKING_STDATOMIC) */ static int -sys_threads_initialize(void) +subsys_threads_initialize(void) { tor_threads_init(); return 0; @@ -123,5 +123,5 @@ const subsys_fns_t sys_threads = { .name = "threads", .supported = true, .level = -95, - .initialize = sys_threads_initialize, + .initialize = subsys_threads_initialize, }; diff --git a/src/lib/time/time_sys.c b/src/lib/time/time_sys.c index 2303874f29..b29ca35e69 100644 --- a/src/lib/time/time_sys.c +++ b/src/lib/time/time_sys.c @@ -12,7 +12,7 @@ #include "lib/time/compat_time.h" static int -init_time_sys(void) +subsys_time_initialize(void) { monotime_init(); return 0; @@ -22,5 +22,5 @@ const subsys_fns_t sys_time = { .name = "time", .level = -90, .supported = true, - .initialize = init_time_sys, + .initialize = subsys_time_initialize, }; diff --git a/src/lib/tls/tortls.c b/src/lib/tls/tortls.c index fdeea9e0d4..654cacacf7 100644 --- a/src/lib/tls/tortls.c +++ b/src/lib/tls/tortls.c @@ -443,8 +443,14 @@ tor_tls_verify(int severity, tor_tls_t *tls, crypto_pk_t **identity) return rv; } +static void +subsys_tortls_shutdown(void) +{ + tor_tls_free_all(); +} + const subsys_fns_t sys_tortls = { .name = "tortls", .level = -50, - .shutdown = tor_tls_free_all + .shutdown = subsys_tortls_shutdown }; diff --git a/src/lib/wallclock/approx_time.c b/src/lib/wallclock/approx_time.c index c7a7ae9bd7..0b0ef382c2 100644 --- a/src/lib/wallclock/approx_time.c +++ b/src/lib/wallclock/approx_time.c @@ -45,7 +45,7 @@ update_approx_time(time_t now) #endif /* !defined(TIME_IS_FAST) */ static int -init_wallclock_subsys(void) +subsys_wallclock_initialize(void) { update_approx_time(time(NULL)); return 0; @@ -55,5 +55,5 @@ const subsys_fns_t sys_wallclock = { .name = "wallclock", .supported = true, .level = -99, - .initialize = init_wallclock_subsys, + .initialize = subsys_wallclock_initialize, }; From ba722e47995e106b46d848263638fa3009687cd9 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 9 Nov 2018 11:15:27 -0500 Subject: [PATCH 0131/2557] Add list of levels in subsystem_list.c --- src/app/main/subsystem_list.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/app/main/subsystem_list.c b/src/app/main/subsystem_list.c index 62c87005c6..190e6579d8 100644 --- a/src/app/main/subsystem_list.c +++ b/src/app/main/subsystem_list.c @@ -25,7 +25,7 @@ * Global list of the subsystems in Tor, in the order of their initialization. **/ const subsys_fns_t *tor_subsystems[] = { - &sys_winprocess, + &sys_winprocess, /* -100 */ &sys_torerr, &sys_wallclock, &sys_threads, From 60d10812368458cb88aa9f9d628c49766d4bb490 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 9 Nov 2018 11:56:26 -0500 Subject: [PATCH 0132/2557] Log before performing a subsystem operation --- src/app/main/subsysmgr.c | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/src/app/main/subsysmgr.c b/src/app/main/subsysmgr.c index 05803ee946..abd2edd10b 100644 --- a/src/app/main/subsysmgr.c +++ b/src/app/main/subsysmgr.c @@ -7,6 +7,8 @@ #include "app/main/subsysmgr.h" #include "lib/err/torerr.h" +#include "lib/log/log.h" + #include #include #include @@ -85,8 +87,13 @@ subsystems_init_upto(int target_level) if (sys_initialized[i]) continue; int r = 0; - if (sys->initialize) + if (sys->initialize) { + // Note that the logging subsystem is designed so that it does no harm + // to log a message in an uninitialized state. These messages will be + // discarded for now, however. + log_debug(LD_GENERAL, "Initializing %s", sys->name); r = sys->initialize(); + } if (r < 0) { fprintf(stderr, "BUG: subsystem %s (at %u) initialization failed.\n", sys->name, i); @@ -123,8 +130,10 @@ subsystems_shutdown_downto(int target_level) break; if (! sys_initialized[i]) continue; - if (sys->shutdown) + if (sys->shutdown) { + log_debug(LD_GENERAL, "Shutting down %s", sys->name); sys->shutdown(); + } sys_initialized[i] = false; } } @@ -143,8 +152,10 @@ subsystems_prefork(void) continue; if (! sys_initialized[i]) continue; - if (sys->prefork) + if (sys->prefork) { + log_debug(LD_GENERAL, "Pre-fork: %s", sys->name); sys->prefork(); + } } } @@ -162,13 +173,15 @@ subsystems_postfork(void) continue; if (! sys_initialized[i]) continue; - if (sys->postfork) + if (sys->postfork) { + log_debug(LD_GENERAL, "Post-fork: %s", sys->name); sys->postfork(); + } } } /** - * Run thread-clanup code on all subsystems that declare any + * Run thread-cleanup code on all subsystems that declare any **/ void subsystems_thread_cleanup(void) @@ -181,7 +194,9 @@ subsystems_thread_cleanup(void) continue; if (! sys_initialized[i]) continue; - if (sys->thread_cleanup) + if (sys->thread_cleanup) { + log_debug(LD_GENERAL, "Thread cleanup: %s", sys->name); sys->thread_cleanup(); + } } } From 91355c0fac0f90ba286edb3d3f12d71d250be16d Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 9 Nov 2018 22:17:18 -0500 Subject: [PATCH 0133/2557] Annotate subsystem list with their levels. --- src/app/main/subsystem_list.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/app/main/subsystem_list.c b/src/app/main/subsystem_list.c index 190e6579d8..8640329e92 100644 --- a/src/app/main/subsystem_list.c +++ b/src/app/main/subsystem_list.c @@ -26,15 +26,15 @@ **/ const subsys_fns_t *tor_subsystems[] = { &sys_winprocess, /* -100 */ - &sys_torerr, - &sys_wallclock, - &sys_threads, - &sys_logging, - &sys_time, - &sys_network, - &sys_compress, - &sys_crypto, - &sys_tortls, + &sys_torerr, /* -100 */ + &sys_wallclock, /* -99 */ + &sys_threads, /* -95 */ + &sys_logging, /* -90 */ + &sys_time, /* -90 */ + &sys_network, /* -90 */ + &sys_compress, /* -70 */ + &sys_crypto, /* -60 */ + &sys_tortls, /* -50 */ }; const unsigned n_tor_subsystems = ARRAY_LENGTH(tor_subsystems); From 100136ca8624151605601d80b63746cfaeb6df47 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Sun, 11 Nov 2018 20:24:07 +0200 Subject: [PATCH 0134/2557] Create new periodic event for pruning old info about Tor routers --- changes/bug27929 | 5 +++++ src/core/mainloop/mainloop.c | 32 ++++++++++++++++++++++++++------ 2 files changed, 31 insertions(+), 6 deletions(-) create mode 100644 changes/bug27929 diff --git a/changes/bug27929 b/changes/bug27929 new file mode 100644 index 0000000000..a97d18f202 --- /dev/null +++ b/changes/bug27929 @@ -0,0 +1,5 @@ + o Minor bugfixes (periodic events): + - Refrain from calling routerlist_remove_old_routers() from + check_descriptor_callback(). Instead, create a new periodic + event that will run every minute even if Tor is not configured + as onion router. Fixes bug 27929; bugfix on 0.2.8.1-alpha. diff --git a/src/core/mainloop/mainloop.c b/src/core/mainloop/mainloop.c index a9f1429787..be19136130 100644 --- a/src/core/mainloop/mainloop.c +++ b/src/core/mainloop/mainloop.c @@ -1359,6 +1359,7 @@ CALLBACK(heartbeat); CALLBACK(hs_service); CALLBACK(launch_descriptor_fetches); CALLBACK(launch_reachability_tests); +CALLBACK(prune_old_routers); CALLBACK(reachability_warnings); CALLBACK(record_bridge_stats); CALLBACK(rend_cache_failure_clean); @@ -1391,6 +1392,8 @@ STATIC periodic_event_item_t periodic_events[] = { CALLBACK(retry_listeners, PERIODIC_EVENT_ROLE_ALL, PERIODIC_EVENT_FLAG_NEED_NET), CALLBACK(save_state, PERIODIC_EVENT_ROLE_ALL, 0), + CALLBACK(prune_old_routers, PERIODIC_EVENT_ROLE_ALL, + PERIODIC_EVENT_FLAG_NEED_NET), CALLBACK(rotate_x509_certificate, PERIODIC_EVENT_ROLE_ALL, 0), CALLBACK(write_stats_file, PERIODIC_EVENT_ROLE_ALL, 0), @@ -1454,6 +1457,7 @@ static periodic_event_item_t *fetch_networkstatus_event=NULL; static periodic_event_item_t *launch_descriptor_fetches_event=NULL; static periodic_event_item_t *check_dns_honesty_event=NULL; static periodic_event_item_t *save_state_event=NULL; +static periodic_event_item_t *prune_old_routers_event=NULL; /** Reset all the periodic events so we'll do all our actions again as if we * just started up. @@ -1556,6 +1560,7 @@ initialize_periodic_events(void) STMT_BEGIN name ## _event = find_periodic_event( #name ); STMT_END NAMED_CALLBACK(check_descriptor); + NAMED_CALLBACK(prune_old_routers); NAMED_CALLBACK(dirvote); NAMED_CALLBACK(fetch_networkstatus); NAMED_CALLBACK(launch_descriptor_fetches); @@ -2220,6 +2225,27 @@ retry_dns_callback(time_t now, const or_options_t *options) return RETRY_DNS_INTERVAL; } +/** + * Periodic callback: prune routerlist of old information about Tor network. + */ +static int +prune_old_routers_callback(time_t now, const or_options_t *options) +{ +#define ROUTERLIST_PRUNING_INTERVAL (60) // 1 minute. + (void)now; + (void)options; + + if (!net_is_disabled()) { + /* If any networkstatus documents are no longer recent, we need to + * update all the descriptors' running status. */ + /* Remove dead routers. */ + log_debug(LD_GENERAL, "Pruning routerlist..."); + routerlist_remove_old_routers(); + } + + return ROUTERLIST_PRUNING_INTERVAL; +} + /** Periodic callback: consider rebuilding or and re-uploading our descriptor * (if we've passed our internal checks). */ static int @@ -2239,12 +2265,6 @@ check_descriptor_callback(time_t now, const or_options_t *options) check_descriptor_ipaddress_changed(now); mark_my_descriptor_dirty_if_too_old(now); consider_publishable_server(0); - /* If any networkstatus documents are no longer recent, we need to - * update all the descriptors' running status. */ - /* Remove dead routers. */ - /* XXXX This doesn't belong here, but it was here in the pre- - * XXXX refactoring code. */ - routerlist_remove_old_routers(); } return CHECK_DESCRIPTOR_INTERVAL; From ec93385cb235a9aafc7bd3bd83a440b3f35ff6fd Mon Sep 17 00:00:00 2001 From: Neel Chauhan Date: Tue, 13 Nov 2018 10:33:51 -0500 Subject: [PATCH 0135/2557] Comment for rend_cache_failure in feature/rend/rendcache.c: "usuable" should be "usable" --- src/feature/rend/rendcache.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/feature/rend/rendcache.c b/src/feature/rend/rendcache.c index 848386b97d..b851e71959 100644 --- a/src/feature/rend/rendcache.c +++ b/src/feature/rend/rendcache.c @@ -45,7 +45,7 @@ STATIC digestmap_t *rend_cache_v2_dir = NULL; * looked up in this cache and if present, it is discarded from the fetched * descriptor. At the end, all IP(s) in the cache, for a specific service * ID, that were NOT present in the descriptor are removed from this cache. - * Which means that if at least one IP was not in this cache, thus usuable, + * Which means that if at least one IP was not in this cache, thus usable, * it's considered a new descriptor so we keep it. Else, if all IPs were in * this cache, we discard the descriptor as it's considered unusable. * From c9f9c9bc497bbfdd5e1acdc37f84da5f3f7396c2 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 6 Nov 2018 17:52:44 -0500 Subject: [PATCH 0136/2557] Make memarea use smartlist_core, not container. --- src/lib/memarea/.may_include | 2 +- src/lib/memarea/memarea.c | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/lib/memarea/.may_include b/src/lib/memarea/.may_include index 814652a93c..a1edaf2231 100644 --- a/src/lib/memarea/.may_include +++ b/src/lib/memarea/.may_include @@ -1,7 +1,7 @@ orconfig.h lib/arch/*.h lib/cc/*.h -lib/container/*.h lib/log/*.h lib/malloc/*.h lib/memarea/*.h +lib/smartlist_core/*.h \ No newline at end of file diff --git a/src/lib/memarea/memarea.c b/src/lib/memarea/memarea.c index dd7c48c079..96d94c89d9 100644 --- a/src/lib/memarea/memarea.c +++ b/src/lib/memarea/memarea.c @@ -16,7 +16,8 @@ #include "lib/arch/bytes.h" #include "lib/cc/torint.h" -#include "lib/container/smartlist.h" +#include "lib/smartlist_core/smartlist_core.h" +#include "lib/smartlist_core/smartlist_foreach.h" #include "lib/log/log.h" #include "lib/log/util_bug.h" #include "lib/malloc/malloc.h" From f6b8c7da66bf93a9505b397661616cc4af2a34f6 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 6 Nov 2018 18:05:14 -0500 Subject: [PATCH 0137/2557] Move buffers.c out of lib/containers to resolve a circularity. --- .gitignore | 2 ++ Makefile.am | 2 ++ src/app/main/main.c | 2 +- src/core/mainloop/connection.c | 2 +- src/core/mainloop/mainloop.c | 2 +- src/core/or/circuitlist.c | 2 +- src/core/or/connection_edge.c | 2 +- src/core/or/connection_or.c | 2 +- src/core/or/or.h | 2 +- src/core/or/relay.c | 2 +- src/core/or/scheduler.c | 2 +- src/core/or/scheduler_kist.c | 2 +- src/core/proto/proto_cell.c | 2 +- src/core/proto/proto_control0.c | 2 +- src/core/proto/proto_ext_or.c | 2 +- src/core/proto/proto_http.c | 2 +- src/core/proto/proto_socks.c | 2 +- src/feature/control/control.c | 2 +- src/feature/stats/geoip_stats.c | 2 +- src/include.am | 1 + src/lib/buf/.may_include | 10 ++++++++++ src/lib/{container => buf}/buffers.c | 2 +- src/lib/{container => buf}/buffers.h | 0 src/lib/buf/include.am | 17 +++++++++++++++++ src/lib/compress/.may_include | 1 + src/lib/compress/compress_buf.c | 2 +- src/lib/container/.may_include | 3 --- src/lib/container/include.am | 2 -- src/lib/net/.may_include | 1 + src/lib/net/buffers_net.c | 2 +- src/lib/tls/.may_include | 1 + src/lib/tls/buffers_tls.c | 2 +- src/test/fuzz/fuzz_http.c | 2 +- src/test/fuzz/fuzz_http_connect.c | 2 +- src/test/fuzz/fuzz_socks.c | 2 +- src/test/test.c | 2 +- src/test/test_buffers.c | 2 +- src/test/test_channelpadding.c | 2 +- src/test/test_channeltls.c | 2 +- src/test/test_extorport.c | 2 +- src/test/test_helpers.c | 2 +- src/test/test_oom.c | 2 +- src/test/test_proto_http.c | 2 +- src/test/test_proto_misc.c | 2 +- src/test/test_routerlist.c | 2 +- src/test/test_socks.c | 2 +- src/test/test_util.c | 2 +- 47 files changed, 71 insertions(+), 41 deletions(-) create mode 100644 src/lib/buf/.may_include rename src/lib/{container => buf}/buffers.c (99%) rename src/lib/{container => buf}/buffers.h (100%) create mode 100644 src/lib/buf/include.am diff --git a/.gitignore b/.gitignore index ee2de376a6..e5d021f30d 100644 --- a/.gitignore +++ b/.gitignore @@ -155,6 +155,8 @@ uptime-*.json # /src/lib /src/lib/libcurve25519_donna.a +/src/lib/libtor-buf.a +/src/lib/libtor-buf-testing.a /src/lib/libtor-compress.a /src/lib/libtor-compress-testing.a /src/lib/libtor-container.a diff --git a/Makefile.am b/Makefile.am index cb76edfa2f..99dec36e36 100644 --- a/Makefile.am +++ b/Makefile.am @@ -42,6 +42,7 @@ endif TOR_UTIL_LIBS = \ src/lib/libtor-geoip.a \ src/lib/libtor-process.a \ + src/lib/libtor-buf.a \ src/lib/libtor-time.a \ src/lib/libtor-fs.a \ src/lib/libtor-encoding.a \ @@ -72,6 +73,7 @@ if UNITTESTS_ENABLED TOR_UTIL_TESTING_LIBS = \ src/lib/libtor-geoip-testing.a \ src/lib/libtor-process-testing.a \ + src/lib/libtor-buf-testing.a \ src/lib/libtor-time-testing.a \ src/lib/libtor-fs-testing.a \ src/lib/libtor-encoding-testing.a \ diff --git a/src/app/main/main.c b/src/app/main/main.c index b8dcb852d2..653a393fe4 100644 --- a/src/app/main/main.c +++ b/src/app/main/main.c @@ -67,7 +67,7 @@ #include "feature/stats/predict_ports.h" #include "feature/stats/rephist.h" #include "lib/compress/compress.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "lib/crypt_ops/crypto_rand.h" #include "lib/crypt_ops/crypto_s2k.h" #include "lib/geoip/geoip.h" diff --git a/src/core/mainloop/connection.c b/src/core/mainloop/connection.c index 1198a01ad9..8058077cd5 100644 --- a/src/core/mainloop/connection.c +++ b/src/core/mainloop/connection.c @@ -57,7 +57,7 @@ #define CONNECTION_PRIVATE #include "core/or/or.h" #include "feature/client/bridges.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "lib/tls/buffers_tls.h" #include "lib/err/backtrace.h" diff --git a/src/core/mainloop/mainloop.c b/src/core/mainloop/mainloop.c index 7eff82fee4..8edee9d29d 100644 --- a/src/core/mainloop/mainloop.c +++ b/src/core/mainloop/mainloop.c @@ -95,7 +95,7 @@ #include "feature/stats/geoip_stats.h" #include "feature/stats/predict_ports.h" #include "feature/stats/rephist.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "lib/crypt_ops/crypto_rand.h" #include "lib/err/backtrace.h" #include "lib/tls/buffers_tls.h" diff --git a/src/core/or/circuitlist.c b/src/core/or/circuitlist.c index 35efc6541f..0aa21000a1 100644 --- a/src/core/or/circuitlist.c +++ b/src/core/or/circuitlist.c @@ -94,7 +94,7 @@ #include "lib/compress/compress_lzma.h" #include "lib/compress/compress_zlib.h" #include "lib/compress/compress_zstd.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "ht.h" diff --git a/src/core/or/connection_edge.c b/src/core/or/connection_edge.c index 58aefcf8f2..a69c907319 100644 --- a/src/core/or/connection_edge.c +++ b/src/core/or/connection_edge.c @@ -97,7 +97,7 @@ #include "feature/rend/rendservice.h" #include "feature/stats/predict_ports.h" #include "feature/stats/rephist.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "lib/crypt_ops/crypto_util.h" #include "core/or/cell_st.h" diff --git a/src/core/or/connection_or.c b/src/core/or/connection_or.c index 65f4e28c92..f231fda123 100644 --- a/src/core/or/connection_or.c +++ b/src/core/or/connection_or.c @@ -22,7 +22,7 @@ **/ #include "core/or/or.h" #include "feature/client/bridges.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" /* * Define this so we get channel internal functions, since we're implementing * part of a subclass (channel_tls_t). diff --git a/src/core/or/or.h b/src/core/or/or.h index acf092c8dc..0c4995cfd6 100644 --- a/src/core/or/or.h +++ b/src/core/or/or.h @@ -26,7 +26,7 @@ #include "lib/cc/compat_compiler.h" #include "lib/cc/torint.h" #include "lib/container/map.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "lib/container/smartlist.h" #include "lib/crypt_ops/crypto_cipher.h" #include "lib/crypt_ops/crypto_rsa.h" diff --git a/src/core/or/relay.c b/src/core/or/relay.c index 510d96c648..2e92f2a55d 100644 --- a/src/core/or/relay.c +++ b/src/core/or/relay.c @@ -49,7 +49,7 @@ #include "core/or/or.h" #include "feature/client/addressmap.h" #include "lib/err/backtrace.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "core/or/channel.h" #include "feature/client/circpathbias.h" #include "core/or/circuitbuild.h" diff --git a/src/core/or/scheduler.c b/src/core/or/scheduler.c index 326e0d65d9..937e7e45db 100644 --- a/src/core/or/scheduler.c +++ b/src/core/or/scheduler.c @@ -9,7 +9,7 @@ #define SCHEDULER_KIST_PRIVATE #include "core/or/scheduler.h" #include "core/mainloop/mainloop.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #define TOR_CHANNEL_INTERNAL_ #include "core/or/channeltls.h" #include "lib/evloop/compat_libevent.h" diff --git a/src/core/or/scheduler_kist.c b/src/core/or/scheduler_kist.c index f112ea6357..3ed0f1a5e2 100644 --- a/src/core/or/scheduler_kist.c +++ b/src/core/or/scheduler_kist.c @@ -4,7 +4,7 @@ #define SCHEDULER_KIST_PRIVATE #include "core/or/or.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "app/config/config.h" #include "core/mainloop/connection.h" #include "feature/nodelist/networkstatus.h" diff --git a/src/core/proto/proto_cell.c b/src/core/proto/proto_cell.c index 49b07663d7..70278cd488 100644 --- a/src/core/proto/proto_cell.c +++ b/src/core/proto/proto_cell.c @@ -5,7 +5,7 @@ /* See LICENSE for licensing information */ #include "core/or/or.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "core/proto/proto_cell.h" #include "core/or/connection_or.h" diff --git a/src/core/proto/proto_control0.c b/src/core/proto/proto_control0.c index d26c69753a..a770a061a7 100644 --- a/src/core/proto/proto_control0.c +++ b/src/core/proto/proto_control0.c @@ -5,7 +5,7 @@ /* See LICENSE for licensing information */ #include "core/or/or.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "core/proto/proto_control0.h" /** Return 1 iff buf looks more like it has an (obsolete) v0 controller diff --git a/src/core/proto/proto_ext_or.c b/src/core/proto/proto_ext_or.c index 04bec4e6b9..fe36f6b396 100644 --- a/src/core/proto/proto_ext_or.c +++ b/src/core/proto/proto_ext_or.c @@ -5,7 +5,7 @@ /* See LICENSE for licensing information */ #include "core/or/or.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "feature/relay/ext_orport.h" #include "core/proto/proto_ext_or.h" diff --git a/src/core/proto/proto_http.c b/src/core/proto/proto_http.c index 37298d1284..4ce9ba02f5 100644 --- a/src/core/proto/proto_http.c +++ b/src/core/proto/proto_http.c @@ -6,7 +6,7 @@ #define PROTO_HTTP_PRIVATE #include "core/or/or.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "core/proto/proto_http.h" /** Return true if cmd looks like a HTTP (proxy) request. */ diff --git a/src/core/proto/proto_socks.c b/src/core/proto/proto_socks.c index e23da7730b..4071f34f0d 100644 --- a/src/core/proto/proto_socks.c +++ b/src/core/proto/proto_socks.c @@ -6,7 +6,7 @@ #include "core/or/or.h" #include "feature/client/addressmap.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "core/mainloop/connection.h" #include "feature/control/control.h" #include "app/config/config.h" diff --git a/src/feature/control/control.c b/src/feature/control/control.c index b31b448e96..11e722c3a8 100644 --- a/src/feature/control/control.c +++ b/src/feature/control/control.c @@ -87,7 +87,7 @@ #include "feature/rend/rendservice.h" #include "feature/stats/geoip_stats.h" #include "feature/stats/predict_ports.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "lib/crypt_ops/crypto_rand.h" #include "lib/crypt_ops/crypto_util.h" #include "lib/encoding/confline.h" diff --git a/src/feature/stats/geoip_stats.c b/src/feature/stats/geoip_stats.c index 1a4f8ddfb0..3106c6c82c 100644 --- a/src/feature/stats/geoip_stats.c +++ b/src/feature/stats/geoip_stats.c @@ -30,7 +30,7 @@ #include "core/or/or.h" #include "ht.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "app/config/config.h" #include "feature/control/control.h" #include "feature/client/dnsserv.h" diff --git a/src/include.am b/src/include.am index 8279499936..9070a69a03 100644 --- a/src/include.am +++ b/src/include.am @@ -1,5 +1,6 @@ include src/ext/include.am include src/lib/arch/include.am +include src/lib/buf/include.am include src/lib/err/include.am include src/lib/cc/include.am include src/lib/ctime/include.am diff --git a/src/lib/buf/.may_include b/src/lib/buf/.may_include new file mode 100644 index 0000000000..c4be73bce2 --- /dev/null +++ b/src/lib/buf/.may_include @@ -0,0 +1,10 @@ +orconfig.h + +lib/buf/*.h +lib/cc/*.h +lib/ctime/*.h +lib/malloc/*.h +lib/testsupport/*.h +lib/log/*.h +lib/string/*.h +lib/time/*.h diff --git a/src/lib/container/buffers.c b/src/lib/buf/buffers.c similarity index 99% rename from src/lib/container/buffers.c rename to src/lib/buf/buffers.c index 87d782d7ef..495c5ec453 100644 --- a/src/lib/container/buffers.c +++ b/src/lib/buf/buffers.c @@ -25,7 +25,7 @@ #define BUFFERS_PRIVATE #include "orconfig.h" #include -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "lib/cc/torint.h" #include "lib/log/log.h" #include "lib/log/util_bug.h" diff --git a/src/lib/container/buffers.h b/src/lib/buf/buffers.h similarity index 100% rename from src/lib/container/buffers.h rename to src/lib/buf/buffers.h diff --git a/src/lib/buf/include.am b/src/lib/buf/include.am new file mode 100644 index 0000000000..3338c3dbdb --- /dev/null +++ b/src/lib/buf/include.am @@ -0,0 +1,17 @@ + +noinst_LIBRARIES += src/lib/libtor-buf.a + +if UNITTESTS_ENABLED +noinst_LIBRARIES += src/lib/libtor-buf-testing.a +endif + +src_lib_libtor_buf_a_SOURCES = \ + src/lib/buf/buffers.c + +src_lib_libtor_buf_testing_a_SOURCES = \ + $(src_lib_libtor_buf_a_SOURCES) +src_lib_libtor_buf_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) +src_lib_libtor_buf_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) + +noinst_HEADERS += \ + src/lib/buf/buffers.h diff --git a/src/lib/compress/.may_include b/src/lib/compress/.may_include index 4870259ec9..6cd80086e6 100644 --- a/src/lib/compress/.may_include +++ b/src/lib/compress/.may_include @@ -1,5 +1,6 @@ orconfig.h lib/arch/*.h +lib/buf/*.h lib/cc/*.h lib/compress/*.h lib/container/*.h diff --git a/src/lib/compress/compress_buf.c b/src/lib/compress/compress_buf.c index 63ee9e0102..ecf76ee078 100644 --- a/src/lib/compress/compress_buf.c +++ b/src/lib/compress/compress_buf.c @@ -11,7 +11,7 @@ #define BUFFERS_PRIVATE #include "lib/cc/compat_compiler.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "lib/compress/compress.h" #include "lib/log/util_bug.h" diff --git a/src/lib/container/.may_include b/src/lib/container/.may_include index 90de5eda40..76e5843728 100644 --- a/src/lib/container/.may_include +++ b/src/lib/container/.may_include @@ -11,8 +11,5 @@ lib/testsupport/testsupport.h lib/intmath/*.h lib/log/*.h -# XXXX I am unsure about this one. It's only here for buffers.c -lib/time/*.h - ht.h siphash.h diff --git a/src/lib/container/include.am b/src/lib/container/include.am index e6492098b5..032e4033da 100644 --- a/src/lib/container/include.am +++ b/src/lib/container/include.am @@ -7,7 +7,6 @@ endif src_lib_libtor_container_a_SOURCES = \ src/lib/container/bloomfilt.c \ - src/lib/container/buffers.c \ src/lib/container/map.c \ src/lib/container/order.c \ src/lib/container/smartlist.c @@ -20,7 +19,6 @@ src_lib_libtor_container_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) noinst_HEADERS += \ src/lib/container/bitarray.h \ src/lib/container/bloomfilt.h \ - src/lib/container/buffers.h \ src/lib/container/handles.h \ src/lib/container/map.h \ src/lib/container/order.h \ diff --git a/src/lib/net/.may_include b/src/lib/net/.may_include index f93f0e1552..d34aaed2ca 100644 --- a/src/lib/net/.may_include +++ b/src/lib/net/.may_include @@ -3,6 +3,7 @@ siphash.h ht.h lib/arch/*.h +lib/buf/*.h lib/cc/*.h lib/container/*.h lib/ctime/*.h diff --git a/src/lib/net/buffers_net.c b/src/lib/net/buffers_net.c index c52ea2784e..b0936e9928 100644 --- a/src/lib/net/buffers_net.c +++ b/src/lib/net/buffers_net.c @@ -11,7 +11,7 @@ #define BUFFERS_PRIVATE #include "lib/net/buffers_net.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "lib/log/log.h" #include "lib/log/util_bug.h" #include "lib/net/nettypes.h" diff --git a/src/lib/tls/.may_include b/src/lib/tls/.may_include index 79301bc318..069181b70e 100644 --- a/src/lib/tls/.may_include +++ b/src/lib/tls/.may_include @@ -1,6 +1,7 @@ orconfig.h lib/arch/*.h +lib/buf/*.h lib/cc/*.h lib/container/*.h lib/crypt_ops/*.h diff --git a/src/lib/tls/buffers_tls.c b/src/lib/tls/buffers_tls.c index 69ae4f7fc0..b4059292ea 100644 --- a/src/lib/tls/buffers_tls.c +++ b/src/lib/tls/buffers_tls.c @@ -12,7 +12,7 @@ #define BUFFERS_PRIVATE #include "orconfig.h" #include -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "lib/tls/buffers_tls.h" #include "lib/cc/torint.h" #include "lib/log/log.h" diff --git a/src/test/fuzz/fuzz_http.c b/src/test/fuzz/fuzz_http.c index 06483282bc..4341bfabae 100644 --- a/src/test/fuzz/fuzz_http.c +++ b/src/test/fuzz/fuzz_http.c @@ -8,7 +8,7 @@ #include "core/or/or.h" #include "lib/err/backtrace.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "app/config/config.h" #include "core/mainloop/connection.h" #include "feature/dircache/dircache.h" diff --git a/src/test/fuzz/fuzz_http_connect.c b/src/test/fuzz/fuzz_http_connect.c index ca007a2c7f..e03d9e29d8 100644 --- a/src/test/fuzz/fuzz_http_connect.c +++ b/src/test/fuzz/fuzz_http_connect.c @@ -8,7 +8,7 @@ #include "core/or/or.h" #include "lib/err/backtrace.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "app/config/config.h" #include "core/mainloop/connection.h" #include "core/or/connection_edge.h" diff --git a/src/test/fuzz/fuzz_socks.c b/src/test/fuzz/fuzz_socks.c index 14c25304b1..2d93bea924 100644 --- a/src/test/fuzz/fuzz_socks.c +++ b/src/test/fuzz/fuzz_socks.c @@ -6,7 +6,7 @@ #define BUFFERS_PRIVATE #include "core/or/or.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "lib/err/backtrace.h" #include "lib/log/log.h" #include "core/proto/proto_socks.h" diff --git a/src/test/test.c b/src/test/test.c index 17b736d305..a654030fac 100644 --- a/src/test/test.c +++ b/src/test/test.c @@ -37,7 +37,7 @@ #include "core/or/or.h" #include "lib/err/backtrace.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "core/or/circuitlist.h" #include "core/or/circuitstats.h" #include "lib/compress/compress.h" diff --git a/src/test/test_buffers.c b/src/test/test_buffers.c index 477066f699..85e7b8d90a 100644 --- a/src/test/test_buffers.c +++ b/src/test/test_buffers.c @@ -6,7 +6,7 @@ #define BUFFERS_PRIVATE #define PROTO_HTTP_PRIVATE #include "core/or/or.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "lib/tls/buffers_tls.h" #include "lib/tls/tortls.h" #include "lib/compress/compress.h" diff --git a/src/test/test_channelpadding.c b/src/test/test_channelpadding.c index 0fd60d0a92..bdd7c5f0a6 100644 --- a/src/test/test_channelpadding.c +++ b/src/test/test_channelpadding.c @@ -21,7 +21,7 @@ #include "test/log_test_helpers.h" #include "lib/tls/tortls.h" #include "lib/evloop/timers.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "core/or/cell_st.h" #include "feature/nodelist/networkstatus_st.h" diff --git a/src/test/test_channeltls.c b/src/test/test_channeltls.c index 787a30a85d..44d623561b 100644 --- a/src/test/test_channeltls.c +++ b/src/test/test_channeltls.c @@ -8,7 +8,7 @@ #define TOR_CHANNEL_INTERNAL_ #include "core/or/or.h" #include "lib/net/address.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "core/or/channel.h" #include "core/or/channeltls.h" #include "core/mainloop/connection.h" diff --git a/src/test/test_extorport.c b/src/test/test_extorport.c index 71be9131c7..432a9ea5e3 100644 --- a/src/test/test_extorport.c +++ b/src/test/test_extorport.c @@ -5,7 +5,7 @@ #define EXT_ORPORT_PRIVATE #define MAINLOOP_PRIVATE #include "core/or/or.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "core/mainloop/connection.h" #include "core/or/connection_or.h" #include "app/config/config.h" diff --git a/src/test/test_helpers.c b/src/test/test_helpers.c index 6ac73bff5c..b7bda16494 100644 --- a/src/test/test_helpers.c +++ b/src/test/test_helpers.c @@ -14,7 +14,7 @@ #include "orconfig.h" #include "core/or/or.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "app/config/config.h" #include "app/config/confparse.h" #include "core/mainloop/connection.h" diff --git a/src/test/test_oom.c b/src/test/test_oom.c index 313a6b3114..f84dc0764b 100644 --- a/src/test/test_oom.c +++ b/src/test/test_oom.c @@ -8,7 +8,7 @@ #define CIRCUITLIST_PRIVATE #define CONNECTION_PRIVATE #include "core/or/or.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "core/or/circuitlist.h" #include "lib/evloop/compat_libevent.h" #include "core/mainloop/connection.h" diff --git a/src/test/test_proto_http.c b/src/test/test_proto_http.c index 1cfa0a752c..b4e8278423 100644 --- a/src/test/test_proto_http.c +++ b/src/test/test_proto_http.c @@ -8,7 +8,7 @@ #include "core/or/or.h" #include "test/test.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "core/proto/proto_http.h" #include "test/log_test_helpers.h" diff --git a/src/test/test_proto_misc.c b/src/test/test_proto_misc.c index 1fcb763421..f7f6f69667 100644 --- a/src/test/test_proto_misc.c +++ b/src/test/test_proto_misc.c @@ -8,7 +8,7 @@ #include "core/or/or.h" #include "test/test.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "core/or/connection_or.h" #include "feature/relay/ext_orport.h" #include "core/proto/proto_cell.h" diff --git a/src/test/test_routerlist.c b/src/test/test_routerlist.c index 1071a095fe..167b7a35ce 100644 --- a/src/test/test_routerlist.c +++ b/src/test/test_routerlist.c @@ -46,7 +46,7 @@ #include "feature/nodelist/routerstatus_st.h" #include "lib/encoding/confline.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "test/test.h" #include "test/test_dir_common.h" diff --git a/src/test/test_socks.c b/src/test/test_socks.c index 7f6d8a48f1..d430f4329b 100644 --- a/src/test/test_socks.c +++ b/src/test/test_socks.c @@ -4,7 +4,7 @@ /* See LICENSE for licensing information */ #include "core/or/or.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "app/config/config.h" #include "core/mainloop/connection.h" #include "core/proto/proto_socks.h" diff --git a/src/test/test_util.c b/src/test/test_util.c index bcface64fd..2b4d64e42e 100644 --- a/src/test/test_util.c +++ b/src/test/test_util.c @@ -13,7 +13,7 @@ #define SUBPROCESS_PRIVATE #include "lib/testsupport/testsupport.h" #include "core/or/or.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "app/config/config.h" #include "feature/control/control.h" #include "feature/client/transports.h" From c0a7527eb8590b39f11dd8b0ae18794dfc63a934 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 6 Nov 2018 18:14:16 -0500 Subject: [PATCH 0138/2557] Remove dependency on lib/net from lib/sandbox. This was trivial, and the easiest way to remove the remaining .may_include circularities. --- src/app/main/main.c | 1 + src/lib/sandbox/.may_include | 1 - src/lib/sandbox/sandbox.c | 2 -- 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/app/main/main.c b/src/app/main/main.c index 653a393fe4..ec86c46397 100644 --- a/src/app/main/main.c +++ b/src/app/main/main.c @@ -1402,6 +1402,7 @@ tor_run_main(const tor_main_configuration_t *tor_cfg) tor_free_all(0); return -1; } + tor_make_getaddrinfo_cache_active(); // registering libevent rng #ifdef HAVE_EVUTIL_SECURE_RNG_SET_URANDOM_DEVICE_FILE diff --git a/src/lib/sandbox/.may_include b/src/lib/sandbox/.may_include index 84906dfb3d..e8ba5bb73c 100644 --- a/src/lib/sandbox/.may_include +++ b/src/lib/sandbox/.may_include @@ -5,7 +5,6 @@ lib/container/*.h lib/err/*.h lib/log/*.h lib/malloc/*.h -lib/net/*.h lib/sandbox/*.h lib/sandbox/*.inc lib/string/*.h diff --git a/src/lib/sandbox/sandbox.c b/src/lib/sandbox/sandbox.c index 6f074bb4e1..4d832f106d 100644 --- a/src/lib/sandbox/sandbox.c +++ b/src/lib/sandbox/sandbox.c @@ -38,7 +38,6 @@ #include "lib/err/torerr.h" #include "lib/log/log.h" #include "lib/cc/torint.h" -#include "lib/net/resolve.h" #include "lib/malloc/malloc.h" #include "lib/string/scanf.h" @@ -1553,7 +1552,6 @@ install_syscall_filter(sandbox_cfg_t* cfg) // marking the sandbox as active sandbox_active = 1; - tor_make_getaddrinfo_cache_active(); end: seccomp_release(ctx); From e429e31ad1c3000d814f3172fcd3f7c433ee3219 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 6 Nov 2018 20:27:43 -0500 Subject: [PATCH 0139/2557] Normalize .may_include to always have paths, and paths to include --- src/lib/container/.may_include | 6 +++--- src/lib/container/bloomfilt.c | 2 +- src/lib/container/map.c | 2 +- src/lib/container/map.h | 2 +- src/lib/crypt_ops/.may_include | 4 ++-- src/lib/crypt_ops/crypto_init.c | 2 +- src/lib/crypt_ops/digestset.c | 2 +- src/lib/evloop/.may_include | 4 ++-- src/lib/evloop/timers.c | 3 ++- src/lib/evloop/workqueue.c | 2 +- src/lib/fs/.may_include | 2 +- src/lib/malloc/.may_include | 2 +- src/lib/net/.may_include | 4 ++-- src/lib/net/address.c | 2 +- src/lib/net/resolve.c | 4 ++-- src/lib/process/.may_include | 2 +- src/lib/process/waitpid.c | 2 +- src/lib/sandbox/.may_include | 6 +++--- src/lib/sandbox/sandbox.c | 6 +++--- src/lib/smartlist_core/.may_include | 2 +- src/lib/string/.may_include | 4 ++-- src/lib/string/compat_string.c | 4 ++-- src/lib/term/.may_include | 3 +-- src/lib/term/getpass.c | 2 +- src/lib/tls/.may_include | 5 ++--- src/lib/tls/tortls_openssl.c | 2 +- 26 files changed, 40 insertions(+), 41 deletions(-) diff --git a/src/lib/container/.may_include b/src/lib/container/.may_include index 76e5843728..81507527d3 100644 --- a/src/lib/container/.may_include +++ b/src/lib/container/.may_include @@ -7,9 +7,9 @@ lib/malloc/*.h lib/err/*.h lib/smartlist_core/*.h lib/string/*.h -lib/testsupport/testsupport.h +lib/testsupport/*.h lib/intmath/*.h lib/log/*.h -ht.h -siphash.h +ext/ht.h +ext/siphash.h diff --git a/src/lib/container/bloomfilt.c b/src/lib/container/bloomfilt.c index ea2d2917c7..a64fcb9300 100644 --- a/src/lib/container/bloomfilt.c +++ b/src/lib/container/bloomfilt.c @@ -14,7 +14,7 @@ #include "lib/container/bloomfilt.h" #include "lib/intmath/bits.h" #include "lib/log/util_bug.h" -#include "siphash.h" +#include "ext/siphash.h" /** How many bloom-filter bits we set per address. This is twice the * BLOOMFILT_N_HASHES value, since we split the siphash output into two 32-bit diff --git a/src/lib/container/map.c b/src/lib/container/map.c index 85e074e10a..137e316920 100644 --- a/src/lib/container/map.c +++ b/src/lib/container/map.c @@ -21,7 +21,7 @@ #include #include -#include "ht.h" +#include "ext/ht.h" /** Helper: Declare an entry type and a map type to implement a mapping using * ht.h. The map type will be called maptype. The key part of each diff --git a/src/lib/container/map.h b/src/lib/container/map.h index ff71622682..4f427fe767 100644 --- a/src/lib/container/map.h +++ b/src/lib/container/map.h @@ -15,7 +15,7 @@ #include "lib/testsupport/testsupport.h" #include "lib/cc/torint.h" -#include "siphash.h" +#include "ext/siphash.h" #define DECLARE_MAP_FNS(maptype, keytype, prefix) \ typedef struct maptype maptype; \ diff --git a/src/lib/crypt_ops/.may_include b/src/lib/crypt_ops/.may_include index 352fde858c..0739699686 100644 --- a/src/lib/crypt_ops/.may_include +++ b/src/lib/crypt_ops/.may_include @@ -13,7 +13,7 @@ lib/intmath/*.h lib/sandbox/*.h lib/string/*.h lib/subsys/*.h -lib/testsupport/testsupport.h +lib/testsupport/*.h lib/thread/*.h lib/log/*.h @@ -22,4 +22,4 @@ trunnel/pwbox.h keccak-tiny/*.h ed25519/*.h -siphash.h +ext/siphash.h diff --git a/src/lib/crypt_ops/crypto_init.c b/src/lib/crypt_ops/crypto_init.c index 4c4cc3e43b..f9943939fb 100644 --- a/src/lib/crypt_ops/crypto_init.c +++ b/src/lib/crypt_ops/crypto_init.c @@ -24,7 +24,7 @@ #include "lib/subsys/subsys.h" -#include "siphash.h" +#include "ext/siphash.h" /** Boolean: has our crypto library been initialized? (early phase) */ static int crypto_early_initialized_ = 0; diff --git a/src/lib/crypt_ops/digestset.c b/src/lib/crypt_ops/digestset.c index 89dd377a9c..84516e0172 100644 --- a/src/lib/crypt_ops/digestset.c +++ b/src/lib/crypt_ops/digestset.c @@ -11,7 +11,7 @@ #include "lib/crypt_ops/crypto_rand.h" #include "lib/defs/digest_sizes.h" #include "lib/crypt_ops/digestset.h" -#include "siphash.h" +#include "ext/siphash.h" /* Wrap our hash function to have the signature that the bloom filter * needs. */ diff --git a/src/lib/evloop/.may_include b/src/lib/evloop/.may_include index 30af508914..273de7bb94 100644 --- a/src/lib/evloop/.may_include +++ b/src/lib/evloop/.may_include @@ -12,5 +12,5 @@ lib/testsupport/*.h lib/thread/*.h lib/time/*.h -src/ext/timeouts/timeout.c -tor_queue.h \ No newline at end of file +ext/timeouts/timeout.c +ext/tor_queue.h \ No newline at end of file diff --git a/src/lib/evloop/timers.c b/src/lib/evloop/timers.c index 3603bf1a7d..6743b6af51 100644 --- a/src/lib/evloop/timers.c +++ b/src/lib/evloop/timers.c @@ -80,7 +80,8 @@ struct timeout_cb { * use 32-bit math. */ #define WHEEL_BIT 5 #endif -#include "src/ext/timeouts/timeout.c" + +#include "ext/timeouts/timeout.c" static struct timeouts *global_timeouts = NULL; static struct mainloop_event_t *global_timer_event = NULL; diff --git a/src/lib/evloop/workqueue.c b/src/lib/evloop/workqueue.c index 5471f87b04..b36a02da5e 100644 --- a/src/lib/evloop/workqueue.c +++ b/src/lib/evloop/workqueue.c @@ -36,7 +36,7 @@ #include "lib/net/socket.h" #include "lib/thread/threads.h" -#include "tor_queue.h" +#include "ext/tor_queue.h" #include #include diff --git a/src/lib/fs/.may_include b/src/lib/fs/.may_include index b1e49fc891..c192e6181c 100644 --- a/src/lib/fs/.may_include +++ b/src/lib/fs/.may_include @@ -13,4 +13,4 @@ lib/malloc/*.h lib/memarea/*.h lib/sandbox/*.h lib/string/*.h -lib/testsupport/testsupport.h +lib/testsupport/*.h diff --git a/src/lib/malloc/.may_include b/src/lib/malloc/.may_include index cc62bb1013..7686bf862a 100644 --- a/src/lib/malloc/.may_include +++ b/src/lib/malloc/.may_include @@ -3,4 +3,4 @@ orconfig.h lib/cc/*.h lib/err/*.h lib/malloc/*.h -lib/testsupport/testsupport.h +lib/testsupport/*.h diff --git a/src/lib/net/.may_include b/src/lib/net/.may_include index d34aaed2ca..e4368f799b 100644 --- a/src/lib/net/.may_include +++ b/src/lib/net/.may_include @@ -1,6 +1,6 @@ orconfig.h -siphash.h -ht.h +ext/siphash.h +ext/ht.h lib/arch/*.h lib/buf/*.h diff --git a/src/lib/net/address.c b/src/lib/net/address.c index a351b9df28..240201d7b6 100644 --- a/src/lib/net/address.c +++ b/src/lib/net/address.c @@ -53,7 +53,7 @@ #include "lib/string/printf.h" #include "lib/string/util_string.h" -#include "siphash.h" +#include "ext/siphash.h" #ifdef HAVE_SYS_TIME_H #include diff --git a/src/lib/net/resolve.c b/src/lib/net/resolve.c index 7c8df3e307..01f7882964 100644 --- a/src/lib/net/resolve.c +++ b/src/lib/net/resolve.c @@ -16,8 +16,8 @@ #include "lib/string/parse_int.h" #include "lib/string/util_string.h" -#include "siphash.h" -#include "ht.h" +#include "ext/siphash.h" +#include "ext/ht.h" #ifdef HAVE_SYS_TYPES_H #include diff --git a/src/lib/process/.may_include b/src/lib/process/.may_include index a2d57a52f3..3a5d849fe7 100644 --- a/src/lib/process/.may_include +++ b/src/lib/process/.may_include @@ -15,4 +15,4 @@ lib/subsys/*.h lib/testsupport/*.h lib/thread/*.h -ht.h \ No newline at end of file +ext/ht.h \ No newline at end of file diff --git a/src/lib/process/waitpid.c b/src/lib/process/waitpid.c index 32ba4530da..46d30bf50e 100644 --- a/src/lib/process/waitpid.c +++ b/src/lib/process/waitpid.c @@ -16,7 +16,7 @@ #include "lib/log/log.h" #include "lib/log/util_bug.h" #include "lib/malloc/malloc.h" -#include "ht.h" +#include "ext/ht.h" #ifdef HAVE_SYS_WAIT_H #include diff --git a/src/lib/sandbox/.may_include b/src/lib/sandbox/.may_include index e8ba5bb73c..853dae7880 100644 --- a/src/lib/sandbox/.may_include +++ b/src/lib/sandbox/.may_include @@ -9,6 +9,6 @@ lib/sandbox/*.h lib/sandbox/*.inc lib/string/*.h -ht.h -siphash.h -tor_queue.h +ext/ht.h +ext/siphash.h +ext/tor_queue.h diff --git a/src/lib/sandbox/sandbox.c b/src/lib/sandbox/sandbox.c index 4d832f106d..9477818553 100644 --- a/src/lib/sandbox/sandbox.c +++ b/src/lib/sandbox/sandbox.c @@ -41,9 +41,9 @@ #include "lib/malloc/malloc.h" #include "lib/string/scanf.h" -#include "tor_queue.h" -#include "ht.h" -#include "siphash.h" +#include "ext/tor_queue.h" +#include "ext/ht.h" +#include "ext/siphash.h" #define DEBUGGING_CLOSE diff --git a/src/lib/smartlist_core/.may_include b/src/lib/smartlist_core/.may_include index a8507761a4..2f0c8d341e 100644 --- a/src/lib/smartlist_core/.may_include +++ b/src/lib/smartlist_core/.may_include @@ -4,4 +4,4 @@ lib/malloc/*.h lib/err/*.h lib/string/*.h lib/smartlist_core/*.h -lib/testsupport/testsupport.h +lib/testsupport/*.h diff --git a/src/lib/string/.may_include b/src/lib/string/.may_include index ec5c769831..1fb9127f19 100644 --- a/src/lib/string/.may_include +++ b/src/lib/string/.may_include @@ -6,5 +6,5 @@ lib/malloc/*.h lib/ctime/*.h lib/string/*.h -strlcat.c -strlcpy.c +ext/strlcat.c +ext/strlcpy.c diff --git a/src/lib/string/compat_string.c b/src/lib/string/compat_string.c index eae82fdae0..b3f1e0fd96 100644 --- a/src/lib/string/compat_string.c +++ b/src/lib/string/compat_string.c @@ -14,10 +14,10 @@ /* Inline the strl functions if the platform doesn't have them. */ #ifndef HAVE_STRLCPY -#include "strlcpy.c" +#include "ext/strlcpy.c" #endif #ifndef HAVE_STRLCAT -#include "strlcat.c" +#include "ext/strlcat.c" #endif #include diff --git a/src/lib/term/.may_include b/src/lib/term/.may_include index c93a06e59e..306fa57b7a 100644 --- a/src/lib/term/.may_include +++ b/src/lib/term/.may_include @@ -5,5 +5,4 @@ lib/log/*.h lib/term/*.h lib/malloc/*.h -# From src/ext -tor_readpassphrase.h +ext/tor_readpassphrase.h diff --git a/src/lib/term/getpass.c b/src/lib/term/getpass.c index 27a27179b6..a473fb765b 100644 --- a/src/lib/term/getpass.c +++ b/src/lib/term/getpass.c @@ -36,7 +36,7 @@ SecureZeroMemory(PVOID ptr, SIZE_T cnt) #elif defined(HAVE_READPASSPHRASE_H) #include #else -#include "tor_readpassphrase.h" +#include "ext/tor_readpassphrase.h" #endif /* defined(_WIN32) || ... */ #include diff --git a/src/lib/tls/.may_include b/src/lib/tls/.may_include index 069181b70e..c550bde024 100644 --- a/src/lib/tls/.may_include +++ b/src/lib/tls/.may_include @@ -13,7 +13,6 @@ lib/malloc/*.h lib/net/*.h lib/string/*.h lib/subsys/*.h -lib/testsupport/testsupport.h +lib/testsupport/*.h lib/tls/*.h - -ciphers.inc +lib/tls/*.inc diff --git a/src/lib/tls/tortls_openssl.c b/src/lib/tls/tortls_openssl.c index 63f6259a6c..ddeaabc416 100644 --- a/src/lib/tls/tortls_openssl.c +++ b/src/lib/tls/tortls_openssl.c @@ -461,7 +461,7 @@ static const char UNRESTRICTED_SERVER_CIPHER_LIST[] = /** List of ciphers that clients should advertise, omitting items that * our OpenSSL doesn't know about. */ static const char CLIENT_CIPHER_LIST[] = -#include "ciphers.inc" +#include "lib/tls/ciphers.inc" /* Tell it not to use SSLv2 ciphers, so that it can select an SSLv3 version * of any cipher we say. */ "!SSLv2" From d32795bb6ebdfbf05180d7552d4cd18cb33ddcee Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 6 Nov 2018 20:34:47 -0500 Subject: [PATCH 0140/2557] Make "ext" participate in may_include. Also, resolve a circular dependency involving the use of lib/log by csiphash.c. --- src/ext/.may_include | 10 ++++++++++ src/ext/csiphash.c | 10 +++++----- src/ext/readpassphrase.c | 2 +- 3 files changed, 16 insertions(+), 6 deletions(-) create mode 100644 src/ext/.may_include diff --git a/src/ext/.may_include b/src/ext/.may_include new file mode 100644 index 0000000000..1eafff2eeb --- /dev/null +++ b/src/ext/.may_include @@ -0,0 +1,10 @@ + +orconfig.h + +lib/err/*.h +lib/cc/*.h + +tinytest*.h +ext/siphash.h +ext/byteorder.h +ext/tor_readpassphrase.h \ No newline at end of file diff --git a/src/ext/csiphash.c b/src/ext/csiphash.c index a6a9846db4..af8559a476 100644 --- a/src/ext/csiphash.c +++ b/src/ext/csiphash.c @@ -30,12 +30,12 @@ */ #include "lib/cc/torint.h" -#include "lib/log/util_bug.h" +#include "lib/err/torerr.h" -#include "siphash.h" +#include "ext/siphash.h" #include #include -#include "byteorder.h" +#include "ext/byteorder.h" #define ROTATE(x, b) (uint64_t)( ((x) << (b)) | ( (x) >> (64 - (b))) ) @@ -112,13 +112,13 @@ static int the_siphash_key_is_set = 0; static struct sipkey the_siphash_key; uint64_t siphash24g(const void *src, unsigned long src_sz) { - tor_assert(the_siphash_key_is_set); + raw_assert(the_siphash_key_is_set); return siphash24(src, src_sz, &the_siphash_key); } void siphash_set_global_key(const struct sipkey *key) { - tor_assert(! the_siphash_key_is_set); + raw_assert(! the_siphash_key_is_set); the_siphash_key.k0 = key->k0; the_siphash_key.k1 = key->k1; the_siphash_key_is_set = 1; diff --git a/src/ext/readpassphrase.c b/src/ext/readpassphrase.c index e0df05d7b7..16611af1e2 100644 --- a/src/ext/readpassphrase.c +++ b/src/ext/readpassphrase.c @@ -30,7 +30,7 @@ #include #include #include -#include "tor_readpassphrase.h" +#include "ext/tor_readpassphrase.h" #include #include #include From 4ecd5da3065d83af0743669dfcadd3cb01356cee Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 6 Nov 2018 20:38:09 -0500 Subject: [PATCH 0141/2557] Add .may_include to ext/timeouts. --- src/ext/timeouts/.may_include | 6 ++++++ src/ext/timeouts/timeout.c | 5 ++--- src/ext/timeouts/timeout.h | 4 ++-- 3 files changed, 10 insertions(+), 5 deletions(-) create mode 100644 src/ext/timeouts/.may_include diff --git a/src/ext/timeouts/.may_include b/src/ext/timeouts/.may_include new file mode 100644 index 0000000000..42f44befd4 --- /dev/null +++ b/src/ext/timeouts/.may_include @@ -0,0 +1,6 @@ +orconfig.h + +ext/tor_queue.h +timeout-bitops.c +timeout-debug.h +timeout.h diff --git a/src/ext/timeouts/timeout.c b/src/ext/timeouts/timeout.c index d4b514d2c5..07d06772c5 100644 --- a/src/ext/timeouts/timeout.c +++ b/src/ext/timeouts/timeout.c @@ -38,7 +38,7 @@ #include /* errno */ -#include "tor_queue.h" /* TAILQ(3) */ +#include "ext/tor_queue.h" /* TAILQ(3) */ #include "timeout.h" @@ -531,7 +531,7 @@ static timeout_t timeouts_int(struct timeouts *T) { timeout = MIN(_timeout, timeout); } - relmask <<= WHEEL_BIT; + relmask <<= WHEEL_BIT; relmask |= WHEEL_MASK; } @@ -751,4 +751,3 @@ TIMEOUT_PUBLIC int timeout_v_abi(void) { TIMEOUT_PUBLIC int timeout_v_api(void) { return TIMEOUT_V_API; } /* timeout_version() */ - diff --git a/src/ext/timeouts/timeout.h b/src/ext/timeouts/timeout.h index b35874e153..1ed309fd08 100644 --- a/src/ext/timeouts/timeout.h +++ b/src/ext/timeouts/timeout.h @@ -31,7 +31,7 @@ #include /* PRIu64 PRIx64 PRIX64 uint64_t */ -#include "tor_queue.h" /* TAILQ(3) */ +#include "ext/tor_queue.h" /* TAILQ(3) */ /* @@ -147,7 +147,7 @@ TIMEOUT_PUBLIC struct timeout *timeout_init(struct timeout *, int); #ifndef TIMEOUT_DISABLE_RELATIVE_ACCESS TIMEOUT_PUBLIC bool timeout_pending(struct timeout *); /* true if on timing wheel, false otherwise */ - + TIMEOUT_PUBLIC bool timeout_expired(struct timeout *); /* true if on expired queue, false otherwise */ From c3c8c926bfc8e44e28f3a404e3452f014f6bcbaa Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 7 Nov 2018 10:30:46 -0500 Subject: [PATCH 0142/2557] Update the check-includes script to enforce some naming and no-circularity rules --- scripts/maint/checkIncludes.py | 64 ++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/scripts/maint/checkIncludes.py b/scripts/maint/checkIncludes.py index 46a3f39638..ecda3fc565 100755 --- a/scripts/maint/checkIncludes.py +++ b/scripts/maint/checkIncludes.py @@ -33,6 +33,9 @@ else: def open_file(fname): return open(fname, 'r', encoding='utf-8') +def warn(msg): + print(msg, file=sys.stderr) + def err(msg): """ Declare that an error has happened, and remember that there has been an error. """ @@ -48,14 +51,34 @@ def fname_is_c(fname): INCLUDE_PATTERN = re.compile(r'\s*#\s*include\s+"([^"]*)"') RULES_FNAME = ".may_include" +ALLOWED_PATTERNS = [ + re.compile(r'^.*\*\.(h|inc)$'), + re.compile(r'^.*/.*\.h$'), + re.compile(r'^ext/.*\.c$'), + re.compile(r'^orconfig.h$'), + re.compile(r'^micro-revision.i$'), +] + +def pattern_is_normal(s): + for p in ALLOWED_PATTERNS: + if p.match(s): + return True + return False + class Rules(object): """ A 'Rules' object is the parsed version of a .may_include file. """ def __init__(self, dirpath): self.dirpath = dirpath + if dirpath.startswith("src/"): + self.incpath = dirpath[4:] + else: + self.incpath = dirpath self.patterns = [] self.usedPatterns = set() def addPattern(self, pattern): + if not pattern_is_normal(pattern): + warn("Unusual pattern {} in {}".format(pattern, self.dirpath)) self.patterns.append(pattern) def includeOk(self, path): @@ -86,6 +109,20 @@ class Rules(object): if p not in self.usedPatterns: print("Pattern {} in {} was never used.".format(p, self.dirpath)) + def getAllowedDirectories(self): + allowed = [] + for p in self.patterns: + m = re.match(r'^(.*)/\*\.(h|inc)$', p) + if m: + allowed.append(m.group(1)) + continue + m = re.match(r'^(.*)/[^/]*$', p) + if m: + allowed.append(m.group(1)) + continue + + return allowed + def load_include_rules(fname): """ Read a rules file from 'fname', and return it as a Rules object. """ result = Rules(os.path.split(fname)[0]) @@ -99,6 +136,8 @@ def load_include_rules(fname): list_unused = False +uses_dirs = { } + for dirpath, dirnames, fnames in os.walk("src"): if ".may_include" in fnames: rules = load_include_rules(os.path.join(dirpath, RULES_FNAME)) @@ -108,8 +147,33 @@ for dirpath, dirnames, fnames in os.walk("src"): if list_unused: rules.noteUnusedRules() + uses_dirs[rules.incpath] = rules.getAllowedDirectories() + if trouble: err( """To change which includes are allowed in a C file, edit the {} files in its enclosing directory.""".format(RULES_FNAME)) sys.exit(1) + +all_levels = [] + +n = 0 +while uses_dirs: + n += 0 + cur_level = [] + for k in list(uses_dirs): + uses_dirs[k] = [ d for d in uses_dirs[k] + if (d in uses_dirs and d != k)] + if uses_dirs[k] == []: + cur_level.append(k) + for k in cur_level: + del uses_dirs[k] + n += 1 + if cur_level: + print(n, cur_level) + if n > 100: + break + +if uses_dirs: + print("Circular dependencies in here somewhere:", uses_dirs) + sys.exit(1) From fae29f7b424460353b3a5ab44e1394fdc28cc4c4 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 7 Nov 2018 10:59:23 -0500 Subject: [PATCH 0143/2557] Make the topological-sort output off by default --- scripts/maint/checkIncludes.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/maint/checkIncludes.py b/scripts/maint/checkIncludes.py index ecda3fc565..3afd9bbebe 100755 --- a/scripts/maint/checkIncludes.py +++ b/scripts/maint/checkIncludes.py @@ -135,6 +135,7 @@ def load_include_rules(fname): return result list_unused = False +log_sorted_levels = False uses_dirs = { } @@ -169,11 +170,12 @@ while uses_dirs: for k in cur_level: del uses_dirs[k] n += 1 - if cur_level: + if cur_level and log_sorted_levels: print(n, cur_level) if n > 100: break if uses_dirs: - print("Circular dependencies in here somewhere:", uses_dirs) + print("There are circular .may_include dependencies in here somewhere:", + uses_dirs) sys.exit(1) From 5090fecaca61c87d52559342d6488ee8bb8fd20b Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 7 Nov 2018 11:02:03 -0500 Subject: [PATCH 0144/2557] changes file for no-circular-dependencies stuff (28362) --- changes/ticket28362 | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 changes/ticket28362 diff --git a/changes/ticket28362 b/changes/ticket28362 new file mode 100644 index 0000000000..4ac22d50f2 --- /dev/null +++ b/changes/ticket28362 @@ -0,0 +1,6 @@ + o Code simplification and refactoring: + - The .may_include files that we use to describe our + directory-by-directory dependency structure now describe a noncircular + dependency graph over the directories that they cover. + Our checkIncludes.py tool now enforces this. + Closes ticket 28362. From 6ca29ea4094c7cb5696cd41d3994dd2fb230fe9a Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 7 Nov 2018 12:14:24 -0500 Subject: [PATCH 0145/2557] Add libtor-buf-testing to build.rs --- src/rust/build.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/rust/build.rs b/src/rust/build.rs index bf566c56bf..5626b35f75 100644 --- a/src/rust/build.rs +++ b/src/rust/build.rs @@ -149,8 +149,9 @@ pub fn main() { cfg.component("tor-sandbox-testing"); cfg.component("tor-encoding-testing"); cfg.component("tor-fs-testing"); - cfg.component("tor-time-testing"); cfg.component("tor-net-testing"); + cfg.component("tor-buf-testing"); + cfg.component("tor-time-testing"); cfg.component("tor-thread-testing"); cfg.component("tor-memarea-testing"); cfg.component("tor-log-testing"); From e420154ce77d9169bc72c373bfc7e29cb7245723 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 7 Nov 2018 14:39:20 -0500 Subject: [PATCH 0146/2557] Add an include to main.c --- src/app/main/main.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/app/main/main.c b/src/app/main/main.c index ec86c46397..59ab4ce408 100644 --- a/src/app/main/main.c +++ b/src/app/main/main.c @@ -71,6 +71,7 @@ #include "lib/crypt_ops/crypto_rand.h" #include "lib/crypt_ops/crypto_s2k.h" #include "lib/geoip/geoip.h" +#include "lib/net/resolve.h" #include "lib/process/waitpid.h" From eaff47352a1ef607f6d97e9b35bd6d5eefdfcb26 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 13 Nov 2018 17:02:15 -0500 Subject: [PATCH 0147/2557] Make sure sandbox-related getaddrinfo() functions always exist. --- src/lib/net/resolve.c | 9 +++++++++ src/lib/net/resolve.h | 2 +- src/lib/sandbox/sandbox.c | 5 ----- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/lib/net/resolve.c b/src/lib/net/resolve.c index 01f7882964..95c1b171b5 100644 --- a/src/lib/net/resolve.c +++ b/src/lib/net/resolve.c @@ -421,4 +421,13 @@ tor_make_getaddrinfo_cache_active(void) { sandbox_getaddrinfo_is_active = 1; } +#else +void +sandbox_disable_getaddrinfo_cache(void) +{ +} +void +tor_make_getaddrinfo_cache_active(void) +{ +} #endif diff --git a/src/lib/net/resolve.h b/src/lib/net/resolve.h index bf870c44c4..39157aaa67 100644 --- a/src/lib/net/resolve.h +++ b/src/lib/net/resolve.h @@ -42,7 +42,6 @@ int tor_getaddrinfo(const char *name, const char *servname, struct addrinfo **res); void tor_freeaddrinfo(struct addrinfo *addrinfo); void tor_free_getaddrinfo_cache(void); -void tor_make_getaddrinfo_cache_active(void); #else /* !(defined(USE_SANDBOX_GETADDRINFO)) */ #define tor_getaddrinfo(name, servname, hints, res) \ getaddrinfo((name),(servname), (hints),(res)) @@ -54,5 +53,6 @@ void tor_make_getaddrinfo_cache_active(void); #endif /* defined(USE_SANDBOX_GETADDRINFO) */ void sandbox_disable_getaddrinfo_cache(void); +void tor_make_getaddrinfo_cache_active(void); #endif diff --git a/src/lib/sandbox/sandbox.c b/src/lib/sandbox/sandbox.c index 9477818553..ea738b273e 100644 --- a/src/lib/sandbox/sandbox.c +++ b/src/lib/sandbox/sandbox.c @@ -1798,9 +1798,4 @@ sandbox_is_active(void) return 0; } -void -sandbox_disable_getaddrinfo_cache(void) -{ -} - #endif /* !defined(USE_LIBSECCOMP) */ From 2070765c7c062c505358d0f1c83f2846181d1667 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 5 Nov 2018 16:09:13 -0500 Subject: [PATCH 0148/2557] Use macros to make the periodic event table less verbose. --- src/core/mainloop/mainloop.c | 78 ++++++++++++++++-------------------- 1 file changed, 35 insertions(+), 43 deletions(-) diff --git a/src/core/mainloop/mainloop.c b/src/core/mainloop/mainloop.c index 7eff82fee4..12820888f2 100644 --- a/src/core/mainloop/mainloop.c +++ b/src/core/mainloop/mainloop.c @@ -1370,74 +1370,66 @@ CALLBACK(write_stats_file); #undef CALLBACK /* Now we declare an array of periodic_event_item_t for each periodic event */ -#define CALLBACK(name, r, f) PERIODIC_EVENT(name, r, f) +#define CALLBACK(name, r, f) \ + PERIODIC_EVENT(name, PERIODIC_EVENT_ROLE_ ## r, f) +#define FL(name) (PERIODIC_EVENT_FLAG_ ## name) STATIC periodic_event_item_t periodic_events[] = { /* Everyone needs to run those. */ - CALLBACK(add_entropy, PERIODIC_EVENT_ROLE_ALL, 0), - CALLBACK(check_expired_networkstatus, PERIODIC_EVENT_ROLE_ALL, 0), - CALLBACK(clean_caches, PERIODIC_EVENT_ROLE_ALL, 0), - CALLBACK(fetch_networkstatus, PERIODIC_EVENT_ROLE_ALL, - PERIODIC_EVENT_FLAG_NEED_NET), - CALLBACK(heartbeat, PERIODIC_EVENT_ROLE_ALL, 0), - CALLBACK(launch_descriptor_fetches, PERIODIC_EVENT_ROLE_ALL, - PERIODIC_EVENT_FLAG_NEED_NET), - CALLBACK(reset_padding_counts, PERIODIC_EVENT_ROLE_ALL, 0), - CALLBACK(retry_listeners, PERIODIC_EVENT_ROLE_ALL, - PERIODIC_EVENT_FLAG_NEED_NET), - CALLBACK(save_state, PERIODIC_EVENT_ROLE_ALL, 0), - CALLBACK(rotate_x509_certificate, PERIODIC_EVENT_ROLE_ALL, 0), - CALLBACK(write_stats_file, PERIODIC_EVENT_ROLE_ALL, 0), + CALLBACK(add_entropy, ALL, 0), + CALLBACK(check_expired_networkstatus, ALL, 0), + CALLBACK(clean_caches, ALL, 0), + CALLBACK(fetch_networkstatus, ALL, 0), + CALLBACK(heartbeat, ALL, 0), + CALLBACK(launch_descriptor_fetches, ALL, FL(NEED_NET)), + CALLBACK(reset_padding_counts, ALL, 0), + CALLBACK(retry_listeners, ALL, FL(NEED_NET)), + CALLBACK(save_state, ALL, 0), + CALLBACK(rotate_x509_certificate, ALL, 0), + CALLBACK(write_stats_file, ALL, 0), /* Routers (bridge and relay) only. */ - CALLBACK(check_descriptor, PERIODIC_EVENT_ROLE_ROUTER, - PERIODIC_EVENT_FLAG_NEED_NET), - CALLBACK(check_ed_keys, PERIODIC_EVENT_ROLE_ROUTER, 0), - CALLBACK(check_for_reachability_bw, PERIODIC_EVENT_ROLE_ROUTER, - PERIODIC_EVENT_FLAG_NEED_NET), - CALLBACK(check_onion_keys_expiry_time, PERIODIC_EVENT_ROLE_ROUTER, 0), - CALLBACK(expire_old_ciruits_serverside, PERIODIC_EVENT_ROLE_ROUTER, - PERIODIC_EVENT_FLAG_NEED_NET), - CALLBACK(reachability_warnings, PERIODIC_EVENT_ROLE_ROUTER, - PERIODIC_EVENT_FLAG_NEED_NET), - CALLBACK(retry_dns, PERIODIC_EVENT_ROLE_ROUTER, 0), - CALLBACK(rotate_onion_key, PERIODIC_EVENT_ROLE_ROUTER, 0), + CALLBACK(check_descriptor, ROUTER, FL(NEED_NET)), + CALLBACK(check_ed_keys, ROUTER, 0), + CALLBACK(check_for_reachability_bw, ROUTER, FL(NEED_NET)), + CALLBACK(check_onion_keys_expiry_time, ROUTER, 0), + CALLBACK(expire_old_ciruits_serverside, ROUTER, FL(NEED_NET)), + CALLBACK(reachability_warnings, ROUTER, FL(NEED_NET)), + CALLBACK(retry_dns, ROUTER, 0), + CALLBACK(rotate_onion_key, ROUTER, 0), /* Authorities (bridge and directory) only. */ - CALLBACK(downrate_stability, PERIODIC_EVENT_ROLE_AUTHORITIES, 0), - CALLBACK(launch_reachability_tests, PERIODIC_EVENT_ROLE_AUTHORITIES, - PERIODIC_EVENT_FLAG_NEED_NET), - CALLBACK(save_stability, PERIODIC_EVENT_ROLE_AUTHORITIES, 0), + CALLBACK(downrate_stability, AUTHORITIES, 0), + CALLBACK(launch_reachability_tests, AUTHORITIES, FL(NEED_NET)), + CALLBACK(save_stability, AUTHORITIES, 0), /* Directory authority only. */ - CALLBACK(check_authority_cert, PERIODIC_EVENT_ROLE_DIRAUTH, 0), - CALLBACK(dirvote, PERIODIC_EVENT_ROLE_DIRAUTH, PERIODIC_EVENT_FLAG_NEED_NET), + CALLBACK(check_authority_cert, DIRAUTH, 0), + CALLBACK(dirvote, DIRAUTH, FL(NEED_NET)), /* Relay only. */ - CALLBACK(check_canonical_channels, PERIODIC_EVENT_ROLE_RELAY, - PERIODIC_EVENT_FLAG_NEED_NET), - CALLBACK(check_dns_honesty, PERIODIC_EVENT_ROLE_RELAY, - PERIODIC_EVENT_FLAG_NEED_NET), + CALLBACK(check_canonical_channels, RELAY, FL(NEED_NET)), + CALLBACK(check_dns_honesty, RELAY, FL(NEED_NET)), /* Hidden Service service only. */ - CALLBACK(hs_service, PERIODIC_EVENT_ROLE_HS_SERVICE, - PERIODIC_EVENT_FLAG_NEED_NET), + CALLBACK(hs_service, HS_SERVICE, FL(NEED_NET)), /* Bridge only. */ - CALLBACK(record_bridge_stats, PERIODIC_EVENT_ROLE_BRIDGE, 0), + CALLBACK(record_bridge_stats, BRIDGE, 0), /* Client only. */ - CALLBACK(rend_cache_failure_clean, PERIODIC_EVENT_ROLE_CLIENT, 0), + CALLBACK(rend_cache_failure_clean, CLIENT, 0), /* Bridge Authority only. */ - CALLBACK(write_bridge_ns, PERIODIC_EVENT_ROLE_BRIDGEAUTH, 0), + CALLBACK(write_bridge_ns, BRIDGEAUTH, 0), /* Directory server only. */ - CALLBACK(clean_consdiffmgr, PERIODIC_EVENT_ROLE_DIRSERVER, 0), + CALLBACK(clean_consdiffmgr, DIRSERVER, 0), END_OF_PERIODIC_EVENTS }; #undef CALLBACK +#undef FL /* These are pointers to members of periodic_events[] that are used to * implement particular callbacks. We keep them separate here so that we From 6d84972eb8e27d5e9f1adea36fcc9a9879d718ad Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 5 Nov 2018 16:24:10 -0500 Subject: [PATCH 0149/2557] Add a function to schedule a periodic event once, then disable it --- src/core/mainloop/mainloop.c | 6 +++++- src/core/mainloop/periodic.c | 22 +++++++++++++++++----- src/core/mainloop/periodic.h | 7 ++++++- 3 files changed, 28 insertions(+), 7 deletions(-) diff --git a/src/core/mainloop/mainloop.c b/src/core/mainloop/mainloop.c index 12820888f2..42f6fb50c7 100644 --- a/src/core/mainloop/mainloop.c +++ b/src/core/mainloop/mainloop.c @@ -1600,7 +1600,11 @@ rescan_periodic_events(const or_options_t *options) periodic_event_enable(item); } else { log_debug(LD_GENERAL, "Disabling periodic event %s", item->name); - periodic_event_disable(item); + if (item->flags & PERIODIC_EVENT_FLAG_FLUSH_ON_DISABLE) { + periodic_event_flush_and_disable(item); + } else { + periodic_event_disable(item); + } } } } diff --git a/src/core/mainloop/periodic.c b/src/core/mainloop/periodic.c index c1785eb38f..c290c3744e 100644 --- a/src/core/mainloop/periodic.c +++ b/src/core/mainloop/periodic.c @@ -45,10 +45,6 @@ periodic_event_dispatch(mainloop_event_t *ev, void *data) periodic_event_item_t *event = data; tor_assert(ev == event->ev); - if (BUG(!periodic_event_is_enabled(event))) { - return; - } - time_t now = time(NULL); update_current_time(now); const or_options_t *options = get_options(); @@ -57,7 +53,7 @@ periodic_event_dispatch(mainloop_event_t *ev, void *data) int next_interval = 0; if (!periodic_event_is_enabled(event)) { - /* The event got disabled from inside its callback; no need to + /* The event got disabled from inside its callback, or before: no need to * reschedule. */ return; } @@ -172,3 +168,19 @@ periodic_event_disable(periodic_event_item_t *event) mainloop_event_cancel(event->ev); event->enabled = 0; } + +/** + * Disable an event, then schedule it to run once. + * Do nothing if the event was already disabled. + */ +void +periodic_event_flush_and_disable(periodic_event_item_t *event) +{ + tor_assert(event); + if (!periodic_event_is_enabled(event)) + return; + + periodic_event_disable(event); + + mainloop_event_activate(event->ev); +} diff --git a/src/core/mainloop/periodic.h b/src/core/mainloop/periodic.h index 4c8c3c96cc..7c71be7bc4 100644 --- a/src/core/mainloop/periodic.h +++ b/src/core/mainloop/periodic.h @@ -39,6 +39,11 @@ * the net_is_disabled() check. */ #define PERIODIC_EVENT_FLAG_NEED_NET (1U << 0) +/* Indicate that it the event is enabled, it event needs to be run once before + * it becomes disabled. + */ +#define PERIODIC_EVENT_FLAG_FLUSH_ON_DISABLE (1U << 1) + /** Callback function for a periodic event to take action. The return value * influences the next time the function will get called. Return * PERIODIC_EVENT_NO_UPDATE to not update last_action_time and be polled @@ -83,6 +88,6 @@ void periodic_event_destroy(periodic_event_item_t *event); void periodic_event_reschedule(periodic_event_item_t *event); void periodic_event_enable(periodic_event_item_t *event); void periodic_event_disable(periodic_event_item_t *event); +void periodic_event_flush_and_disable(periodic_event_item_t *event); #endif /* !defined(TOR_PERIODIC_H) */ - From b9a88bd53ae79a29c292275381bc7dbaa3804034 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 6 Nov 2018 07:27:31 -0500 Subject: [PATCH 0150/2557] Add new "ALL" and "NET_PARTICIPANT" roles for periodic events The previous "ALL" role was the OR of a bunch of other roles, which is a mistake: it's better if "ALL" means "all". The "NET_PARTICIPANT" role refers to the anything that is actively building circuits, downloading directory information, and participating in the Tor network. For now, it is set to !net_is_disabled(), but we're going to use it to implement a new "extra dormant mode". Closes ticket 28336. --- src/core/mainloop/mainloop.c | 36 ++++++++++++++++++------- src/core/mainloop/periodic.h | 7 +++-- src/test/test_periodic_event.c | 49 +++++++++++++++++++++------------- 3 files changed, 60 insertions(+), 32 deletions(-) diff --git a/src/core/mainloop/mainloop.c b/src/core/mainloop/mainloop.c index 42f6fb50c7..3c3f441a91 100644 --- a/src/core/mainloop/mainloop.c +++ b/src/core/mainloop/mainloop.c @@ -1377,16 +1377,27 @@ CALLBACK(write_stats_file); STATIC periodic_event_item_t periodic_events[] = { /* Everyone needs to run those. */ CALLBACK(add_entropy, ALL, 0), - CALLBACK(check_expired_networkstatus, ALL, 0), - CALLBACK(clean_caches, ALL, 0), - CALLBACK(fetch_networkstatus, ALL, 0), CALLBACK(heartbeat, ALL, 0), - CALLBACK(launch_descriptor_fetches, ALL, FL(NEED_NET)), - CALLBACK(reset_padding_counts, ALL, 0), + + /* XXXX Do we have a reason to do this on a callback? */ CALLBACK(retry_listeners, ALL, FL(NEED_NET)), - CALLBACK(save_state, ALL, 0), - CALLBACK(rotate_x509_certificate, ALL, 0), - CALLBACK(write_stats_file, ALL, 0), + + /* We need to do these if we're participating in the Tor network. */ + CALLBACK(check_expired_networkstatus, NET_PARTICIPANT, 0), + CALLBACK(fetch_networkstatus, NET_PARTICIPANT, 0), + CALLBACK(launch_descriptor_fetches, NET_PARTICIPANT, FL(NEED_NET)), + CALLBACK(rotate_x509_certificate, NET_PARTICIPANT, 0), + + /* We need to do these if we're participating in the Tor network, and + * immediately before we stop. */ + CALLBACK(clean_caches, NET_PARTICIPANT, FL(FLUSH_ON_DISABLE)), + CALLBACK(save_state, NET_PARTICIPANT, FL(FLUSH_ON_DISABLE)), + + /* XXXX investigate this. */ + CALLBACK(write_stats_file, NET_PARTICIPANT, FL(FLUSH_ON_DISABLE)), + + /* XXXX investigate this. */ + CALLBACK(reset_padding_counts, NET_PARTICIPANT, FL(FLUSH_ON_DISABLE)), /* Routers (bridge and relay) only. */ CALLBACK(check_descriptor, ROUTER, FL(NEED_NET)), @@ -1418,7 +1429,8 @@ STATIC periodic_event_item_t periodic_events[] = { CALLBACK(record_bridge_stats, BRIDGE, 0), /* Client only. */ - CALLBACK(rend_cache_failure_clean, CLIENT, 0), + /* XXXX this could be restricted to CLIENT even. */ + CALLBACK(rend_cache_failure_clean, NET_PARTICIPANT, FL(FLUSH_ON_DISABLE)), /* Bridge Authority only. */ CALLBACK(write_bridge_ns, BRIDGEAUTH, 0), @@ -1477,7 +1489,7 @@ get_my_roles(const or_options_t *options) { tor_assert(options); - int roles = 0; + int roles = PERIODIC_EVENT_ROLE_ALL; int is_bridge = options->BridgeRelay; int is_relay = server_mode(options); int is_dirauth = authdir_mode_v3(options); @@ -1492,6 +1504,9 @@ get_my_roles(const or_options_t *options) options->ControlPort_set || options->OwningControllerFD != UINT64_MAX; + /* We actually want a better definition here for our work on dormancy. */ + int is_net_participant = ! net_is_disabled(); + if (is_bridge) roles |= PERIODIC_EVENT_ROLE_BRIDGE; if (is_client) roles |= PERIODIC_EVENT_ROLE_CLIENT; if (is_relay) roles |= PERIODIC_EVENT_ROLE_RELAY; @@ -1499,6 +1514,7 @@ get_my_roles(const or_options_t *options) if (is_bridgeauth) roles |= PERIODIC_EVENT_ROLE_BRIDGEAUTH; if (is_hidden_service) roles |= PERIODIC_EVENT_ROLE_HS_SERVICE; if (is_dirserver) roles |= PERIODIC_EVENT_ROLE_DIRSERVER; + if (is_net_participant) roles |= PERIODIC_EVENT_ROLE_NET_PARTICIPANT; return roles; } diff --git a/src/core/mainloop/periodic.h b/src/core/mainloop/periodic.h index 7c71be7bc4..23459ff2b3 100644 --- a/src/core/mainloop/periodic.h +++ b/src/core/mainloop/periodic.h @@ -16,6 +16,9 @@ #define PERIODIC_EVENT_ROLE_HS_SERVICE (1U << 5) #define PERIODIC_EVENT_ROLE_DIRSERVER (1U << 6) +#define PERIODIC_EVENT_ROLE_NET_PARTICIPANT (1U << 7) +#define PERIODIC_EVENT_ROLE_ALL (1U << 8) + /* Helper macro to make it a bit less annoying to defined groups of roles that * are often used. */ @@ -25,10 +28,6 @@ /* Authorities that is both bridge and directory. */ #define PERIODIC_EVENT_ROLE_AUTHORITIES \ (PERIODIC_EVENT_ROLE_BRIDGEAUTH | PERIODIC_EVENT_ROLE_DIRAUTH) -/* All roles. */ -#define PERIODIC_EVENT_ROLE_ALL \ - (PERIODIC_EVENT_ROLE_AUTHORITIES | PERIODIC_EVENT_ROLE_CLIENT | \ - PERIODIC_EVENT_ROLE_HS_SERVICE | PERIODIC_EVENT_ROLE_ROUTER) /* * Event flags which can change the behavior of an event. diff --git a/src/test/test_periodic_event.c b/src/test/test_periodic_event.c index 86dedd85d8..f63adf8e3a 100644 --- a/src/test/test_periodic_event.c +++ b/src/test/test_periodic_event.c @@ -19,6 +19,7 @@ #include "feature/hibernate/hibernate.h" #include "feature/hs/hs_service.h" #include "core/mainloop/mainloop.h" +#include "core/mainloop/netstatus.h" #include "core/mainloop/periodic.h" /** Helper function: This is replaced in some tests for the event callbacks so @@ -59,7 +60,9 @@ test_pe_initialize(void *arg) tt_u64_op(item->last_action_time, OP_EQ, 0); /* Every event must have role(s) assign to it. This is done statically. */ tt_u64_op(item->roles, OP_NE, 0); - tt_uint_op(periodic_event_is_enabled(item), OP_EQ, 0); + int should_be_enabled = (item->roles & PERIODIC_EVENT_ROLE_ALL) && + !(item->flags & PERIODIC_EVENT_FLAG_NEED_NET); + tt_uint_op(periodic_event_is_enabled(item), OP_EQ, should_be_enabled); } done: @@ -106,13 +109,12 @@ test_pe_launch(void *arg) /* Now that we've initialized, rescan the list to launch. */ periodic_events_on_new_options(options); + int mask = PERIODIC_EVENT_ROLE_CLIENT|PERIODIC_EVENT_ROLE_ALL| + PERIODIC_EVENT_ROLE_NET_PARTICIPANT; for (int i = 0; periodic_events[i].name; ++i) { periodic_event_item_t *item = &periodic_events[i]; - if (item->roles & PERIODIC_EVENT_ROLE_CLIENT) { - tt_int_op(periodic_event_is_enabled(item), OP_EQ, 1); - } else { - tt_int_op(periodic_event_is_enabled(item), OP_EQ, 0); - } + int should_be_enabled = !!(item->roles & mask); + tt_int_op(periodic_event_is_enabled(item), OP_EQ, should_be_enabled); // enabled or not, the event has not yet been run. tt_u64_op(item->last_action_time, OP_EQ, 0); } @@ -124,7 +126,8 @@ test_pe_launch(void *arg) unsigned roles = get_my_roles(options); tt_uint_op(roles, OP_EQ, - PERIODIC_EVENT_ROLE_RELAY|PERIODIC_EVENT_ROLE_DIRSERVER); + PERIODIC_EVENT_ROLE_RELAY|PERIODIC_EVENT_ROLE_DIRSERVER| + PERIODIC_EVENT_ROLE_ALL|PERIODIC_EVENT_ROLE_NET_PARTICIPANT); for (int i = 0; periodic_events[i].name; ++i) { periodic_event_item_t *item = &periodic_events[i]; @@ -144,17 +147,21 @@ test_pe_launch(void *arg) /* Disable everything and we'll enable them ALL. */ options->SocksPort_set = 0; options->ORPort_set = 0; + options->DisableNetwork = 1; periodic_events_on_new_options(options); for (int i = 0; periodic_events[i].name; ++i) { periodic_event_item_t *item = &periodic_events[i]; - tt_int_op(periodic_event_is_enabled(item), OP_EQ, 0); + int should_be_enabled = (item->roles & PERIODIC_EVENT_ROLE_ALL) && + !(item->flags & PERIODIC_EVENT_FLAG_NEED_NET); + tt_int_op(periodic_event_is_enabled(item), OP_EQ, should_be_enabled); } /* Enable everything. */ options->SocksPort_set = 1; options->ORPort_set = 1; options->BridgeRelay = 1; options->AuthoritativeDir = 1; options->V3AuthoritativeDir = 1; options->BridgeAuthoritativeDir = 1; + options->DisableNetwork = 0; register_dummy_hidden_service(&service); periodic_events_on_new_options(options); /* Note down the reference because we need to remove this service from the @@ -188,41 +195,46 @@ test_pe_get_roles(void *arg) or_options_t *options = get_options_mutable(); tt_assert(options); + const int ALL = PERIODIC_EVENT_ROLE_ALL; + /* Nothing configured, should be no roles. */ + tt_assert(net_is_disabled()); roles = get_my_roles(options); - tt_int_op(roles, OP_EQ, 0); + tt_int_op(roles, OP_EQ, ALL); /* Indicate we have a SocksPort, roles should be come Client. */ options->SocksPort_set = 1; roles = get_my_roles(options); - tt_int_op(roles, OP_EQ, PERIODIC_EVENT_ROLE_CLIENT); + tt_int_op(roles, OP_EQ, PERIODIC_EVENT_ROLE_CLIENT|ALL); /* Now, we'll add a ORPort so should now be a Relay + Client. */ options->ORPort_set = 1; roles = get_my_roles(options); tt_int_op(roles, OP_EQ, (PERIODIC_EVENT_ROLE_CLIENT | PERIODIC_EVENT_ROLE_RELAY | - PERIODIC_EVENT_ROLE_DIRSERVER)); + PERIODIC_EVENT_ROLE_DIRSERVER | ALL)); /* Now add a Bridge. */ options->BridgeRelay = 1; roles = get_my_roles(options); tt_int_op(roles, OP_EQ, (PERIODIC_EVENT_ROLE_CLIENT | PERIODIC_EVENT_ROLE_RELAY | - PERIODIC_EVENT_ROLE_BRIDGE | PERIODIC_EVENT_ROLE_DIRSERVER)); + PERIODIC_EVENT_ROLE_BRIDGE | PERIODIC_EVENT_ROLE_DIRSERVER | + ALL)); tt_assert(roles & PERIODIC_EVENT_ROLE_ROUTER); /* Unset client so we can solely test Router role. */ options->SocksPort_set = 0; roles = get_my_roles(options); tt_int_op(roles, OP_EQ, - PERIODIC_EVENT_ROLE_ROUTER | PERIODIC_EVENT_ROLE_DIRSERVER); + PERIODIC_EVENT_ROLE_ROUTER | PERIODIC_EVENT_ROLE_DIRSERVER | + ALL); /* Reset options so we can test authorities. */ options->SocksPort_set = 0; options->ORPort_set = 0; options->BridgeRelay = 0; roles = get_my_roles(options); - tt_int_op(roles, OP_EQ, 0); + tt_int_op(roles, OP_EQ, ALL); /* Now upgrade to Dirauth. */ options->DirPort_set = 1; @@ -230,7 +242,7 @@ test_pe_get_roles(void *arg) options->V3AuthoritativeDir = 1; roles = get_my_roles(options); tt_int_op(roles, OP_EQ, - PERIODIC_EVENT_ROLE_DIRAUTH|PERIODIC_EVENT_ROLE_DIRSERVER); + PERIODIC_EVENT_ROLE_DIRAUTH|PERIODIC_EVENT_ROLE_DIRSERVER|ALL); tt_assert(roles & PERIODIC_EVENT_ROLE_AUTHORITIES); /* Now Bridge Authority. */ @@ -238,7 +250,7 @@ test_pe_get_roles(void *arg) options->BridgeAuthoritativeDir = 1; roles = get_my_roles(options); tt_int_op(roles, OP_EQ, - PERIODIC_EVENT_ROLE_BRIDGEAUTH|PERIODIC_EVENT_ROLE_DIRSERVER); + PERIODIC_EVENT_ROLE_BRIDGEAUTH|PERIODIC_EVENT_ROLE_DIRSERVER|ALL); tt_assert(roles & PERIODIC_EVENT_ROLE_AUTHORITIES); /* Move that bridge auth to become a relay. */ @@ -246,7 +258,7 @@ test_pe_get_roles(void *arg) roles = get_my_roles(options); tt_int_op(roles, OP_EQ, (PERIODIC_EVENT_ROLE_BRIDGEAUTH | PERIODIC_EVENT_ROLE_RELAY - | PERIODIC_EVENT_ROLE_DIRSERVER)); + | PERIODIC_EVENT_ROLE_DIRSERVER|ALL)); tt_assert(roles & PERIODIC_EVENT_ROLE_AUTHORITIES); /* And now an Hidden service. */ @@ -257,7 +269,8 @@ test_pe_get_roles(void *arg) remove_service(get_hs_service_map(), &service); tt_int_op(roles, OP_EQ, (PERIODIC_EVENT_ROLE_BRIDGEAUTH | PERIODIC_EVENT_ROLE_RELAY | - PERIODIC_EVENT_ROLE_HS_SERVICE | PERIODIC_EVENT_ROLE_DIRSERVER)); + PERIODIC_EVENT_ROLE_HS_SERVICE | PERIODIC_EVENT_ROLE_DIRSERVER | + ALL)); tt_assert(roles & PERIODIC_EVENT_ROLE_AUTHORITIES); done: From db53bfe8f74dad1b45ba381a5ee3366148a30237 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 6 Nov 2018 11:14:50 -0500 Subject: [PATCH 0151/2557] Annotate 1/s callback elements with NET_PARTICIPANT status. --- src/core/mainloop/mainloop.c | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/src/core/mainloop/mainloop.c b/src/core/mainloop/mainloop.c index 3c3f441a91..7e5e5d0efb 100644 --- a/src/core/mainloop/mainloop.c +++ b/src/core/mainloop/mainloop.c @@ -1379,8 +1379,9 @@ STATIC periodic_event_item_t periodic_events[] = { CALLBACK(add_entropy, ALL, 0), CALLBACK(heartbeat, ALL, 0), - /* XXXX Do we have a reason to do this on a callback? */ - CALLBACK(retry_listeners, ALL, FL(NEED_NET)), + /* XXXX Do we have a reason to do this on a callback? Does it do any good at + * all? For now, if we're dormant, we can let our listeners decay. */ + CALLBACK(retry_listeners, NET_PARTICIPANT, FL(NEED_NET)), /* We need to do these if we're participating in the Tor network. */ CALLBACK(check_expired_networkstatus, NET_PARTICIPANT, 0), @@ -1393,10 +1394,10 @@ STATIC periodic_event_item_t periodic_events[] = { CALLBACK(clean_caches, NET_PARTICIPANT, FL(FLUSH_ON_DISABLE)), CALLBACK(save_state, NET_PARTICIPANT, FL(FLUSH_ON_DISABLE)), - /* XXXX investigate this. */ + /* XXXX investigate this. ??? */ CALLBACK(write_stats_file, NET_PARTICIPANT, FL(FLUSH_ON_DISABLE)), - /* XXXX investigate this. */ + /* XXXX investigate this. ???? */ CALLBACK(reset_padding_counts, NET_PARTICIPANT, FL(FLUSH_ON_DISABLE)), /* Routers (bridge and relay) only. */ @@ -1423,13 +1424,13 @@ STATIC periodic_event_item_t periodic_events[] = { CALLBACK(check_dns_honesty, RELAY, FL(NEED_NET)), /* Hidden Service service only. */ - CALLBACK(hs_service, HS_SERVICE, FL(NEED_NET)), + CALLBACK(hs_service, HS_SERVICE, FL(NEED_NET)), // XXXX break this down more /* Bridge only. */ CALLBACK(record_bridge_stats, BRIDGE, 0), /* Client only. */ - /* XXXX this could be restricted to CLIENT even. */ + /* XXXX this could be restricted to CLIENT+NET_PARTICIPANT */ CALLBACK(rend_cache_failure_clean, NET_PARTICIPANT, FL(FLUSH_ON_DISABLE)), /* Bridge Authority only. */ @@ -1730,18 +1731,24 @@ run_scheduled_events(time_t now) * expired; or if our bandwidth limits are exhausted and we * should hibernate; or if it's time to wake up from hibernation. */ + // TODO: Refactor or rewrite, or NET_PARTICIPANT. Needs separate wakeup + // handling. consider_hibernation(now); /* Maybe enough time elapsed for us to reconsider a circuit. */ + // TODO: NET_PARTICIPANT circuit_upgrade_circuits_from_guard_wait(); if (options->UseBridges && !net_is_disabled()) { /* Note: this check uses net_is_disabled(), not should_delay_dir_fetches() * -- the latter is only for fetching consensus-derived directory info. */ + // TODO: client+NET_PARTICIPANT. + // Also, schedule this rather than probing 1x / sec fetch_bridge_descriptors(options, now); } if (accounting_is_enabled(options)) { + // TODO: refactor or rewrite; NET_PARTICIPANT accounting_run_housekeeping(now); } @@ -1752,6 +1759,7 @@ run_scheduled_events(time_t now) */ /* (If our circuit build timeout can ever become lower than a second (which * it can't, currently), we should do this more often.) */ + // TODO: All expire stuff can become NET_PARTICIPANT, FLUSH_ON_DISABLE circuit_expire_building(); circuit_expire_waiting_for_better_guard(); @@ -1760,10 +1768,12 @@ run_scheduled_events(time_t now) * Do this before step 4, so we can put them back into pending * state to be picked up by the new circuit. */ + // TODO: All expire stuff can become NET_PARTICIPANT, FLUSH_ON_DISABLE connection_ap_expire_beginning(); /* 3c. And expire connections that we've held open for too long. */ + // TODO: All expire stuff can become NET_PARTICIPANT, FLUSH_ON_DISABLE connection_expire_held_open(); /* 4. Every second, we try a new circuit if there are no valid @@ -1773,19 +1783,24 @@ run_scheduled_events(time_t now) */ const int have_dir_info = router_have_minimum_dir_info(); if (have_dir_info && !net_is_disabled()) { + // TODO: NET_PARTICIPANT. circuit_build_needed_circs(now); } else { + // TODO: NET_PARTICIPANT, FLUSH_ON_DISABLE circuit_expire_old_circs_as_needed(now); } /* 5. We do housekeeping for each connection... */ + // TODO: NET_PARTICIPANT channel_update_bad_for_new_circs(NULL, 0); int i; for (i=0;i Date: Tue, 13 Nov 2018 08:22:58 -0500 Subject: [PATCH 0152/2557] Move control_per_second_events() into a callback with its own role Part of making extra-dormant mode work; closes ticket 28421. --- src/core/mainloop/mainloop.c | 29 +++++++++++++++++++++++------ src/core/mainloop/periodic.h | 5 +++-- src/feature/control/control.c | 2 +- src/test/test_periodic_event.c | 3 ++- 4 files changed, 29 insertions(+), 10 deletions(-) diff --git a/src/core/mainloop/mainloop.c b/src/core/mainloop/mainloop.c index 7e5e5d0efb..a9d5d8155a 100644 --- a/src/core/mainloop/mainloop.c +++ b/src/core/mainloop/mainloop.c @@ -1366,6 +1366,7 @@ CALLBACK(save_stability); CALLBACK(save_state); CALLBACK(write_bridge_ns); CALLBACK(write_stats_file); +CALLBACK(control_per_second_events); #undef CALLBACK @@ -1439,6 +1440,9 @@ STATIC periodic_event_item_t periodic_events[] = { /* Directory server only. */ CALLBACK(clean_consdiffmgr, DIRSERVER, 0), + /* Controller with per-second events only. */ + CALLBACK(control_per_second_events, CONTROLEV, 0), + END_OF_PERIODIC_EVENTS }; #undef CALLBACK @@ -1498,6 +1502,8 @@ get_my_roles(const or_options_t *options) int is_hidden_service = !!hs_service_get_num_services() || !!rend_num_services(); int is_dirserver = dir_server_mode(options); + int sending_control_events = control_any_per_second_event_enabled(); + /* We also consider tor to have the role of a client if the ControlPort is * set because a lot of things can be done over the control port which * requires tor to have basic functionnalities. */ @@ -1516,6 +1522,7 @@ get_my_roles(const or_options_t *options) if (is_hidden_service) roles |= PERIODIC_EVENT_ROLE_HS_SERVICE; if (is_dirserver) roles |= PERIODIC_EVENT_ROLE_DIRSERVER; if (is_net_participant) roles |= PERIODIC_EVENT_ROLE_NET_PARTICIPANT; + if (sending_control_events) roles |= PERIODIC_EVENT_ROLE_CONTROLEV; return roles; } @@ -2524,6 +2531,21 @@ hs_service_callback(time_t now, const or_options_t *options) return 1; } +/* + * Periodic callback: Send once-per-second events to the controller(s). + * This is called every second. + */ +static int +control_per_second_events_callback(time_t now, const or_options_t *options) +{ + (void) options; + (void) now; + + control_per_second_events(); + + return 1; +} + /** Timer: used to invoke second_elapsed_callback() once per second. */ static periodic_timer_t *second_timer = NULL; @@ -2546,8 +2568,7 @@ reschedule_per_second_timer(void) tor_assert(second_timer); } - const bool run_per_second_events = - control_any_per_second_event_enabled() || ! net_is_completely_disabled(); + const bool run_per_second_events = ! net_is_completely_disabled(); if (run_per_second_events) { periodic_timer_launch(second_timer, &one_second); @@ -2640,10 +2661,6 @@ second_elapsed_callback(periodic_timer_t *timer, void *arg) */ update_current_time(now); - // TODO XXXX Turn this into a separate event. - /* Maybe some controller events are ready to fire */ - control_per_second_events(); - run_scheduled_events(now); } diff --git a/src/core/mainloop/periodic.h b/src/core/mainloop/periodic.h index 23459ff2b3..52d5450ee8 100644 --- a/src/core/mainloop/periodic.h +++ b/src/core/mainloop/periodic.h @@ -15,9 +15,10 @@ #define PERIODIC_EVENT_ROLE_BRIDGEAUTH (1U << 4) #define PERIODIC_EVENT_ROLE_HS_SERVICE (1U << 5) #define PERIODIC_EVENT_ROLE_DIRSERVER (1U << 6) +#define PERIODIC_EVENT_ROLE_CONTROLEV (1U << 7) -#define PERIODIC_EVENT_ROLE_NET_PARTICIPANT (1U << 7) -#define PERIODIC_EVENT_ROLE_ALL (1U << 8) +#define PERIODIC_EVENT_ROLE_NET_PARTICIPANT (1U << 8) +#define PERIODIC_EVENT_ROLE_ALL (1U << 9) /* Helper macro to make it a bit less annoying to defined groups of roles that * are often used. */ diff --git a/src/feature/control/control.c b/src/feature/control/control.c index b31b448e96..a5b6ab3bf5 100644 --- a/src/feature/control/control.c +++ b/src/feature/control/control.c @@ -368,7 +368,7 @@ control_update_global_event_mask(void) control_get_bytes_rw_last_sec(&r, &w); } if (any_old_per_sec_events != control_any_per_second_event_enabled()) { - reschedule_per_second_timer(); + rescan_periodic_events(get_options()); } #undef NEWLY_ENABLED diff --git a/src/test/test_periodic_event.c b/src/test/test_periodic_event.c index f63adf8e3a..6a3e320b2e 100644 --- a/src/test/test_periodic_event.c +++ b/src/test/test_periodic_event.c @@ -172,7 +172,8 @@ test_pe_launch(void *arg) for (int i = 0; periodic_events[i].name; ++i) { periodic_event_item_t *item = &periodic_events[i]; - tt_int_op(periodic_event_is_enabled(item), OP_EQ, 1); + tt_int_op(periodic_event_is_enabled(item), OP_EQ, + (item->roles != PERIODIC_EVENT_ROLE_CONTROLEV)); } done: From e535ec8542a1d42243c0b6ae28036aec8262269b Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 13 Nov 2018 08:36:38 -0500 Subject: [PATCH 0153/2557] Remove run_scheduled_events() as a separate function. (There was nothing else in second_elapsed_callbck() that couldn't go here.) --- src/core/mainloop/mainloop.c | 35 +++++++++++------------------------ 1 file changed, 11 insertions(+), 24 deletions(-) diff --git a/src/core/mainloop/mainloop.c b/src/core/mainloop/mainloop.c index a9d5d8155a..ebf9735d49 100644 --- a/src/core/mainloop/mainloop.c +++ b/src/core/mainloop/mainloop.c @@ -1727,13 +1727,22 @@ safe_timer_diff(time_t now, time_t next) } /** Perform regular maintenance tasks. This function gets run once per - * second by second_elapsed_callback(). + * second. */ static void -run_scheduled_events(time_t now) +second_elapsed_callback(periodic_timer_t *timer, void *arg) { + (void) timer; + (void) arg; + const time_t now = time(NULL); const or_options_t *options = get_options(); + /* We don't need to do this once-per-second any more: time-updating is + * only in this callback _because it is a callback_. It should be fine + * to disable this callback, and the time will still get updated. + */ + update_current_time(now); + /* 0. See if we've been asked to shut down and our timeout has * expired; or if our bandwidth limits are exhausted and we * should hibernate; or if it's time to wake up from hibernation. @@ -2642,28 +2651,6 @@ update_current_time(time_t now) current_second = now; } -/** Libevent callback: invoked once every second. */ -static void -second_elapsed_callback(periodic_timer_t *timer, void *arg) -{ - /* XXXX This could be sensibly refactored into multiple callbacks, and we - * could use Libevent's timers for this rather than checking the current - * time against a bunch of timeouts every second. */ - time_t now; - (void)timer; - (void)arg; - - now = time(NULL); - - /* We don't need to do this once-per-second any more: time-updating is - * only in this callback _because it is a callback_. It should be fine - * to disable this callback, and the time will still get updated. - */ - update_current_time(now); - - run_scheduled_events(now); -} - #ifdef HAVE_SYSTEMD_209 static periodic_timer_t *systemd_watchdog_timer = NULL; From 303e5c70e08d72614ea95fd0c5aad7e05852b6b7 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 13 Nov 2018 09:04:11 -0500 Subject: [PATCH 0154/2557] Move the responsibility for delayed shutdown into the mainloop This is part of 28422, so we don't have to call consider_hibernation() once per second when we're dormant. This commit does not remove delayed shutdown from hibernate.c: it uses it as a backup shutdown mechanism, in case the regular shutdown timer mechanism fails for some reason. --- src/core/mainloop/mainloop.c | 36 ++++++++++++++++++++++++++----- src/core/mainloop/mainloop.h | 2 ++ src/feature/hibernate/hibernate.c | 20 +++++++++++------ 3 files changed, 47 insertions(+), 11 deletions(-) diff --git a/src/core/mainloop/mainloop.c b/src/core/mainloop/mainloop.c index ebf9735d49..d5f3cb13f9 100644 --- a/src/core/mainloop/mainloop.c +++ b/src/core/mainloop/mainloop.c @@ -1703,6 +1703,30 @@ mainloop_schedule_postloop_cleanup(void) mainloop_event_activate(postloop_cleanup_ev); } +/** Event to run 'scheduled_shutdown_cb' */ +static mainloop_event_t *scheduled_shutdown_ev=NULL; + +/** Callback: run a scheduled shutdown */ +static void +scheduled_shutdown_cb(mainloop_event_t *ev, void *arg) +{ + (void)ev; + (void)arg; + log_notice(LD_GENERAL, "Clean shutdown finished. Exiting."); + tor_shutdown_event_loop_and_exit(0); +} + +/** Schedule the mainloop to exit after delay_sec seconds. */ +void +mainloop_schedule_shutdown(int delay_sec) +{ + const struct timeval delay_tv = { delay_sec, 0 }; + if (! scheduled_shutdown_ev) { + scheduled_shutdown_ev = mainloop_event_new(scheduled_shutdown_cb, NULL); + } + mainloop_event_schedule(scheduled_shutdown_ev, &delay_tv); +} + #define LONGEST_TIMER_PERIOD (30 * 86400) /** Helper: Return the number of seconds between now and next, * clipped to the range [1 second, LONGEST_TIMER_PERIOD]. */ @@ -1743,12 +1767,13 @@ second_elapsed_callback(periodic_timer_t *timer, void *arg) */ update_current_time(now); - /* 0. See if we've been asked to shut down and our timeout has - * expired; or if our bandwidth limits are exhausted and we - * should hibernate; or if it's time to wake up from hibernation. + /* 0. See if our bandwidth limits are exhausted and we should hibernate; or + * if it's time to wake up from hibernation; or if we have a scheduled + * shutdown and it's time to run it. + * + * Note: we have redundant mechanisms to handle the */ - // TODO: Refactor or rewrite, or NET_PARTICIPANT. Needs separate wakeup - // handling. + // TODO: NET_PARTICIPANT. consider_hibernation(now); /* Maybe enough time elapsed for us to reconsider a circuit. */ @@ -2936,6 +2961,7 @@ tor_mainloop_free_all(void) mainloop_event_free(schedule_active_linked_connections_event); mainloop_event_free(postloop_cleanup_ev); mainloop_event_free(handle_deferred_signewnym_ev); + mainloop_event_free(scheduled_shutdown_ev); #ifdef HAVE_SYSTEMD_209 periodic_timer_free(systemd_watchdog_timer); diff --git a/src/core/mainloop/mainloop.h b/src/core/mainloop/mainloop.h index 632733d9a6..6f7b716858 100644 --- a/src/core/mainloop/mainloop.h +++ b/src/core/mainloop/mainloop.h @@ -86,6 +86,8 @@ void reschedule_per_second_timer(void); void do_signewnym(time_t); time_t get_last_signewnym_time(void); +void mainloop_schedule_shutdown(int delay_sec); + void tor_init_connection_lists(void); void initialize_mainloop_events(void); void tor_mainloop_free_all(void); diff --git a/src/feature/hibernate/hibernate.c b/src/feature/hibernate/hibernate.c index 6f8795cecc..968c39dd6d 100644 --- a/src/feature/hibernate/hibernate.c +++ b/src/feature/hibernate/hibernate.c @@ -66,8 +66,9 @@ static hibernate_state_t hibernate_state = HIBERNATE_STATE_INITIAL; /** If are hibernating, when do we plan to wake up? Set to 0 if we * aren't hibernating. */ static time_t hibernate_end_time = 0; -/** If we are shutting down, when do we plan finally exit? Set to 0 if - * we aren't shutting down. */ +/** If we are shutting down, when do we plan finally exit? Set to 0 if we + * aren't shutting down. (This is obsolete; scheduled shutdowns are supposed + * to happen from mainloop_schedule_shutdown() now.) */ static time_t shutdown_time = 0; /** A timed event that we'll use when it's time to wake up from @@ -867,7 +868,13 @@ hibernate_begin(hibernate_state_t new_state, time_t now) log_notice(LD_GENERAL,"Interrupt: we have stopped accepting new " "connections, and will shut down in %d seconds. Interrupt " "again to exit now.", options->ShutdownWaitLength); - shutdown_time = time(NULL) + options->ShutdownWaitLength; + /* We add an arbitrary delay here so that even if something goes wrong + * with the mainloop shutdown code, we can still shutdown from + * consider_hibernation() if we call it... but so that the + * mainloop_schedule_shutdown() mechanism will be the first one called. + */ + shutdown_time = time(NULL) + options->ShutdownWaitLength + 5; + mainloop_schedule_shutdown(options->ShutdownWaitLength); #ifdef HAVE_SYSTEMD /* tell systemd that we may need more than the default 90 seconds to shut * down so they don't kill us. add some extra time to actually finish @@ -1096,11 +1103,12 @@ consider_hibernation(time_t now) hibernate_state_t prev_state = hibernate_state; /* If we're in 'exiting' mode, then we just shut down after the interval - * elapses. */ + * elapses. The mainloop was supposed to catch this via + * mainloop_schedule_shutdown(), but apparently it didn't. */ if (hibernate_state == HIBERNATE_STATE_EXITING) { tor_assert(shutdown_time); if (shutdown_time <= now) { - log_notice(LD_GENERAL, "Clean shutdown finished. Exiting."); + log_notice(LD_BUG, "Mainloop did not catch shutdown event; exiting."); tor_shutdown_event_loop_and_exit(0); } return; /* if exiting soon, don't worry about bandwidth limits */ @@ -1112,7 +1120,7 @@ consider_hibernation(time_t now) if (hibernate_end_time > now && accounting_enabled) { /* If we're hibernating, don't wake up until it's time, regardless of * whether we're in a new interval. */ - return ; + return; } else { hibernate_end_time_elapsed(now); } From 4bf79fa4fa99fe66f6b1ad413cbf475325803e04 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 13 Nov 2018 09:30:51 -0500 Subject: [PATCH 0155/2557] Turn second_elapsed_callback into a normal periodic event. --- src/app/config/config.c | 3 -- src/core/mainloop/mainloop.c | 81 +++++++------------------------ src/core/mainloop/mainloop.h | 1 - src/feature/hibernate/hibernate.c | 2 - 4 files changed, 17 insertions(+), 70 deletions(-) diff --git a/src/app/config/config.c b/src/app/config/config.c index 45a23d67d5..8aa0c1f4bd 100644 --- a/src/app/config/config.c +++ b/src/app/config/config.c @@ -1993,9 +1993,6 @@ options_act(const or_options_t *old_options) finish_daemon(options->DataDirectory); } - /* See whether we need to enable/disable our once-a-second timer. */ - reschedule_per_second_timer(); - /* We want to reinit keys as needed before we do much of anything else: keys are important, and other things can depend on them. */ if (transition_affects_workers || diff --git a/src/core/mainloop/mainloop.c b/src/core/mainloop/mainloop.c index d5f3cb13f9..176399b333 100644 --- a/src/core/mainloop/mainloop.c +++ b/src/core/mainloop/mainloop.c @@ -205,7 +205,6 @@ static void connection_start_reading_from_linked_conn(connection_t *conn); static int connection_should_read_from_linked_conn(connection_t *conn); static void conn_read_callback(evutil_socket_t fd, short event, void *_conn); static void conn_write_callback(evutil_socket_t fd, short event, void *_conn); -static void second_elapsed_callback(periodic_timer_t *timer, void *args); static void shutdown_did_not_work_callback(evutil_socket_t fd, short event, void *arg) ATTR_NORETURN; @@ -1367,6 +1366,7 @@ CALLBACK(save_state); CALLBACK(write_bridge_ns); CALLBACK(write_stats_file); CALLBACK(control_per_second_events); +CALLBACK(second_elapsed); #undef CALLBACK @@ -1380,6 +1380,11 @@ STATIC periodic_event_item_t periodic_events[] = { CALLBACK(add_entropy, ALL, 0), CALLBACK(heartbeat, ALL, 0), + /* This is a legacy catch-all callback that runs once per second if + * we are online and active. */ + CALLBACK(second_elapsed, NET_PARTICIPANT, + FL(NEED_NET)|FL(FLUSH_ON_DISABLE)), + /* XXXX Do we have a reason to do this on a callback? Does it do any good at * all? For now, if we're dormant, we can let our listeners decay. */ CALLBACK(retry_listeners, NET_PARTICIPANT, FL(NEED_NET)), @@ -1753,43 +1758,30 @@ safe_timer_diff(time_t now, time_t next) /** Perform regular maintenance tasks. This function gets run once per * second. */ -static void -second_elapsed_callback(periodic_timer_t *timer, void *arg) +static int +second_elapsed_callback(time_t now, const or_options_t *options) { - (void) timer; - (void) arg; - const time_t now = time(NULL); - const or_options_t *options = get_options(); - - /* We don't need to do this once-per-second any more: time-updating is - * only in this callback _because it is a callback_. It should be fine - * to disable this callback, and the time will still get updated. - */ - update_current_time(now); - - /* 0. See if our bandwidth limits are exhausted and we should hibernate; or - * if it's time to wake up from hibernation; or if we have a scheduled - * shutdown and it's time to run it. + /* 0. See if our bandwidth limits are exhausted and we should hibernate * - * Note: we have redundant mechanisms to handle the + * Note: we have redundant mechanisms to handle the case where it's + * time to wake up from hibernation; or where we have a scheduled + * shutdown and it's time to run it, but this will also handle those. */ - // TODO: NET_PARTICIPANT. consider_hibernation(now); /* Maybe enough time elapsed for us to reconsider a circuit. */ - // TODO: NET_PARTICIPANT circuit_upgrade_circuits_from_guard_wait(); if (options->UseBridges && !net_is_disabled()) { /* Note: this check uses net_is_disabled(), not should_delay_dir_fetches() * -- the latter is only for fetching consensus-derived directory info. */ - // TODO: client+NET_PARTICIPANT. + // TODO: client // Also, schedule this rather than probing 1x / sec fetch_bridge_descriptors(options, now); } if (accounting_is_enabled(options)) { - // TODO: refactor or rewrite; NET_PARTICIPANT + // TODO: refactor or rewrite? accounting_run_housekeeping(now); } @@ -1809,12 +1801,10 @@ second_elapsed_callback(periodic_timer_t *timer, void *arg) * Do this before step 4, so we can put them back into pending * state to be picked up by the new circuit. */ - // TODO: All expire stuff can become NET_PARTICIPANT, FLUSH_ON_DISABLE connection_ap_expire_beginning(); /* 3c. And expire connections that we've held open for too long. */ - // TODO: All expire stuff can become NET_PARTICIPANT, FLUSH_ON_DISABLE connection_expire_held_open(); /* 4. Every second, we try a new circuit if there are no valid @@ -1824,26 +1814,24 @@ second_elapsed_callback(periodic_timer_t *timer, void *arg) */ const int have_dir_info = router_have_minimum_dir_info(); if (have_dir_info && !net_is_disabled()) { - // TODO: NET_PARTICIPANT. circuit_build_needed_circs(now); } else { - // TODO: NET_PARTICIPANT, FLUSH_ON_DISABLE circuit_expire_old_circs_as_needed(now); } /* 5. We do housekeeping for each connection... */ - // TODO: NET_PARTICIPANT channel_update_bad_for_new_circs(NULL, 0); int i; for (i=0;i Date: Tue, 13 Nov 2018 10:43:08 -0500 Subject: [PATCH 0156/2557] reset_padding_counts is only once per 24h; it can be all. --- src/core/mainloop/mainloop.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/core/mainloop/mainloop.c b/src/core/mainloop/mainloop.c index 176399b333..6df51062a7 100644 --- a/src/core/mainloop/mainloop.c +++ b/src/core/mainloop/mainloop.c @@ -1376,9 +1376,11 @@ CALLBACK(second_elapsed); #define FL(name) (PERIODIC_EVENT_FLAG_ ## name) STATIC periodic_event_item_t periodic_events[] = { - /* Everyone needs to run those. */ + /* Everyone needs to run these. They need to have very long timeouts for + * that to be safe. */ CALLBACK(add_entropy, ALL, 0), CALLBACK(heartbeat, ALL, 0), + CALLBACK(reset_padding_counts, ALL, 0), /* This is a legacy catch-all callback that runs once per second if * we are online and active. */ @@ -1403,9 +1405,6 @@ STATIC periodic_event_item_t periodic_events[] = { /* XXXX investigate this. ??? */ CALLBACK(write_stats_file, NET_PARTICIPANT, FL(FLUSH_ON_DISABLE)), - /* XXXX investigate this. ???? */ - CALLBACK(reset_padding_counts, NET_PARTICIPANT, FL(FLUSH_ON_DISABLE)), - /* Routers (bridge and relay) only. */ CALLBACK(check_descriptor, ROUTER, FL(NEED_NET)), CALLBACK(check_ed_keys, ROUTER, 0), From ccbb36048f5167b9d5011b7c8b0d2c346ce567e8 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 13 Nov 2018 10:44:04 -0500 Subject: [PATCH 0157/2557] write_stats_file() is indeed NET_PARTICIPANT; remove comment. --- src/core/mainloop/mainloop.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/core/mainloop/mainloop.c b/src/core/mainloop/mainloop.c index 6df51062a7..e67ebdb4a8 100644 --- a/src/core/mainloop/mainloop.c +++ b/src/core/mainloop/mainloop.c @@ -1401,8 +1401,6 @@ STATIC periodic_event_item_t periodic_events[] = { * immediately before we stop. */ CALLBACK(clean_caches, NET_PARTICIPANT, FL(FLUSH_ON_DISABLE)), CALLBACK(save_state, NET_PARTICIPANT, FL(FLUSH_ON_DISABLE)), - - /* XXXX investigate this. ??? */ CALLBACK(write_stats_file, NET_PARTICIPANT, FL(FLUSH_ON_DISABLE)), /* Routers (bridge and relay) only. */ From 2c15b6538123047c258987b00475fa658ca14878 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 13 Nov 2018 15:33:46 -0500 Subject: [PATCH 0158/2557] Make the NET_PARTICIPANT role dependent on user activity This patch implements all of 28337, except for the part where we turn off the role if we've been idle for a long time. --- src/app/main/main.c | 15 +++++++ src/core/mainloop/connection.c | 3 ++ src/core/mainloop/mainloop.c | 34 ++++++++++++++- src/core/mainloop/mainloop.h | 1 + src/core/mainloop/netstatus.c | 73 +++++++++++++++++++++++++++++++++ src/core/mainloop/netstatus.h | 7 ++++ src/core/or/or.h | 2 + src/feature/client/dnsserv.c | 4 ++ src/feature/control/control.c | 2 + src/test/test_compat_libevent.c | 1 - src/test/test_periodic_event.c | 11 ++++- 11 files changed, 149 insertions(+), 4 deletions(-) diff --git a/src/app/main/main.c b/src/app/main/main.c index b8dcb852d2..03b3a95d03 100644 --- a/src/app/main/main.c +++ b/src/app/main/main.c @@ -303,6 +303,19 @@ process_signal(int sig) log_heartbeat(time(NULL)); control_event_signal(sig); break; + case SIGACTIVE: + /* "SIGACTIVE" counts as ersatz user activity. */ + note_user_activity(approx_time()); + control_event_signal(sig); + break; + case SIGDORMANT: + /* "SIGDORMANT" means to ignore past user activity */ + log_notice(LD_GENERAL, "Going dormant because of controller request."); + reset_user_activity(0); + set_network_participation(false); + schedule_rescan_periodic_events(); + control_event_signal(sig); + break; } } @@ -472,6 +485,8 @@ static struct { { SIGNEWNYM, 0, NULL }, { SIGCLEARDNSCACHE, 0, NULL }, { SIGHEARTBEAT, 0, NULL }, + { SIGACTIVE, 0, NULL }, + { SIGDORMANT, 0, NULL }, { -1, -1, NULL } }; diff --git a/src/core/mainloop/connection.c b/src/core/mainloop/connection.c index 1198a01ad9..e0f1680c91 100644 --- a/src/core/mainloop/connection.c +++ b/src/core/mainloop/connection.c @@ -1874,6 +1874,9 @@ connection_init_accepted_conn(connection_t *conn, TO_ENTRY_CONN(conn)->nym_epoch = get_signewnym_epoch(); TO_ENTRY_CONN(conn)->socks_request->listener_type = listener->base_.type; + /* Any incoming connection on an entry port counts as user activity. */ + note_user_activity(approx_time()); + switch (TO_CONN(listener)->type) { case CONN_TYPE_AP_LISTENER: conn->state = AP_CONN_STATE_SOCKS_WAIT; diff --git a/src/core/mainloop/mainloop.c b/src/core/mainloop/mainloop.c index e67ebdb4a8..f18db2898a 100644 --- a/src/core/mainloop/mainloop.c +++ b/src/core/mainloop/mainloop.c @@ -1513,8 +1513,7 @@ get_my_roles(const or_options_t *options) options->ControlPort_set || options->OwningControllerFD != UINT64_MAX; - /* We actually want a better definition here for our work on dormancy. */ - int is_net_participant = ! net_is_disabled(); + int is_net_participant = is_participating_on_network(); if (is_bridge) roles |= PERIODIC_EVENT_ROLE_BRIDGE; if (is_client) roles |= PERIODIC_EVENT_ROLE_CLIENT; @@ -1592,6 +1591,30 @@ teardown_periodic_events(void) periodic_events_initialized = 0; } +static mainloop_event_t *rescan_periodic_events_ev = NULL; + +/** Callback: rescan the periodic event list. */ +static void +rescan_periodic_events_cb(mainloop_event_t *event, void *arg) +{ + (void)event; + (void)arg; + rescan_periodic_events(get_options()); +} + +/** + * Schedule an event that will rescan which periodic events should run. + **/ +void +schedule_rescan_periodic_events(void) +{ + if (!rescan_periodic_events_ev) { + rescan_periodic_events_ev = + mainloop_event_new(rescan_periodic_events_cb, NULL); + } + mainloop_event_activate(rescan_periodic_events_ev); +} + /** Do a pass at all our periodic events, disable those we don't need anymore * and enable those we need now using the given options. */ void @@ -2714,6 +2737,12 @@ initialize_mainloop_events(void) int do_main_loop(void) { + /* For now, starting Tor always counts as user activity. Later, we might + * have an option to control this. + */ + reset_user_activity(approx_time()); + set_network_participation(true); + /* initialize the periodic events first, so that code that depends on the * events being present does not assert. */ @@ -2912,6 +2941,7 @@ tor_mainloop_free_all(void) mainloop_event_free(postloop_cleanup_ev); mainloop_event_free(handle_deferred_signewnym_ev); mainloop_event_free(scheduled_shutdown_ev); + mainloop_event_free(rescan_periodic_events_ev); #ifdef HAVE_SYSTEMD_209 periodic_timer_free(systemd_watchdog_timer); diff --git a/src/core/mainloop/mainloop.h b/src/core/mainloop/mainloop.h index be642d81f9..7f27ef9a52 100644 --- a/src/core/mainloop/mainloop.h +++ b/src/core/mainloop/mainloop.h @@ -65,6 +65,7 @@ void reschedule_or_state_save(void); void reschedule_dirvote(const or_options_t *options); void mainloop_schedule_postloop_cleanup(void); void rescan_periodic_events(const or_options_t *options); +void schedule_rescan_periodic_events(void); void update_current_time(time_t now); diff --git a/src/core/mainloop/netstatus.c b/src/core/mainloop/netstatus.c index f026474494..ed7c952dcd 100644 --- a/src/core/mainloop/netstatus.c +++ b/src/core/mainloop/netstatus.c @@ -6,6 +6,7 @@ #include "core/or/or.h" #include "core/mainloop/netstatus.h" +#include "core/mainloop/mainloop.h" #include "app/config/config.h" #include "feature/hibernate/hibernate.h" @@ -26,3 +27,75 @@ net_is_completely_disabled(void) { return get_options()->DisableNetwork || we_are_fully_hibernating(); } + +/** + * The time at which we've last seen "user activity" -- that is, any activity + * that should keep us as a participant on the network. + */ +static time_t last_user_activity_seen = 0; + +/** + * True iff we are currently a "network participant" -- that is, we + * are building circuits, fetching directory information, and so on. + **/ +static bool participating_on_network = false; + +/** + * Record the fact that we have seen "user activity" at the time now. Move + * "last activity seen" time forwards, but never backwards. + * + * If we were previously not participating on the network, set our + * participation status to true, and launch periodic events as appropriate. + **/ +void +note_user_activity(time_t now) +{ + last_user_activity_seen = MAX(now, last_user_activity_seen); + + if (! participating_on_network) { + log_notice(LD_GENERAL, "Tor is no longer dormant."); + set_network_participation(true); + schedule_rescan_periodic_events(); + } +} + +/** + * Change the time at which "user activitiy" was last seen to now. + * + * Unlike note_user_actity, this function sets the time without checking + * whether it is in the past, and without causing any rescan of periodic events + * or change in participation status. + */ +void +reset_user_activity(time_t now) +{ + last_user_activity_seen = now; +} + +/** + * Return the most recent time at which we recorded "user activity". + **/ +time_t +get_last_user_activity_time(void) +{ + return last_user_activity_seen; +} + +/** + * Set the field that remembers whether we are currently participating on the + * network. Does not schedule or un-schedule periodic events. + **/ +void +set_network_participation(bool participation) +{ + participating_on_network = participation; +} + +/** + * Return true iff we are currently participating on the network. + **/ +bool +is_participating_on_network(void) +{ + return participating_on_network; +} diff --git a/src/core/mainloop/netstatus.h b/src/core/mainloop/netstatus.h index e9310c2929..58c994fd14 100644 --- a/src/core/mainloop/netstatus.h +++ b/src/core/mainloop/netstatus.h @@ -10,4 +10,11 @@ int net_is_disabled(void); int net_is_completely_disabled(void); +void note_user_activity(time_t now); +void reset_user_activity(time_t now); +time_t get_last_user_activity_time(void); + +void set_network_participation(bool participation); +bool is_participating_on_network(void); + #endif diff --git a/src/core/or/or.h b/src/core/or/or.h index acf092c8dc..e4b374b122 100644 --- a/src/core/or/or.h +++ b/src/core/or/or.h @@ -97,6 +97,8 @@ struct curve25519_public_key_t; #define SIGNEWNYM 129 #define SIGCLEARDNSCACHE 130 #define SIGHEARTBEAT 131 +#define SIGACTIVE 132 +#define SIGDORMANT 133 #if (SIZEOF_CELL_T != 0) /* On Irix, stdlib.h defines a cell_t type, so we need to make sure diff --git a/src/feature/client/dnsserv.c b/src/feature/client/dnsserv.c index ea4951f915..e5abe5c6aa 100644 --- a/src/feature/client/dnsserv.c +++ b/src/feature/client/dnsserv.c @@ -28,6 +28,7 @@ #include "core/or/connection_edge.h" #include "feature/control/control.h" #include "core/mainloop/mainloop.h" +#include "core/mainloop/netstatus.h" #include "core/or/policies.h" #include "feature/control/control_connection_st.h" @@ -213,6 +214,9 @@ dnsserv_launch_request(const char *name, int reverse, edge_connection_t *conn; char *q_name; + /* Launching a request for a user counts as user activity. */ + note_user_activity(approx_time()); + /* Make a new dummy AP connection, and attach the request to it. */ entry_conn = entry_connection_new(CONN_TYPE_AP, AF_INET); entry_conn->entry_cfg.dns_request = 1; diff --git a/src/feature/control/control.c b/src/feature/control/control.c index a5b6ab3bf5..1344d66a3d 100644 --- a/src/feature/control/control.c +++ b/src/feature/control/control.c @@ -1681,6 +1681,8 @@ static const struct signal_t signal_table[] = { { SIGNEWNYM, "NEWNYM" }, { SIGCLEARDNSCACHE, "CLEARDNSCACHE"}, { SIGHEARTBEAT, "HEARTBEAT"}, + { SIGACTIVE, "ACTIVE" }, + { SIGDORMANT, "DORMANT" }, { 0, NULL }, }; diff --git a/src/test/test_compat_libevent.c b/src/test/test_compat_libevent.c index 3f505d013b..ade76bdb07 100644 --- a/src/test/test_compat_libevent.c +++ b/src/test/test_compat_libevent.c @@ -187,4 +187,3 @@ struct testcase_t compat_libevent_tests[] = { TT_FORK, NULL, NULL }, END_OF_TESTCASES }; - diff --git a/src/test/test_periodic_event.c b/src/test/test_periodic_event.c index 6a3e320b2e..f3d518eb7b 100644 --- a/src/test/test_periodic_event.c +++ b/src/test/test_periodic_event.c @@ -51,6 +51,8 @@ test_pe_initialize(void *arg) * need to run the main loop and then wait for a second delaying the unit * tests. Instead, we'll test the callback work indepedently elsewhere. */ initialize_periodic_events(); + set_network_participation(false); + rescan_periodic_events(get_options()); /* Validate that all events have been set up. */ for (int i = 0; periodic_events[i].name; ++i) { @@ -82,6 +84,8 @@ test_pe_launch(void *arg) * network gets enabled. */ consider_hibernation(time(NULL)); + set_network_participation(true); + /* Hack: We'll set a dumb fn() of each events so they don't get called when * dispatching them. We just want to test the state of the callbacks, not * the whole code path. */ @@ -93,6 +97,7 @@ test_pe_launch(void *arg) options = get_options_mutable(); options->SocksPort_set = 1; periodic_events_on_new_options(options); + #if 0 /* Lets make sure that before intialization, we can't scan the periodic * events list and launch them. Lets try by being a Client. */ @@ -148,6 +153,7 @@ test_pe_launch(void *arg) options->SocksPort_set = 0; options->ORPort_set = 0; options->DisableNetwork = 1; + set_network_participation(false); periodic_events_on_new_options(options); for (int i = 0; periodic_events[i].name; ++i) { @@ -162,6 +168,7 @@ test_pe_launch(void *arg) options->BridgeRelay = 1; options->AuthoritativeDir = 1; options->V3AuthoritativeDir = 1; options->BridgeAuthoritativeDir = 1; options->DisableNetwork = 0; + set_network_participation(true); register_dummy_hidden_service(&service); periodic_events_on_new_options(options); /* Note down the reference because we need to remove this service from the @@ -195,8 +202,10 @@ test_pe_get_roles(void *arg) or_options_t *options = get_options_mutable(); tt_assert(options); + set_network_participation(true); - const int ALL = PERIODIC_EVENT_ROLE_ALL; + const int ALL = PERIODIC_EVENT_ROLE_ALL | + PERIODIC_EVENT_ROLE_NET_PARTICIPANT; /* Nothing configured, should be no roles. */ tt_assert(net_is_disabled()); From ce6209cee4a113c6a224f0c98244852354ccdb40 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 13 Nov 2018 15:51:53 -0500 Subject: [PATCH 0159/2557] Add a periodic event to become dormant. This event makes us become dormant if we have seen no activity in a long time. Note that being any kind of a server, or running an onion service, always counts as being active. Note that right now, just having an open stream that Tor did not open on its own (for a directory request) counts as "being active", so if you have an idle ssh connection, that will keep Tor from becoming dormant. Many of the features here should become configurable; I'd like feedback on which. --- src/core/mainloop/connection.c | 10 +++++++ src/core/mainloop/connection.h | 1 + src/core/mainloop/mainloop.c | 50 ++++++++++++++++++++++++++++++++++ 3 files changed, 61 insertions(+) diff --git a/src/core/mainloop/connection.c b/src/core/mainloop/connection.c index e0f1680c91..25224fd999 100644 --- a/src/core/mainloop/connection.c +++ b/src/core/mainloop/connection.c @@ -4429,6 +4429,16 @@ connection_get_by_type_state(int type, int state) CONN_GET_TEMPLATE(conn, conn->type == type && conn->state == state); } +/** + * Return a connection of type type that is not an internally linked + * connection, and is not marked for close. + **/ +connection_t * +connection_get_by_type_nonlinked(int type) +{ + CONN_GET_TEMPLATE(conn, conn->type == type && !conn->linked); +} + /** Return a connection of type type that has rendquery equal * to rendquery, and that is not marked for close. If state * is non-zero, conn must be of that state too. diff --git a/src/core/mainloop/connection.h b/src/core/mainloop/connection.h index b569bb038e..9f1a23c6f2 100644 --- a/src/core/mainloop/connection.h +++ b/src/core/mainloop/connection.h @@ -240,6 +240,7 @@ size_t connection_get_outbuf_len(connection_t *conn); connection_t *connection_get_by_global_id(uint64_t id); connection_t *connection_get_by_type(int type); +connection_t *connection_get_by_type_nonlinked(int type); MOCK_DECL(connection_t *,connection_get_by_type_addr_port_purpose,(int type, const tor_addr_t *addr, uint16_t port, int purpose)); diff --git a/src/core/mainloop/mainloop.c b/src/core/mainloop/mainloop.c index f18db2898a..e6dee94fcc 100644 --- a/src/core/mainloop/mainloop.c +++ b/src/core/mainloop/mainloop.c @@ -1367,6 +1367,7 @@ CALLBACK(write_bridge_ns); CALLBACK(write_stats_file); CALLBACK(control_per_second_events); CALLBACK(second_elapsed); +CALLBACK(check_network_participation); #undef CALLBACK @@ -1396,6 +1397,7 @@ STATIC periodic_event_item_t periodic_events[] = { CALLBACK(fetch_networkstatus, NET_PARTICIPANT, 0), CALLBACK(launch_descriptor_fetches, NET_PARTICIPANT, FL(NEED_NET)), CALLBACK(rotate_x509_certificate, NET_PARTICIPANT, 0), + CALLBACK(check_network_participation, NET_PARTICIPANT, 0), /* We need to do these if we're participating in the Tor network, and * immediately before we stop. */ @@ -1998,6 +2000,54 @@ add_entropy_callback(time_t now, const or_options_t *options) return ENTROPY_INTERVAL; } +/** Periodic callback: if there has been no network usage in a while, + * enter a dormant state. */ +static int +check_network_participation_callback(time_t now, const or_options_t *options) +{ + /* If we're a server, we can't become dormant. */ + if (server_mode(options)) { + goto found_activity; + } + + /* If we're running an onion service, we can't become dormant. */ + /* XXXX this would be nice to change, so that we can be dormant with a + * service. */ + if (hs_service_get_num_services() || rend_num_services()) { + goto found_activity; + } + + /* XXXX Add an option to never become dormant. */ + + /* If we have any currently open entry streams other than "linked" + * connections used for directory requests, those count as user activity. + */ + /* XXXX make this configurable? */ + if (connection_get_by_type_nonlinked(CONN_TYPE_AP) != NULL) { + goto found_activity; + } + + /* XXXX Make this configurable? */ +/** How often do we check whether we have had network activity? */ +#define CHECK_PARTICIPATION_INTERVAL (5*60) + + /** Become dormant if there has been no user activity in this long. */ + /* XXXX make this configurable! */ +#define BECOME_DORMANT_AFTER_INACTIVITY (24*60*60) + if (get_last_user_activity_time() + BECOME_DORMANT_AFTER_INACTIVITY >= now) { + log_notice(LD_GENERAL, "No user activity in a long time: becoming" + " dormant."); + set_network_participation(false); + rescan_periodic_events(options); + } + + return CHECK_PARTICIPATION_INTERVAL; + + found_activity: + note_user_activity(now); + return CHECK_PARTICIPATION_INTERVAL; +} + /** * Periodic callback: if we're an authority, make sure we test * the routers on the network for reachability. From d0e6abd0876f0d4adb0b82f5528220a81b34962e Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 13 Nov 2018 15:57:18 -0500 Subject: [PATCH 0160/2557] Reset dormancy status when the clock jumps. --- src/core/mainloop/mainloop.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/core/mainloop/mainloop.c b/src/core/mainloop/mainloop.c index e6dee94fcc..2a68e8c098 100644 --- a/src/core/mainloop/mainloop.c +++ b/src/core/mainloop/mainloop.c @@ -2674,6 +2674,11 @@ update_current_time(time_t now) if (seconds_elapsed < -NUM_JUMPED_SECONDS_BEFORE_WARN) { // moving back in time is always a bad sign. circuit_note_clock_jumped(seconds_elapsed, false); + + /* Don't go dormant just because we jumped in time. */ + if (is_participating_on_network()) { + reset_user_activity(now); + } } else if (seconds_elapsed >= NUM_JUMPED_SECONDS_BEFORE_WARN) { /* Compare the monotonic clock to the result of time(). */ const int32_t monotime_msec_passed = @@ -2695,6 +2700,11 @@ update_current_time(time_t now) if (clock_jumped || seconds_elapsed >= NUM_IDLE_SECONDS_BEFORE_WARN) { circuit_note_clock_jumped(seconds_elapsed, ! clock_jumped); } + + /* Don't go dormant just because we jumped in time. */ + if (is_participating_on_network()) { + reset_user_activity(now); + } } else if (seconds_elapsed > 0) { stats_n_seconds_working += seconds_elapsed; } From 2f28cd1dc8e797b140271e5c33b9e4f823f7f2d8 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 14 Nov 2018 13:42:09 -0500 Subject: [PATCH 0161/2557] Rename and fix docs on FLUSH_ON_DISABLE Also rename "...flush_and_disable()" to "...schedule_and_disable()" --- src/core/mainloop/mainloop.c | 16 ++++++++-------- src/core/mainloop/periodic.c | 2 +- src/core/mainloop/periodic.h | 6 +++--- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/core/mainloop/mainloop.c b/src/core/mainloop/mainloop.c index 2a68e8c098..9f45f3c869 100644 --- a/src/core/mainloop/mainloop.c +++ b/src/core/mainloop/mainloop.c @@ -1386,7 +1386,7 @@ STATIC periodic_event_item_t periodic_events[] = { /* This is a legacy catch-all callback that runs once per second if * we are online and active. */ CALLBACK(second_elapsed, NET_PARTICIPANT, - FL(NEED_NET)|FL(FLUSH_ON_DISABLE)), + FL(NEED_NET)|FL(RUN_ON_DISABLE)), /* XXXX Do we have a reason to do this on a callback? Does it do any good at * all? For now, if we're dormant, we can let our listeners decay. */ @@ -1401,9 +1401,9 @@ STATIC periodic_event_item_t periodic_events[] = { /* We need to do these if we're participating in the Tor network, and * immediately before we stop. */ - CALLBACK(clean_caches, NET_PARTICIPANT, FL(FLUSH_ON_DISABLE)), - CALLBACK(save_state, NET_PARTICIPANT, FL(FLUSH_ON_DISABLE)), - CALLBACK(write_stats_file, NET_PARTICIPANT, FL(FLUSH_ON_DISABLE)), + CALLBACK(clean_caches, NET_PARTICIPANT, FL(RUN_ON_DISABLE)), + CALLBACK(save_state, NET_PARTICIPANT, FL(RUN_ON_DISABLE)), + CALLBACK(write_stats_file, NET_PARTICIPANT, FL(RUN_ON_DISABLE)), /* Routers (bridge and relay) only. */ CALLBACK(check_descriptor, ROUTER, FL(NEED_NET)), @@ -1436,7 +1436,7 @@ STATIC periodic_event_item_t periodic_events[] = { /* Client only. */ /* XXXX this could be restricted to CLIENT+NET_PARTICIPANT */ - CALLBACK(rend_cache_failure_clean, NET_PARTICIPANT, FL(FLUSH_ON_DISABLE)), + CALLBACK(rend_cache_failure_clean, NET_PARTICIPANT, FL(RUN_ON_DISABLE)), /* Bridge Authority only. */ CALLBACK(write_bridge_ns, BRIDGEAUTH, 0), @@ -1651,8 +1651,8 @@ rescan_periodic_events(const or_options_t *options) periodic_event_enable(item); } else { log_debug(LD_GENERAL, "Disabling periodic event %s", item->name); - if (item->flags & PERIODIC_EVENT_FLAG_FLUSH_ON_DISABLE) { - periodic_event_flush_and_disable(item); + if (item->flags & PERIODIC_EVENT_FLAG_RUN_ON_DISABLE) { + periodic_event_schedule_and_disable(item); } else { periodic_event_disable(item); } @@ -1814,7 +1814,7 @@ second_elapsed_callback(time_t now, const or_options_t *options) */ /* (If our circuit build timeout can ever become lower than a second (which * it can't, currently), we should do this more often.) */ - // TODO: All expire stuff can become NET_PARTICIPANT, FLUSH_ON_DISABLE + // TODO: All expire stuff can become NET_PARTICIPANT, RUN_ON_DISABLE circuit_expire_building(); circuit_expire_waiting_for_better_guard(); diff --git a/src/core/mainloop/periodic.c b/src/core/mainloop/periodic.c index c290c3744e..9f9b178e43 100644 --- a/src/core/mainloop/periodic.c +++ b/src/core/mainloop/periodic.c @@ -174,7 +174,7 @@ periodic_event_disable(periodic_event_item_t *event) * Do nothing if the event was already disabled. */ void -periodic_event_flush_and_disable(periodic_event_item_t *event) +periodic_event_schedule_and_disable(periodic_event_item_t *event) { tor_assert(event); if (!periodic_event_is_enabled(event)) diff --git a/src/core/mainloop/periodic.h b/src/core/mainloop/periodic.h index 52d5450ee8..05ba4297f3 100644 --- a/src/core/mainloop/periodic.h +++ b/src/core/mainloop/periodic.h @@ -39,10 +39,10 @@ * the net_is_disabled() check. */ #define PERIODIC_EVENT_FLAG_NEED_NET (1U << 0) -/* Indicate that it the event is enabled, it event needs to be run once before +/* Indicate that if the event is enabled, it needs to be run once before * it becomes disabled. */ -#define PERIODIC_EVENT_FLAG_FLUSH_ON_DISABLE (1U << 1) +#define PERIODIC_EVENT_FLAG_RUN_ON_DISABLE (1U << 1) /** Callback function for a periodic event to take action. The return value * influences the next time the function will get called. Return @@ -88,6 +88,6 @@ void periodic_event_destroy(periodic_event_item_t *event); void periodic_event_reschedule(periodic_event_item_t *event); void periodic_event_enable(periodic_event_item_t *event); void periodic_event_disable(periodic_event_item_t *event); -void periodic_event_flush_and_disable(periodic_event_item_t *event); +void periodic_event_schedule_and_disable(periodic_event_item_t *event); #endif /* !defined(TOR_PERIODIC_H) */ From 53ccdb6945f0d4a9b27a9939211a3c9125ca4427 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 14 Nov 2018 15:05:05 -0500 Subject: [PATCH 0162/2557] Make sure that we are always a net participant when being a server Otherwise, if we're dormant, and we set ORPort, nothing makes us become non-dormant. --- src/core/mainloop/mainloop.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/core/mainloop/mainloop.c b/src/core/mainloop/mainloop.c index 9f45f3c869..2d12e26485 100644 --- a/src/core/mainloop/mainloop.c +++ b/src/core/mainloop/mainloop.c @@ -1515,7 +1515,8 @@ get_my_roles(const or_options_t *options) options->ControlPort_set || options->OwningControllerFD != UINT64_MAX; - int is_net_participant = is_participating_on_network(); + int is_net_participant = is_participating_on_network() || + is_relay || is_hidden_service; if (is_bridge) roles |= PERIODIC_EVENT_ROLE_BRIDGE; if (is_client) roles |= PERIODIC_EVENT_ROLE_CLIENT; From ce1b018ab742492e96b5bdae049a13c2faaefdbf Mon Sep 17 00:00:00 2001 From: rl1987 Date: Sun, 11 Nov 2018 17:44:41 +0200 Subject: [PATCH 0163/2557] Fix shellcheck SC2006 warnings in 'coverage' script --- scripts/test/coverage | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/test/coverage b/scripts/test/coverage index 59d468ee1e..e611a4be1b 100755 --- a/scripts/test/coverage +++ b/scripts/test/coverage @@ -8,19 +8,19 @@ dst=$1 for fn in src/core/*/*.c src/feature/*/*.c src/app/*/*.c src/lib/*/*.c; do - BN=`basename $fn` - DN=`dirname $fn` - F=`echo $BN | sed -e 's/\.c$//;'` + BN=$(basename $fn) + DN=$(dirname $fn) + F=$(echo $BN | sed -e 's/\.c$//;') GC="${BN}.gcov" # Figure out the object file names - ONS=`echo ${DN}/src_*-${F}.o` + ONS=$(echo ${DN}/src_*-${F}.o) ONS_WILDCARD_LITERAL="${DN}/src_*-${F}.o" # If the wildcard didn't expand, no files if [ "$ONS" != "${ONS_WILDCARD_LITERAL}" ] then for on in $ONS; do # We should have a gcno file - GCNO=`echo $on | sed -e 's/\.o$/\.gcno/;'` + GCNO=$(echo $on | sed -e 's/\.o$/\.gcno/;') if [ -e $GCNO ] then # No need to test for gcda, since gcov assumes no execution From 7c04b00e65d67a38f119f4d3e50e8a47236e202f Mon Sep 17 00:00:00 2001 From: rl1987 Date: Sun, 11 Nov 2018 17:50:43 +0200 Subject: [PATCH 0164/2557] Fix most instances of SC2086 warning --- scripts/test/coverage | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/scripts/test/coverage b/scripts/test/coverage index e611a4be1b..180a2d6dcb 100755 --- a/scripts/test/coverage +++ b/scripts/test/coverage @@ -8,9 +8,9 @@ dst=$1 for fn in src/core/*/*.c src/feature/*/*.c src/app/*/*.c src/lib/*/*.c; do - BN=$(basename $fn) - DN=$(dirname $fn) - F=$(echo $BN | sed -e 's/\.c$//;') + BN=$(basename "$fn") + DN=$(dirname "$fn") + F=$(echo "$BN" | sed -e 's/\.c$//;') GC="${BN}.gcov" # Figure out the object file names ONS=$(echo ${DN}/src_*-${F}.o) @@ -20,18 +20,18 @@ for fn in src/core/*/*.c src/feature/*/*.c src/app/*/*.c src/lib/*/*.c; do then for on in $ONS; do # We should have a gcno file - GCNO=$(echo $on | sed -e 's/\.o$/\.gcno/;') - if [ -e $GCNO ] + GCNO=$(echo "$on" | sed -e 's/\.o$/\.gcno/;') + if [ -e "$GCNO" ] then # No need to test for gcda, since gcov assumes no execution # if it's absent - rm -f $GC - gcov -o $on $fn - if [ -e $GC ] + rm -f "$GC" + gcov -o "$on" "$fn" + if [ -e "$GC" ] then if [ -d "$dst" ] then - mv $GC $dst/$GC + mv "$GC" "$dst"/"$GC" fi else echo "gcov -o $on $fn didn't make a .gcov file" From 4fc584f20e5aae8f7b89f169690c17d5bce84d41 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Sun, 11 Nov 2018 18:04:07 +0200 Subject: [PATCH 0165/2557] Fix one last SC2086 --- scripts/test/coverage | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/test/coverage b/scripts/test/coverage index 180a2d6dcb..7a417cdc89 100755 --- a/scripts/test/coverage +++ b/scripts/test/coverage @@ -13,7 +13,7 @@ for fn in src/core/*/*.c src/feature/*/*.c src/app/*/*.c src/lib/*/*.c; do F=$(echo "$BN" | sed -e 's/\.c$//;') GC="${BN}.gcov" # Figure out the object file names - ONS=$(echo ${DN}/src_*-${F}.o) + ONS=$(echo "${DN}"/src_*-"${F}".o) ONS_WILDCARD_LITERAL="${DN}/src_*-${F}.o" # If the wildcard didn't expand, no files if [ "$ONS" != "${ONS_WILDCARD_LITERAL}" ] From 8cb817cc5e0cc36cc0218da53bc4c1f277ccd8cc Mon Sep 17 00:00:00 2001 From: rl1987 Date: Sun, 11 Nov 2018 18:56:20 +0200 Subject: [PATCH 0166/2557] Add changes file --- changes/ticket28008 | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 changes/ticket28008 diff --git a/changes/ticket28008 b/changes/ticket28008 new file mode 100644 index 0000000000..1f0de1a14d --- /dev/null +++ b/changes/ticket28008 @@ -0,0 +1,3 @@ + o Code simplification and refactoring: + - Fix shellcheck warnings in scripts/test/coverage. Resolves issue + 28008. From d9f7cb3f6202291d38d611165f70e0c1c33b3e92 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Sun, 11 Nov 2018 19:18:49 +0200 Subject: [PATCH 0167/2557] Fix shellcheck issues in cov-diff - SC2231, SC2006 and SC2086 --- changes/ticket28009 | 3 +++ scripts/test/cov-diff | 6 +++--- 2 files changed, 6 insertions(+), 3 deletions(-) create mode 100644 changes/ticket28009 diff --git a/changes/ticket28009 b/changes/ticket28009 new file mode 100644 index 0000000000..1d986d4211 --- /dev/null +++ b/changes/ticket28009 @@ -0,0 +1,3 @@ + o Code simplification and refactoring: + - Fix shellcheck warnings in cov-diff script. Resolves issue + 28009. diff --git a/scripts/test/cov-diff b/scripts/test/cov-diff index 6179dff63e..f3ca856888 100755 --- a/scripts/test/cov-diff +++ b/scripts/test/cov-diff @@ -7,9 +7,9 @@ DIRA="$1" DIRB="$2" -for B in $DIRB/*; do - A=$DIRA/`basename $B` - if [ -f $A ]; then +for B in "$DIRB"/*; do + A=$DIRA/$(basename "$B") + if [ -f "$A" ]; then perl -pe 's/^\s*\!*\d+(\*?):/ 1$1:/; s/^([^:]+:)[\d\s]+:/$1/; s/^ *-:(Runs|Programs):.*//;' "$A" > "$A.tmp" else cat /dev/null > "$A.tmp" From bf82389e198a0cc6c7e078cada7f35a9cbf65fe1 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 16 Nov 2018 11:32:52 -0500 Subject: [PATCH 0168/2557] Copy 0.3.5.5-alpha changelog into master --- ChangeLog | 67 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/ChangeLog b/ChangeLog index cdf7249059..4c776f7163 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,70 @@ +Changes in version 0.3.5.5-alpha - 2018-11-16 + Tor 0.3.5.5-alpha includes numerous bugfixes on earlier releases, + including several that we hope to backport to older release series in + the future. + + o Major bugfixes (OpenSSL, portability): + - Fix our usage of named groups when running as a TLS 1.3 client in + OpenSSL 1.1.1. Previously, we only initialized EC groups when + running as a relay, which caused clients to fail to negotiate TLS + 1.3 with relays. Fixes bug 28245; bugfix on 0.2.9.15 (when TLS 1.3 + support was added). + + o Minor features (geoip): + - Update geoip and geoip6 to the November 6 2018 Maxmind GeoLite2 + Country database. Closes ticket 28395. + + o Minor bugfixes (compilation): + - Initialize a variable unconditionally in aes_new_cipher(), since + some compilers cannot tell that we always initialize it before + use. Fixes bug 28413; bugfix on 0.2.9.3-alpha. + + o Minor bugfixes (connection, relay): + - Avoid a logging a BUG() stacktrace when closing connection held + open because the write side is rate limited but not the read side. + Now, the connection read side is simply shut down until Tor is + able to flush the connection and close it. Fixes bug 27750; bugfix + on 0.3.4.1-alpha. + + o Minor bugfixes (continuous integration, Windows): + - Manually configure the zstd compiler options, when building using + mingw on Appveyor Windows CI. The MSYS2 mingw zstd package does + not come with a pkg-config file. Fixes bug 28454; bugfix + on 0.3.4.1-alpha. + - Stop using an external OpenSSL install, and stop installing MSYS2 + packages, when building using mingw on Appveyor Windows CI. Fixes + bug 28399; bugfix on 0.3.4.1-alpha. + + o Minor bugfixes (documentation): + - Make Doxygen work again after the code movement in the 0.3.5 + source tree. Fixes bug 28435; bugfix on 0.3.5.1-alpha. + + o Minor bugfixes (Linux seccomp2 sandbox): + - Permit the "shutdown()" system call, which is apparently used by + OpenSSL under some circumstances. Fixes bug 28183; bugfix + on 0.2.5.1-alpha. + + o Minor bugfixes (logging): + - Stop talking about the Named flag in log messages. Clients have + ignored the Named flag since 0.3.2. Fixes bug 28441; bugfix + on 0.3.2.1-alpha. + + o Minor bugfixes (memory leaks): + - Fix a harmless memory leak in libtorrunner.a. Fixes bug 28419; + bugfix on 0.3.3.1-alpha. Patch from Martin Kepplinger. + + o Minor bugfixes (onion services): + - On an intro point for a version 3 onion service, stop closing + introduction circuits on an NACK. This lets the client decide + whether to reuse the circuit or discard it. Previously, we closed + intro circuits when sending NACKs. Fixes bug 27841; bugfix on + 0.3.2.1-alpha. Patch by Neel Chaunan. + - When replacing a descriptor in the client cache, make sure to + close all client introduction circuits for the old descriptor, so + we don't end up with unusable leftover circuits. Fixes bug 27471; + bugfix on 0.3.2.1-alpha. + + Changes in version 0.3.5.4-alpha - 2018-11-08 Tor 0.3.5.4-alpha includes numerous bugfixes on earlier versions and improves our continuous integration support. It continues our attempts From 411780d563e4d08e12d8aef0bbe8915f48011cca Mon Sep 17 00:00:00 2001 From: rl1987 Date: Sat, 17 Nov 2018 10:19:25 +0200 Subject: [PATCH 0169/2557] Make ROUTERLIST_PRUNING_INTERVAL 1 hr. --- src/core/mainloop/mainloop.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/mainloop/mainloop.c b/src/core/mainloop/mainloop.c index be19136130..0b9d4e0889 100644 --- a/src/core/mainloop/mainloop.c +++ b/src/core/mainloop/mainloop.c @@ -2231,7 +2231,7 @@ retry_dns_callback(time_t now, const or_options_t *options) static int prune_old_routers_callback(time_t now, const or_options_t *options) { -#define ROUTERLIST_PRUNING_INTERVAL (60) // 1 minute. +#define ROUTERLIST_PRUNING_INTERVAL (60*60) // 1 hour. (void)now; (void)options; From c8c4c3dffa71b4bbc9e7cabfee2124fb5e19ad39 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Sat, 17 Nov 2018 10:27:10 +0200 Subject: [PATCH 0170/2557] fixup! Make ROUTERLIST_PRUNING_INTERVAL 1 hr. --- changes/bug27929 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changes/bug27929 b/changes/bug27929 index a97d18f202..dab57a2eca 100644 --- a/changes/bug27929 +++ b/changes/bug27929 @@ -1,5 +1,5 @@ o Minor bugfixes (periodic events): - Refrain from calling routerlist_remove_old_routers() from check_descriptor_callback(). Instead, create a new periodic - event that will run every minute even if Tor is not configured + event that will run once every hour even if Tor is not configured as onion router. Fixes bug 27929; bugfix on 0.2.8.1-alpha. From 83be4d2bbd7a4ed584f42d043558a4026c4a449d Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 28 Aug 2018 11:34:31 -0400 Subject: [PATCH 0171/2557] Backend for compact node-family representation. This representation is meant to save memory in microdescriptors -- we can't use it in routerinfo_t yet, since those families need to be encoded losslessly for directory voting to work. This representation saves memory in three ways: 1. It uses only one allocation per family. (The old way used a smartlist (2 allocs) plus one strdup per entry.) 2. It stores identity digests in binary, not hex. 3. It keeps families in a canonical format, memoizes, and reference-counts them. Part of #27359. --- src/core/include.am | 3 + src/feature/nodelist/nodefamily.c | 373 +++++++++++++++++++++++++++ src/feature/nodelist/nodefamily.h | 47 ++++ src/feature/nodelist/nodefamily_st.h | 48 ++++ src/test/test_nodelist.c | 251 +++++++++++++++++- 5 files changed, 721 insertions(+), 1 deletion(-) create mode 100644 src/feature/nodelist/nodefamily.c create mode 100644 src/feature/nodelist/nodefamily.h create mode 100644 src/feature/nodelist/nodefamily_st.h diff --git a/src/core/include.am b/src/core/include.am index 1b8ef2ac58..003ef84216 100644 --- a/src/core/include.am +++ b/src/core/include.am @@ -106,6 +106,7 @@ LIBTOR_APP_A_SOURCES = \ src/feature/nodelist/microdesc.c \ src/feature/nodelist/networkstatus.c \ src/feature/nodelist/nickname.c \ + src/feature/nodelist/nodefamily.c \ src/feature/nodelist/nodelist.c \ src/feature/nodelist/node_select.c \ src/feature/nodelist/routerinfo.c \ @@ -340,6 +341,8 @@ noinst_HEADERS += \ src/feature/nodelist/networkstatus_voter_info_st.h \ src/feature/nodelist/nickname.h \ src/feature/nodelist/node_st.h \ + src/feature/nodelist/nodefamily.h \ + src/feature/nodelist/nodefamily_st.h \ src/feature/nodelist/nodelist.h \ src/feature/nodelist/node_select.h \ src/feature/nodelist/routerinfo.h \ diff --git a/src/feature/nodelist/nodefamily.c b/src/feature/nodelist/nodefamily.c new file mode 100644 index 0000000000..6b504c0ac4 --- /dev/null +++ b/src/feature/nodelist/nodefamily.c @@ -0,0 +1,373 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file nodefamily.c + * \brief Code to manipulate encoded, reference-counted node families. We + * use these tricks to save space, since these families would otherwise + * require a large number of tiny allocations. + **/ + +#include "core/or/or.h" +#include "feature/nodelist/nickname.h" +#include "feature/nodelist/nodefamily.h" +#include "feature/nodelist/nodefamily_st.h" +#include "feature/nodelist/nodelist.h" +#include "feature/relay/router.h" +#include "feature/nodelist/routerlist.h" + +#include "ht.h" +#include "siphash.h" + +#include "lib/container/smartlist.h" +#include "lib/ctime/di_ops.h" +#include "lib/defs/digest_sizes.h" +#include "lib/log/util_bug.h" + +#include +#include + +/** + * Allocate and return a blank node family with space to hold n_members + * members. + */ +static nodefamily_t * +nodefamily_alloc(int n_members) +{ + size_t alloc_len = offsetof(nodefamily_t, family_members) + + NODEFAMILY_ARRAY_SIZE(n_members); + nodefamily_t *nf = tor_malloc_zero(alloc_len); + nf->n_members = n_members; + return nf; +} + +/** + * Hashtable hash implementation. + */ +static inline unsigned int +nodefamily_hash(const nodefamily_t *nf) +{ + return (unsigned) siphash24g(nf->family_members, + NODEFAMILY_ARRAY_SIZE(nf->n_members)); +} + +/** + * Hashtable equality implementation. + */ +static inline unsigned int +nodefamily_eq(const nodefamily_t *a, const nodefamily_t *b) +{ + return (a->n_members == b->n_members) && + fast_memeq(a->family_members, b->family_members, + NODEFAMILY_ARRAY_SIZE(a->n_members)); +} + +static HT_HEAD(nodefamily_map, nodefamily_t) the_node_families + = HT_INITIALIZER(); + +HT_PROTOTYPE(nodefamily_map, nodefamily_t, ht_ent, nodefamily_hash, + nodefamily_eq) +HT_GENERATE2(nodefamily_map, nodefamily_t, ht_ent, nodefamily_hash, + node_family_eq, 0.6, tor_reallocarray_, tor_free_) + +/** + * Parse the family declaration in s, returning the canonical + * nodefamily_t for its members. Return NULL on error. + * + * If rsa_id_self is provided, it is a DIGEST_LEN-byte digest + * for the router that declared this family: insert it into the + * family declaration if it is not there already. + * + * If NF_WARN_MALFORMED is set in flags, warn about any + * elements that we can't parse. (By default, we log at info.) + * + * If NF_REJECT_MALFORMED is set in flags, treat any unparseable + * elements as an error. (By default, we simply omit them.) + **/ +nodefamily_t * +nodefamily_parse(const char *s, const uint8_t *rsa_id_self, + unsigned flags) +{ + smartlist_t *sl = smartlist_new(); + smartlist_split_string(sl, s, NULL, SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); + nodefamily_t *result = nodefamily_from_members(sl, rsa_id_self, flags); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_free(sl); + return result; +} + +/** + * qsort helper for encoded nodefamily elements. + **/ +static int +compare_members(const void *a, const void *b) +{ + return fast_memcmp(a, b, NODEFAMILY_MEMBER_LEN); +} + +/** + * Parse the member strings in members, returning their canonical + * nodefamily_t. Return NULL on error. + * + * If rsa_id_self is provided, it is a DIGEST_LEN-byte digest + * for the router that declared this family: insert it into the + * family declaration if it is not there already. + * + * The flags element is interpreted as in nodefamily_parse(). + **/ +nodefamily_t * +nodefamily_from_members(const smartlist_t *members, + const uint8_t *rsa_id_self, + unsigned flags) +{ + const int n_self = rsa_id_self ? 1 : 0; + int n_bad_elements = 0; + int n_members = smartlist_len(members) + n_self; + nodefamily_t *tmp = nodefamily_alloc(n_members); + uint8_t *ptr = NODEFAMILY_MEMBER_PTR(tmp, 0); + + SMARTLIST_FOREACH_BEGIN(members, const char *, cp) { + bool bad_element = true; + if (is_legal_nickname(cp)) { + ptr[0] = NODEFAMILY_BY_NICKNAME; + tor_assert(strlen(cp) < DIGEST_LEN); // guaranteed by is_legal_nickname + memcpy(ptr+1, cp, strlen(cp)); + bad_element = false; + } else if (is_legal_hexdigest(cp)) { + char digest_buf[DIGEST_LEN]; + char nn_buf[MAX_NICKNAME_LEN+1]; + char nn_char=0; + if (hex_digest_nickname_decode(cp, digest_buf, &nn_char, nn_buf)==0) { + bad_element = false; + ptr[0] = NODEFAMILY_BY_RSA_ID; + memcpy(ptr+1, digest_buf, DIGEST_LEN); + } + } + + if (bad_element) { + const int severity = (flags & NF_WARN_MALFORMED) ? LOG_WARN : LOG_INFO; + log_fn(severity, LD_GENERAL, + "Bad element %s while parsing a node family.", + escaped(cp)); + ++n_bad_elements; + } else { + ptr += NODEFAMILY_MEMBER_LEN; + } + } SMARTLIST_FOREACH_END(cp); + + if (n_bad_elements && (flags & NF_REJECT_MALFORMED)) + goto err; + + if (rsa_id_self) { + /* Add self. */ + ptr[0] = NODEFAMILY_BY_RSA_ID; + memcpy(ptr+1, rsa_id_self, DIGEST_LEN); + } + + n_members -= n_bad_elements; + + /* Sort tmp into canonical order. */ + qsort(tmp->family_members, n_members, NODEFAMILY_MEMBER_LEN, + compare_members); + + /* Remove duplicates. */ + int i; + for (i = 0; i < n_members-1; ++i) { + uint8_t *thisptr = NODEFAMILY_MEMBER_PTR(tmp, i); + uint8_t *nextptr = NODEFAMILY_MEMBER_PTR(tmp, i+1); + if (fast_memeq(thisptr, nextptr, NODEFAMILY_MEMBER_LEN)) { + memmove(thisptr, nextptr, (n_members-i-1)*NODEFAMILY_MEMBER_LEN); + --n_members; + --i; + } + } + int n_members_alloc = tmp->n_members; + tmp->n_members = n_members; + + /* See if we already allocated this family. */ + nodefamily_t *found = HT_FIND(nodefamily_map, &the_node_families, tmp); + if (found) { + /* If we did, great: incref it and return it. */ + ++found->refcnt; + tor_free(tmp); + return found; + } else { + /* If not, insert it into the hashtable. */ + if (n_members_alloc != n_members) { + /* Compact the family if needed */ + nodefamily_t *tmp2 = nodefamily_alloc(n_members); + memcpy(tmp2->family_members, tmp->family_members, + n_members * NODEFAMILY_MEMBER_LEN); + tor_free(tmp); + tmp = tmp2; + } + + tmp->refcnt = 1; + HT_INSERT(nodefamily_map, &the_node_families, tmp); + return tmp; + } + + err: + tor_free(tmp); + return NULL; +} + +/** + * Drop our reference to family, freeing it if there are no more + * references. + */ +void +nodefamily_free_(nodefamily_t *family) +{ + if (family == NULL) + return; + + --family->refcnt; + + if (family->refcnt == 0) { + HT_REMOVE(nodefamily_map, &the_node_families, family); + tor_free(family); + } +} + +/** + * Return true iff family contains the SHA1 RSA1024 identity + * rsa_id. + */ +bool +nodefamily_contains_rsa_id(const nodefamily_t *family, + const uint8_t *rsa_id) +{ + if (family == NULL) + return false; + + unsigned i; + for (i = 0; i < family->n_members; ++i) { + const uint8_t *ptr = NODEFAMILY_MEMBER_PTR(family, i); + if (ptr[0] == NODEFAMILY_BY_RSA_ID && + fast_memeq(ptr+1, rsa_id, DIGEST_LEN)) { + return true; + } + } + return false; +} + +/** + * Return true iff family contains the nickname name. + */ +bool +nodefamily_contains_nickname(const nodefamily_t *family, + const char *name) +{ + if (family == NULL) + return false; + + unsigned i; + for (i = 0; i < family->n_members; ++i) { + const uint8_t *ptr = NODEFAMILY_MEMBER_PTR(family, i); + // note that the strcasecmp() is safe because there is always at least one + // NUL in the encoded nickname, because all legal nicknames are less than + // DIGEST_LEN bytes long. + if (ptr[0] == NODEFAMILY_BY_NICKNAME && !strcasecmp((char*)ptr+1, name)) { + return true; + } + } + return false; +} + +/** + * Return true if family contains the nickname or the RSA ID for + * node + **/ +bool +nodefamily_contains_node(const nodefamily_t *family, + const node_t *node) +{ + return + nodefamily_contains_nickname(family, node_get_nickname(node)) + || + nodefamily_contains_rsa_id(family, node_get_rsa_id_digest(node)); +} + +/** + * Look up every entry in family, and add add the corresponding + * node_t to out. + **/ +void +nodefamily_add_nodes_to_smartlist(const nodefamily_t *family, + smartlist_t *out) +{ + if (!family) + return; + + unsigned i; + for (i = 0; i < family->n_members; ++i) { + const uint8_t *ptr = NODEFAMILY_MEMBER_PTR(family, i); + const node_t *node = NULL; + switch (ptr[0]) { + case NODEFAMILY_BY_NICKNAME: + node = node_get_by_nickname((char*)ptr+1, NNF_NO_WARN_UNNAMED); + break; + case NODEFAMILY_BY_RSA_ID: + node = node_get_by_id((char*)ptr+1); + break; + default: + /* LCOV_EXCL_START */ + tor_assert_nonfatal_unreached(); + break; + /* LCOV_EXCL_STOP */ + } + if (node) + smartlist_add(out, (void *)node); + } +} + +/** + * Encode family as a space-separated string. + */ +char * +nodefamily_format(const nodefamily_t *family) +{ + if (!family) + return tor_strdup(""); + + unsigned i; + smartlist_t *sl = smartlist_new(); + for (i = 0; i < family->n_members; ++i) { + const uint8_t *ptr = NODEFAMILY_MEMBER_PTR(family, i); + switch (ptr[0]) { + case NODEFAMILY_BY_NICKNAME: + smartlist_add_strdup(sl, (char*)ptr+1); + break; + case NODEFAMILY_BY_RSA_ID: { + char buf[HEX_DIGEST_LEN+2]; + buf[0]='$'; + base16_encode(buf+1, sizeof(buf)-1, (char*)ptr+1, DIGEST_LEN); + smartlist_add_strdup(sl, buf); + break; + } + default: + /* LCOV_EXCL_START */ + tor_assert_nonfatal_unreached(); + break; + /* LCOV_EXCL_STOP */ + } + } + + char *result = smartlist_join_strings(sl, " ", 0, NULL); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_free(sl); + return result; +} + +/** + * Free all storage held in the nodefamily map. + **/ +void +nodefamily_free_all(void) +{ + HT_CLEAR(nodefamily_map, &the_node_families); +} diff --git a/src/feature/nodelist/nodefamily.h b/src/feature/nodelist/nodefamily.h new file mode 100644 index 0000000000..342f161a07 --- /dev/null +++ b/src/feature/nodelist/nodefamily.h @@ -0,0 +1,47 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file nodefamily.h + * \brief Header file for nodefamily.c. + **/ + +#ifndef TOR_NODEFAMILY_H +#define TOR_NODEFAMILY_H + +#include "lib/malloc/malloc.h" +#include + +typedef struct nodefamily_t nodefamily_t; +struct node_t; +struct smartlist_t; + +#define NF_WARN_MALFORMED (1u<<0) +#define NF_REJECT_MALFORMED (1u<<1) + +nodefamily_t *nodefamily_parse(const char *s, + const uint8_t *rsa_id_self, + unsigned flags); +nodefamily_t *nodefamily_from_members(const struct smartlist_t *members, + const uint8_t *rsa_id_self, + unsigned flags); +void nodefamily_free_(nodefamily_t *family); +#define nodefamily_free(family) \ + FREE_AND_NULL(nodefamily_t, nodefamily_free_, (family)) + +bool nodefamily_contains_rsa_id(const nodefamily_t *family, + const uint8_t *rsa_id); +bool nodefamily_contains_nickname(const nodefamily_t *family, + const char *name); +bool nodefamily_contains_node(const nodefamily_t *family, + const struct node_t *node); +void nodefamily_add_nodes_to_smartlist(const nodefamily_t *family, + struct smartlist_t *out); +char *nodefamily_format(const nodefamily_t *family); + +void nodefamily_free_all(void); + +#endif diff --git a/src/feature/nodelist/nodefamily_st.h b/src/feature/nodelist/nodefamily_st.h new file mode 100644 index 0000000000..f88ada494a --- /dev/null +++ b/src/feature/nodelist/nodefamily_st.h @@ -0,0 +1,48 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef TOR_NODEFAMILY_ST_H +#define TOR_NODEFAMILY_ST_H + +#include "orconfig.h" +#include "ht.h" + +struct nodefamily_t { + /** Entry for this nodefamily_t within the hashtable. */ + HT_ENTRY(nodefamily_t) ht_ent; + /** Reference count. (The hashtable is not treated as a reference */ + uint32_t refcnt; + /** Number of items encoded in family_members. */ + uint32_t n_members; + /* A byte-array encoding the members of this family. We encode each member + * as one byte to indicate whether it's a nickname or a fingerprint, plus + * DIGEST_LEN bytes of data. The entries are lexically sorted. + */ + uint8_t family_members[FLEXIBLE_ARRAY_MEMBER]; +}; + +#define NODEFAMILY_MEMBER_LEN (1+DIGEST_LEN) + +/** Tag byte, indicates that the following bytes are a NUL-padded nickname. + */ +#define NODEFAMILY_BY_NICKNAME 0 +/** Tag byte, indicates that the following bytes are a RSA1024 SHA1 ID. + */ +#define NODEFAMILY_BY_RSA_ID 1 + +/** + * Number of bytes to allocate in the array for a nodefamily_t with N members. + **/ +#define NODEFAMILY_ARRAY_SIZE(n) \ + ((n) * NODEFAMILY_MEMBER_LEN) + +/** + * Pointer to the i'th member of nf, as encoded. + */ +#define NODEFAMILY_MEMBER_PTR(nf, i) \ + (&((nf)->family_members[(i) * NODEFAMILY_MEMBER_LEN])) + +#endif diff --git a/src/test/test_nodelist.c b/src/test/test_nodelist.c index 1af6db68ec..2dbd949b7d 100644 --- a/src/test/test_nodelist.c +++ b/src/test/test_nodelist.c @@ -9,12 +9,14 @@ #include "core/or/or.h" #include "lib/crypt_ops/crypto_rand.h" #include "feature/nodelist/networkstatus.h" +#include "feature/nodelist/nodefamily.h" #include "feature/nodelist/nodelist.h" #include "feature/nodelist/torcert.h" #include "feature/nodelist/microdesc_st.h" #include "feature/nodelist/networkstatus_st.h" #include "feature/nodelist/node_st.h" +#include "feature/nodelist/nodefamily_st.h" #include "feature/nodelist/routerinfo_st.h" #include "feature/nodelist/routerstatus_st.h" @@ -231,6 +233,251 @@ test_nodelist_ed_id(void *arg) #undef N_NODES } +static void +test_nodelist_nodefamily(void *arg) +{ + (void)arg; + /* hex ID digests */ + const char h1[] = "5B435D6869206861206C65207363617270652070"; + const char h2[] = "75C3B220616E6461726520696E206769726F2061"; + const char h3[] = "2074726F766172206461206D616E67696172652C"; + const char h4[] = "206D656E747265206E6F6E2076616C65206C2769"; + const char h5[] = "6E766572736F2E202D2D5072696D6F204C657669"; + + /* binary ID digests */ + uint8_t d1[DIGEST_LEN], d2[DIGEST_LEN], d3[DIGEST_LEN], d4[DIGEST_LEN], + d5[DIGEST_LEN]; + base16_decode((char*)d1, sizeof(d1), h1, strlen(h1)); + base16_decode((char*)d2, sizeof(d2), h2, strlen(h2)); + base16_decode((char*)d3, sizeof(d3), h3, strlen(h3)); + base16_decode((char*)d4, sizeof(d4), h4, strlen(h4)); + base16_decode((char*)d5, sizeof(d5), h5, strlen(h5)); + + char *enc=NULL, *enc2=NULL; + + nodefamily_t *nf1 = NULL; + nodefamily_t *nf2 = NULL; + nodefamily_t *nf3 = NULL; + + /* Make sure that sorting and de-duplication work. */ + tor_asprintf(&enc, "$%s hello", h1); + nf1 = nodefamily_parse(enc, NULL, 0); + tt_assert(nf1); + tor_free(enc); + + tor_asprintf(&enc, "hello hello $%s hello", h1); + nf2 = nodefamily_parse(enc, NULL, 0); + tt_assert(nf2); + tt_ptr_op(nf1, OP_EQ, nf2); + tor_free(enc); + + tor_asprintf(&enc, "%s $%s hello", h1, h1); + nf3 = nodefamily_parse(enc, NULL, 0); + tt_assert(nf3); + tt_ptr_op(nf1, OP_EQ, nf3); + tor_free(enc); + + tt_assert(nodefamily_contains_rsa_id(nf1, d1)); + tt_assert(! nodefamily_contains_rsa_id(nf1, d2)); + tt_assert(nodefamily_contains_nickname(nf1, "hello")); + tt_assert(nodefamily_contains_nickname(nf1, "HELLO")); + tt_assert(! nodefamily_contains_nickname(nf1, "goodbye")); + + tt_int_op(nf1->refcnt, OP_EQ, 3); + nodefamily_free(nf3); + tt_int_op(nf1->refcnt, OP_EQ, 2); + + /* Try parsing with a provided self RSA digest. */ + nf3 = nodefamily_parse("hello ", d1, 0); + tt_assert(nf3); + tt_ptr_op(nf1, OP_EQ, nf3); + + /* Do we get the expected result when we re-encode? */ + tor_asprintf(&enc, "hello $%s", h1); + enc2 = nodefamily_format(nf1); + tt_str_op(enc2, OP_EQ, enc); + tor_free(enc2); + tor_free(enc); + + /* Make sure that we get a different result if we give a different digest. */ + nodefamily_free(nf3); + tor_asprintf(&enc, "hello $%s hello", h3); + nf3 = nodefamily_parse(enc, NULL, 0); + tt_assert(nf3); + tt_ptr_op(nf1, OP_NE, nf3); + tor_free(enc); + + tt_assert(nodefamily_contains_rsa_id(nf3, d3)); + tt_assert(! nodefamily_contains_rsa_id(nf3, d2)); + tt_assert(! nodefamily_contains_rsa_id(nf3, d1)); + tt_assert(nodefamily_contains_nickname(nf3, "hello")); + tt_assert(! nodefamily_contains_nickname(nf3, "goodbye")); + + nodefamily_free(nf1); + nodefamily_free(nf2); + nodefamily_free(nf3); + + /* Try one with several digests, all with nicknames appended, in different + formats. */ + tor_asprintf(&enc, "%s $%s $%s=res $%s~ist", h1, h2, h3, h4); + nf1 = nodefamily_parse(enc, d5, 0); + tt_assert(nf1); + tt_assert(nodefamily_contains_rsa_id(nf1, d1)); + tt_assert(nodefamily_contains_rsa_id(nf1, d2)); + tt_assert(nodefamily_contains_rsa_id(nf1, d3)); + tt_assert(nodefamily_contains_rsa_id(nf1, d4)); + tt_assert(nodefamily_contains_rsa_id(nf1, d5)); + /* Nicknames aren't preserved when ids are present, since node naming is + * deprecated */ + tt_assert(! nodefamily_contains_nickname(nf3, "res")); + tor_free(enc); + tor_asprintf(&enc, "$%s $%s $%s $%s $%s", h4, h3, h1, h5, h2); + enc2 = nodefamily_format(nf1); + tt_str_op(enc, OP_EQ, enc2); + tor_free(enc); + tor_free(enc2); + + /* Try ones where we parse the empty string. */ + nf2 = nodefamily_parse("", NULL, 0); + nf3 = nodefamily_parse("", d4, 0); + tt_assert(nf2); + tt_assert(nf3); + tt_ptr_op(nf2, OP_NE, nf3); + + tt_assert(! nodefamily_contains_rsa_id(nf2, d4)); + tt_assert(nodefamily_contains_rsa_id(nf3, d4)); + tt_assert(! nodefamily_contains_rsa_id(nf2, d5)); + tt_assert(! nodefamily_contains_rsa_id(nf3, d5)); + tt_assert(! nodefamily_contains_nickname(nf2, "fred")); + tt_assert(! nodefamily_contains_nickname(nf3, "bosco")); + + /* The NULL family should contain nothing. */ + tt_assert(! nodefamily_contains_rsa_id(NULL, d4)); + tt_assert(! nodefamily_contains_rsa_id(NULL, d5)); + + done: + tor_free(enc); + tor_free(enc2); + nodefamily_free(nf1); + nodefamily_free(nf2); + nodefamily_free(nf3); +} + +static void +test_nodelist_nodefamily_parse_err(void *arg) +{ + (void)arg; + nodefamily_t *nf1 = NULL; + char *enc = NULL; + const char *semibogus = + "sdakljfdslkfjdsaklfjdkl9sdf " // too long for nickname + "$jkASDFLkjsadfjhkl " // not hex + "$7468696e67732d696e2d7468656d73656c766573 " // ok + "reticulatogranulate "// ok + "$73656d69616e7468726f706f6c6f676963616c6c79 " // too long for hex + "$616273656e746d696e6465646e6573736573" // too short for hex + ; + + setup_capture_of_logs(LOG_WARN); + + // We only get two items when we parse this. + for (int reject = 0; reject <= 1; ++reject) { + for (int log_at_warn = 0; log_at_warn <= 1; ++log_at_warn) { + unsigned flags = log_at_warn ? NF_WARN_MALFORMED : 0; + flags |= reject ? NF_REJECT_MALFORMED : 0; + nf1 = nodefamily_parse(semibogus, NULL, flags); + if (reject) { + tt_assert(nf1 == NULL); + } else { + tt_assert(nf1); + enc = nodefamily_format(nf1); + tt_str_op(enc, OP_EQ, + "reticulatogranulate " + "$7468696E67732D696E2D7468656D73656C766573"); + tor_free(enc); + } + + if (log_at_warn) { + expect_log_msg_containing("$616273656e746d696e6465646e6573736573"); + expect_log_msg_containing("sdakljfdslkfjdsaklfjdkl9sdf"); + } else { + tt_int_op(mock_saved_log_n_entries(), OP_EQ, 0); + } + mock_clean_saved_logs(); + } + } + + done: + tor_free(enc); + nodefamily_free(nf1); + teardown_capture_of_logs(); +} + +static const node_t * +mock_node_get_by_id(const char *id) +{ + if (fast_memeq(id, "!!!!!!!!!!!!!!!!!!!!", DIGEST_LEN)) + return NULL; + + // use tor_free, not node_free. + node_t *fake_node = tor_malloc_zero(sizeof(node_t)); + memcpy(fake_node->identity, id, DIGEST_LEN); + return fake_node; +} + +static const node_t * +mock_node_get_by_nickname(const char *nn, unsigned flags) +{ + (void)flags; + if (!strcmp(nn, "nonesuch")) + return NULL; + + // use tor_free, not node_free. + node_t *fake_node = tor_malloc_zero(sizeof(node_t)); + strlcpy(fake_node->identity, nn, DIGEST_LEN); + return fake_node; +} + +static void +test_nodelist_nodefamily_lookup(void *arg) +{ + (void)arg; + MOCK(node_get_by_nickname, mock_node_get_by_nickname); + MOCK(node_get_by_id, mock_node_get_by_id); + smartlist_t *sl = smartlist_new(); + nodefamily_t *nf1 = NULL; + char *mem_op_hex_tmp = NULL; + + // 'null' is allowed. + nodefamily_add_nodes_to_smartlist(NULL, sl); + tt_int_op(smartlist_len(sl), OP_EQ, 0); + + // Try a real family + nf1 = nodefamily_parse("$EEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEE " + "$2121212121212121212121212121212121212121 " + "$3333333333333333333333333333333333333333 " + "erewhon nonesuch", NULL, 0); + tt_assert(nf1); + nodefamily_add_nodes_to_smartlist(nf1, sl); + // There were 5 elements; 2 were dropped because the mocked lookup failed. + tt_int_op(smartlist_len(sl), OP_EQ, 3); + + const node_t *n = smartlist_get(sl, 0); + tt_str_op(n->identity, OP_EQ, "erewhon"); + n = smartlist_get(sl, 1); + test_memeq_hex(n->identity, "3333333333333333333333333333333333333333"); + n = smartlist_get(sl, 2); + test_memeq_hex(n->identity, "EEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEE"); + + done: + UNMOCK(node_get_by_nickname); + UNMOCK(node_get_by_id); + SMARTLIST_FOREACH(sl, node_t *, fake_node, tor_free(fake_node)); + smartlist_free(sl); + nodefamily_free(nf1); + tor_free(mem_op_hex_tmp); +} + #define NODE(name, flags) \ { #name, test_nodelist_##name, (flags), NULL, NULL } @@ -239,6 +486,8 @@ struct testcase_t nodelist_tests[] = { NODE(node_get_verbose_nickname_not_named, TT_FORK), NODE(node_is_dir, TT_FORK), NODE(ed_id, TT_FORK), + NODE(nodefamily, TT_FORK), + NODE(nodefamily_parse_err, TT_FORK), + NODE(nodefamily_lookup, TT_FORK), END_OF_TESTCASES }; - From 426c9561c5f5bc5f38a42f3e46437db59fcdc7c0 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 23 Oct 2018 19:55:12 -0400 Subject: [PATCH 0172/2557] Use nodefamily_t in microdescriptors. Closes ticket 27359. --- changes/ticket27359 | 3 + src/feature/dirparse/microdesc_parse.c | 16 ++-- src/feature/nodelist/microdesc.c | 6 +- src/feature/nodelist/microdesc_st.h | 5 +- src/feature/nodelist/nodelist.c | 110 ++++++++++++++++--------- src/feature/nodelist/nodelist.h | 1 - src/test/test_microdesc.c | 8 +- 7 files changed, 89 insertions(+), 60 deletions(-) create mode 100644 changes/ticket27359 diff --git a/changes/ticket27359 b/changes/ticket27359 new file mode 100644 index 0000000000..bddc90634d --- /dev/null +++ b/changes/ticket27359 @@ -0,0 +1,3 @@ + o Minor features (memory usage): + - Store microdescriptor family lists with a more compact representation + to save memory. Closes ticket 27359. diff --git a/src/feature/dirparse/microdesc_parse.c b/src/feature/dirparse/microdesc_parse.c index aebff5a35f..8ad9626377 100644 --- a/src/feature/dirparse/microdesc_parse.c +++ b/src/feature/dirparse/microdesc_parse.c @@ -18,6 +18,7 @@ #include "feature/dirparse/routerparse.h" #include "feature/nodelist/microdesc.h" #include "feature/nodelist/nickname.h" +#include "feature/nodelist/nodefamily.h" #include "feature/relay/router.h" #include "lib/crypt_ops/crypto_curve25519.h" #include "lib/crypt_ops/crypto_ed25519.h" @@ -32,7 +33,7 @@ static token_rule_t microdesc_token_table[] = { T01("ntor-onion-key", K_ONION_KEY_NTOR, GE(1), NO_OBJ ), T0N("id", K_ID, GE(2), NO_OBJ ), T0N("a", K_A, GE(1), NO_OBJ ), - T01("family", K_FAMILY, ARGS, NO_OBJ ), + T01("family", K_FAMILY, CONCAT_ARGS, NO_OBJ ), T01("p", K_P, CONCAT_ARGS, NO_OBJ ), T01("p6", K_P6, CONCAT_ARGS, NO_OBJ ), A01("@last-listed", A_LAST_LISTED, CONCAT_ARGS, NO_OBJ ), @@ -222,16 +223,9 @@ microdescs_parse_from_string(const char *s, const char *eos, } if ((tok = find_opt_by_keyword(tokens, K_FAMILY))) { - int i; - md->family = smartlist_new(); - for (i=0;in_args;++i) { - if (!is_legal_nickname_or_hexdigest(tok->args[i])) { - log_warn(LD_DIR, "Illegal nickname %s in family line", - escaped(tok->args[i])); - goto next; - } - smartlist_add_strdup(md->family, tok->args[i]); - } + md->family = nodefamily_parse(tok->args[0], + NULL, + NF_WARN_MALFORMED); } if ((tok = find_opt_by_keyword(tokens, K_P))) { diff --git a/src/feature/nodelist/microdesc.c b/src/feature/nodelist/microdesc.c index 146c772daf..f331d5e109 100644 --- a/src/feature/nodelist/microdesc.c +++ b/src/feature/nodelist/microdesc.c @@ -23,6 +23,7 @@ #include "feature/nodelist/dirlist.h" #include "feature/nodelist/microdesc.h" #include "feature/nodelist/networkstatus.h" +#include "feature/nodelist/nodefamily.h" #include "feature/nodelist/nodelist.h" #include "feature/nodelist/routerlist.h" #include "feature/relay/router.h" @@ -882,10 +883,7 @@ microdesc_free_(microdesc_t *md, const char *fname, int lineno) if (md->body && md->saved_location != SAVED_IN_CACHE) tor_free(md->body); - if (md->family) { - SMARTLIST_FOREACH(md->family, char *, cp, tor_free(cp)); - smartlist_free(md->family); - } + nodefamily_free(md->family); short_policy_free(md->exit_policy); short_policy_free(md->ipv6_exit_policy); diff --git a/src/feature/nodelist/microdesc_st.h b/src/feature/nodelist/microdesc_st.h index d23da13137..30c896181d 100644 --- a/src/feature/nodelist/microdesc_st.h +++ b/src/feature/nodelist/microdesc_st.h @@ -9,6 +9,7 @@ struct curve25519_public_key_t; struct ed25519_public_key_t; +struct nodefamily_t; struct short_policy_t; /** A microdescriptor is the smallest amount of information needed to build a @@ -69,8 +70,8 @@ struct microdesc_t { tor_addr_t ipv6_addr; /** As routerinfo_t.ipv6_orport */ uint16_t ipv6_orport; - /** As routerinfo_t.family */ - smartlist_t *family; + /** As routerinfo_t.family, with readable members parsed. */ + struct nodefamily_t *family; /** IPv4 exit policy summary */ struct short_policy_t *exit_policy; /** IPv6 exit policy summary */ diff --git a/src/feature/nodelist/nodelist.c b/src/feature/nodelist/nodelist.c index a98a5c8655..3994c8d072 100644 --- a/src/feature/nodelist/nodelist.c +++ b/src/feature/nodelist/nodelist.c @@ -59,6 +59,7 @@ #include "feature/nodelist/microdesc.h" #include "feature/nodelist/networkstatus.h" #include "feature/nodelist/node_select.h" +#include "feature/nodelist/nodefamily.h" #include "feature/nodelist/nodelist.h" #include "feature/nodelist/routerlist.h" #include "feature/nodelist/routerset.h" @@ -1504,19 +1505,6 @@ node_is_me(const node_t *node) return router_digest_is_me(node->identity); } -/** Return node declared family (as a list of names), or NULL if - * the node didn't declare a family. */ -const smartlist_t * -node_get_declared_family(const node_t *node) -{ - if (node->ri && node->ri->declared_family) - return node->ri->declared_family; - else if (node->md && node->md->family) - return node->md->family; - else - return NULL; -} - /* Does this node have a valid IPv6 address? * Prefer node_has_ipv6_orport() or node_has_ipv6_dirport() for * checking specific ports. */ @@ -1905,6 +1893,61 @@ node_in_nickname_smartlist(const smartlist_t *lst, const node_t *node) return 0; } +/** Return true iff n1's declared family contains n2. */ +static int +node_family_contains(const node_t *n1, const node_t *n2) +{ + if (n1->ri && n1->ri->declared_family) { + return node_in_nickname_smartlist(n1->ri->declared_family, n2); + } else if (n1->md) { + return nodefamily_contains_node(n1->md->family, n2); + } else { + return 0; + } +} + +/** + * Return true iff node has declared a nonempty family. + **/ +static bool +node_has_declared_family(const node_t *node) +{ + if (node->ri && node->ri->declared_family && + smartlist_len(node->ri->declared_family)) { + return true; + } + + if (node->md && node->md->family) { + return true; + } + + return false; +} + +/** + * Add to out every node_t that is listed by node as being in + * its family. (Note that these nodes are not in node's family unless they + * also agree that node is in their family.) + **/ +static void +node_lookup_declared_family(smartlist_t *out, const node_t *node) +{ + if (node->ri && node->ri->declared_family && + smartlist_len(node->ri->declared_family)) { + SMARTLIST_FOREACH_BEGIN(node->ri->declared_family, const char *, name) { + const node_t *n2 = node_get_by_nickname(name, NNF_NO_WARN_UNNAMED); + if (n2) { + smartlist_add(out, (node_t *)n2); + } + } SMARTLIST_FOREACH_END(name); + return; + } + + if (node->md && node->md->family) { + nodefamily_add_nodes_to_smartlist(node->md->family, out); + } +} + /** Return true iff r1 and r2 are in the same family, but not the same * router. */ int @@ -1922,14 +1965,9 @@ nodes_in_same_family(const node_t *node1, const node_t *node2) } /* Are they in the same family because the agree they are? */ - { - const smartlist_t *f1, *f2; - f1 = node_get_declared_family(node1); - f2 = node_get_declared_family(node2); - if (f1 && f2 && - node_in_nickname_smartlist(f1, node2) && - node_in_nickname_smartlist(f2, node1)) - return 1; + if (node_family_contains(node1, node2) && + node_family_contains(node2, node1)) { + return 1; } /* Are they in the same option because the user says they are? */ @@ -1957,13 +1995,10 @@ void nodelist_add_node_and_family(smartlist_t *sl, const node_t *node) { const smartlist_t *all_nodes = nodelist_get_list(); - const smartlist_t *declared_family; const or_options_t *options = get_options(); tor_assert(node); - declared_family = node_get_declared_family(node); - /* Let's make sure that we have the node itself, if it's a real node. */ { const node_t *real_node = node_get_by_id(node->identity); @@ -1984,25 +2019,20 @@ nodelist_add_node_and_family(smartlist_t *sl, const node_t *node) } SMARTLIST_FOREACH_END(node2); } - /* Now, add all nodes in the declared_family of this node, if they + /* Now, add all nodes in the declared family of this node, if they * also declare this node to be in their family. */ - if (declared_family) { + if (node_has_declared_family(node)) { + smartlist_t *declared_family = smartlist_new(); + node_lookup_declared_family(declared_family, node); + /* Add every r such that router declares familyness with node, and node * declares familyhood with router. */ - SMARTLIST_FOREACH_BEGIN(declared_family, const char *, name) { - const node_t *node2; - const smartlist_t *family2; - if (!(node2 = node_get_by_nickname(name, NNF_NO_WARN_UNNAMED))) - continue; - if (!(family2 = node_get_declared_family(node2))) - continue; - SMARTLIST_FOREACH_BEGIN(family2, const char *, name2) { - if (node_nickname_matches(node, name2)) { - smartlist_add(sl, (void*)node2); - break; - } - } SMARTLIST_FOREACH_END(name2); - } SMARTLIST_FOREACH_END(name); + SMARTLIST_FOREACH_BEGIN(declared_family, const node_t *, node2) { + if (node_family_contains(node2, node)) { + smartlist_add(sl, (void*)node2); + } + } SMARTLIST_FOREACH_END(node2); + smartlist_free(declared_family); } /* If the user declared any families locally, honor those too. */ diff --git a/src/feature/nodelist/nodelist.h b/src/feature/nodelist/nodelist.h index cf7658cf9d..87ea544db2 100644 --- a/src/feature/nodelist/nodelist.h +++ b/src/feature/nodelist/nodelist.h @@ -68,7 +68,6 @@ const char *node_get_platform(const node_t *node); uint32_t node_get_prim_addr_ipv4h(const node_t *node); void node_get_address_string(const node_t *node, char *cp, size_t len); long node_get_declared_uptime(const node_t *node); -const smartlist_t *node_get_declared_family(const node_t *node); const struct ed25519_public_key_t *node_get_ed25519_id(const node_t *node); int node_ed25519_id_matches(const node_t *node, const struct ed25519_public_key_t *id); diff --git a/src/test/test_microdesc.c b/src/test/test_microdesc.c index 8ede2690ec..3318408d53 100644 --- a/src/test/test_microdesc.c +++ b/src/test/test_microdesc.c @@ -11,6 +11,7 @@ #include "feature/dirparse/routerparse.h" #include "feature/nodelist/microdesc.h" #include "feature/nodelist/networkstatus.h" +#include "feature/nodelist/nodefamily.h" #include "feature/nodelist/routerlist.h" #include "feature/nodelist/torcert.h" @@ -70,6 +71,7 @@ test_md_cache(void *data) const char *test_md3_noannotation = strchr(test_md3, '\n')+1; time_t time1, time2, time3; char *fn = NULL, *s = NULL; + char *encoded_family = NULL; (void)data; options = get_options_mutable(); @@ -172,8 +174,9 @@ test_md_cache(void *data) tt_ptr_op(md1->family, OP_EQ, NULL); tt_ptr_op(md3->family, OP_NE, NULL); - tt_int_op(smartlist_len(md3->family), OP_EQ, 3); - tt_str_op(smartlist_get(md3->family, 0), OP_EQ, "nodeX"); + + encoded_family = nodefamily_format(md3->family); + tt_str_op(encoded_family, OP_EQ, "nodeX nodeY nodeZ"); /* Now rebuild the cache! */ tt_int_op(microdesc_cache_rebuild(mc, 1), OP_EQ, 0); @@ -254,6 +257,7 @@ test_md_cache(void *data) smartlist_free(wanted); tor_free(s); tor_free(fn); + tor_free(encoded_family); } static const char truncated_md[] = From aa1d767e6b98efa4ffa7d39dba2272949aae2f65 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Sat, 17 Nov 2018 12:46:59 -0500 Subject: [PATCH 0173/2557] Aim for 100% test coverage on nodefamily.c --- src/test/test_nodelist.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/test/test_nodelist.c b/src/test/test_nodelist.c index 2dbd949b7d..7e0df56df5 100644 --- a/src/test/test_nodelist.c +++ b/src/test/test_nodelist.c @@ -259,6 +259,10 @@ test_nodelist_nodefamily(void *arg) nodefamily_t *nf2 = NULL; nodefamily_t *nf3 = NULL; + enc = nodefamily_format(NULL); + tt_str_op(enc, OP_EQ, ""); + tor_free(enc); + /* Make sure that sorting and de-duplication work. */ tor_asprintf(&enc, "$%s hello", h1); nf1 = nodefamily_parse(enc, NULL, 0); @@ -361,6 +365,7 @@ test_nodelist_nodefamily(void *arg) nodefamily_free(nf1); nodefamily_free(nf2); nodefamily_free(nf3); + nodefamily_free_all(); } static void From 4f9548f89339fe1fab1863ac2b2c72d09f7224ef Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Sat, 17 Nov 2018 12:55:23 -0500 Subject: [PATCH 0174/2557] Expose more nodelist.c functions to tests --- src/feature/nodelist/nodelist.c | 10 +++++----- src/feature/nodelist/nodelist.h | 10 ++++++++-- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/src/feature/nodelist/nodelist.c b/src/feature/nodelist/nodelist.c index 3994c8d072..d070f31c12 100644 --- a/src/feature/nodelist/nodelist.c +++ b/src/feature/nodelist/nodelist.c @@ -1870,7 +1870,7 @@ addrs_in_same_network_family(const tor_addr_t *a1, * (case-insensitive), or if node's identity key digest * matches a hexadecimal value stored in nickname. Return * false otherwise. */ -static int +STATIC int node_nickname_matches(const node_t *node, const char *nickname) { const char *n = node_get_nickname(node); @@ -1882,7 +1882,7 @@ node_nickname_matches(const node_t *node, const char *nickname) } /** Return true iff node is named by some nickname in lst. */ -static inline int +STATIC int node_in_nickname_smartlist(const smartlist_t *lst, const node_t *node) { if (!lst) return 0; @@ -1894,7 +1894,7 @@ node_in_nickname_smartlist(const smartlist_t *lst, const node_t *node) } /** Return true iff n1's declared family contains n2. */ -static int +STATIC int node_family_contains(const node_t *n1, const node_t *n2) { if (n1->ri && n1->ri->declared_family) { @@ -1909,7 +1909,7 @@ node_family_contains(const node_t *n1, const node_t *n2) /** * Return true iff node has declared a nonempty family. **/ -static bool +STATIC bool node_has_declared_family(const node_t *node) { if (node->ri && node->ri->declared_family && @@ -1929,7 +1929,7 @@ node_has_declared_family(const node_t *node) * its family. (Note that these nodes are not in node's family unless they * also agree that node is in their family.) **/ -static void +STATIC void node_lookup_declared_family(smartlist_t *out, const node_t *node) { if (node->ri && node->ri->declared_family && diff --git a/src/feature/nodelist/nodelist.h b/src/feature/nodelist/nodelist.h index 87ea544db2..32300eb00c 100644 --- a/src/feature/nodelist/nodelist.h +++ b/src/feature/nodelist/nodelist.h @@ -154,10 +154,16 @@ int count_loading_descriptors_progress(void); #ifdef NODELIST_PRIVATE +STATIC int node_nickname_matches(const node_t *node, const char *nickname); +STATIC int node_in_nickname_smartlist(const smartlist_t *lst, + const node_t *node); +STATIC int node_family_contains(const node_t *n1, const node_t *n2); +STATIC bool node_has_declared_family(const node_t *node); +STATIC void node_lookup_declared_family(smartlist_t *out, const node_t *node); + #ifdef TOR_UNIT_TESTS -STATIC void -node_set_hsdir_index(node_t *node, const networkstatus_t *ns); +STATIC void node_set_hsdir_index(node_t *node, const networkstatus_t *ns); #endif /* defined(TOR_UNIT_TESTS) */ From 0e762c0cf5260040c38e93b4d4204be3f6746301 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Sat, 17 Nov 2018 13:35:52 -0500 Subject: [PATCH 0175/2557] Test new functions in nodelist.c --- src/test/test_nodelist.c | 129 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 129 insertions(+) diff --git a/src/test/test_nodelist.c b/src/test/test_nodelist.c index 7e0df56df5..0287be3305 100644 --- a/src/test/test_nodelist.c +++ b/src/test/test_nodelist.c @@ -6,6 +6,8 @@ * \brief Unit tests for nodelist related functions. **/ +#define NODELIST_PRIVATE + #include "core/or/or.h" #include "lib/crypt_ops/crypto_rand.h" #include "feature/nodelist/networkstatus.h" @@ -483,6 +485,131 @@ test_nodelist_nodefamily_lookup(void *arg) tor_free(mem_op_hex_tmp); } +static void +test_nodelist_nickname_matches(void *arg) +{ + (void)arg; + node_t mock_node; + routerstatus_t mock_rs; + memset(&mock_node, 0, sizeof(mock_node)); + memset(&mock_rs, 0, sizeof(mock_rs)); + + strlcpy(mock_rs.nickname, "evilgeniuses", sizeof(mock_rs.nickname)); + mock_node.rs = &mock_rs; + memcpy(mock_node.identity, ".forabettertomorrow.", DIGEST_LEN); + +#define match(x) tt_assert(node_nickname_matches(&mock_node, (x))) +#define no_match(x) tt_assert(! node_nickname_matches(&mock_node, (x))) + + match("evilgeniuses"); + match("EvilGeniuses"); + match("EvilGeniuses"); + match("2e666f7261626574746572746f6d6f72726f772e"); + match("2E666F7261626574746572746F6D6F72726F772E"); + match("$2e666f7261626574746572746f6d6f72726f772e"); + match("$2E666F7261626574746572746F6D6F72726F772E"); + match("$2E666F7261626574746572746F6D6F72726F772E~evilgeniuses"); + match("$2E666F7261626574746572746F6D6F72726F772E~EVILGENIUSES"); + + no_match("evilgenius"); + no_match("evilgeniuseses"); + no_match("evil.genius"); + no_match("$2E666F7261626574746572746F6D6F72726FFFFF"); + no_match("2E666F7261626574746572746F6D6F72726FFFFF"); + no_match("$2E666F7261626574746572746F6D6F72726F772E~fred"); + no_match("$2E666F7261626574746572746F6D6F72726F772E=EVILGENIUSES"); + done: + ; +} + +static void +test_nodelist_node_nodefamily(void *arg) +{ + (void)arg; + node_t mock_node1; + routerstatus_t mock_rs; + microdesc_t mock_md; + + node_t mock_node2; + routerinfo_t mock_ri; + + smartlist_t *nodes=smartlist_new(); + + memset(&mock_node1, 0, sizeof(mock_node1)); + memset(&mock_node2, 0, sizeof(mock_node2)); + memset(&mock_rs, 0, sizeof(mock_rs)); + memset(&mock_md, 0, sizeof(mock_md)); + memset(&mock_ri, 0, sizeof(mock_ri)); + + mock_node1.rs = &mock_rs; + mock_node1.md = &mock_md; + + mock_node2.ri = &mock_ri; + + strlcpy(mock_rs.nickname, "nodeone", sizeof(mock_rs.nickname)); + mock_ri.nickname = tor_strdup("nodetwo"); + + memcpy(mock_node1.identity, "NodeOneNode1NodeOne1", DIGEST_LEN); + memcpy(mock_node2.identity, "SecondNodeWe'reTestn", DIGEST_LEN); + + // empty families. + tt_assert(! node_family_contains(&mock_node1, &mock_node2)); + tt_assert(! node_family_contains(&mock_node2, &mock_node1)); + + // Families contain nodes, but not these nodes + mock_ri.declared_family = smartlist_new(); + smartlist_add(mock_ri.declared_family, (char*)"NodeThree"); + mock_md.family = nodefamily_parse("NodeFour", NULL, 0); + tt_assert(! node_family_contains(&mock_node1, &mock_node2)); + tt_assert(! node_family_contains(&mock_node2, &mock_node1)); + + // Families contain one another. + smartlist_add(mock_ri.declared_family, (char*) + "4e6f64654f6e654e6f6465314e6f64654f6e6531"); + tt_assert(! node_family_contains(&mock_node1, &mock_node2)); + tt_assert(node_family_contains(&mock_node2, &mock_node1)); + + nodefamily_free(mock_md.family); + mock_md.family = nodefamily_parse( + "NodeFour " + "5365636f6e644e6f64655765277265546573746e", NULL, 0); + tt_assert(node_family_contains(&mock_node1, &mock_node2)); + tt_assert(node_family_contains(&mock_node2, &mock_node1)); + + // Try looking up families now. + MOCK(node_get_by_nickname, mock_node_get_by_nickname); + MOCK(node_get_by_id, mock_node_get_by_id); + + node_lookup_declared_family(nodes, &mock_node1); + tt_int_op(smartlist_len(nodes), OP_EQ, 2); + const node_t *n = smartlist_get(nodes, 0); + tt_str_op(n->identity, OP_EQ, "NodeFour"); + n = smartlist_get(nodes, 1); + tt_mem_op(n->identity, OP_EQ, "SecondNodeWe'reTestn", DIGEST_LEN); + + // free, try the other one. + SMARTLIST_FOREACH(nodes, node_t *, x, tor_free(x)); + smartlist_clear(nodes); + + node_lookup_declared_family(nodes, &mock_node2); + tt_int_op(smartlist_len(nodes), OP_EQ, 2); + n = smartlist_get(nodes, 0); + tt_str_op(n->identity, OP_EQ, "NodeThree"); + n = smartlist_get(nodes, 1); + // This gets a truncated hex hex ID since it was looked up by name + tt_str_op(n->identity, OP_EQ, "4e6f64654f6e654e6f6"); + + done: + UNMOCK(node_get_by_nickname); + UNMOCK(node_get_by_id); + smartlist_free(mock_ri.declared_family); + nodefamily_free(mock_md.family); + tor_free(mock_ri.nickname); + // use tor_free, these aren't real nodes + SMARTLIST_FOREACH(nodes, node_t *, x, tor_free(x)); + smartlist_free(nodes); +} + #define NODE(name, flags) \ { #name, test_nodelist_##name, (flags), NULL, NULL } @@ -494,5 +621,7 @@ struct testcase_t nodelist_tests[] = { NODE(nodefamily, TT_FORK), NODE(nodefamily_parse_err, TT_FORK), NODE(nodefamily_lookup, TT_FORK), + NODE(nickname_matches, 0), + NODE(node_nodefamily, TT_FORK), END_OF_TESTCASES }; From 71651ea4aa507ee50865bd9584873e0c9a422c14 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 7 Apr 2015 16:36:05 +0200 Subject: [PATCH 0176/2557] Complain if net.inet.ip.random_id is not set on FreeBSD-based servers Apparently a couple of operators haven't gotten the memos [0] yet and it looks like FreeBSD's default value will not change any time soon [1]. [0]: https://lists.torproject.org/pipermail/tor-relays/2014-March/004199.html https://lists.torproject.org/pipermail/tor-relays/2014-November/005687.html https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=195828 [1]: https://lists.freebsd.org/pipermail/freebsd-net/2015-April/041942.html --- src/app/config/config.c | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/app/config/config.c b/src/app/config/config.c index 45a23d67d5..56fca15499 100644 --- a/src/app/config/config.c +++ b/src/app/config/config.c @@ -157,6 +157,10 @@ #include "core/or/connection_st.h" #include "core/or/port_cfg_st.h" +#ifdef __FreeBSD__ +#include +#endif + #ifdef HAVE_SYSTEMD # if defined(__COVERITY__) && !defined(__INCLUDE_LEVEL__) /* Systemd's use of gcc's __INCLUDE_LEVEL__ extension macro appears to confuse @@ -3383,6 +3387,23 @@ options_validate(or_options_t *old_options, or_options_t *options, if (ContactInfo && !string_is_utf8(ContactInfo, strlen(ContactInfo))) REJECT("ContactInfo config option must be UTF-8."); +#ifdef __FreeBSD__ + if (server_mode(options)) { + int random_id_state; + size_t state_size = sizeof(random_id_state); + + if (sysctlbyname("net.inet.ip.random_id", &random_id_state, + &state_size, NULL, 0)) { + log_warn(LD_CONFIG, + "Failed to figure out if IP ids are randomized."); + } else if (random_id_state == 0) { + log_warn(LD_CONFIG, "Looks like IP ids are not randomized. " + "Please consider setting the net.inet.ip.random_id sysctl, " + "so your relay makes it harder to figure out how busy it is."); + } + } +#endif + /* Special case on first boot if no Log options are given. */ if (!options->Logs && !options->RunAsDaemon && !from_setconf) { if (quiet_level == 0) From d020124138cc0d16e685bfd35dd388a4db7f68af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Mon, 19 Nov 2018 16:33:06 +0100 Subject: [PATCH 0177/2557] Add changes file for #28518. See: https://bugs.torproject.org/28518 --- changes/bug28518 | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 changes/bug28518 diff --git a/changes/bug28518 b/changes/bug28518 new file mode 100644 index 0000000000..d7ebab29bb --- /dev/null +++ b/changes/bug28518 @@ -0,0 +1,4 @@ + o Minor features (FreeBSD): + - Warn relay operators if the "net.inet.ip.random_id" sysctl (IP ID + randomization) is disabled on their relay if it is running on FreeBSD + based operating systems. Closes ticket 28518. From ffee0a6384e751486bb4ca2752b6a00527b923ca Mon Sep 17 00:00:00 2001 From: rl1987 Date: Tue, 20 Nov 2018 15:40:52 +0200 Subject: [PATCH 0178/2557] Add pre-push git hook to prevent fixup and squash commits from ending up in master --- changes/ticket27993 | 3 +++ scripts/maint/pre-push | 56 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+) create mode 100644 changes/ticket27993 create mode 100755 scripts/maint/pre-push diff --git a/changes/ticket27993 b/changes/ticket27993 new file mode 100644 index 0000000000..78ee7c2054 --- /dev/null +++ b/changes/ticket27993 @@ -0,0 +1,3 @@ + o Minor features (developer tooling): + - Provide git hook script to prevent "fixup!" and "squash!" commits from + ending up in master. Closes ticket 27993. diff --git a/scripts/maint/pre-push b/scripts/maint/pre-push new file mode 100755 index 0000000000..2cf1837b8d --- /dev/null +++ b/scripts/maint/pre-push @@ -0,0 +1,56 @@ +#!/bin/sh + +# git pre-push hook script to prevent "fixup!" and "squash!" commit +# from ending up in master, or in any branch if CUR_BRANCH check is removed. +# It is meant to be placed in .git/hooks directory. +# +# The following sample script was used as starting point: +# https://github.com/git/git/blob/master/templates/hooks--pre-push.sample + +z40=0000000000000000000000000000000000000000 + +CUR_BRANCH=$(git rev-parse --abbrev-ref HEAD) +if [ "$CUR_BRANCH" != "master" ] +then + exit 0 +fi + +echo "Running pre-push hook" + +# shellcheck disable=SC2034 +while read -r local_ref local_sha remote_ref remote_sha +do + if [ "$local_sha" = $z40 ] + then + # Handle delete + : + else + if [ "$remote_sha" = $z40 ] + then + # New branch, examine all commits + range="$local_sha" + else + # Update to existing branch, examine new commits + range="$remote_sha..$local_sha" + fi + + # Check for fixup! commit + commit=$(git rev-list -n 1 --grep '^fixup!' "$range") + if [ -n "$commit" ] + then + echo >&2 "Found fixup! commit in $local_ref, not pushing" + exit 1 + fi + + # Check for squash! commit + commit=$(git rev-list -n 1 --grep '^squash!' "$range") + if [ -n "$commit" ] + then + echo >&2 "Found squash! commit in $local_ref, not pushing" + exit 1 + fi + fi +done + +exit 0 + From 7dd515b0e0a12cfd9a324a878f7ca6b6b84b24cb Mon Sep 17 00:00:00 2001 From: rl1987 Date: Tue, 20 Nov 2018 16:22:47 +0200 Subject: [PATCH 0179/2557] Silence SC1117 warnings --- scripts/maint/updateRustDependencies.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/maint/updateRustDependencies.sh b/scripts/maint/updateRustDependencies.sh index d2680856b8..1e927dd0b1 100755 --- a/scripts/maint/updateRustDependencies.sh +++ b/scripts/maint/updateRustDependencies.sh @@ -27,19 +27,19 @@ VENDORED="$TOPLEVEL/src/ext/rust/crates" CARGO=$(which cargo) if ! test -f "$TOML" ; then - printf "Error: Couldn't find workspace Cargo.toml in expected location: %s\n" "$TOML" + printf "Error: Couldn't find workspace Cargo.toml in expected location: %s\\n" "$TOML" fi if ! test -d "$VENDORED" ; then - printf "Error: Couldn't find directory for Rust dependencies! Expected location: %s\n" "$VENDORED" + printf "Error: Couldn't find directory for Rust dependencies! Expected location: %s\\n" "$VENDORED" fi if test -z "$CARGO" ; then - printf "Error: cargo must be installed and in your \$PATH\n" + printf "Error: cargo must be installed and in your \$PATH\\n" fi if test -z $(cargo --list | grep vendor) ; then - printf "Error: cargo-vendor not installed\n" + printf "Error: cargo-vendor not installed\\n" fi $CARGO vendor -v --locked --explicit-version --no-delete --sync $TOML $VENDORED From b2053cfc44f4876a52f8d71f2308077c6e39498d Mon Sep 17 00:00:00 2001 From: rl1987 Date: Tue, 20 Nov 2018 16:37:30 +0200 Subject: [PATCH 0180/2557] Also disallow fixup/squash commits in maint-* and release-* --- scripts/maint/pre-push | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/maint/pre-push b/scripts/maint/pre-push index 2cf1837b8d..26c48c4e21 100755 --- a/scripts/maint/pre-push +++ b/scripts/maint/pre-push @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # git pre-push hook script to prevent "fixup!" and "squash!" commit # from ending up in master, or in any branch if CUR_BRANCH check is removed. @@ -10,7 +10,8 @@ z40=0000000000000000000000000000000000000000 CUR_BRANCH=$(git rev-parse --abbrev-ref HEAD) -if [ "$CUR_BRANCH" != "master" ] +if [ "$CUR_BRANCH" != "master" ] && [[ $CUR_BRANCH != release-* ]] && + [[ $CUR_BRANCH != maint-* ]] then exit 0 fi From 27e982b470f4d401a9be687609897ab3320591e1 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Tue, 20 Nov 2018 16:55:35 +0200 Subject: [PATCH 0181/2557] Fix SC2046 and SC2086 warnings --- scripts/maint/updateRustDependencies.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/maint/updateRustDependencies.sh b/scripts/maint/updateRustDependencies.sh index 1e927dd0b1..0b303d5382 100755 --- a/scripts/maint/updateRustDependencies.sh +++ b/scripts/maint/updateRustDependencies.sh @@ -20,8 +20,8 @@ set -e -HERE=$(dirname $(realpath $0)) -TOPLEVEL=$(dirname $(dirname $HERE)) +HERE=$(dirname "$(realpath "$0")") +TOPLEVEL=$(dirname "$(dirname "$HERE")") TOML="$TOPLEVEL/src/rust/Cargo.toml" VENDORED="$TOPLEVEL/src/ext/rust/crates" CARGO=$(which cargo) @@ -38,8 +38,8 @@ if test -z "$CARGO" ; then printf "Error: cargo must be installed and in your \$PATH\\n" fi -if test -z $(cargo --list | grep vendor) ; then +if test -z "$(cargo --list | grep vendor)" ; then printf "Error: cargo-vendor not installed\\n" fi -$CARGO vendor -v --locked --explicit-version --no-delete --sync $TOML $VENDORED +$CARGO vendor -v --locked --explicit-version --no-delete --sync "$TOML" "$VENDORED" From ba2a9988ccd40b9c2ef1f16cb7960191b4131d78 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Tue, 20 Nov 2018 16:57:57 +0200 Subject: [PATCH 0182/2557] Fix SC2230 warning --- scripts/maint/updateRustDependencies.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/maint/updateRustDependencies.sh b/scripts/maint/updateRustDependencies.sh index 0b303d5382..6d0587351f 100755 --- a/scripts/maint/updateRustDependencies.sh +++ b/scripts/maint/updateRustDependencies.sh @@ -24,7 +24,7 @@ HERE=$(dirname "$(realpath "$0")") TOPLEVEL=$(dirname "$(dirname "$HERE")") TOML="$TOPLEVEL/src/rust/Cargo.toml" VENDORED="$TOPLEVEL/src/ext/rust/crates" -CARGO=$(which cargo) +CARGO=$(command -v cargo) if ! test -f "$TOML" ; then printf "Error: Couldn't find workspace Cargo.toml in expected location: %s\\n" "$TOML" From befcd6ab7f53e8d5d825949547e2530fdc91b177 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Tue, 20 Nov 2018 17:00:32 +0200 Subject: [PATCH 0183/2557] Add changes file for #28012 --- changes/ticket28012 | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 changes/ticket28012 diff --git a/changes/ticket28012 b/changes/ticket28012 new file mode 100644 index 0000000000..b2fe83e02a --- /dev/null +++ b/changes/ticket28012 @@ -0,0 +1,3 @@ + o Code simplification and refactoring: + - Fix issues that shellcheck found in updateRustDependencies.sh. + Resolves ticket 28012. From 469f47ef8dc8b18104108f0437c860ec88fca6ad Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 21 Nov 2018 07:38:38 -0500 Subject: [PATCH 0184/2557] Fix a fun heisenbug in memoize_protover_flags() After we clear the protover map for getting full, we need to re-create it, since we are about to use it. This is a bugfix for bug 28558. It is a bugfix for the code from ticket 27225, which is not in any released Tor. Found by Google OSS-Fuzz, as issue 11475. --- src/core/or/versions.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/core/or/versions.c b/src/core/or/versions.c index 6f8eea7a67..5d4effcaf8 100644 --- a/src/core/or/versions.c +++ b/src/core/or/versions.c @@ -399,6 +399,7 @@ memoize_protover_summary(protover_summary_flags_t *out, if (strmap_size(protover_summary_map) >= MAX_PROTOVER_SUMMARY_MAP_LEN) { protover_summary_cache_free_all(); + protover_summary_map = strmap_new(); } const protover_summary_flags_t *cached = From 6d2c2be291e682bff985a357f3703adf1cd03774 Mon Sep 17 00:00:00 2001 From: Colin Childs Date: Wed, 21 Nov 2018 20:36:08 -0600 Subject: [PATCH 0185/2557] Add CrackerboxPalace fallback --- scripts/maint/fallback.whitelist | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/maint/fallback.whitelist b/scripts/maint/fallback.whitelist index 94db9272fd..23c7be58a5 100644 --- a/scripts/maint/fallback.whitelist +++ b/scripts/maint/fallback.whitelist @@ -1070,3 +1070,6 @@ # Email sent directly to Phoul 192.42.116.16:80 orport=443 id=81B75D534F91BFB7C57AB67DA10BCEF622582AE8 + +# https://lists.torproject.org/pipermail/tor-relays/2018-November/016610.html +24.117.194.80:80 orport=443 id=B6C4C9A43658F686F8892CA5666717532F72979C From 3741f9e524a2d3bd7239ca865d6169fd1e3dddb5 Mon Sep 17 00:00:00 2001 From: teor Date: Wed, 21 Nov 2018 20:15:21 +1000 Subject: [PATCH 0186/2557] Fix a comment typo in test_hs_common.c --- src/test/test_hs_common.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/test_hs_common.c b/src/test/test_hs_common.c index 95f7ed14ba..6198573f22 100644 --- a/src/test/test_hs_common.c +++ b/src/test/test_hs_common.c @@ -630,7 +630,7 @@ test_disaster_srv(void *arg) get_disaster_srv(1, srv_one); get_disaster_srv(2, srv_two); - /* Check that the cached ones where updated */ + /* Check that the cached ones were updated */ tt_mem_op(cached_disaster_srv_one, OP_EQ, srv_one, DIGEST256_LEN); tt_mem_op(cached_disaster_srv_two, OP_EQ, srv_two, DIGEST256_LEN); From 997a8b0ca7a4f4eafa2cd09582c4bcc0507e2fa6 Mon Sep 17 00:00:00 2001 From: Taylor R Campbell Date: Thu, 22 Nov 2018 00:53:57 +0000 Subject: [PATCH 0187/2557] Create a temporary directory for tor's DataDirectory in test_rebind. Fixes #28562. While here, put the argument count test and usage message _before_ we attempt to read from sys.argv. --- src/test/test_rebind.py | 10 +++++++--- src/test/test_rebind.sh | 15 ++++++++++++++- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/src/test/test_rebind.py b/src/test/test_rebind.py index c63341a681..2f366b2716 100644 --- a/src/test/test_rebind.py +++ b/src/test/test_rebind.py @@ -67,12 +67,19 @@ socks_port = pick_random_port() assert control_port != 0 assert socks_port != 0 +if len(sys.argv) < 3: + fail('Usage: %s ' % sys.argv[0]) + if not os.path.exists(sys.argv[1]): fail('ERROR: cannot find tor at %s' % sys.argv[1]) +if not os.path.exists(sys.argv[2]): + fail('ERROR: cannot find datadir at %s' % sys.argv[2]) tor_path = sys.argv[1] +data_dir = sys.argv[2] tor_process = subprocess.Popen([tor_path, + '-DataDirectory', data_dir, '-ControlPort', '127.0.0.1:{}'.format(control_port), '-SOCKSPort', '127.0.0.1:{}'.format(socks_port), '-FetchServerDescriptors', '0'], @@ -82,9 +89,6 @@ tor_process = subprocess.Popen([tor_path, if tor_process == None: fail('ERROR: running tor failed') -if len(sys.argv) < 2: - fail('Usage: %s ' % sys.argv[0]) - wait_for_log('Opened Control listener on') try_connecting_to_socksport() diff --git a/src/test/test_rebind.sh b/src/test/test_rebind.sh index 76eb9f2e4d..498072de35 100755 --- a/src/test/test_rebind.sh +++ b/src/test/test_rebind.sh @@ -14,6 +14,19 @@ fi exitcode=0 -"${PYTHON:-python}" "${abs_top_srcdir:-.}/src/test/test_rebind.py" "${TESTING_TOR_BINARY}" || exitcode=1 +tmpdir= +clean () { test -n "$tmpdir" && test -d "$tmpdir" && rm -rf "$tmpdir" || :; } +trap clean EXIT HUP INT TERM + +tmpdir="`mktemp -d -t tor_rebind_test.XXXXXX`" +if [ -z "$tmpdir" ]; then + echo >&2 mktemp failed + exit 2 +elif [ ! -d "$tmpdir" ]; then + echo >&2 mktemp failed to make a directory + exit 3 +fi + +"${PYTHON:-python}" "${abs_top_srcdir:-.}/src/test/test_rebind.py" "${TESTING_TOR_BINARY}" "$tmpdir" || exitcode=1 exit ${exitcode} From 0e9a963b6b87282011fe204e81b5c2530153a935 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Sat, 24 Nov 2018 10:30:15 -0500 Subject: [PATCH 0188/2557] Revise nodefamily.c to match proposal 298 Prop298 says that family entries should be formatted with $hexids in uppercase, nicknames in lower case, $hexid~names truncated, and everything sorted lexically. These changes implement that ordering for nodefamily.c. We don't _strictly speaking_ need to nodefamily.c formatting use this for prop298 microdesc generation, but it seems silly to have two separate canonicalization algorithms. --- src/feature/nodelist/nodefamily.c | 2 ++ src/feature/nodelist/nodefamily_st.h | 8 ++++---- src/test/test_microdesc.c | 2 +- src/test/test_nodelist.c | 18 +++++++++--------- 4 files changed, 16 insertions(+), 14 deletions(-) diff --git a/src/feature/nodelist/nodefamily.c b/src/feature/nodelist/nodefamily.c index 6b504c0ac4..29659ed93d 100644 --- a/src/feature/nodelist/nodefamily.c +++ b/src/feature/nodelist/nodefamily.c @@ -135,6 +135,7 @@ nodefamily_from_members(const smartlist_t *members, ptr[0] = NODEFAMILY_BY_NICKNAME; tor_assert(strlen(cp) < DIGEST_LEN); // guaranteed by is_legal_nickname memcpy(ptr+1, cp, strlen(cp)); + tor_strlower((char*) ptr+1); bad_element = false; } else if (is_legal_hexdigest(cp)) { char digest_buf[DIGEST_LEN]; @@ -346,6 +347,7 @@ nodefamily_format(const nodefamily_t *family) char buf[HEX_DIGEST_LEN+2]; buf[0]='$'; base16_encode(buf+1, sizeof(buf)-1, (char*)ptr+1, DIGEST_LEN); + tor_strupper(buf); smartlist_add_strdup(sl, buf); break; } diff --git a/src/feature/nodelist/nodefamily_st.h b/src/feature/nodelist/nodefamily_st.h index f88ada494a..a498b4b3b9 100644 --- a/src/feature/nodelist/nodefamily_st.h +++ b/src/feature/nodelist/nodefamily_st.h @@ -26,12 +26,12 @@ struct nodefamily_t { #define NODEFAMILY_MEMBER_LEN (1+DIGEST_LEN) -/** Tag byte, indicates that the following bytes are a NUL-padded nickname. - */ -#define NODEFAMILY_BY_NICKNAME 0 /** Tag byte, indicates that the following bytes are a RSA1024 SHA1 ID. */ -#define NODEFAMILY_BY_RSA_ID 1 +#define NODEFAMILY_BY_RSA_ID 0 +/** Tag byte, indicates that the following bytes are a NUL-padded nickname. + */ +#define NODEFAMILY_BY_NICKNAME 1 /** * Number of bytes to allocate in the array for a nodefamily_t with N members. diff --git a/src/test/test_microdesc.c b/src/test/test_microdesc.c index 3318408d53..debb11155a 100644 --- a/src/test/test_microdesc.c +++ b/src/test/test_microdesc.c @@ -176,7 +176,7 @@ test_md_cache(void *data) tt_ptr_op(md3->family, OP_NE, NULL); encoded_family = nodefamily_format(md3->family); - tt_str_op(encoded_family, OP_EQ, "nodeX nodeY nodeZ"); + tt_str_op(encoded_family, OP_EQ, "nodex nodey nodez"); /* Now rebuild the cache! */ tt_int_op(microdesc_cache_rebuild(mc, 1), OP_EQ, 0); diff --git a/src/test/test_nodelist.c b/src/test/test_nodelist.c index 0287be3305..afbcc60ac0 100644 --- a/src/test/test_nodelist.c +++ b/src/test/test_nodelist.c @@ -299,7 +299,7 @@ test_nodelist_nodefamily(void *arg) tt_ptr_op(nf1, OP_EQ, nf3); /* Do we get the expected result when we re-encode? */ - tor_asprintf(&enc, "hello $%s", h1); + tor_asprintf(&enc, "$%s hello", h1); enc2 = nodefamily_format(nf1); tt_str_op(enc2, OP_EQ, enc); tor_free(enc2); @@ -399,8 +399,8 @@ test_nodelist_nodefamily_parse_err(void *arg) tt_assert(nf1); enc = nodefamily_format(nf1); tt_str_op(enc, OP_EQ, - "reticulatogranulate " - "$7468696E67732D696E2D7468656D73656C766573"); + "$7468696E67732D696E2D7468656D73656C766573 " + "reticulatogranulate"); tor_free(enc); } @@ -470,11 +470,11 @@ test_nodelist_nodefamily_lookup(void *arg) tt_int_op(smartlist_len(sl), OP_EQ, 3); const node_t *n = smartlist_get(sl, 0); - tt_str_op(n->identity, OP_EQ, "erewhon"); - n = smartlist_get(sl, 1); test_memeq_hex(n->identity, "3333333333333333333333333333333333333333"); - n = smartlist_get(sl, 2); + n = smartlist_get(sl, 1); test_memeq_hex(n->identity, "EEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEE"); + n = smartlist_get(sl, 2); + tt_str_op(n->identity, OP_EQ, "erewhon"); done: UNMOCK(node_get_by_nickname); @@ -583,9 +583,9 @@ test_nodelist_node_nodefamily(void *arg) node_lookup_declared_family(nodes, &mock_node1); tt_int_op(smartlist_len(nodes), OP_EQ, 2); const node_t *n = smartlist_get(nodes, 0); - tt_str_op(n->identity, OP_EQ, "NodeFour"); - n = smartlist_get(nodes, 1); tt_mem_op(n->identity, OP_EQ, "SecondNodeWe'reTestn", DIGEST_LEN); + n = smartlist_get(nodes, 1); + tt_str_op(n->identity, OP_EQ, "nodefour"); // free, try the other one. SMARTLIST_FOREACH(nodes, node_t *, x, tor_free(x)); @@ -594,9 +594,9 @@ test_nodelist_node_nodefamily(void *arg) node_lookup_declared_family(nodes, &mock_node2); tt_int_op(smartlist_len(nodes), OP_EQ, 2); n = smartlist_get(nodes, 0); + // This gets a truncated hex hex ID since it was looked up by name tt_str_op(n->identity, OP_EQ, "NodeThree"); n = smartlist_get(nodes, 1); - // This gets a truncated hex hex ID since it was looked up by name tt_str_op(n->identity, OP_EQ, "4e6f64654f6e654e6f6"); done: From d29e3a02d57aef402a1aaf9747ef44393b043d98 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Sat, 24 Nov 2018 10:53:38 -0500 Subject: [PATCH 0189/2557] Add a function to canonicalize nodefamilies per prop298 This is the same as the regular canonical nodefamily format, except that unrecognized elements are preserved. --- src/feature/nodelist/nodefamily.c | 45 +++++++++++++++++++++++++++++-- src/feature/nodelist/nodefamily.h | 5 +++- src/test/test_nodelist.c | 31 +++++++++++++++++++++ 3 files changed, 78 insertions(+), 3 deletions(-) diff --git a/src/feature/nodelist/nodefamily.c b/src/feature/nodelist/nodefamily.c index 29659ed93d..944ad54755 100644 --- a/src/feature/nodelist/nodefamily.c +++ b/src/feature/nodelist/nodefamily.c @@ -93,12 +93,46 @@ nodefamily_parse(const char *s, const uint8_t *rsa_id_self, { smartlist_t *sl = smartlist_new(); smartlist_split_string(sl, s, NULL, SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); - nodefamily_t *result = nodefamily_from_members(sl, rsa_id_self, flags); + nodefamily_t *result = nodefamily_from_members(sl, rsa_id_self, flags, NULL); SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); smartlist_free(sl); return result; } +/** + * Canonicalize the family list s, returning a newly allocated string. + * + * The canonicalization rules are fully specified in dir-spec.txt, but, + * briefly: $hexid entries are put in caps, $hexid[=~]foo entries are + * truncated, nicknames are put into lowercase, unrecognized entries are left + * alone, and everything is sorted. + **/ +char * +nodefamily_canonicalize(const char *s, const uint8_t *rsa_id_self, + unsigned flags) +{ + smartlist_t *sl = smartlist_new(); + smartlist_t *result_members = smartlist_new(); + smartlist_split_string(sl, s, NULL, SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); + nodefamily_t *nf = nodefamily_from_members(sl, rsa_id_self, flags, + result_members); + + char *formatted = nodefamily_format(nf); + smartlist_split_string(result_members, formatted, NULL, + SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); + smartlist_sort_strings(result_members); + char *combined = smartlist_join_strings(result_members, " ", 0, NULL); + + nodefamily_free(nf); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_free(sl); + SMARTLIST_FOREACH(result_members, char *, cp, tor_free(cp)); + smartlist_free(result_members); + tor_free(formatted); + + return combined; +} + /** * qsort helper for encoded nodefamily elements. **/ @@ -117,11 +151,15 @@ compare_members(const void *a, const void *b) * family declaration if it is not there already. * * The flags element is interpreted as in nodefamily_parse(). + * + * If unrecognized is provided, fill it copies of any unrecognized + * members. (Note that malformed $hexids are not considered unrecognized.) **/ nodefamily_t * nodefamily_from_members(const smartlist_t *members, const uint8_t *rsa_id_self, - unsigned flags) + unsigned flags, + smartlist_t *unrecognized_out) { const int n_self = rsa_id_self ? 1 : 0; int n_bad_elements = 0; @@ -146,6 +184,9 @@ nodefamily_from_members(const smartlist_t *members, ptr[0] = NODEFAMILY_BY_RSA_ID; memcpy(ptr+1, digest_buf, DIGEST_LEN); } + } else { + if (unrecognized_out) + smartlist_add_strdup(unrecognized_out, cp); } if (bad_element) { diff --git a/src/feature/nodelist/nodefamily.h b/src/feature/nodelist/nodefamily.h index 342f161a07..ea1076876d 100644 --- a/src/feature/nodelist/nodefamily.h +++ b/src/feature/nodelist/nodefamily.h @@ -27,7 +27,8 @@ nodefamily_t *nodefamily_parse(const char *s, unsigned flags); nodefamily_t *nodefamily_from_members(const struct smartlist_t *members, const uint8_t *rsa_id_self, - unsigned flags); + unsigned flags, + smartlist_t *unrecognized_out); void nodefamily_free_(nodefamily_t *family); #define nodefamily_free(family) \ FREE_AND_NULL(nodefamily_t, nodefamily_free_, (family)) @@ -41,6 +42,8 @@ bool nodefamily_contains_node(const nodefamily_t *family, void nodefamily_add_nodes_to_smartlist(const nodefamily_t *family, struct smartlist_t *out); char *nodefamily_format(const nodefamily_t *family); +char *nodefamily_canonicalize(const char *s, const uint8_t *rsa_id_self, + unsigned flags); void nodefamily_free_all(void); diff --git a/src/test/test_nodelist.c b/src/test/test_nodelist.c index afbcc60ac0..ed919f4edf 100644 --- a/src/test/test_nodelist.c +++ b/src/test/test_nodelist.c @@ -610,6 +610,36 @@ test_nodelist_node_nodefamily(void *arg) smartlist_free(nodes); } +static void +test_nodelist_nodefamily_canonicalize(void *arg) +{ + (void)arg; + char *c = NULL; + + c = nodefamily_canonicalize("", NULL, 0); + tt_str_op(c, OP_EQ, ""); + tor_free(c); + + uint8_t own_id[20]; + memset(own_id, 0, sizeof(own_id)); + c = nodefamily_canonicalize( + "alice BOB caroL %potrzebie !!!@#@# " + "$bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb=fred " + "ffffffffffffffffffffffffffffffffffffffff " + "$cccccccccccccccccccccccccccccccccccccccc ", own_id, 0); + tt_str_op(c, OP_EQ, + "!!!@#@# " + "$0000000000000000000000000000000000000000 " + "$BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB " + "$CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC " + "$FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF " + "%potrzebie " + "alice bob carol"); + + done: + tor_free(c); +} + #define NODE(name, flags) \ { #name, test_nodelist_##name, (flags), NULL, NULL } @@ -623,5 +653,6 @@ struct testcase_t nodelist_tests[] = { NODE(nodefamily_lookup, TT_FORK), NODE(nickname_matches, 0), NODE(node_nodefamily, TT_FORK), + NODE(nodefamily_canonicalize, 0), END_OF_TESTCASES }; From 0a0c612b79cb5230cc70922634a37f73a1c87c10 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Sat, 24 Nov 2018 11:51:41 -0500 Subject: [PATCH 0190/2557] Add a consensus method in which md families get canonicalized. Implements prop298. Closes ticket 28266. --- changes/ticket28266 | 5 +++++ src/feature/dirauth/dirvote.c | 18 +++++++++++++++--- src/feature/dirauth/dirvote.h | 8 +++++++- src/test/test_microdesc.c | 33 +++++++++++++++++++++++++++++++++ 4 files changed, 60 insertions(+), 4 deletions(-) create mode 100644 changes/ticket28266 diff --git a/changes/ticket28266 b/changes/ticket28266 new file mode 100644 index 0000000000..d90dc74f76 --- /dev/null +++ b/changes/ticket28266 @@ -0,0 +1,5 @@ + o Minor features (directory authority): + - Directory authorities support a new consensus algorithm, + under which microdescriptor entries are encoded in a canonical + form. This improves their compressibility in transit and on the client. + Closes ticket 28266; implements proposal 298. diff --git a/src/feature/dirauth/dirvote.c b/src/feature/dirauth/dirvote.c index 066a9e6e8a..6c4e12cd51 100644 --- a/src/feature/dirauth/dirvote.c +++ b/src/feature/dirauth/dirvote.c @@ -28,6 +28,7 @@ #include "feature/nodelist/fmt_routerstatus.h" #include "feature/nodelist/microdesc.h" #include "feature/nodelist/networkstatus.h" +#include "feature/nodelist/nodefamily.h" #include "feature/nodelist/nodelist.h" #include "feature/nodelist/routerlist.h" #include "feature/relay/router.h" @@ -3799,8 +3800,16 @@ dirvote_create_microdescriptor(const routerinfo_t *ri, int consensus_method) smartlist_add_asprintf(chunks, "a %s\n", fmt_addrport(&ri->ipv6_addr, ri->ipv6_orport)); - if (family) - smartlist_add_asprintf(chunks, "family %s\n", family); + if (family) { + if (consensus_method < MIN_METHOD_FOR_CANONICAL_FAMILIES_IN_MICRODESCS) { + smartlist_add_asprintf(chunks, "family %s\n", family); + } else { + const uint8_t *id = (const uint8_t *)ri->cache_info.identity_digest; + char *canonical_family = nodefamily_canonicalize(family, id, 0); + smartlist_add_asprintf(chunks, "family %s\n", canonical_family); + tor_free(canonical_family); + } + } if (summary && strcmp(summary, "reject 1-65535")) smartlist_add_asprintf(chunks, "p %s\n", summary); @@ -3898,7 +3907,10 @@ static const struct consensus_method_range_t { int high; } microdesc_consensus_methods[] = { {MIN_SUPPORTED_CONSENSUS_METHOD, MIN_METHOD_FOR_NO_A_LINES_IN_MICRODESC - 1}, - {MIN_METHOD_FOR_NO_A_LINES_IN_MICRODESC, MAX_SUPPORTED_CONSENSUS_METHOD}, + {MIN_METHOD_FOR_NO_A_LINES_IN_MICRODESC, + MIN_METHOD_FOR_CANONICAL_FAMILIES_IN_MICRODESCS - 1}, + {MIN_METHOD_FOR_CANONICAL_FAMILIES_IN_MICRODESCS, + MAX_SUPPORTED_CONSENSUS_METHOD}, {-1, -1} }; diff --git a/src/feature/dirauth/dirvote.h b/src/feature/dirauth/dirvote.h index a21e9f3455..6afb6047ff 100644 --- a/src/feature/dirauth/dirvote.h +++ b/src/feature/dirauth/dirvote.h @@ -57,7 +57,7 @@ #define MIN_SUPPORTED_CONSENSUS_METHOD 25 /** The highest consensus method that we currently support. */ -#define MAX_SUPPORTED_CONSENSUS_METHOD 28 +#define MAX_SUPPORTED_CONSENSUS_METHOD 29 /** Lowest consensus method where authorities vote on required/recommended * protocols. */ @@ -79,6 +79,12 @@ * addresses. See #23828 and #20916. */ #define MIN_METHOD_FOR_NO_A_LINES_IN_MICRODESC 28 +/** + * Lowest consensus method where microdescriptor lines are put in canonical + * form for improved compressibility and ease of storage. See proposal 298. + **/ +#define MIN_METHOD_FOR_CANONICAL_FAMILIES_IN_MICRODESCS 29 + /** Default bandwidth to clip unmeasured bandwidths to using method >= * MIN_METHOD_TO_CLIP_UNMEASURED_BW. (This is not a consensus method; do not * get confused with the above macros.) */ diff --git a/src/test/test_microdesc.c b/src/test/test_microdesc.c index debb11155a..fd79aee6be 100644 --- a/src/test/test_microdesc.c +++ b/src/test/test_microdesc.c @@ -421,6 +421,28 @@ static const char test_md2_21[] = "ntor-onion-key hbxdRnfVUJJY7+KcT4E3Rs7/zuClbN3hJrjSBiEGMgI=\n" "id ed25519 wqfLzgfCtRfYNg88LsL1QpzxS0itapJ1aj6TbnByx/Q\n"; +static const char test_md2_withfamily_28[] = + "onion-key\n" + "-----BEGIN RSA PUBLIC KEY-----\n" + "MIGJAoGBAL2R8EfubUcahxha4u02P4VAR0llQIMwFAmrHPjzcK7apcQgDOf2ovOA\n" + "+YQnJFxlpBmCoCZC6ssCi+9G0mqo650lFuTMP5I90BdtjotfzESfTykHLiChyvhd\n" + "l0dlqclb2SU/GKem/fLRXH16aNi72CdSUu/1slKs/70ILi34QixRAgMBAAE=\n" + "-----END RSA PUBLIC KEY-----\n" + "ntor-onion-key hbxdRnfVUJJY7+KcT4E3Rs7/zuClbN3hJrjSBiEGMgI=\n" + "family OtherNode !Strange\n" + "id ed25519 wqfLzgfCtRfYNg88LsL1QpzxS0itapJ1aj6TbnByx/Q\n"; + +static const char test_md2_withfamily_29[] = + "onion-key\n" + "-----BEGIN RSA PUBLIC KEY-----\n" + "MIGJAoGBAL2R8EfubUcahxha4u02P4VAR0llQIMwFAmrHPjzcK7apcQgDOf2ovOA\n" + "+YQnJFxlpBmCoCZC6ssCi+9G0mqo650lFuTMP5I90BdtjotfzESfTykHLiChyvhd\n" + "l0dlqclb2SU/GKem/fLRXH16aNi72CdSUu/1slKs/70ILi34QixRAgMBAAE=\n" + "-----END RSA PUBLIC KEY-----\n" + "ntor-onion-key hbxdRnfVUJJY7+KcT4E3Rs7/zuClbN3hJrjSBiEGMgI=\n" + "family !Strange $B7E27F104213C36F13E7E9829182845E495997A0 othernode\n" + "id ed25519 wqfLzgfCtRfYNg88LsL1QpzxS0itapJ1aj6TbnByx/Q\n"; + static void test_md_generate(void *arg) { @@ -451,6 +473,17 @@ test_md_generate(void *arg) tt_assert(ed25519_pubkey_eq(md->ed25519_identity_pkey, &ri->cache_info.signing_key_cert->signing_key)); + // Try family encoding. + microdesc_free(md); + ri->declared_family = smartlist_new(); + smartlist_add_strdup(ri->declared_family, "OtherNode !Strange"); + md = dirvote_create_microdescriptor(ri, 28); + tt_str_op(md->body, OP_EQ, test_md2_withfamily_28); + + microdesc_free(md); + md = dirvote_create_microdescriptor(ri, 29); + tt_str_op(md->body, OP_EQ, test_md2_withfamily_29); + done: microdesc_free(md); routerinfo_free(ri); From f82eb6269ff738d136aff66cff59b51b9c56f731 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Sat, 24 Nov 2018 12:07:54 -0500 Subject: [PATCH 0191/2557] Extract get_declared_family() into its own function. This will help as we refactor it. --- src/feature/relay/router.c | 114 +++++++++++++++++++++---------------- 1 file changed, 66 insertions(+), 48 deletions(-) diff --git a/src/feature/relay/router.c b/src/feature/relay/router.c index 9d61ced11c..3f153e3a2d 100644 --- a/src/feature/relay/router.c +++ b/src/feature/relay/router.c @@ -1807,6 +1807,71 @@ router_check_descriptor_address_consistency(uint32_t ipv4h_desc_addr) CONN_TYPE_DIR_LISTENER); } +/** + * Return a new smartlist containing the family members configured in + * options. Warn about invalid or missing entries. Return NULL + * if this relay should not declare a family. + **/ +static smartlist_t * +get_declared_family(const or_options_t *options) +{ + if (!options->MyFamily) + return NULL; + + if (options->BridgeRelay) + return NULL; + + if (!warned_nonexistent_family) + warned_nonexistent_family = smartlist_new(); + + smartlist_t *declared_family = smartlist_new(); + config_line_t *family; + for (family = options->MyFamily; family; family = family->next) { + char *name = family->value; + const node_t *member; + if (!strcasecmp(name, options->Nickname)) + continue; /* Don't list ourself, that's redundant */ + else + member = node_get_by_nickname(name, 0); + if (!member) { + int is_legal = is_legal_nickname_or_hexdigest(name); + if (!smartlist_contains_string(warned_nonexistent_family, name) && + !is_legal_hexdigest(name)) { + if (is_legal) + log_warn(LD_CONFIG, + "I have no descriptor for the router named \"%s\" in my " + "declared family; I'll use the nickname as is, but " + "this may confuse clients.", name); + else + log_warn(LD_CONFIG, "There is a router named \"%s\" in my " + "declared family, but that isn't a legal nickname. " + "Skipping it.", escaped(name)); + smartlist_add_strdup(warned_nonexistent_family, name); + } + if (is_legal) { + smartlist_add_strdup(declared_family, name); + } + } else if (router_digest_is_me(member->identity)) { + /* Don't list ourself in our own family; that's redundant */ + /* XXX shouldn't be possible */ + } else { + char *fp = tor_malloc(HEX_DIGEST_LEN+2); + fp[0] = '$'; + base16_encode(fp+1,HEX_DIGEST_LEN+1, + member->identity, DIGEST_LEN); + smartlist_add(declared_family, fp); + if (smartlist_contains_string(warned_nonexistent_family, name)) + smartlist_string_remove(warned_nonexistent_family, name); + } + } + + /* remove duplicates from the list */ + smartlist_sort_strings(declared_family); + smartlist_uniq_strings(declared_family); + + return declared_family; +} + /** Build a fresh routerinfo, signed server descriptor, and extra-info document * for this OR. Set r to the generated routerinfo, e to the generated * extra-info document. Return 0 on success, -1 on temporary error. Failure to @@ -1921,54 +1986,7 @@ router_build_fresh_descriptor(routerinfo_t **r, extrainfo_t **e) tor_free(p_tmp); } - if (options->MyFamily && ! options->BridgeRelay) { - if (!warned_nonexistent_family) - warned_nonexistent_family = smartlist_new(); - ri->declared_family = smartlist_new(); - config_line_t *family; - for (family = options->MyFamily; family; family = family->next) { - char *name = family->value; - const node_t *member; - if (!strcasecmp(name, options->Nickname)) - continue; /* Don't list ourself, that's redundant */ - else - member = node_get_by_nickname(name, 0); - if (!member) { - int is_legal = is_legal_nickname_or_hexdigest(name); - if (!smartlist_contains_string(warned_nonexistent_family, name) && - !is_legal_hexdigest(name)) { - if (is_legal) - log_warn(LD_CONFIG, - "I have no descriptor for the router named \"%s\" in my " - "declared family; I'll use the nickname as is, but " - "this may confuse clients.", name); - else - log_warn(LD_CONFIG, "There is a router named \"%s\" in my " - "declared family, but that isn't a legal nickname. " - "Skipping it.", escaped(name)); - smartlist_add_strdup(warned_nonexistent_family, name); - } - if (is_legal) { - smartlist_add_strdup(ri->declared_family, name); - } - } else if (router_digest_is_me(member->identity)) { - /* Don't list ourself in our own family; that's redundant */ - /* XXX shouldn't be possible */ - } else { - char *fp = tor_malloc(HEX_DIGEST_LEN+2); - fp[0] = '$'; - base16_encode(fp+1,HEX_DIGEST_LEN+1, - member->identity, DIGEST_LEN); - smartlist_add(ri->declared_family, fp); - if (smartlist_contains_string(warned_nonexistent_family, name)) - smartlist_string_remove(warned_nonexistent_family, name); - } - } - - /* remove duplicates from the list */ - smartlist_sort_strings(ri->declared_family); - smartlist_uniq_strings(ri->declared_family); - } + ri->declared_family = get_declared_family(options); /* Now generate the extrainfo. */ ei = tor_malloc_zero(sizeof(extrainfo_t)); From 05dee063c8d74437c792bdee2432293f97b6307c Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Sat, 24 Nov 2018 16:35:58 -0500 Subject: [PATCH 0192/2557] Emit router families in canonical form This patch has routers use the same canonicalization logic as authorities when encoding their family lists. Additionally, they now warn if any router in their list is given by nickname, since that's error-prone. This patch also adds some long-overdue tests for family formatting. --- changes/ticket28266 | 5 ++ src/feature/relay/router.c | 135 ++++++++++++++++++++++++-------- src/feature/relay/router.h | 4 + src/test/test_router.c | 155 +++++++++++++++++++++++++++++++++++++ 4 files changed, 266 insertions(+), 33 deletions(-) diff --git a/changes/ticket28266 b/changes/ticket28266 index d90dc74f76..e0bc171080 100644 --- a/changes/ticket28266 +++ b/changes/ticket28266 @@ -3,3 +3,8 @@ under which microdescriptor entries are encoded in a canonical form. This improves their compressibility in transit and on the client. Closes ticket 28266; implements proposal 298. + + o Minor features (relay): + - When listing relay families, list them in canonical form including the + relay's own identity, and try to give a more useful set of warnings. + Part of ticket 28266 and proposal 298. diff --git a/src/feature/relay/router.c b/src/feature/relay/router.c index 3f153e3a2d..d57fcc3a44 100644 --- a/src/feature/relay/router.c +++ b/src/feature/relay/router.c @@ -30,6 +30,7 @@ #include "feature/nodelist/dirlist.h" #include "feature/nodelist/networkstatus.h" #include "feature/nodelist/nickname.h" +#include "feature/nodelist/nodefamily.h" #include "feature/nodelist/nodelist.h" #include "feature/nodelist/routerlist.h" #include "feature/nodelist/torcert.h" @@ -340,6 +341,16 @@ set_server_identity_key(crypto_pk_t *k) } } +#ifdef TOR_UNIT_TESTS +/** Testing only -- set the server's RSA identity digest to + * be digest */ +void +set_server_identity_key_digest_testing(const uint8_t *digest) +{ + memcpy(server_identitykey_digest, digest, DIGEST_LEN); +} +#endif + /** Make sure that we have set up our identity keys to match or not match as * appropriate, and die with an assertion if we have not. */ static void @@ -1691,10 +1702,6 @@ router_get_descriptor_gen_reason(void) return desc_gen_reason; } -/** A list of nicknames that we've warned about including in our family - * declaration verbatim rather than as digests. */ -static smartlist_t *warned_nonexistent_family = NULL; - static int router_guess_address_from_dir_headers(uint32_t *guess); /** Make a current best guess at our address, either because @@ -1807,13 +1814,17 @@ router_check_descriptor_address_consistency(uint32_t ipv4h_desc_addr) CONN_TYPE_DIR_LISTENER); } +/** A list of nicknames that we've warned about including in our family, + * for one reason or another. */ +static smartlist_t *warned_family = NULL; + /** * Return a new smartlist containing the family members configured in * options. Warn about invalid or missing entries. Return NULL * if this relay should not declare a family. **/ -static smartlist_t * -get_declared_family(const or_options_t *options) +STATIC smartlist_t * +get_my_declared_family(const or_options_t *options) { if (!options->MyFamily) return NULL; @@ -1821,55 +1832,112 @@ get_declared_family(const or_options_t *options) if (options->BridgeRelay) return NULL; - if (!warned_nonexistent_family) - warned_nonexistent_family = smartlist_new(); + if (!warned_family) + warned_family = smartlist_new(); smartlist_t *declared_family = smartlist_new(); config_line_t *family; + + /* First we try to get the whole family in the form of hexdigests. */ for (family = options->MyFamily; family; family = family->next) { char *name = family->value; const node_t *member; - if (!strcasecmp(name, options->Nickname)) - continue; /* Don't list ourself, that's redundant */ + if (options->Nickname && !strcasecmp(name, options->Nickname)) + continue; /* Don't list ourself by nickname, that's redundant */ else member = node_get_by_nickname(name, 0); + if (!member) { + /* This node doesn't seem to exist, so warn about it if it is not + * a hexdigest. */ int is_legal = is_legal_nickname_or_hexdigest(name); - if (!smartlist_contains_string(warned_nonexistent_family, name) && + if (!smartlist_contains_string(warned_family, name) && !is_legal_hexdigest(name)) { if (is_legal) log_warn(LD_CONFIG, - "I have no descriptor for the router named \"%s\" in my " - "declared family; I'll use the nickname as is, but " - "this may confuse clients.", name); + "There is a router named %s in my declared family, but " + "I have no descriptor for it. I'll use the nickname " + "as is, but this may confuse clients. Please list it " + "by identity digest instead.", escaped(name)); else - log_warn(LD_CONFIG, "There is a router named \"%s\" in my " - "declared family, but that isn't a legal nickname. " + log_warn(LD_CONFIG, "There is a router named %s in my declared " + "family, but that isn't a legal digest or nickname. " "Skipping it.", escaped(name)); - smartlist_add_strdup(warned_nonexistent_family, name); + smartlist_add_strdup(warned_family, name); } if (is_legal) { smartlist_add_strdup(declared_family, name); } - } else if (router_digest_is_me(member->identity)) { - /* Don't list ourself in our own family; that's redundant */ - /* XXX shouldn't be possible */ } else { + /* List the node by digest. */ char *fp = tor_malloc(HEX_DIGEST_LEN+2); fp[0] = '$'; base16_encode(fp+1,HEX_DIGEST_LEN+1, member->identity, DIGEST_LEN); smartlist_add(declared_family, fp); - if (smartlist_contains_string(warned_nonexistent_family, name)) - smartlist_string_remove(warned_nonexistent_family, name); + + if (! is_legal_hexdigest(name) && + !smartlist_contains_string(warned_family, name)) { + /* Warn if this node was not specified by hexdigest. */ + log_warn(LD_CONFIG, "There is a router named %s in my declared " + "family, but it wasn't listed by digest. Please consider " + "saying %s instead, if that's what you meant.", + escaped(name), fp); + smartlist_add_strdup(warned_family, name); + } } } - /* remove duplicates from the list */ - smartlist_sort_strings(declared_family); - smartlist_uniq_strings(declared_family); + /* Now declared_family should have the closest we can come to the + * identities that the user wanted. + * + * Unlike older versions of Tor, we _do_ include our own identity: this + * helps microdescriptor compression, and helps in-memory compression + * on clients. */ + nodefamily_t *nf = nodefamily_from_members(declared_family, + router_get_my_id_digest(), + NF_WARN_MALFORMED, + NULL); + SMARTLIST_FOREACH(declared_family, char *, s, tor_free(s)); + smartlist_free(declared_family); + if (!nf) { + return NULL; + } - return declared_family; + char *s = nodefamily_format(nf); + nodefamily_free(nf); + + smartlist_t *result = smartlist_new(); + smartlist_split_string(result, s, NULL, + SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); + tor_free(s); + + if (smartlist_len(result) == 1) { + /* This is a one-element list containing only ourself; instead return + * nothing */ + const char *singleton = smartlist_get(result, 0); + bool is_me = false; + if (singleton[0] == '$') { + char d[DIGEST_LEN]; + int n = base16_decode(d, sizeof(d), singleton+1, strlen(singleton+1)); + if (n == DIGEST_LEN && + fast_memeq(d, router_get_my_id_digest(), DIGEST_LEN)) { + is_me = true; + } + } + if (!is_me) { + // LCOV_EXCL_START + log_warn(LD_BUG, "Found a singleton family list with an element " + "that wasn't us! Element was %s", escaped(singleton)); + // LCOV_EXCL_STOP + } else { + SMARTLIST_FOREACH(result, char *, cp, tor_free(cp)); + smartlist_free(result); + return NULL; + } + } + + return result; } /** Build a fresh routerinfo, signed server descriptor, and extra-info document @@ -1986,7 +2054,7 @@ router_build_fresh_descriptor(routerinfo_t **r, extrainfo_t **e) tor_free(p_tmp); } - ri->declared_family = get_declared_family(options); + ri->declared_family = get_my_declared_family(options); /* Now generate the extrainfo. */ ei = tor_malloc_zero(sizeof(extrainfo_t)); @@ -3077,9 +3145,9 @@ extrainfo_dump_to_string(char **s_out, extrainfo_t *extrainfo, void router_reset_warnings(void) { - if (warned_nonexistent_family) { - SMARTLIST_FOREACH(warned_nonexistent_family, char *, cp, tor_free(cp)); - smartlist_clear(warned_nonexistent_family); + if (warned_family) { + SMARTLIST_FOREACH(warned_family, char *, cp, tor_free(cp)); + smartlist_clear(warned_family); } } @@ -3103,11 +3171,12 @@ router_free_all(void) memwipe(&curve25519_onion_key, 0, sizeof(curve25519_onion_key)); memwipe(&last_curve25519_onion_key, 0, sizeof(last_curve25519_onion_key)); - if (warned_nonexistent_family) { - SMARTLIST_FOREACH(warned_nonexistent_family, char *, cp, tor_free(cp)); - smartlist_free(warned_nonexistent_family); + if (warned_family) { + SMARTLIST_FOREACH(warned_family, char *, cp, tor_free(cp)); + smartlist_free(warned_family); } } + /* From the given RSA key object, convert it to ASN-1 encoded format and set * the newly allocated object in onion_pkey_out. The length of the key is set * in onion_pkey_len_out. */ diff --git a/src/feature/relay/router.h b/src/feature/relay/router.h index 4575172afb..872eed1f46 100644 --- a/src/feature/relay/router.h +++ b/src/feature/relay/router.h @@ -117,6 +117,10 @@ void router_free_all(void); /* Used only by router.c and test.c */ STATIC void get_platform_str(char *platform, size_t len); STATIC int router_write_fingerprint(int hashed); +STATIC smartlist_t *get_my_declared_family(const or_options_t *options); +#ifdef TOR_UNIT_TESTS +void set_server_identity_key_digest_testing(const uint8_t *digest); +#endif #endif #endif /* !defined(TOR_ROUTER_H) */ diff --git a/src/test/test_router.c b/src/test/test_router.c index 921ec42904..b92e96ff3e 100644 --- a/src/test/test_router.c +++ b/src/test/test_router.c @@ -7,16 +7,21 @@ * \brief Unittests for code in router.c **/ +#define CONFIG_PRIVATE +#define ROUTER_PRIVATE #include "core/or/or.h" #include "app/config/config.h" #include "core/mainloop/mainloop.h" #include "feature/hibernate/hibernate.h" +#include "feature/nodelist/node_st.h" +#include "feature/nodelist/nodelist.h" #include "feature/nodelist/routerinfo_st.h" #include "feature/nodelist/routerlist.h" #include "feature/relay/router.h" #include "feature/stats/rephist.h" #include "lib/crypt_ops/crypto_curve25519.h" #include "lib/crypt_ops/crypto_ed25519.h" +#include "lib/encoding/confline.h" /* Test suite stuff */ #include "test/test.h" @@ -231,11 +236,161 @@ test_router_check_descriptor_bandwidth_changed(void *arg) UNMOCK(we_are_hibernating); } +static node_t fake_node; +static const node_t * +mock_node_get_by_nickname(const char *name, unsigned flags) +{ + (void)flags; + if (!strcasecmp(name, "crumpet")) + return &fake_node; + else + return NULL; +} + +static void +test_router_get_my_family(void *arg) +{ + (void)arg; + or_options_t *options = options_new(); + smartlist_t *sl = NULL; + char *join = NULL; + // Overwrite the result of router_get_my_identity_digest(). This + // happens to be okay, but only for testing. + set_server_identity_key_digest_testing( + (const uint8_t*)"holeinthebottomofthe"); + + setup_capture_of_logs(LOG_WARN); + + // No family listed -- so there's no list. + sl = get_my_declared_family(options); + tt_ptr_op(sl, OP_EQ, NULL); + expect_no_log_entry(); + +#define CLEAR() do { \ + if (sl) { \ + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); \ + smartlist_free(sl); \ + } \ + tor_free(join); \ + mock_clean_saved_logs(); \ + } while (0) + + // Add a single nice friendly hex member. This should be enough + // to have our own ID added. + tt_ptr_op(options->MyFamily, OP_EQ, NULL); + config_line_append(&options->MyFamily, "MyFamily", + "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"); + + sl = get_my_declared_family(options); + tt_ptr_op(sl, OP_NE, NULL); + tt_int_op(smartlist_len(sl), OP_EQ, 2); + join = smartlist_join_strings(sl, " ", 0, NULL); + tt_str_op(join, OP_EQ, + "$686F6C65696E746865626F74746F6D6F66746865 " + "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"); + expect_no_log_entry(); + CLEAR(); + + // Add a hex member with a ~. The ~ part should get removed. + config_line_append(&options->MyFamily, "MyFamily", + "$0123456789abcdef0123456789abcdef01234567~Muffin"); + sl = get_my_declared_family(options); + tt_ptr_op(sl, OP_NE, NULL); + tt_int_op(smartlist_len(sl), OP_EQ, 3); + join = smartlist_join_strings(sl, " ", 0, NULL); + tt_str_op(join, OP_EQ, + "$0123456789ABCDEF0123456789ABCDEF01234567 " + "$686F6C65696E746865626F74746F6D6F66746865 " + "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"); + expect_no_log_entry(); + CLEAR(); + + // Nickname lookup will fail, so a nickname will appear verbatim. + config_line_append(&options->MyFamily, "MyFamily", + "BAGEL"); + sl = get_my_declared_family(options); + tt_ptr_op(sl, OP_NE, NULL); + tt_int_op(smartlist_len(sl), OP_EQ, 4); + join = smartlist_join_strings(sl, " ", 0, NULL); + tt_str_op(join, OP_EQ, + "$0123456789ABCDEF0123456789ABCDEF01234567 " + "$686F6C65696E746865626F74746F6D6F66746865 " + "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA " + "bagel"); + expect_single_log_msg_containing( + "There is a router named \"BAGEL\" in my declared family, but " + "I have no descriptor for it."); + CLEAR(); + + // A bogus digest should fail entirely. + config_line_append(&options->MyFamily, "MyFamily", + "$painauchocolat"); + sl = get_my_declared_family(options); + tt_ptr_op(sl, OP_NE, NULL); + tt_int_op(smartlist_len(sl), OP_EQ, 4); + join = smartlist_join_strings(sl, " ", 0, NULL); + tt_str_op(join, OP_EQ, + "$0123456789ABCDEF0123456789ABCDEF01234567 " + "$686F6C65696E746865626F74746F6D6F66746865 " + "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA " + "bagel"); + // "BAGEL" is still there, but it won't make a warning, because we already + // warned about it. + expect_single_log_msg_containing( + "There is a router named \"$painauchocolat\" in my declared " + "family, but that isn't a legal digest or nickname. Skipping it."); + CLEAR(); + + // Let's introduce a node we can look up by nickname + memset(&fake_node, 0, sizeof(fake_node)); + memcpy(fake_node.identity, "whydoyouasknonononon", DIGEST_LEN); + MOCK(node_get_by_nickname, mock_node_get_by_nickname); + + config_line_append(&options->MyFamily, "MyFamily", + "CRUmpeT"); + sl = get_my_declared_family(options); + tt_ptr_op(sl, OP_NE, NULL); + tt_int_op(smartlist_len(sl), OP_EQ, 5); + join = smartlist_join_strings(sl, " ", 0, NULL); + tt_str_op(join, OP_EQ, + "$0123456789ABCDEF0123456789ABCDEF01234567 " + "$686F6C65696E746865626F74746F6D6F66746865 " + "$776879646F796F7561736B6E6F6E6F6E6F6E6F6E " + "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA " + "bagel"); + // "BAGEL" is still there, but it won't make a warning, because we already + // warned about it. Some with "$painauchocolat". + expect_single_log_msg_containing( + "There is a router named \"CRUmpeT\" in my declared " + "family, but it wasn't listed by digest. Please consider saying " + "$776879646F796F7561736B6E6F6E6F6E6F6E6F6E instead, if that's " + "what you meant."); + CLEAR(); + UNMOCK(node_get_by_nickname); + + // Try a singleton list containing only us: It should give us NULL. + config_free_lines(options->MyFamily); + config_line_append(&options->MyFamily, "MyFamily", + "$686F6C65696E746865626F74746F6D6F66746865"); + sl = get_my_declared_family(options); + tt_ptr_op(sl, OP_EQ, NULL); + expect_no_log_entry(); + + done: + or_options_free(options); + teardown_capture_of_logs(); + CLEAR(); + UNMOCK(node_get_by_nickname); + +#undef CLEAR +} + #define ROUTER_TEST(name, flags) \ { #name, test_router_ ## name, flags, NULL, NULL } struct testcase_t router_tests[] = { ROUTER_TEST(check_descriptor_bandwidth_changed, TT_FORK), ROUTER_TEST(dump_router_to_string_no_bridge_distribution_method, TT_FORK), + ROUTER_TEST(get_my_family, TT_FORK), END_OF_TESTCASES }; From b16d6453adde2b13d2aa95a74ec3c09a259f185f Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Sat, 24 Nov 2018 20:03:48 -0500 Subject: [PATCH 0193/2557] Rewrite updateVersions script in Python, add datestamp functionality. This updateVersions.pl script was one of the only essential perl scripts left in out maint system, and was the only one that used autoconf to fill in the script. This script adds a feature to define an APPROX_RELEASE_DATE macro that is updated when the version changes. We'll use this to implement prop297, so that we have an accurate view of when a release date happens. --- Makefile.am | 2 +- configure.ac | 10 ++- doc/HACKING/ReleasingTor.md | 6 +- scripts/maint/updateVersions.pl.in | 59 ------------- scripts/maint/update_versions.py | 133 +++++++++++++++++++++++++++++ 5 files changed, 144 insertions(+), 66 deletions(-) delete mode 100755 scripts/maint/updateVersions.pl.in create mode 100755 scripts/maint/update_versions.py diff --git a/Makefile.am b/Makefile.am index 803e9d00df..cbe94ad937 100644 --- a/Makefile.am +++ b/Makefile.am @@ -416,7 +416,7 @@ endif .PHONY: update-versions update-versions: - $(PERL) $(top_builddir)/scripts/maint/updateVersions.pl + abs_top_srcdir="$(abs_top_srcdir)" $(PYTHON) $(top_srcdir)/scripts/maint/update_versions.py .PHONY: callgraph callgraph: diff --git a/configure.ac b/configure.ac index 31e41c3bbc..7f0d375440 100644 --- a/configure.ac +++ b/configure.ac @@ -8,6 +8,15 @@ AC_INIT([tor],[0.4.0.0-alpha-dev]) AC_CONFIG_SRCDIR([src/app/main/tor_main.c]) AC_CONFIG_MACRO_DIR([m4]) +# DO NOT EDIT THIS DEFINITION BY HAND UNLESS YOU KNOW WHAT YOU'RE DOING. +# +# The update_versions.py script updates this definition when the +# version number changes. Tor uses it to make sure that it +# only shuts down for missing "required protocols" when those protocols +# are listed as required by a consensus after this date. +AC_DEFINE(APPROX_RELEASE_DATE, ["2019-01-15"], # for 0.4.0.0-alpha-dev + [Approximate date when this software was released. (Updated when the version changes.)]) + # "foreign" means we don't follow GNU package layout standards # "1.11" means we require automake version 1.11 or newer # "subdir-objects" means put .o files in the same directory as the .c files @@ -2417,7 +2426,6 @@ AC_CONFIG_FILES([ src/config/torrc.minimal src/rust/.cargo/config scripts/maint/checkOptionDocs.pl - scripts/maint/updateVersions.pl ]) if test "x$asciidoc" = "xtrue" && test "$ASCIIDOC" = "none"; then diff --git a/doc/HACKING/ReleasingTor.md b/doc/HACKING/ReleasingTor.md index b5444afa96..b260cdbb19 100644 --- a/doc/HACKING/ReleasingTor.md +++ b/doc/HACKING/ReleasingTor.md @@ -131,13 +131,9 @@ new Tor release: === III. Making the source release. 1. In `maint-0.?.x`, bump the version number in `configure.ac` and run - `perl scripts/maint/updateVersions.pl` to update version numbers in other + `make update-versions` to update version numbers in other places, and commit. Then merge `maint-0.?.x` into `release-0.?.x`. - (NOTE: To bump the version number, edit `configure.ac`, and then run - either `make`, or `perl scripts/maint/updateVersions.pl`, depending on - your version.) - When you merge the maint branch forward to the next maint branch, or into master, merge it with "-s ours" to avoid a needless version bump. diff --git a/scripts/maint/updateVersions.pl.in b/scripts/maint/updateVersions.pl.in deleted file mode 100755 index 65c51a1f2d..0000000000 --- a/scripts/maint/updateVersions.pl.in +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/perl -w - -$CONFIGURE_IN = '@abs_top_srcdir@/configure.ac'; -$ORCONFIG_H = '@abs_top_srcdir@/src/win32/orconfig.h'; -$TOR_NSI = '@abs_top_srcdir@/contrib/win32build/tor-mingw.nsi.in'; - -$quiet = 1; - -sub demand { - my $fn = shift; - die "Missing file $fn" unless (-f $fn); -} - -demand($CONFIGURE_IN); -demand($ORCONFIG_H); -demand($TOR_NSI); - -# extract version from configure.ac - -open(F, $CONFIGURE_IN) or die "$!"; -$version = undef; -while () { - if (/AC_INIT\(\[tor\],\s*\[([^\]]*)\]\)/) { - $version = $1; - last; - } -} -die "No version found" unless $version; -print "Tor version is $version\n" unless $quiet; -close F; - -sub correctversion { - my ($fn, $defchar) = @_; - undef $/; - open(F, $fn) or die "$!"; - my $s = ; - close F; - if ($s =~ /^$defchar(?:)define\s+VERSION\s+\"([^\"]+)\"/m) { - $oldver = $1; - if ($oldver ne $version) { - print "Version mismatch in $fn: It thinks that the version is $oldver. I think it's $version. Fixing.\n"; - $line = $defchar . "define VERSION \"$version\""; - open(F, ">$fn.bak"); - print F $s; - close F; - $s =~ s/^$defchar(?:)define\s+VERSION.*?$/$line/m; - open(F, ">$fn"); - print F $s; - close F; - } else { - print "$fn has the correct version. Good.\n" unless $quiet; - } - } else { - print "Didn't find a version line in $fn -- uh oh.\n"; - } -} - -correctversion($TOR_NSI, "!"); -correctversion($ORCONFIG_H, "#"); diff --git a/scripts/maint/update_versions.py b/scripts/maint/update_versions.py new file mode 100755 index 0000000000..8067f2c6c8 --- /dev/null +++ b/scripts/maint/update_versions.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python + +from __future__ import print_function + +import io +import os +import re +import sys +import time + +def P(path): + """ + Give 'path' as a path relative to the abs_top_srcdir environment + variable. + """ + return os.path.join( + os.environ.get('abs_top_srcdir', "."), + path) + +def warn(msg): + """ + Print an warning message. + """ + print("WARNING: {}".format(msg), file=sys.stderr) + +def find_version(infile): + """ + Given an open file (or some other iterator of lines) holding a + configure.ac file, find the current version line. + """ + for line in infile: + m = re.search(r'AC_INIT\(\[tor\],\s*\[([^\]]*)\]\)', line) + if m: + return m.group(1) + + return None + +def update_version_in(infile, outfile, regex, versionline): + """ + Copy every line from infile to outfile. If any line matches 'regex', + replace it with 'versionline'. Return True if any line was changed; + false otherwise. + + 'versionline' is either a string -- in which case it is used literally, + or a function that receives the output of 'regex.match'. + """ + found = False + have_changed = False + for line in infile: + m = regex.match(line) + if m: + found = True + oldline = line + if type(versionline) == type(u""): + line = versionline + else: + line = versionline(m) + if not line.endswith("\n"): + line += "\n" + if oldline != line: + have_changed = True + outfile.write(line) + + if not found: + warn("didn't find any version line to replace in {}".format(infile.name)) + + return have_changed + +def replace_on_change(fname, change): + """ + If "change" is true, replace fname with fname.tmp. Otherwise, + delete fname.tmp. Log what we're doing to stderr. + """ + if not change: + print("No change in {}".format(fname)) + os.unlink(fname+".tmp") + else: + print("Updating {}".format(fname)) + os.rename(fname+".tmp", fname) + + +def update_file(fname, + regex, + versionline, + encoding="utf-8"): + """ + Replace any line matching 'regex' in 'fname' with 'versionline'. + Do not modify 'fname' if there are no changes made. Use the + provided encoding to read and write. + """ + with io.open(fname, "r", encoding=encoding) as f, \ + io.open(fname+".tmp", "w", encoding=encoding) as outf: + have_changed = update_version_in(f, outf, regex, versionline) + + replace_on_change(fname, have_changed) + +# Find out our version +with open("configure.ac") as f: + version = find_version(f) + +# If we have no version, we can't proceed. +if version == None: + print("No version found in configure.ac", file=sys.stderr()) + sys.exit(1) + +print("The version is {}".format(version)) + +today = time.strftime("%Y-%m-%d", time.gmtime()) + +# In configure.ac, we replace the definition of APPROX_RELEASE_DATE +# with "{today} for {version}", but only if the version does not match +# what is already there. +def replace_fn(m): + if m.group(1) != version: + # The version changed -- we change the date. + return u'AC_DEFINE(APPROX_RELEASE_DATE, ["{}"], # for {}'.format(today, version) + else: + # No changes. + return m.group(0) +update_file(P("configure.ac"), + re.compile(r'AC_DEFINE\(APPROX_RELEASE_DATE.* for (.*)'), + replace_fn) + +# In tor-mingw.nsi.in, we replace the definition of VERSION. +update_file(P("contrib/win32build/tor-mingw.nsi.in"), + re.compile(r'!define VERSION .*'), + u'!define VERSION "{}"'.format(version), + encoding="iso-8859-1") + +# In src/win32/orconfig.h, we replace the definition of VERSION. +update_file(P("src/win32/orconfig.h"), + re.compile(r'#define VERSION .*'), + u'#define VERSION "{}"'.format(version)) From 7da06e43da96f4253a756af546f27f03141b3784 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Sat, 24 Nov 2018 20:40:48 -0500 Subject: [PATCH 0194/2557] No longer exit for missing required protocolversions on an old consensus Specifically, if the consensus is older than the (estimted or measured) release date for this version of tor, we assume that the required versions may have changed in between that consensus and this release. Implements ticket 27735 and proposal 297. --- changes/prop297 | 7 +++++++ src/core/or/versions.c | 19 +++++++++++++++++++ src/core/or/versions.h | 2 ++ src/feature/nodelist/networkstatus.c | 5 ++++- 4 files changed, 32 insertions(+), 1 deletion(-) create mode 100644 changes/prop297 diff --git a/changes/prop297 b/changes/prop297 new file mode 100644 index 0000000000..4f93b232d2 --- /dev/null +++ b/changes/prop297 @@ -0,0 +1,7 @@ + o Minor features (required protocols): + - Tor no longer exits if it is missing a required protocol, if the + consensus that requires the protocol predates the release date of the + version of Tor. This change prevents Tor releases from exiting because + of an old cached consensus, on the theory that a newer cached + consensus might not require the protocol. Implements proposal 297; + closes ticket 27735. diff --git a/src/core/or/versions.c b/src/core/or/versions.c index 5d4effcaf8..d978935f4f 100644 --- a/src/core/or/versions.c +++ b/src/core/or/versions.c @@ -16,6 +16,25 @@ #include "core/or/tor_version_st.h" +/** + * Return the approximate date when this release came out, or was + * scheduled to come out, according to the APPROX_RELEASE_DATE set in + * configure.ac + **/ +time_t +tor_get_approx_release_date(void) +{ + char tbuf[ISO_TIME_LEN+1]; + tor_snprintf(tbuf, sizeof(tbuf), + "%s 00:00:00", APPROX_RELEASE_DATE); + time_t result = 0; + int r = parse_iso_time(tbuf, &result); + if (BUG(r < 0)) { + result = 0; + } + return result; +} + /** Return VS_RECOMMENDED if myversion is contained in * versionlist. Else, return VS_EMPTY if versionlist has no * entries. Else, return VS_OLD if every member of diff --git a/src/core/or/versions.h b/src/core/or/versions.h index 4fc50a0018..acd8998918 100644 --- a/src/core/or/versions.h +++ b/src/core/or/versions.h @@ -26,6 +26,8 @@ typedef enum version_status_t { VS_UNKNOWN, /**< We have no idea. */ } version_status_t; +time_t tor_get_approx_release_date(void); + version_status_t tor_version_is_obsolete(const char *myversion, const char *versionlist); int tor_version_parse_platform(const char *platform, diff --git a/src/feature/nodelist/networkstatus.c b/src/feature/nodelist/networkstatus.c index f1def9afb1..a25a539cd8 100644 --- a/src/feature/nodelist/networkstatus.c +++ b/src/feature/nodelist/networkstatus.c @@ -2684,6 +2684,9 @@ networkstatus_check_required_protocols(const networkstatus_t *ns, const char *required, *recommended; char *missing = NULL; + const bool consensus_postdates_this_release = + ns->valid_after >= tor_get_approx_release_date(); + tor_assert(warning_out); if (client_mode) { @@ -2701,7 +2704,7 @@ networkstatus_check_required_protocols(const networkstatus_t *ns, "%s on the Tor network. The missing protocols are: %s", func, missing); tor_free(missing); - return 1; + return consensus_postdates_this_release ? 1 : 0; } if (! protover_all_supported(recommended, &missing)) { From 36f808c9364bb2e6781e555697eac977d754e905 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Sun, 25 Nov 2018 10:05:13 -0500 Subject: [PATCH 0195/2557] Vote on the StaleDesc flag from prop293 The StaleDesc flag tells relays that they need to upload a new descriptor soon, or they will drop out of the consensus. --- src/feature/dirauth/dirvote.c | 3 ++- src/feature/dirauth/voteflags.c | 4 ++++ src/feature/dirauth/voteflags.h | 3 +++ src/feature/dirparse/ns_parse.c | 2 ++ src/feature/nodelist/fmt_routerstatus.c | 3 ++- src/feature/nodelist/routerstatus_st.h | 2 ++ 6 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/feature/dirauth/dirvote.c b/src/feature/dirauth/dirvote.c index 066a9e6e8a..aa4242f678 100644 --- a/src/feature/dirauth/dirvote.c +++ b/src/feature/dirauth/dirvote.c @@ -4620,7 +4620,8 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key, v3_out->known_flags = smartlist_new(); smartlist_split_string(v3_out->known_flags, - "Authority Exit Fast Guard Stable V2Dir Valid HSDir", + "Authority Exit Fast Guard Stable V2Dir Valid HSDir " + "StaleDesc", 0, SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); if (vote_on_reachability) smartlist_add_strdup(v3_out->known_flags, "Running"); diff --git a/src/feature/dirauth/voteflags.c b/src/feature/dirauth/voteflags.c index 4a24dcb50d..5adf21ad11 100644 --- a/src/feature/dirauth/voteflags.c +++ b/src/feature/dirauth/voteflags.c @@ -593,6 +593,10 @@ set_routerstatus_from_routerinfo(routerstatus_t *rs, rs->or_port = ri->or_port; rs->dir_port = ri->dir_port; rs->is_v2_dir = ri->supports_tunnelled_dir_requests; + + rs->is_staledesc = + (ri->cache_info.published_on + DESC_IS_STALE_INTERVAL) < now; + if (options->AuthDirHasIPv6Connectivity == 1 && !tor_addr_is_null(&ri->ipv6_addr) && node->last_reachable6 >= now - REACHABLE_TIMEOUT) { diff --git a/src/feature/dirauth/voteflags.h b/src/feature/dirauth/voteflags.h index 2f0e061ea4..743a666cc8 100644 --- a/src/feature/dirauth/voteflags.h +++ b/src/feature/dirauth/voteflags.h @@ -25,6 +25,9 @@ void set_routerstatus_from_routerinfo(routerstatus_t *rs, void dirserv_compute_performance_thresholds(digestmap_t *omit_as_sybil); #ifdef VOTEFLAGS_PRIVATE +/** Any descriptor older than this age causes the authorities to set the + * StaleDesc flag. */ +#define DESC_IS_STALE_INTERVAL (18*60*60) STATIC void dirserv_set_routerstatus_testing(routerstatus_t *rs); #endif diff --git a/src/feature/dirparse/ns_parse.c b/src/feature/dirparse/ns_parse.c index 3fccec1540..e0cdb2d46d 100644 --- a/src/feature/dirparse/ns_parse.c +++ b/src/feature/dirparse/ns_parse.c @@ -434,6 +434,8 @@ routerstatus_parse_entry_from_string(memarea_t *area, rs->is_hs_dir = 1; } else if (!strcmp(tok->args[i], "V2Dir")) { rs->is_v2_dir = 1; + } else if (!strcmp(tok->args[i], "StaleDesc")) { + rs->is_staledesc = 1; } } /* These are implied true by having been included in a consensus made diff --git a/src/feature/nodelist/fmt_routerstatus.c b/src/feature/nodelist/fmt_routerstatus.c index e70aeb2950..b1d4a48038 100644 --- a/src/feature/nodelist/fmt_routerstatus.c +++ b/src/feature/nodelist/fmt_routerstatus.c @@ -135,7 +135,7 @@ routerstatus_format_entry(const routerstatus_t *rs, const char *version, goto done; smartlist_add_asprintf(chunks, - "s%s%s%s%s%s%s%s%s%s%s\n", + "s%s%s%s%s%s%s%s%s%s%s%s\n", /* These must stay in alphabetical order. */ rs->is_authority?" Authority":"", rs->is_bad_exit?" BadExit":"", @@ -145,6 +145,7 @@ routerstatus_format_entry(const routerstatus_t *rs, const char *version, rs->is_hs_dir?" HSDir":"", rs->is_flagged_running?" Running":"", rs->is_stable?" Stable":"", + rs->is_staledesc?" StaleDesc":"", rs->is_v2_dir?" V2Dir":"", rs->is_valid?" Valid":""); diff --git a/src/feature/nodelist/routerstatus_st.h b/src/feature/nodelist/routerstatus_st.h index 714aa27435..ea06587799 100644 --- a/src/feature/nodelist/routerstatus_st.h +++ b/src/feature/nodelist/routerstatus_st.h @@ -47,6 +47,8 @@ struct routerstatus_t { unsigned int is_v2_dir:1; /** True iff this router publishes an open DirPort * or it claims to accept tunnelled dir requests. */ + unsigned int is_staledesc:1; /** True iff the authorities think this router + * should upload a new descriptor soon. */ unsigned int has_bandwidth:1; /**< The vote/consensus had bw info */ unsigned int has_exitsummary:1; /**< The vote/consensus had exit summaries */ From 881b85cf328b5fbcd0108ecb0320374f638b9e47 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Sun, 25 Nov 2018 10:12:20 -0500 Subject: [PATCH 0196/2557] Treat the StaleDesc flag as making our descriptor dirty. Relay side of prop293. --- src/feature/relay/router.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/feature/relay/router.c b/src/feature/relay/router.c index 9d61ced11c..2d4ab9b0a0 100644 --- a/src/feature/relay/router.c +++ b/src/feature/relay/router.c @@ -59,6 +59,7 @@ #include "feature/dircommon/dir_connection_st.h" #include "feature/nodelist/authority_cert_st.h" #include "feature/nodelist/extrainfo_st.h" +#include "feature/nodelist/networkstatus_st.h" #include "feature/nodelist/node_st.h" #include "feature/nodelist/routerinfo_st.h" #include "feature/nodelist/routerstatus_st.h" @@ -2134,7 +2135,9 @@ mark_my_descriptor_dirty_if_too_old(time_t now) /* Now we see whether we want to be retrying frequently or no. The * rule here is that we'll retry frequently if we aren't listed in the * live consensus we have, or if the publication time of the - * descriptor listed for us in the consensus is very old. */ + * descriptor listed for us in the consensus is very old, or if the + * consensus lists us as "stale" and we haven't regenerated since the + * consensus was published. */ ns = networkstatus_get_live_consensus(now); if (ns) { rs = networkstatus_vote_find_entry(ns, server_identitykey_digest); @@ -2142,6 +2145,8 @@ mark_my_descriptor_dirty_if_too_old(time_t now) retry_fast_reason = "not listed in consensus"; else if (rs->published_on < slow_cutoff) retry_fast_reason = "version listed in consensus is quite old"; + else if (rs->is_staledesc && ns->valid_after > desc_clean_since) + retry_fast_reason = "listed as stale in consensus"; } if (retry_fast_reason && desc_clean_since < fast_cutoff) From 439ffcefd57031153e49e605389a1c218e737180 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Sun, 25 Nov 2018 10:19:08 -0500 Subject: [PATCH 0197/2557] changes file for prop293 / ticket 26770 --- changes/ticket26770 | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 changes/ticket26770 diff --git a/changes/ticket26770 b/changes/ticket26770 new file mode 100644 index 0000000000..7f3e92e9dd --- /dev/null +++ b/changes/ticket26770 @@ -0,0 +1,8 @@ + o Minor features (directory authority, relay): + - Authorities now vote on a "StaleDesc" flag to indicate that a relay's + descriptor is so old that the relay should upload again soon. Relays + understand this flag, and treat it as a signal to upload a new + descriptor. This flag will eventually let us remove the 'published' + date from routerstatus entries, and save a great deal of space in our + consensus diffs. Closes ticket 26770; implements proposal 293. + From e92f90019145c09fd8b1292b0198db1c2ba3a9b5 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Mon, 26 Nov 2018 17:32:29 +0200 Subject: [PATCH 0198/2557] Try silencing Coverity false positive CID 1441482 Bugfix on 469f47ef8dc8b18104108f0437c860ec88fca6ad; bug not in any released Tor version. --- src/core/or/versions.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/core/or/versions.c b/src/core/or/versions.c index 5d4effcaf8..a1336c7a78 100644 --- a/src/core/or/versions.c +++ b/src/core/or/versions.c @@ -399,6 +399,7 @@ memoize_protover_summary(protover_summary_flags_t *out, if (strmap_size(protover_summary_map) >= MAX_PROTOVER_SUMMARY_MAP_LEN) { protover_summary_cache_free_all(); + tor_assert(protover_summary_map == NULL); protover_summary_map = strmap_new(); } From 30f8b49d3b7917a15237724a0d0b913d27e876e2 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Fri, 12 Oct 2018 10:09:11 +0300 Subject: [PATCH 0199/2557] Silence SC2034 shellcheck checker for EXTRA_CHECKERS and NOISY_CHECKERS variables --- scripts/test/scan-build.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/test/scan-build.sh b/scripts/test/scan-build.sh index 8d126cbcee..e111566bf7 100755 --- a/scripts/test/scan-build.sh +++ b/scripts/test/scan-build.sh @@ -33,6 +33,7 @@ CHECKERS="\ -enable-checker security.insecureAPI.strcpy \ " +# shellcheck disable=SC2034 # These have high false-positive rates. EXTRA_CHECKERS="\ -enable-checker alpha.security.ArrayBoundV2 \ @@ -40,6 +41,7 @@ EXTRA_CHECKERS="\ -enable-checker alpha.core.CastSize \ " +# shellcheck disable=SC2034 # These don't seem to generate anything useful NOISY_CHECKERS="\ -enable-checker alpha.clone.CloneChecker \ From d6eafd06a9797d2747b0bd5988893c292076ef1f Mon Sep 17 00:00:00 2001 From: rl1987 Date: Fri, 12 Oct 2018 10:37:27 +0300 Subject: [PATCH 0200/2557] Add changes file --- changes/ticket28007 | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 changes/ticket28007 diff --git a/changes/ticket28007 b/changes/ticket28007 new file mode 100644 index 0000000000..1ac87862eb --- /dev/null +++ b/changes/ticket28007 @@ -0,0 +1,3 @@ + o Code simplification and refactoring: + - Cleanup scan-build.sh to silence shellcheck warnings. + Closes ticket 28007. From 59001a69c96a2f14821d312b10482c0854c6d7dd Mon Sep 17 00:00:00 2001 From: rl1987 Date: Mon, 26 Nov 2018 18:03:36 +0200 Subject: [PATCH 0201/2557] Address SC2086 in scan-build.sh We can safely silence SC2086 warning on $CHECKERS, as contents of that is hardcoded into script, and we don't want to require Bash to use Bash array here. Double-quote $OUTPUTARG, as it's value depends on environment variable. --- scripts/test/scan-build.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/test/scan-build.sh b/scripts/test/scan-build.sh index e111566bf7..9a63383804 100755 --- a/scripts/test/scan-build.sh +++ b/scripts/test/scan-build.sh @@ -54,6 +54,7 @@ else OUTPUTARG="" fi +# shellcheck disable=SC2086 scan-build \ $CHECKERS \ ./configure @@ -63,13 +64,15 @@ scan-build \ # Make this not get scanned for dead assignments, since it has lots of # dead assignments we don't care about. +# shellcheck disable=SC2086 scan-build \ $CHECKERS \ -disable-checker deadcode.DeadStores \ make -j5 -k ./src/ext/ed25519/ref10/libed25519_ref10.a +# shellcheck disable=SC2086 scan-build \ - $CHECKERS $OUTPUTARG \ + $CHECKERS "$OUTPUTARG" \ make -j5 -k CHECKERS="\ From 3743f7969587079a2f2bb03d0b7e5038557fd64a Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 15 Nov 2018 13:16:58 -0500 Subject: [PATCH 0202/2557] Add options to control dormant-client feature. The DormantClientTimeout option controls how long Tor will wait before going dormant. It also provides a way to disable the feature by setting DormantClientTimeout to e.g. "50 years". The DormantTimeoutDisabledByIdleStreams option controls whether open but inactive streams count as "client activity". To implement it, I had to make it so that reading or writing on a client stream *always* counts as activity. Closes ticket 28429. --- doc/tor.1.txt | 13 +++++++++++++ src/app/config/config.c | 6 ++++++ src/app/config/or_options_st.h | 10 ++++++++++ src/core/mainloop/mainloop.c | 19 ++++++++++--------- src/core/or/connection_edge.c | 11 +++++++++++ src/test/test_options.c | 1 + 6 files changed, 51 insertions(+), 9 deletions(-) diff --git a/doc/tor.1.txt b/doc/tor.1.txt index b147ad68aa..47bddea097 100644 --- a/doc/tor.1.txt +++ b/doc/tor.1.txt @@ -1789,6 +1789,19 @@ The following options are useful only for clients (that is, if Try this many simultaneous connections to download a consensus before waiting for one to complete, timeout, or error out. (Default: 3) +[[DormantClientTimeout]] **DormantClientTimeout** __N__ **minutes**|**hours**|**days**|**weeks**:: + If Tor spends this much time without any client activity, + enter a dormant state where automatic circuits are not built, and + directory information is not fetched. + Does not affect servers or onion services. Must be at least 10 minutes. + (Default: 24 hours) + +[[DormantTimeoutDisabledByIdleStreams]] **DormantTimeoutDisabledByIdleStreams **0**|**1**:: + If true, then any open client stream (even one not reading or writing) + counts as client activity for the purpose of DormantClientTimeout. + If false, then only network activity counts. (Default: 1) + + SERVER OPTIONS -------------- diff --git a/src/app/config/config.c b/src/app/config/config.c index 8aa0c1f4bd..90eae50fdd 100644 --- a/src/app/config/config.c +++ b/src/app/config/config.c @@ -389,6 +389,8 @@ static config_var_t option_vars_[] = { OBSOLETE("DynamicDHGroups"), VPORT(DNSPort), OBSOLETE("DNSListenAddress"), + V(DormantClientTimeout, INTERVAL, "24 hours"), + V(DormantTimeoutDisabledByIdleStreams, BOOL, "1"), /* DoS circuit creation options. */ V(DoSCircuitCreationEnabled, AUTOBOOL, "auto"), V(DoSCircuitCreationMinConnections, UINT, "0"), @@ -3836,6 +3838,10 @@ options_validate(or_options_t *old_options, or_options_t *options, "default."); } + if (options->DormantClientTimeout < 10*60 && !options->TestingTorNetwork) { + REJECT("DormantClientTimeout is too low. It must be at least 10 minutes."); + } + if (options->PathBiasNoticeRate > 1.0) { tor_asprintf(msg, "PathBiasNoticeRate is too high. " diff --git a/src/app/config/or_options_st.h b/src/app/config/or_options_st.h index 3524b99b53..6cbc86ec18 100644 --- a/src/app/config/or_options_st.h +++ b/src/app/config/or_options_st.h @@ -1072,6 +1072,16 @@ struct or_options_t { /** Autobool: Do we refuse single hop client rendezvous? */ int DoSRefuseSingleHopClientRendezvous; + + /** Interval: how long without activity does it take for a client + * to become dormant? + **/ + int DormantClientTimeout; + + /** Boolean: true if having an idle stream is sufficient to prevent a client + * from becoming dormant. + **/ + int DormantTimeoutDisabledByIdleStreams; }; #endif diff --git a/src/core/mainloop/mainloop.c b/src/core/mainloop/mainloop.c index 2d12e26485..1bd186d856 100644 --- a/src/core/mainloop/mainloop.c +++ b/src/core/mainloop/mainloop.c @@ -2018,24 +2018,25 @@ check_network_participation_callback(time_t now, const or_options_t *options) goto found_activity; } - /* XXXX Add an option to never become dormant. */ - /* If we have any currently open entry streams other than "linked" * connections used for directory requests, those count as user activity. */ - /* XXXX make this configurable? */ - if (connection_get_by_type_nonlinked(CONN_TYPE_AP) != NULL) { - goto found_activity; + if (options->DormantTimeoutDisabledByIdleStreams) { + if (connection_get_by_type_nonlinked(CONN_TYPE_AP) != NULL) { + goto found_activity; + } } /* XXXX Make this configurable? */ /** How often do we check whether we have had network activity? */ #define CHECK_PARTICIPATION_INTERVAL (5*60) - /** Become dormant if there has been no user activity in this long. */ - /* XXXX make this configurable! */ -#define BECOME_DORMANT_AFTER_INACTIVITY (24*60*60) - if (get_last_user_activity_time() + BECOME_DORMANT_AFTER_INACTIVITY >= now) { + /* Become dormant if there has been no user activity in a long time. + * (The funny checks below are in order to prevent overflow.) */ + time_t time_since_last_activity = 0; + if (get_last_user_activity_time() < now) + time_since_last_activity = now - get_last_user_activity_time(); + if (time_since_last_activity >= options->DormantClientTimeout) { log_notice(LD_GENERAL, "No user activity in a long time: becoming" " dormant."); set_network_participation(false); diff --git a/src/core/or/connection_edge.c b/src/core/or/connection_edge.c index 58aefcf8f2..7b51313e8a 100644 --- a/src/core/or/connection_edge.c +++ b/src/core/or/connection_edge.c @@ -62,6 +62,7 @@ #include "app/config/config.h" #include "core/mainloop/connection.h" #include "core/mainloop/mainloop.h" +#include "core/mainloop/netstatus.h" #include "core/or/channel.h" #include "core/or/circuitbuild.h" #include "core/or/circuitlist.h" @@ -297,6 +298,11 @@ connection_edge_process_inbuf(edge_connection_t *conn, int package_partial) } return 0; case AP_CONN_STATE_OPEN: + if (! conn->base_.linked) { + note_user_activity(approx_time()); + } + + /* falls through. */ case EXIT_CONN_STATE_OPEN: if (connection_edge_package_raw_inbuf(conn, package_partial, NULL) < 0) { /* (We already sent an end cell if possible) */ @@ -751,6 +757,11 @@ connection_edge_flushed_some(edge_connection_t *conn) { switch (conn->base_.state) { case AP_CONN_STATE_OPEN: + if (! conn->base_.linked) { + note_user_activity(approx_time()); + } + + /* falls through. */ case EXIT_CONN_STATE_OPEN: connection_edge_consider_sending_sendme(conn); break; diff --git a/src/test/test_options.c b/src/test/test_options.c index f14e620eeb..376d77626f 100644 --- a/src/test/test_options.c +++ b/src/test/test_options.c @@ -425,6 +425,7 @@ get_options_test_data(const char *conf) // with options_init(), but about a dozen tests break when I do that. // Being kinda lame and just fixing the immedate breakage for now.. result->opt->ConnectionPadding = -1; // default must be "auto" + result->opt->DormantClientTimeout = 1800; // must be over 600. rv = config_get_lines(conf, &cl, 1); tt_int_op(rv, OP_EQ, 0); From 55512ef022de39770e0787e9dfc2e29e762652ae Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 19 Nov 2018 15:35:55 -0500 Subject: [PATCH 0203/2557] Test netstatus.c tracking of user participation status --- src/core/mainloop/mainloop.c | 4 +-- src/core/mainloop/mainloop.h | 2 +- src/test/test_mainloop.c | 57 +++++++++++++++++++++++++++++++++++- 3 files changed, 59 insertions(+), 4 deletions(-) diff --git a/src/core/mainloop/mainloop.c b/src/core/mainloop/mainloop.c index 1bd186d856..331f7021a4 100644 --- a/src/core/mainloop/mainloop.c +++ b/src/core/mainloop/mainloop.c @@ -1608,8 +1608,8 @@ rescan_periodic_events_cb(mainloop_event_t *event, void *arg) /** * Schedule an event that will rescan which periodic events should run. **/ -void -schedule_rescan_periodic_events(void) +MOCK_IMPL(void, +schedule_rescan_periodic_events,(void)) { if (!rescan_periodic_events_ev) { rescan_periodic_events_ev = diff --git a/src/core/mainloop/mainloop.h b/src/core/mainloop/mainloop.h index 7f27ef9a52..e5e730fc8e 100644 --- a/src/core/mainloop/mainloop.h +++ b/src/core/mainloop/mainloop.h @@ -65,7 +65,7 @@ void reschedule_or_state_save(void); void reschedule_dirvote(const or_options_t *options); void mainloop_schedule_postloop_cleanup(void); void rescan_periodic_events(const or_options_t *options); -void schedule_rescan_periodic_events(void); +MOCK_DECL(void, schedule_rescan_periodic_events,(void)); void update_current_time(time_t now); diff --git a/src/test/test_mainloop.c b/src/test/test_mainloop.c index 92ce2e9918..94b684fb3a 100644 --- a/src/test/test_mainloop.c +++ b/src/test/test_mainloop.c @@ -11,6 +11,7 @@ #include "core/or/or.h" #include "core/mainloop/mainloop.h" +#include "core/mainloop/netstatus.h" static const uint64_t BILLION = 1000000000; @@ -131,12 +132,66 @@ test_mainloop_update_time_jumps(void *arg) monotime_disable_test_mocking(); } +static int schedule_rescan_called = 0; +static void +mock_schedule_rescan_periodic_events(void) +{ + ++schedule_rescan_called; +} + +static void +test_mainloop_user_activity(void *arg) +{ + (void)arg; + const time_t start = 1542658829; + update_approx_time(start); + + MOCK(schedule_rescan_periodic_events, mock_schedule_rescan_periodic_events); + + reset_user_activity(start); + tt_i64_op(get_last_user_activity_time(), OP_EQ, start); + + set_network_participation(false); + + // reset can move backwards and forwards, but does not change network + // participation. + reset_user_activity(start-10); + tt_i64_op(get_last_user_activity_time(), OP_EQ, start-10); + reset_user_activity(start+10); + tt_i64_op(get_last_user_activity_time(), OP_EQ, start+10); + + tt_int_op(schedule_rescan_called, OP_EQ, 0); + tt_int_op(false, OP_EQ, is_participating_on_network()); + + // "note" can only move forward. Calling it from a non-participating + // state makes us rescan the periodic callbacks and set participation. + note_user_activity(start+20); + tt_i64_op(get_last_user_activity_time(), OP_EQ, start+20); + tt_int_op(true, OP_EQ, is_participating_on_network()); + tt_int_op(schedule_rescan_called, OP_EQ, 1); + + // Calling it again will move us forward, but not call rescan again. + note_user_activity(start+25); + tt_i64_op(get_last_user_activity_time(), OP_EQ, start+25); + tt_int_op(true, OP_EQ, is_participating_on_network()); + tt_int_op(schedule_rescan_called, OP_EQ, 1); + + // We won't move backwards. + note_user_activity(start+20); + tt_i64_op(get_last_user_activity_time(), OP_EQ, start+25); + tt_int_op(true, OP_EQ, is_participating_on_network()); + tt_int_op(schedule_rescan_called, OP_EQ, 1); + + done: + UNMOCK(schedule_rescan_periodic_events); +} + #define MAINLOOP_TEST(name) \ { #name, test_mainloop_## name , TT_FORK, NULL, NULL } struct testcase_t mainloop_tests[] = { MAINLOOP_TEST(update_time_normal), MAINLOOP_TEST(update_time_jumps), + MAINLOOP_TEST(user_activity), END_OF_TESTCASES }; - From 02843c4a4e2fab9c5d9cdb95c425c37ff3d1a4ae Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 19 Nov 2018 16:31:50 -0500 Subject: [PATCH 0204/2557] Test for check_network_participation_callback() --- src/core/mainloop/connection.c | 4 +- src/core/mainloop/connection.h | 2 +- src/core/mainloop/mainloop.c | 3 +- src/core/mainloop/mainloop.h | 3 ++ src/feature/hs/hs_service.c | 4 +- src/feature/hs/hs_service.h | 2 +- src/test/test_mainloop.c | 88 ++++++++++++++++++++++++++++++++++ 7 files changed, 98 insertions(+), 8 deletions(-) diff --git a/src/core/mainloop/connection.c b/src/core/mainloop/connection.c index 25224fd999..c1c7c3678b 100644 --- a/src/core/mainloop/connection.c +++ b/src/core/mainloop/connection.c @@ -4433,8 +4433,8 @@ connection_get_by_type_state(int type, int state) * Return a connection of type type that is not an internally linked * connection, and is not marked for close. **/ -connection_t * -connection_get_by_type_nonlinked(int type) +MOCK_IMPL(connection_t *, +connection_get_by_type_nonlinked,(int type)) { CONN_GET_TEMPLATE(conn, conn->type == type && !conn->linked); } diff --git a/src/core/mainloop/connection.h b/src/core/mainloop/connection.h index 9f1a23c6f2..07b8df4138 100644 --- a/src/core/mainloop/connection.h +++ b/src/core/mainloop/connection.h @@ -240,7 +240,7 @@ size_t connection_get_outbuf_len(connection_t *conn); connection_t *connection_get_by_global_id(uint64_t id); connection_t *connection_get_by_type(int type); -connection_t *connection_get_by_type_nonlinked(int type); +MOCK_DECL(connection_t *,connection_get_by_type_nonlinked,(int type)); MOCK_DECL(connection_t *,connection_get_by_type_addr_port_purpose,(int type, const tor_addr_t *addr, uint16_t port, int purpose)); diff --git a/src/core/mainloop/mainloop.c b/src/core/mainloop/mainloop.c index 331f7021a4..42df1038a8 100644 --- a/src/core/mainloop/mainloop.c +++ b/src/core/mainloop/mainloop.c @@ -1367,7 +1367,6 @@ CALLBACK(write_bridge_ns); CALLBACK(write_stats_file); CALLBACK(control_per_second_events); CALLBACK(second_elapsed); -CALLBACK(check_network_participation); #undef CALLBACK @@ -2003,7 +2002,7 @@ add_entropy_callback(time_t now, const or_options_t *options) /** Periodic callback: if there has been no network usage in a while, * enter a dormant state. */ -static int +STATIC int check_network_participation_callback(time_t now, const or_options_t *options) { /* If we're a server, we can't become dormant. */ diff --git a/src/core/mainloop/mainloop.h b/src/core/mainloop/mainloop.h index e5e730fc8e..14e80ebb21 100644 --- a/src/core/mainloop/mainloop.h +++ b/src/core/mainloop/mainloop.h @@ -104,6 +104,9 @@ STATIC void close_closeable_connections(void); STATIC void initialize_periodic_events(void); STATIC void teardown_periodic_events(void); STATIC int get_my_roles(const or_options_t *); +STATIC int check_network_participation_callback(time_t now, + const or_options_t *options); + #ifdef TOR_UNIT_TESTS extern smartlist_t *connection_array; diff --git a/src/feature/hs/hs_service.c b/src/feature/hs/hs_service.c index c288e28e80..ee0b64a969 100644 --- a/src/feature/hs/hs_service.c +++ b/src/feature/hs/hs_service.c @@ -3667,8 +3667,8 @@ hs_service_lookup_current_desc(const ed25519_public_key_t *pk) } /* Return the number of service we have configured and usable. */ -unsigned int -hs_service_get_num_services(void) +MOCK_IMPL(unsigned int, +hs_service_get_num_services,(void)) { if (hs_service_map == NULL) { return 0; diff --git a/src/feature/hs/hs_service.h b/src/feature/hs/hs_service.h index a8a9faaea9..be1155bcd1 100644 --- a/src/feature/hs/hs_service.h +++ b/src/feature/hs/hs_service.h @@ -310,7 +310,7 @@ hs_service_t *hs_service_new(const or_options_t *options); void hs_service_free_(hs_service_t *service); #define hs_service_free(s) FREE_AND_NULL(hs_service_t, hs_service_free_, (s)) -unsigned int hs_service_get_num_services(void); +MOCK_DECL(unsigned int, hs_service_get_num_services,(void)); void hs_service_stage_services(const smartlist_t *service_list); int hs_service_load_all_keys(void); int hs_service_get_version_from_key(const hs_service_t *service); diff --git a/src/test/test_mainloop.c b/src/test/test_mainloop.c index 94b684fb3a..8dfd5f619a 100644 --- a/src/test/test_mainloop.c +++ b/src/test/test_mainloop.c @@ -6,13 +6,21 @@ * \brief Tests for functions closely related to the Tor main loop */ +#define CONFIG_PRIVATE +#define MAINLOOP_PRIVATE + #include "test/test.h" #include "test/log_test_helpers.h" #include "core/or/or.h" +#include "core/mainloop/connection.h" #include "core/mainloop/mainloop.h" #include "core/mainloop/netstatus.h" +#include "feature/hs/hs_service.h" + +#include "app/config/config.h" + static const uint64_t BILLION = 1000000000; static void @@ -186,6 +194,85 @@ test_mainloop_user_activity(void *arg) UNMOCK(schedule_rescan_periodic_events); } +static unsigned int +mock_get_num_services(void) +{ + return 1; +} + +static connection_t * +mock_connection_gbtu(int type) +{ + (void) type; + return (void *)"hello fellow connections"; +} + +static void +test_mainloop_check_participation(void *arg) +{ + (void)arg; + or_options_t *options = options_new(); + const time_t start = 1542658829; + const time_t ONE_DAY = 24*60*60; + + // Suppose we've been idle for a day or two + reset_user_activity(start - 2*ONE_DAY); + set_network_participation(true); + check_network_participation_callback(start, options); + tt_int_op(is_participating_on_network(), OP_EQ, false); + tt_i64_op(get_last_user_activity_time(), OP_EQ, start-2*ONE_DAY); + + // suppose we've been idle for 2 days... but we are a server. + reset_user_activity(start - 2*ONE_DAY); + options->ORPort_set = 1; + set_network_participation(true); + check_network_participation_callback(start+2, options); + tt_int_op(is_participating_on_network(), OP_EQ, true); + tt_i64_op(get_last_user_activity_time(), OP_EQ, start+2); + options->ORPort_set = 0; + + // idle for 2 days, but we have a hidden service. + reset_user_activity(start - 2*ONE_DAY); + set_network_participation(true); + MOCK(hs_service_get_num_services, mock_get_num_services); + check_network_participation_callback(start+3, options); + tt_int_op(is_participating_on_network(), OP_EQ, true); + tt_i64_op(get_last_user_activity_time(), OP_EQ, start+3); + UNMOCK(hs_service_get_num_services); + + // idle for 2 days but we have at least one user connection + MOCK(connection_get_by_type_nonlinked, mock_connection_gbtu); + reset_user_activity(start - 2*ONE_DAY); + set_network_participation(true); + options->DormantTimeoutDisabledByIdleStreams = 1; + check_network_participation_callback(start+10, options); + tt_int_op(is_participating_on_network(), OP_EQ, true); + tt_i64_op(get_last_user_activity_time(), OP_EQ, start+10); + + // as above, but DormantTimeoutDisabledByIdleStreams is not set + reset_user_activity(start - 2*ONE_DAY); + set_network_participation(true); + options->DormantTimeoutDisabledByIdleStreams = 0; + check_network_participation_callback(start+13, options); + tt_int_op(is_participating_on_network(), OP_EQ, false); + tt_i64_op(get_last_user_activity_time(), OP_EQ, start-2*ONE_DAY); + UNMOCK(connection_get_by_type_nonlinked); + options->DormantTimeoutDisabledByIdleStreams = 1; + + // idle for 2 days but DormantClientTimeout is 3 days + reset_user_activity(start - 2*ONE_DAY); + set_network_participation(true); + options->DormantClientTimeout = ONE_DAY * 3; + check_network_participation_callback(start+30, options); + tt_int_op(is_participating_on_network(), OP_EQ, true); + tt_i64_op(get_last_user_activity_time(), OP_EQ, start-2*ONE_DAY); + + done: + or_options_free(options); + UNMOCK(hs_service_get_num_services); + UNMOCK(connection_get_by_type_nonlinked); +} + #define MAINLOOP_TEST(name) \ { #name, test_mainloop_## name , TT_FORK, NULL, NULL } @@ -193,5 +280,6 @@ struct testcase_t mainloop_tests[] = { MAINLOOP_TEST(update_time_normal), MAINLOOP_TEST(update_time_jumps), MAINLOOP_TEST(user_activity), + MAINLOOP_TEST(check_participation), END_OF_TESTCASES }; From e12fdeb18199925070f4e99e60f2c5bda7f6af82 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 26 Nov 2018 16:39:44 -0500 Subject: [PATCH 0205/2557] Changes file for "Dormant Mode" (28335, 2149). --- changes/ticket28335 | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 changes/ticket28335 diff --git a/changes/ticket28335 b/changes/ticket28335 new file mode 100644 index 0000000000..eecf7c7fd9 --- /dev/null +++ b/changes/ticket28335 @@ -0,0 +1,7 @@ + o Major features (client): + - When Tor is running as a client, and it is unused for a long time, it + can now enter a "dormant" state. When Tor is dormant, it avoids + network activity and CPU wakeups until it is reawoken either by a user + request or by a controller command. For more information, see + the configuration options starting with "Dormant". Implements tickets + 2149 and 28335. From 2a3eef4404415f1bb90a86c9cc396b6dee89039c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Mon, 10 Sep 2018 13:05:01 +0200 Subject: [PATCH 0206/2557] Remove unused `int pid` member of `managed_proxy_t`. See: https://bugs.torproject.org/28179 --- src/feature/client/transports.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/feature/client/transports.h b/src/feature/client/transports.h index d304dcd485..b80875c95c 100644 --- a/src/feature/client/transports.h +++ b/src/feature/client/transports.h @@ -97,8 +97,6 @@ typedef struct { /* A pointer to the process handle of this managed proxy. */ struct process_handle_t *process_handle; - int pid; /* The Process ID this managed proxy is using. */ - /** Boolean: We are re-parsing our config, and we are going to * remove this managed proxy if we don't find it any transport * plugins that use it. */ From 5f26ae833eea79e56cb8cb8133b16a9cb0944ae4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Mon, 10 Sep 2018 13:23:59 +0200 Subject: [PATCH 0207/2557] Refactor read_to_chunk() such that it supports both pipes and sockets. See: https://bugs.torproject.org/28179 --- src/lib/net/buffers_net.c | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/src/lib/net/buffers_net.c b/src/lib/net/buffers_net.c index c52ea2784e..bd420510ab 100644 --- a/src/lib/net/buffers_net.c +++ b/src/lib/net/buffers_net.c @@ -21,6 +21,7 @@ #endif #include +#include #ifdef PARANOIA /** Helper: If PARANOIA is defined, assert that the buffer in local variable @@ -30,27 +31,33 @@ #define check() STMT_NIL #endif /* defined(PARANOIA) */ -/** Read up to at_most bytes from the socket fd into +/** Read up to at_most bytes from the file descriptor fd into * chunk (which must be on buf). If we get an EOF, set * *reached_eof to 1. Return -1 on error, 0 on eof or blocking, * and the number of bytes read otherwise. */ static inline int read_to_chunk(buf_t *buf, chunk_t *chunk, tor_socket_t fd, size_t at_most, - int *reached_eof, int *socket_error) + int *reached_eof, int *error, bool is_socket) { ssize_t read_result; if (at_most > CHUNK_REMAINING_CAPACITY(chunk)) at_most = CHUNK_REMAINING_CAPACITY(chunk); - read_result = tor_socket_recv(fd, CHUNK_WRITE_PTR(chunk), at_most, 0); + + if (is_socket) + read_result = tor_socket_recv(fd, CHUNK_WRITE_PTR(chunk), at_most, 0); + else + read_result = read(fd, CHUNK_WRITE_PTR(chunk), at_most); if (read_result < 0) { int e = tor_socket_errno(fd); if (!ERRNO_IS_EAGAIN(e)) { /* it's a real error */ #ifdef _WIN32 if (e == WSAENOBUFS) - log_warn(LD_NET,"recv() failed: WSAENOBUFS. Not enough ram?"); + log_warn(LD_NET, "%s() failed: WSAENOBUFS. Not enough ram?", + is_socket ? "recv" : "read"); #endif - *socket_error = e; + if (error) + *error = e; return -1; } return 0; /* would block. */ @@ -108,7 +115,7 @@ buf_read_from_socket(buf_t *buf, tor_socket_t s, size_t at_most, readlen = cap; } - r = read_to_chunk(buf, chunk, s, readlen, reached_eof, socket_error); + r = read_to_chunk(buf, chunk, s, readlen, reached_eof, socket_error, true); check(); if (r < 0) return r; /* Error */ From 340260281a514ee92b4c40ee5ddf8728ac580a40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Mon, 10 Sep 2018 13:28:01 +0200 Subject: [PATCH 0208/2557] Refactor flush_chunk() to work on pipes as well as sockets. See: https://bugs.torproject.org/28179 --- src/lib/net/buffers_net.c | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/lib/net/buffers_net.c b/src/lib/net/buffers_net.c index bd420510ab..fc133a01fe 100644 --- a/src/lib/net/buffers_net.c +++ b/src/lib/net/buffers_net.c @@ -129,22 +129,26 @@ buf_read_from_socket(buf_t *buf, tor_socket_t s, size_t at_most, } /** Helper for buf_flush_to_socket(): try to write sz bytes from chunk - * chunk of buffer buf onto socket s. On success, deduct - * the bytes written from *buf_flushlen. Return the number of bytes - * written on success, 0 on blocking, -1 on failure. + * chunk of buffer buf onto file descriptor fd. On + * success, deduct the bytes written from *buf_flushlen. Return the + * number of bytes written on success, 0 on blocking, -1 on failure. */ static inline int -flush_chunk(tor_socket_t s, buf_t *buf, chunk_t *chunk, size_t sz, - size_t *buf_flushlen) +flush_chunk(tor_socket_t fd, buf_t *buf, chunk_t *chunk, size_t sz, + size_t *buf_flushlen, bool is_socket) { ssize_t write_result; if (sz > chunk->datalen) sz = chunk->datalen; - write_result = tor_socket_send(s, chunk->data, sz, 0); + + if (is_socket) + write_result = tor_socket_send(fd, chunk->data, sz, 0); + else + write_result = write(fd, chunk->data, sz); if (write_result < 0) { - int e = tor_socket_errno(s); + int e = tor_socket_errno(fd); if (!ERRNO_IS_EAGAIN(e)) { /* it's a real error */ #ifdef _WIN32 if (e == WSAENOBUFS) @@ -195,7 +199,7 @@ buf_flush_to_socket(buf_t *buf, tor_socket_t s, size_t sz, else flushlen0 = buf->head->datalen; - r = flush_chunk(s, buf, buf->head, flushlen0, buf_flushlen); + r = flush_chunk(s, buf, buf->head, flushlen0, buf_flushlen, true); check(); if (r < 0) return r; From c71f9df07bb5f9f01a47a66f245c7794cb2bd839 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Mon, 10 Sep 2018 14:29:27 +0200 Subject: [PATCH 0209/2557] Refactor buf_flush_to_socket() into buf_flush_to_fd(). This patch refactors buf_flush_to_socket() into buf_flush_to_fd() and creates a specialization function for buf_flush_to_socket() that makes use of buf_flush_to_fd(). See: https://bugs.torproject.org/28179 --- src/lib/net/buffers_net.c | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/src/lib/net/buffers_net.c b/src/lib/net/buffers_net.c index fc133a01fe..2ffca8b7da 100644 --- a/src/lib/net/buffers_net.c +++ b/src/lib/net/buffers_net.c @@ -166,15 +166,15 @@ flush_chunk(tor_socket_t fd, buf_t *buf, chunk_t *chunk, size_t sz, } } -/** Write data from buf to the socket s. Write at most +/** Write data from buf to the file descriptor fd. Write at most * sz bytes, decrement *buf_flushlen by * the number of bytes actually written, and remove the written bytes * from the buffer. Return the number of bytes written on success, * -1 on failure. Return 0 if write() would block. */ -int -buf_flush_to_socket(buf_t *buf, tor_socket_t s, size_t sz, - size_t *buf_flushlen) +static int +buf_flush_to_fd(buf_t *buf, int fd, size_t sz, + size_t *buf_flushlen, bool is_socket) { /* XXXX It's stupid to overload the return values for these functions: * "error status" and "number of bytes flushed" are not mutually exclusive. @@ -182,7 +182,7 @@ buf_flush_to_socket(buf_t *buf, tor_socket_t s, size_t sz, int r; size_t flushed = 0; tor_assert(buf_flushlen); - tor_assert(SOCKET_OK(s)); + tor_assert(SOCKET_OK(fd)); if (BUG(*buf_flushlen > buf->datalen)) { *buf_flushlen = buf->datalen; } @@ -199,7 +199,7 @@ buf_flush_to_socket(buf_t *buf, tor_socket_t s, size_t sz, else flushlen0 = buf->head->datalen; - r = flush_chunk(s, buf, buf->head, flushlen0, buf_flushlen, true); + r = flush_chunk(fd, buf, buf->head, flushlen0, buf_flushlen, is_socket); check(); if (r < 0) return r; @@ -211,3 +211,16 @@ buf_flush_to_socket(buf_t *buf, tor_socket_t s, size_t sz, tor_assert(flushed < INT_MAX); return (int)flushed; } + +/** Write data from buf to the socket s. Write at most + * sz bytes, decrement *buf_flushlen by + * the number of bytes actually written, and remove the written bytes + * from the buffer. Return the number of bytes written on success, + * -1 on failure. Return 0 if write() would block. + */ +int +buf_flush_to_socket(buf_t *buf, tor_socket_t s, size_t sz, + size_t *buf_flushlen) +{ + return buf_flush_to_fd(buf, s, sz, buf_flushlen, true); +} From 771930b84cb4aeeb75e175368108697bafd94a2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Mon, 10 Sep 2018 14:35:38 +0200 Subject: [PATCH 0210/2557] Refactor buf_read_from_socket() into buf_read_from_fd(). This patch refactors buf_read_from_socket() into buf_read_from_fd(), and creates a specialized function for buf_read_from_socket(), which uses buf_read_from_fd(). See: https://bugs.torproject.org/28179 --- src/lib/net/buffers_net.c | 35 +++++++++++++++++++++++++---------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/src/lib/net/buffers_net.c b/src/lib/net/buffers_net.c index 2ffca8b7da..7c81096e38 100644 --- a/src/lib/net/buffers_net.c +++ b/src/lib/net/buffers_net.c @@ -75,16 +75,17 @@ read_to_chunk(buf_t *buf, chunk_t *chunk, tor_socket_t fd, size_t at_most, } } -/** Read from socket s, writing onto end of buf. Read at most - * at_most bytes, growing the buffer as necessary. If recv() returns 0 - * (because of EOF), set *reached_eof to 1 and return 0. Return -1 on - * error; else return the number of bytes read. +/** Read from file descriptor fd, writing onto end of buf. Read + * at most at_most bytes, growing the buffer as necessary. If recv() + * returns 0 (because of EOF), set *reached_eof to 1 and return 0. + * Return -1 on error; else return the number of bytes read. */ /* XXXX indicate "read blocked" somehow? */ -int -buf_read_from_socket(buf_t *buf, tor_socket_t s, size_t at_most, - int *reached_eof, - int *socket_error) +static int +buf_read_from_fd(buf_t *buf, int fd, size_t at_most, + int *reached_eof, + int *socket_error, + bool is_socket) { /* XXXX It's stupid to overload the return values for these functions: * "error status" and "number of bytes read" are not mutually exclusive. @@ -94,7 +95,7 @@ buf_read_from_socket(buf_t *buf, tor_socket_t s, size_t at_most, check(); tor_assert(reached_eof); - tor_assert(SOCKET_OK(s)); + tor_assert(SOCKET_OK(fd)); if (BUG(buf->datalen >= INT_MAX)) return -1; @@ -115,7 +116,8 @@ buf_read_from_socket(buf_t *buf, tor_socket_t s, size_t at_most, readlen = cap; } - r = read_to_chunk(buf, chunk, s, readlen, reached_eof, socket_error, true); + r = read_to_chunk(buf, chunk, fd, readlen, + reached_eof, socket_error, is_socket); check(); if (r < 0) return r; /* Error */ @@ -224,3 +226,16 @@ buf_flush_to_socket(buf_t *buf, tor_socket_t s, size_t sz, { return buf_flush_to_fd(buf, s, sz, buf_flushlen, true); } + +/** Read from socket s, writing onto end of buf. Read at most + * at_most bytes, growing the buffer as necessary. If recv() returns 0 + * (because of EOF), set *reached_eof to 1 and return 0. Return -1 on + * error; else return the number of bytes read. + */ +int +buf_read_from_socket(buf_t *buf, tor_socket_t s, size_t at_most, + int *reached_eof, + int *socket_error) +{ + return buf_read_from_fd(buf, s, at_most, reached_eof, socket_error, true); +} From 31b3a6577c89492e94836f6e3b4bfc7051a3dc7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Mon, 10 Sep 2018 14:42:29 +0200 Subject: [PATCH 0211/2557] Add buf_flush_to_pipe() and buf_read_from_pipe(). This patch adds two new functions: buf_flush_to_pipe() and buf_read_from_pipe(), which makes use of our new buf_flush_to_fd() and buf_read_from_fd() functions. See: https://bugs.torproject.org/28179 --- src/lib/net/buffers_net.c | 26 ++++++++++++++++++++++++++ src/lib/net/buffers_net.h | 7 +++++++ 2 files changed, 33 insertions(+) diff --git a/src/lib/net/buffers_net.c b/src/lib/net/buffers_net.c index 7c81096e38..1b65819dbb 100644 --- a/src/lib/net/buffers_net.c +++ b/src/lib/net/buffers_net.c @@ -239,3 +239,29 @@ buf_read_from_socket(buf_t *buf, tor_socket_t s, size_t at_most, { return buf_read_from_fd(buf, s, at_most, reached_eof, socket_error, true); } + +/** Write data from buf to the pipe fd. Write at most + * sz bytes, decrement *buf_flushlen by + * the number of bytes actually written, and remove the written bytes + * from the buffer. Return the number of bytes written on success, + * -1 on failure. Return 0 if write() would block. + */ +int +buf_flush_to_pipe(buf_t *buf, int fd, size_t sz, + size_t *buf_flushlen) +{ + return buf_flush_to_fd(buf, fd, sz, buf_flushlen, false); +} + +/** Read from pipe fd, writing onto end of buf. Read at most + * at_most bytes, growing the buffer as necessary. If read() returns 0 + * (because of EOF), set *reached_eof to 1 and return 0. Return -1 on + * error; else return the number of bytes read. + */ +int +buf_read_from_pipe(buf_t *buf, int fd, size_t at_most, + int *reached_eof, + int *socket_error) +{ + return buf_read_from_fd(buf, fd, at_most, reached_eof, socket_error, false); +} diff --git a/src/lib/net/buffers_net.h b/src/lib/net/buffers_net.h index 417f6f9413..8911b082a2 100644 --- a/src/lib/net/buffers_net.h +++ b/src/lib/net/buffers_net.h @@ -24,4 +24,11 @@ int buf_read_from_socket(struct buf_t *buf, tor_socket_t s, size_t at_most, int buf_flush_to_socket(struct buf_t *buf, tor_socket_t s, size_t sz, size_t *buf_flushlen); +int buf_read_from_pipe(struct buf_t *buf, int fd, size_t at_most, + int *reached_eof, + int *socket_error); + +int buf_flush_to_pipe(struct buf_t *buf, int fd, size_t sz, + size_t *buf_flushlen); + #endif /* !defined(TOR_BUFFERS_H) */ From 2b41b857bdabffeafbbf748189b22cd6ee818394 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Mon, 10 Sep 2018 15:15:45 +0200 Subject: [PATCH 0212/2557] Add LD_PROCESS as log domain. See: https://bugs.torproject.org/28179 --- doc/tor.1.txt | 3 ++- src/lib/log/log.c | 2 +- src/lib/log/log.h | 4 +++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/doc/tor.1.txt b/doc/tor.1.txt index e94382b66b..42fe8f1bcc 100644 --- a/doc/tor.1.txt +++ b/doc/tor.1.txt @@ -669,7 +669,8 @@ GENERAL OPTIONS + The currently recognized domains are: general, crypto, net, config, fs, protocol, mm, http, app, control, circ, rend, bug, dir, dirserv, or, edge, - acct, hist, handshake, heartbeat, channel, sched, guard, consdiff, and dos. + acct, hist, handshake, heartbeat, channel, sched, guard, consdiff, dos, and + process. Domain names are case-insensitive. + + For example, "`Log [handshake]debug [~net,~mm]info notice stdout`" sends diff --git a/src/lib/log/log.c b/src/lib/log/log.c index 46107fe848..861de6e963 100644 --- a/src/lib/log/log.c +++ b/src/lib/log/log.c @@ -1268,7 +1268,7 @@ static const char *domain_list[] = { "GENERAL", "CRYPTO", "NET", "CONFIG", "FS", "PROTOCOL", "MM", "HTTP", "APP", "CONTROL", "CIRC", "REND", "BUG", "DIR", "DIRSERV", "OR", "EDGE", "ACCT", "HIST", "HANDSHAKE", "HEARTBEAT", "CHANNEL", - "SCHED", "GUARD", "CONSDIFF", "DOS", NULL + "SCHED", "GUARD", "CONSDIFF", "DOS", "PROCESS", NULL }; /** Return a bitmask for the log domain for which domain is the name, diff --git a/src/lib/log/log.h b/src/lib/log/log.h index 493409756f..1cd6087eb8 100644 --- a/src/lib/log/log.h +++ b/src/lib/log/log.h @@ -107,8 +107,10 @@ #define LD_CONSDIFF (1u<<24) /** Denial of Service mitigation. */ #define LD_DOS (1u<<25) +/** Processes */ +#define LD_PROCESS (1u<<26) /** Number of logging domains in the code. */ -#define N_LOGGING_DOMAINS 26 +#define N_LOGGING_DOMAINS 27 /** This log message is not safe to send to a callback-based logger * immediately. Used as a flag, not a log domain. */ From bd29b3531a48b41f6eee5b4a29769a13ef199298 Mon Sep 17 00:00:00 2001 From: teor Date: Thu, 29 Nov 2018 01:06:19 +1000 Subject: [PATCH 0213/2557] Dir: Refactor ns expiry check to remove duplicate code Instead of checking NS_EXPIRY_SLOP, use networkstatus_consensus_reasonably_live(). Preparation for 28591. --- src/core/mainloop/mainloop.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/core/mainloop/mainloop.c b/src/core/mainloop/mainloop.c index 7eff82fee4..af2bc8262f 100644 --- a/src/core/mainloop/mainloop.c +++ b/src/core/mainloop/mainloop.c @@ -2028,11 +2028,9 @@ check_expired_networkstatus_callback(time_t now, const or_options_t *options) (void)options; /* Check whether our networkstatus has expired. */ networkstatus_t *ns = networkstatus_get_latest_consensus(); - /*XXXX RD: This value needs to be the same as REASONABLY_LIVE_TIME in - * networkstatus_get_reasonably_live_consensus(), but that value is way - * way too high. Arma: is the bridge issue there resolved yet? -NM */ -#define NS_EXPIRY_SLOP (24*60*60) - if (ns && ns->valid_until < (now - NS_EXPIRY_SLOP) && + /* Use reasonably live consensuses until they are no longer reasonably live. + */ + if (ns && !networkstatus_consensus_reasonably_live(ns, now) && router_have_minimum_dir_info()) { router_dir_info_changed(); } From 7a45bc74a430ff64f0696c77f075f9c830822413 Mon Sep 17 00:00:00 2001 From: teor Date: Thu, 29 Nov 2018 00:46:39 +1000 Subject: [PATCH 0214/2557] Dir: when Tor's clock is behind, use a future consensus to bootstrap When Tor's clock is behind the clocks on the authorities, allow Tor to bootstrap successfully. Fixes bug 28591; bugfix on 0.2.0.9-alpha. --- changes/bug28591 | 4 ++++ src/feature/nodelist/microdesc.c | 10 ++++---- src/feature/nodelist/networkstatus.c | 35 +++++++++++++++++++--------- src/feature/nodelist/networkstatus.h | 2 ++ src/test/test_entrynodes.c | 32 +++++++++++++++++-------- src/test/test_routerlist.c | 1 - 6 files changed, 58 insertions(+), 26 deletions(-) create mode 100644 changes/bug28591 diff --git a/changes/bug28591 b/changes/bug28591 new file mode 100644 index 0000000000..3a1c96ac16 --- /dev/null +++ b/changes/bug28591 @@ -0,0 +1,4 @@ + o Minor bugfixes (client, bootstrap): + - When Tor's clock is behind the clocks on the authorities, allow Tor to + bootstrap successfully. Fixes bug 28591; bugfix on 0.2.0.9-alpha. + diff --git a/src/feature/nodelist/microdesc.c b/src/feature/nodelist/microdesc.c index 3f5085412f..8c5a9ee611 100644 --- a/src/feature/nodelist/microdesc.c +++ b/src/feature/nodelist/microdesc.c @@ -110,8 +110,9 @@ microdesc_note_outdated_dirserver(const char *relay_digest) /* If we have a reasonably live consensus, then most of our dirservers should * still be caching all the microdescriptors in it. Reasonably live - * consensuses are up to a day old. But microdescriptors expire 7 days after - * the last consensus that referenced them. */ + * consensuses are up to a day old (or a day in the future). But + * microdescriptors expire 7 days after the last consensus that referenced + * them. */ if (!networkstatus_get_reasonably_live_consensus(approx_time(), FLAV_MICRODESC)) { return; @@ -544,8 +545,8 @@ microdesc_cache_clean(microdesc_cache_t *cache, time_t cutoff, int force) size_t bytes_dropped = 0; time_t now = time(NULL); - /* If we don't know a live consensus, don't believe last_listed values: we - * might be starting up after being down for a while. */ + /* If we don't know a reasonably live consensus, don't believe last_listed + * values: we might be starting up after being down for a while. */ if (! force && ! networkstatus_get_reasonably_live_consensus(now, FLAV_MICRODESC)) return; @@ -973,6 +974,7 @@ update_microdesc_downloads(time_t now) if (directory_too_idle_to_fetch_descriptors(options, now)) return; + /* Give up if we don't have a reasonably live consensus. */ consensus = networkstatus_get_reasonably_live_consensus(now, FLAV_MICRODESC); if (!consensus) return; diff --git a/src/feature/nodelist/networkstatus.c b/src/feature/nodelist/networkstatus.c index 51e720a984..a40c3dfb22 100644 --- a/src/feature/nodelist/networkstatus.c +++ b/src/feature/nodelist/networkstatus.c @@ -1377,7 +1377,7 @@ networkstatus_get_dl_status_by_flavor_running,(consensus_flavor_t flavor)) } /** Return the most recent consensus that we have downloaded, or NULL if we - * don't have one. */ + * don't have one. May return future or expired consensuses. */ MOCK_IMPL(networkstatus_t *, networkstatus_get_latest_consensus,(void)) { @@ -1388,7 +1388,7 @@ networkstatus_get_latest_consensus,(void)) } /** Return the latest consensus we have whose flavor matches f, or NULL - * if we don't have one. */ + * if we don't have one. May return future or expired consensuses. */ MOCK_IMPL(networkstatus_t *, networkstatus_get_latest_consensus_by_flavor,(consensus_flavor_t f)) { @@ -1422,10 +1422,11 @@ networkstatus_is_live(const networkstatus_t *ns, time_t now) return (ns->valid_after <= now && now <= ns->valid_until); } -/** Determine if consensus is valid or expired recently enough that - * we can still use it. +/** Determine if consensus is valid, or expired recently enough, or not + * too far in the future, so that we can still use it. * - * Return 1 if the consensus is reasonably live, or 0 if it is too old. + * Return 1 if the consensus is reasonably live, or 0 if it is too old or + * too new. */ int networkstatus_consensus_reasonably_live(const networkstatus_t *consensus, @@ -1434,29 +1435,42 @@ networkstatus_consensus_reasonably_live(const networkstatus_t *consensus, if (BUG(!consensus)) return 0; - return networkstatus_valid_until_is_reasonably_live(consensus->valid_until, + return networkstatus_valid_after_is_reasonably_live(consensus->valid_after, + now) && + networkstatus_valid_until_is_reasonably_live(consensus->valid_until, now); } +#define REASONABLY_LIVE_TIME (24*60*60) + +/** As networkstatus_consensus_reasonably_live, but takes a valid_after + * time, and checks to see if it is in the past, or not too far in the future. + */ +int +networkstatus_valid_after_is_reasonably_live(time_t valid_after, + time_t now) +{ + return (now >= valid_after - REASONABLY_LIVE_TIME); +} + /** As networkstatus_consensus_reasonably_live, but takes a valid_until - * time rather than an entire consensus. */ + * time, and checks to see if it is in the future, or not too far in the past. + */ int networkstatus_valid_until_is_reasonably_live(time_t valid_until, time_t now) { -#define REASONABLY_LIVE_TIME (24*60*60) return (now <= valid_until + REASONABLY_LIVE_TIME); } /** As networkstatus_get_live_consensus(), but is way more tolerant of expired - * consensuses. */ + * and future consensuses. */ MOCK_IMPL(networkstatus_t *, networkstatus_get_reasonably_live_consensus,(time_t now, int flavor)) { networkstatus_t *consensus = networkstatus_get_latest_consensus_by_flavor(flavor); if (consensus && - consensus->valid_after <= now && networkstatus_consensus_reasonably_live(consensus, now)) return consensus; else @@ -2082,7 +2096,6 @@ networkstatus_set_current_consensus(const char *consensus, nodelist_set_consensus(c); - /* XXXXNM Microdescs: needs a non-ns variant. ???? NM*/ update_consensus_networkstatus_fetch_time(now); /* Change the cell EWMA settings */ diff --git a/src/feature/nodelist/networkstatus.h b/src/feature/nodelist/networkstatus.h index 7b1a0ff72f..6f1d15d532 100644 --- a/src/feature/nodelist/networkstatus.h +++ b/src/feature/nodelist/networkstatus.h @@ -87,6 +87,8 @@ MOCK_DECL(networkstatus_t *, networkstatus_get_live_consensus,(time_t now)); int networkstatus_is_live(const networkstatus_t *ns, time_t now); int networkstatus_consensus_reasonably_live(const networkstatus_t *consensus, time_t now); +int networkstatus_valid_after_is_reasonably_live(time_t valid_after, + time_t now); int networkstatus_valid_until_is_reasonably_live(time_t valid_until, time_t now); MOCK_DECL(networkstatus_t *,networkstatus_get_reasonably_live_consensus, diff --git a/src/test/test_entrynodes.c b/src/test/test_entrynodes.c index 52b87a84a8..348c642a23 100644 --- a/src/test/test_entrynodes.c +++ b/src/test/test_entrynodes.c @@ -127,6 +127,9 @@ big_fake_network_cleanup(const struct testcase_t *testcase, void *ptr) return 1; /* NOP */ } +#define REASONABLY_FUTURE " reasonably-future" +#define REASONABLY_PAST " reasonably-past" + /* Unittest setup function: Setup a fake network. */ static void * big_fake_network_setup(const struct testcase_t *testcase) @@ -138,9 +141,10 @@ big_fake_network_setup(const struct testcase_t *testcase) const int N_NODES = 271; const char *argument = testcase->setup_data; - int reasonably_live_consensus = 0; + int reasonably_future_consensus = 0, reasonably_past_consensus = 0; if (argument) { - reasonably_live_consensus = strstr(argument, "reasonably-live") != NULL; + reasonably_future_consensus = strstr(argument, REASONABLY_FUTURE) != NULL; + reasonably_past_consensus = strstr(argument, REASONABLY_PAST) != NULL; } big_fake_net_nodes = smartlist_new(); @@ -198,11 +202,15 @@ big_fake_network_setup(const struct testcase_t *testcase) dummy_state = tor_malloc_zero(sizeof(or_state_t)); dummy_consensus = tor_malloc_zero(sizeof(networkstatus_t)); - if (reasonably_live_consensus) { - /* Make the dummy consensus valid from 4 hours ago, but expired an hour + if (reasonably_future_consensus) { + /* Make the dummy consensus valid in 6 hours, and expiring in 7 hours. */ + dummy_consensus->valid_after = approx_time() + 6*3600; + dummy_consensus->valid_until = approx_time() + 7*3600; + } else if (reasonably_past_consensus) { + /* Make the dummy consensus valid from 16 hours ago, but expired 12 hours * ago. */ - dummy_consensus->valid_after = approx_time() - 4*3600; - dummy_consensus->valid_until = approx_time() - 3600; + dummy_consensus->valid_after = approx_time() - 16*3600; + dummy_consensus->valid_until = approx_time() - 12*3600; } else { /* Make the dummy consensus valid for an hour either side of now. */ dummy_consensus->valid_after = approx_time() - 3600; @@ -3035,13 +3043,17 @@ static const struct testcase_setup_t upgrade_circuits = { #define BFN_TEST(name) \ EN_TEST_BASE(name, TT_FORK, &big_fake_network, NULL), \ - { #name "_reasonably_live", test_entry_guard_ ## name, TT_FORK, \ - &big_fake_network, (void*)("reasonably-live") } + { #name "_reasonably_future", test_entry_guard_ ## name, TT_FORK, \ + &big_fake_network, (void*)(REASONABLY_FUTURE) }, \ + { #name "_reasonably_past", test_entry_guard_ ## name, TT_FORK, \ + &big_fake_network, (void*)(REASONABLY_PAST) } #define UPGRADE_TEST(name, arg) \ EN_TEST_BASE(name, TT_FORK, &upgrade_circuits, arg), \ - { #name "_reasonably_live", test_entry_guard_ ## name, TT_FORK, \ - &upgrade_circuits, (void*)(arg " reasonably-live") } + { #name "_reasonably_future", test_entry_guard_ ## name, TT_FORK, \ + &upgrade_circuits, (void*)(arg REASONABLY_FUTURE) }, \ + { #name "_reasonably_past", test_entry_guard_ ## name, TT_FORK, \ + &upgrade_circuits, (void*)(arg REASONABLY_PAST) } struct testcase_t entrynodes_tests[] = { NO_PREFIX_TEST(node_preferred_orport), diff --git a/src/test/test_routerlist.c b/src/test/test_routerlist.c index 67af2fd484..6ba4877b91 100644 --- a/src/test/test_routerlist.c +++ b/src/test/test_routerlist.c @@ -301,7 +301,6 @@ test_router_pick_directory_server_impl(void *arg) tt_assert(!networkstatus_consensus_is_bootstrapping(con_md->valid_until + 24*60*60)); /* These times are outside the test validity period */ - tt_assert(networkstatus_consensus_is_bootstrapping(now)); tt_assert(networkstatus_consensus_is_bootstrapping(now + 2*24*60*60)); tt_assert(networkstatus_consensus_is_bootstrapping(now - 2*24*60*60)); From e2b418bab5c3249fba7b430b942da67ddf8a43dc Mon Sep 17 00:00:00 2001 From: rl1987 Date: Thu, 29 Nov 2018 10:56:56 +0200 Subject: [PATCH 0215/2557] Rename to pre-push.git-hook --- scripts/maint/{pre-push => pre-push.git-hook} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename scripts/maint/{pre-push => pre-push.git-hook} (100%) diff --git a/scripts/maint/pre-push b/scripts/maint/pre-push.git-hook similarity index 100% rename from scripts/maint/pre-push rename to scripts/maint/pre-push.git-hook From 7c6dc2888144e587bc8d62c7b2bddae4fb6606b9 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Thu, 29 Nov 2018 11:10:30 +0200 Subject: [PATCH 0216/2557] Improve comment at the top --- scripts/maint/pre-push.git-hook | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/scripts/maint/pre-push.git-hook b/scripts/maint/pre-push.git-hook index 26c48c4e21..11f062b539 100755 --- a/scripts/maint/pre-push.git-hook +++ b/scripts/maint/pre-push.git-hook @@ -1,8 +1,10 @@ #!/bin/bash -# git pre-push hook script to prevent "fixup!" and "squash!" commit -# from ending up in master, or in any branch if CUR_BRANCH check is removed. -# It is meant to be placed in .git/hooks directory. +# To install this script, copy it into .git/hooks/pre-push path in your +# local copy of git repository. Make sure it has permission to execute. +# +# This is git pre-push hook script to prevent "fixup!" and "squash!" commits +# from ending up in upstream branches (master, release-* or maint-*). # # The following sample script was used as starting point: # https://github.com/git/git/blob/master/templates/hooks--pre-push.sample From 9c90bddc42467396909812746d5b4256adcb5d2d Mon Sep 17 00:00:00 2001 From: rl1987 Date: Thu, 29 Nov 2018 11:10:48 +0200 Subject: [PATCH 0217/2557] Mention --no-verify in error message --- scripts/maint/pre-push.git-hook | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/maint/pre-push.git-hook b/scripts/maint/pre-push.git-hook index 11f062b539..26296023fb 100755 --- a/scripts/maint/pre-push.git-hook +++ b/scripts/maint/pre-push.git-hook @@ -42,6 +42,7 @@ do if [ -n "$commit" ] then echo >&2 "Found fixup! commit in $local_ref, not pushing" + echo >&2 "If you really want to push this, use --no-verify." exit 1 fi @@ -50,6 +51,7 @@ do if [ -n "$commit" ] then echo >&2 "Found squash! commit in $local_ref, not pushing" + echo >&2 "If you really want to push this, use --no-verify." exit 1 fi fi From a51dad4272de8da3d1379e72962a4a8bd500b688 Mon Sep 17 00:00:00 2001 From: David Goulet Date: Thu, 29 Nov 2018 10:02:50 -0500 Subject: [PATCH 0218/2557] test: Fix a warning underflow in rend_cache/clean Because the test is adding entries to the "rend_cache" directly, the rend_cache_increment_allocation() was never called which made the rend_cache_clean() call trigger that underflow warning: rend_cache/clean: [forking] Nov 29 09:55:04.024 [warn] rend_cache_decrement_allocation(): Bug: Underflow in rend_cache_decrement_allocation (on Tor 0.4.0.0-alpha-dev 2240fe63feb9a8cf) The test is still good and valid. Fixes #28660 Signed-off-by: David Goulet --- src/test/test_rendcache.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/test/test_rendcache.c b/src/test/test_rendcache.c index 394e28d785..2ace45d085 100644 --- a/src/test/test_rendcache.c +++ b/src/test/test_rendcache.c @@ -788,7 +788,9 @@ test_rend_cache_clean(void *data) desc_two->pk = pk_generate(1); strmap_set_lc(rend_cache, "foo1", one); + rend_cache_increment_allocation(rend_cache_entry_allocation(one)); strmap_set_lc(rend_cache, "foo2", two); + rend_cache_increment_allocation(rend_cache_entry_allocation(two)); rend_cache_clean(time(NULL), REND_CACHE_TYPE_CLIENT); tt_int_op(strmap_size(rend_cache), OP_EQ, 0); @@ -806,7 +808,9 @@ test_rend_cache_clean(void *data) desc_one->pk = pk_generate(0); desc_two->pk = pk_generate(1); + rend_cache_increment_allocation(rend_cache_entry_allocation(one)); strmap_set_lc(rend_cache, "foo1", one); + rend_cache_increment_allocation(rend_cache_entry_allocation(two)); strmap_set_lc(rend_cache, "foo2", two); rend_cache_clean(time(NULL), REND_CACHE_TYPE_CLIENT); From 1fe6507d29497d7bbd8f42f66f0ddd596078993e Mon Sep 17 00:00:00 2001 From: Taylor Yu Date: Mon, 19 Nov 2018 16:24:01 -0600 Subject: [PATCH 0219/2557] Split bootstrap event reporting out of control.c Part of ticket 27402. --- src/core/include.am | 1 + src/feature/control/control.c | 373 +--------------------- src/feature/control/control.h | 2 + src/feature/control/control_bootstrap.c | 403 ++++++++++++++++++++++++ 4 files changed, 408 insertions(+), 371 deletions(-) create mode 100644 src/feature/control/control_bootstrap.c diff --git a/src/core/include.am b/src/core/include.am index 48d75618ad..3cd6e83ed5 100644 --- a/src/core/include.am +++ b/src/core/include.am @@ -62,6 +62,7 @@ LIBTOR_APP_A_SOURCES = \ src/feature/client/entrynodes.c \ src/feature/client/transports.c \ src/feature/control/control.c \ + src/feature/control/control_bootstrap.c \ src/feature/control/fmt_serverstatus.c \ src/feature/control/getinfo_geoip.c \ src/feature/dirauth/keypin.c \ diff --git a/src/feature/control/control.c b/src/feature/control/control.c index 94679dfd22..060320341c 100644 --- a/src/feature/control/control.c +++ b/src/feature/control/control.c @@ -179,13 +179,6 @@ static uint8_t *authentication_cookie = NULL; */ static smartlist_t *detached_onion_services = NULL; -/** A sufficiently large size to record the last bootstrap phase string. */ -#define BOOTSTRAP_MSG_LEN 1024 - -/** What was the last bootstrap phase message we sent? We keep track - * of this so we can respond to getinfo status/bootstrap-phase queries. */ -static char last_sent_bootstrap_message[BOOTSTRAP_MSG_LEN]; - static void connection_printf_to_buf(control_connection_t *conn, const char *format, ...) CHECK_PRINTF(2,3); @@ -3044,7 +3037,7 @@ getinfo_helper_events(control_connection_t *control_conn, check_whether_orport_reachable(options) ? 1 : 0, check_whether_dirport_reachable(options) ? 1 : 0); } else if (!strcmp(question, "status/bootstrap-phase")) { - *answer = tor_strdup(last_sent_bootstrap_message); + *answer = control_event_boot_last_msg(); } else if (!strcmpstart(question, "status/version/")) { int is_server = server_mode(options); networkstatus_t *c = networkstatus_get_latest_consensus(); @@ -7015,361 +7008,6 @@ monitor_owning_controller_process(const char *process_spec) } } -/** Convert the name of a bootstrapping phase s into strings - * tag and summary suitable for display by the controller. */ -static int -bootstrap_status_to_string(bootstrap_status_t s, const char **tag, - const char **summary) -{ - switch (s) { - case BOOTSTRAP_STATUS_UNDEF: - *tag = "undef"; - *summary = "Undefined"; - break; - case BOOTSTRAP_STATUS_STARTING: - *tag = "starting"; - *summary = "Starting"; - break; - case BOOTSTRAP_STATUS_CONN_DIR: - *tag = "conn_dir"; - *summary = "Connecting to directory server"; - break; - case BOOTSTRAP_STATUS_HANDSHAKE: - *tag = "status_handshake"; - *summary = "Finishing handshake"; - break; - case BOOTSTRAP_STATUS_HANDSHAKE_DIR: - *tag = "handshake_dir"; - *summary = "Finishing handshake with directory server"; - break; - case BOOTSTRAP_STATUS_ONEHOP_CREATE: - *tag = "onehop_create"; - *summary = "Establishing an encrypted directory connection"; - break; - case BOOTSTRAP_STATUS_REQUESTING_STATUS: - *tag = "requesting_status"; - *summary = "Asking for networkstatus consensus"; - break; - case BOOTSTRAP_STATUS_LOADING_STATUS: - *tag = "loading_status"; - *summary = "Loading networkstatus consensus"; - break; - case BOOTSTRAP_STATUS_LOADING_KEYS: - *tag = "loading_keys"; - *summary = "Loading authority key certs"; - break; - case BOOTSTRAP_STATUS_REQUESTING_DESCRIPTORS: - *tag = "requesting_descriptors"; - /* XXXX this appears to incorrectly report internal on most loads */ - *summary = router_have_consensus_path() == CONSENSUS_PATH_INTERNAL ? - "Asking for relay descriptors for internal paths" : - "Asking for relay descriptors"; - break; - /* If we're sure there are no exits in the consensus, - * inform the controller by adding "internal" - * to the status summaries. - * (We only check this while loading descriptors, - * so we may not know in the earlier stages.) - * But if there are exits, we can't be sure whether - * we're creating internal or exit paths/circuits. - * XXXX Or should be use different tags or statuses - * for internal and exit/all? */ - case BOOTSTRAP_STATUS_LOADING_DESCRIPTORS: - *tag = "loading_descriptors"; - *summary = router_have_consensus_path() == CONSENSUS_PATH_INTERNAL ? - "Loading relay descriptors for internal paths" : - "Loading relay descriptors"; - break; - case BOOTSTRAP_STATUS_CONN_OR: - *tag = "conn_or"; - *summary = router_have_consensus_path() == CONSENSUS_PATH_INTERNAL ? - "Connecting to the Tor network internally" : - "Connecting to the Tor network"; - break; - case BOOTSTRAP_STATUS_HANDSHAKE_OR: - *tag = "handshake_or"; - *summary = router_have_consensus_path() == CONSENSUS_PATH_INTERNAL ? - "Finishing handshake with first hop of internal circuit" : - "Finishing handshake with first hop"; - break; - case BOOTSTRAP_STATUS_CIRCUIT_CREATE: - *tag = "circuit_create"; - *summary = router_have_consensus_path() == CONSENSUS_PATH_INTERNAL ? - "Establishing an internal Tor circuit" : - "Establishing a Tor circuit"; - break; - case BOOTSTRAP_STATUS_DONE: - *tag = "done"; - *summary = "Done"; - break; - default: -// log_warn(LD_BUG, "Unrecognized bootstrap status code %d", s); - *tag = *summary = "unknown"; - return -1; - } - return 0; -} - -/** What percentage through the bootstrap process are we? We remember - * this so we can avoid sending redundant bootstrap status events, and - * so we can guess context for the bootstrap messages which are - * ambiguous. It starts at 'undef', but gets set to 'starting' while - * Tor initializes. */ -static int bootstrap_percent = BOOTSTRAP_STATUS_UNDEF; - -/** Like bootstrap_percent, but only takes on the enumerated values in - * bootstrap_status_t. - */ -static int bootstrap_phase = BOOTSTRAP_STATUS_UNDEF; - -/** As bootstrap_percent, but holds the bootstrapping level at which we last - * logged a NOTICE-level message. We use this, plus BOOTSTRAP_PCT_INCREMENT, - * to avoid flooding the log with a new message every time we get a few more - * microdescriptors */ -static int notice_bootstrap_percent = 0; - -/** How many problems have we had getting to the next bootstrapping phase? - * These include failure to establish a connection to a Tor relay, - * failures to finish the TLS handshake, failures to validate the - * consensus document, etc. */ -static int bootstrap_problems = 0; - -/** We only tell the controller once we've hit a threshold of problems - * for the current phase. */ -#define BOOTSTRAP_PROBLEM_THRESHOLD 10 - -/** When our bootstrapping progress level changes, but our bootstrapping - * status has not advanced, we only log at NOTICE when we have made at least - * this much progress. - */ -#define BOOTSTRAP_PCT_INCREMENT 5 - -/** Do the actual logging and notifications for - * control_event_bootstrap(). Doesn't change any state beyond that. - */ -static void -control_event_bootstrap_core(int loglevel, bootstrap_status_t status, - int progress) -{ - char buf[BOOTSTRAP_MSG_LEN]; - const char *tag, *summary; - - bootstrap_status_to_string(status, &tag, &summary); - /* Locally reset status if there's incremental progress */ - if (progress) - status = progress; - - tor_log(loglevel, LD_CONTROL, - "Bootstrapped %d%%: %s", status, summary); - tor_snprintf(buf, sizeof(buf), - "BOOTSTRAP PROGRESS=%d TAG=%s SUMMARY=\"%s\"", - status, tag, summary); - tor_snprintf(last_sent_bootstrap_message, - sizeof(last_sent_bootstrap_message), - "NOTICE %s", buf); - control_event_client_status(LOG_NOTICE, "%s", buf); -} - -/** Called when Tor has made progress at bootstrapping its directory - * information and initial circuits. - * - * status is the new status, that is, what task we will be doing - * next. progress is zero if we just started this task, else it - * represents progress on the task. - */ -void -control_event_bootstrap(bootstrap_status_t status, int progress) -{ - int loglevel = LOG_NOTICE; - - if (bootstrap_percent == BOOTSTRAP_STATUS_DONE) - return; /* already bootstrapped; nothing to be done here. */ - - /* special case for handshaking status, since our TLS handshaking code - * can't distinguish what the connection is going to be for. */ - if (status == BOOTSTRAP_STATUS_HANDSHAKE) { - if (bootstrap_percent < BOOTSTRAP_STATUS_CONN_OR) { - status = BOOTSTRAP_STATUS_HANDSHAKE_DIR; - } else { - status = BOOTSTRAP_STATUS_HANDSHAKE_OR; - } - } - - if (status <= bootstrap_percent) { - /* If there's no new progress, return early. */ - if (!progress || progress <= bootstrap_percent) - return; - /* Log at INFO if not enough progress happened. */ - if (progress < notice_bootstrap_percent + BOOTSTRAP_PCT_INCREMENT) - loglevel = LOG_INFO; - } - - control_event_bootstrap_core(loglevel, status, progress); - - if (status > bootstrap_percent) { - bootstrap_phase = status; /* new milestone reached */ - bootstrap_percent = status; - } - if (progress > bootstrap_percent) { - /* incremental progress within a milestone */ - bootstrap_percent = progress; - bootstrap_problems = 0; /* Progress! Reset our problem counter. */ - } - if (loglevel == LOG_NOTICE && - bootstrap_percent > notice_bootstrap_percent) { - /* Remember that we gave a notice at this level. */ - notice_bootstrap_percent = bootstrap_percent; - } -} - -/** Flag whether we've opened an OR_CONN yet */ -static int bootstrap_first_orconn = 0; - -/** Like bootstrap_phase, but for (possibly deferred) directory progress */ -static int bootstrap_dir_phase = BOOTSTRAP_STATUS_UNDEF; - -/** Like bootstrap_problems, but for (possibly deferred) directory progress */ -static int bootstrap_dir_progress = BOOTSTRAP_STATUS_UNDEF; - -/** Defer directory info bootstrap events until we have successfully - * completed our first connection to a router. */ -void -control_event_boot_dir(bootstrap_status_t status, int progress) -{ - if (status > bootstrap_dir_progress) { - bootstrap_dir_progress = status; - bootstrap_dir_phase = status; - } - if (progress && progress >= bootstrap_dir_progress) { - bootstrap_dir_progress = progress; - } - - /* Don't report unless we have successfully opened at least one OR_CONN */ - if (!bootstrap_first_orconn) - return; - - control_event_bootstrap(status, progress); -} - -/** Set a flag to allow reporting of directory bootstrap progress. - * (Code that reports completion of an OR_CONN calls this.) Also, - * report directory progress so far. */ -void -control_event_boot_first_orconn(void) -{ - bootstrap_first_orconn = 1; - control_event_bootstrap(bootstrap_dir_phase, bootstrap_dir_progress); -} - -/** Called when Tor has failed to make bootstrapping progress in a way - * that indicates a problem. warn gives a human-readable hint - * as to why, and reason provides a controller-facing short - * tag. conn is the connection that caused this problem and - * can be NULL if a connection cannot be easily identified. - */ -void -control_event_bootstrap_problem(const char *warn, const char *reason, - const connection_t *conn, int dowarn) -{ - int status = bootstrap_percent; - const char *tag = "", *summary = ""; - char buf[BOOTSTRAP_MSG_LEN]; - const char *recommendation = "ignore"; - int severity; - char *or_id = NULL, *hostaddr = NULL; - or_connection_t *or_conn = NULL; - - /* bootstrap_percent must not be in "undefined" state here. */ - tor_assert(status >= 0); - - if (bootstrap_percent == 100) - return; /* already bootstrapped; nothing to be done here. */ - - bootstrap_problems++; - - if (bootstrap_problems >= BOOTSTRAP_PROBLEM_THRESHOLD) - dowarn = 1; - - /* Don't warn about our bootstrapping status if we are hibernating or - * shutting down. */ - if (we_are_hibernating()) - dowarn = 0; - - tor_assert(bootstrap_status_to_string(bootstrap_phase, &tag, &summary) == 0); - - severity = dowarn ? LOG_WARN : LOG_INFO; - - if (dowarn) - recommendation = "warn"; - - if (conn && conn->type == CONN_TYPE_OR) { - /* XXX TO_OR_CONN can't deal with const */ - or_conn = TO_OR_CONN((connection_t *)conn); - or_id = tor_strdup(hex_str(or_conn->identity_digest, DIGEST_LEN)); - } else { - or_id = tor_strdup("?"); - } - - if (conn) - tor_asprintf(&hostaddr, "%s:%d", conn->address, (int)conn->port); - else - hostaddr = tor_strdup("?"); - - log_fn(severity, - LD_CONTROL, "Problem bootstrapping. Stuck at %d%%: %s. (%s; %s; " - "count %d; recommendation %s; host %s at %s)", - status, summary, warn, reason, - bootstrap_problems, recommendation, - or_id, hostaddr); - - connection_or_report_broken_states(severity, LD_HANDSHAKE); - - tor_snprintf(buf, sizeof(buf), - "BOOTSTRAP PROGRESS=%d TAG=%s SUMMARY=\"%s\" WARNING=\"%s\" REASON=%s " - "COUNT=%d RECOMMENDATION=%s HOSTID=\"%s\" HOSTADDR=\"%s\"", - bootstrap_percent, tag, summary, warn, reason, bootstrap_problems, - recommendation, - or_id, hostaddr); - - tor_snprintf(last_sent_bootstrap_message, - sizeof(last_sent_bootstrap_message), - "WARN %s", buf); - control_event_client_status(LOG_WARN, "%s", buf); - - tor_free(hostaddr); - tor_free(or_id); -} - -/** Called when Tor has failed to make bootstrapping progress in a way - * that indicates a problem. warn gives a hint as to why, and - * reason provides an "or_conn_end_reason" tag. or_conn - * is the connection that caused this problem. - */ -MOCK_IMPL(void, -control_event_bootstrap_prob_or, (const char *warn, int reason, - or_connection_t *or_conn)) -{ - int dowarn = 0; - - if (or_conn->have_noted_bootstrap_problem) - return; - - or_conn->have_noted_bootstrap_problem = 1; - - if (reason == END_OR_CONN_REASON_NO_ROUTE) - dowarn = 1; - - /* If we are using bridges and all our OR connections are now - closed, it means that we totally failed to connect to our - bridges. Throw a warning. */ - if (get_options()->UseBridges && !any_other_active_or_conns(or_conn)) - dowarn = 1; - - control_event_bootstrap_problem(warn, - orconn_end_reason_to_control_string(reason), - TO_CONN(or_conn), dowarn); -} - /** We just generated a new summary of which countries we've seen clients * from recently. Send a copy to the controller in case it wants to * display it for the user. */ @@ -7886,17 +7524,10 @@ control_free_all(void) mainloop_event_free(flush_queued_events_event); flush_queued_events_event = NULL; } - bootstrap_percent = BOOTSTRAP_STATUS_UNDEF; - bootstrap_phase = BOOTSTRAP_STATUS_UNDEF; - notice_bootstrap_percent = 0; - bootstrap_problems = 0; - bootstrap_first_orconn = 0; - bootstrap_dir_progress = BOOTSTRAP_STATUS_UNDEF; - bootstrap_dir_phase = BOOTSTRAP_STATUS_UNDEF; + control_event_bootstrap_reset(); authentication_cookie_is_set = 0; global_event_mask = 0; disable_log_messages = 0; - memset(last_sent_bootstrap_message, 0, sizeof(last_sent_bootstrap_message)); } #ifdef TOR_UNIT_TESTS diff --git a/src/feature/control/control.h b/src/feature/control/control.h index cd5402d455..4ad245e8e5 100644 --- a/src/feature/control/control.h +++ b/src/feature/control/control.h @@ -200,6 +200,8 @@ void control_event_boot_dir(bootstrap_status_t status, int progress); void control_event_boot_first_orconn(void); void control_event_bootstrap_problem(const char *warn, const char *reason, const connection_t *conn, int dowarn); +char *control_event_boot_last_msg(void); +void control_event_bootstrap_reset(void); void control_event_clients_seen(const char *controller_str); void control_event_transport_launched(const char *mode, diff --git a/src/feature/control/control_bootstrap.c b/src/feature/control/control_bootstrap.c new file mode 100644 index 0000000000..58b342c219 --- /dev/null +++ b/src/feature/control/control_bootstrap.c @@ -0,0 +1,403 @@ +/* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file control_bootstrap.c + * \brief Provide bootstrap progress events for the control port. + */ +#include "core/or/or.h" + +#include "app/config/config.h" +#include "core/mainloop/connection.h" +#include "core/or/connection_or.h" +#include "core/or/connection_st.h" +#include "core/or/or_connection_st.h" +#include "core/or/reasons.h" +#include "feature/control/control.h" +#include "feature/hibernate/hibernate.h" +#include "feature/nodelist/nodelist.h" +#include "lib/malloc/malloc.h" + +/** A sufficiently large size to record the last bootstrap phase string. */ +#define BOOTSTRAP_MSG_LEN 1024 + +/** What was the last bootstrap phase message we sent? We keep track + * of this so we can respond to getinfo status/bootstrap-phase queries. */ +static char last_sent_bootstrap_message[BOOTSTRAP_MSG_LEN]; + +/** Convert the name of a bootstrapping phase s into strings + * tag and summary suitable for display by the controller. */ +static int +bootstrap_status_to_string(bootstrap_status_t s, const char **tag, + const char **summary) +{ + switch (s) { + case BOOTSTRAP_STATUS_UNDEF: + *tag = "undef"; + *summary = "Undefined"; + break; + case BOOTSTRAP_STATUS_STARTING: + *tag = "starting"; + *summary = "Starting"; + break; + case BOOTSTRAP_STATUS_CONN_DIR: + *tag = "conn_dir"; + *summary = "Connecting to directory server"; + break; + case BOOTSTRAP_STATUS_HANDSHAKE: + *tag = "status_handshake"; + *summary = "Finishing handshake"; + break; + case BOOTSTRAP_STATUS_HANDSHAKE_DIR: + *tag = "handshake_dir"; + *summary = "Finishing handshake with directory server"; + break; + case BOOTSTRAP_STATUS_ONEHOP_CREATE: + *tag = "onehop_create"; + *summary = "Establishing an encrypted directory connection"; + break; + case BOOTSTRAP_STATUS_REQUESTING_STATUS: + *tag = "requesting_status"; + *summary = "Asking for networkstatus consensus"; + break; + case BOOTSTRAP_STATUS_LOADING_STATUS: + *tag = "loading_status"; + *summary = "Loading networkstatus consensus"; + break; + case BOOTSTRAP_STATUS_LOADING_KEYS: + *tag = "loading_keys"; + *summary = "Loading authority key certs"; + break; + case BOOTSTRAP_STATUS_REQUESTING_DESCRIPTORS: + *tag = "requesting_descriptors"; + /* XXXX this appears to incorrectly report internal on most loads */ + *summary = router_have_consensus_path() == CONSENSUS_PATH_INTERNAL ? + "Asking for relay descriptors for internal paths" : + "Asking for relay descriptors"; + break; + /* If we're sure there are no exits in the consensus, + * inform the controller by adding "internal" + * to the status summaries. + * (We only check this while loading descriptors, + * so we may not know in the earlier stages.) + * But if there are exits, we can't be sure whether + * we're creating internal or exit paths/circuits. + * XXXX Or should be use different tags or statuses + * for internal and exit/all? */ + case BOOTSTRAP_STATUS_LOADING_DESCRIPTORS: + *tag = "loading_descriptors"; + *summary = router_have_consensus_path() == CONSENSUS_PATH_INTERNAL ? + "Loading relay descriptors for internal paths" : + "Loading relay descriptors"; + break; + case BOOTSTRAP_STATUS_CONN_OR: + *tag = "conn_or"; + *summary = router_have_consensus_path() == CONSENSUS_PATH_INTERNAL ? + "Connecting to the Tor network internally" : + "Connecting to the Tor network"; + break; + case BOOTSTRAP_STATUS_HANDSHAKE_OR: + *tag = "handshake_or"; + *summary = router_have_consensus_path() == CONSENSUS_PATH_INTERNAL ? + "Finishing handshake with first hop of internal circuit" : + "Finishing handshake with first hop"; + break; + case BOOTSTRAP_STATUS_CIRCUIT_CREATE: + *tag = "circuit_create"; + *summary = router_have_consensus_path() == CONSENSUS_PATH_INTERNAL ? + "Establishing an internal Tor circuit" : + "Establishing a Tor circuit"; + break; + case BOOTSTRAP_STATUS_DONE: + *tag = "done"; + *summary = "Done"; + break; + default: +// log_warn(LD_BUG, "Unrecognized bootstrap status code %d", s); + *tag = *summary = "unknown"; + return -1; + } + return 0; +} + +/** What percentage through the bootstrap process are we? We remember + * this so we can avoid sending redundant bootstrap status events, and + * so we can guess context for the bootstrap messages which are + * ambiguous. It starts at 'undef', but gets set to 'starting' while + * Tor initializes. */ +static int bootstrap_percent = BOOTSTRAP_STATUS_UNDEF; + +/** Like bootstrap_percent, but only takes on the enumerated values in + * bootstrap_status_t. + */ +static int bootstrap_phase = BOOTSTRAP_STATUS_UNDEF; + +/** As bootstrap_percent, but holds the bootstrapping level at which we last + * logged a NOTICE-level message. We use this, plus BOOTSTRAP_PCT_INCREMENT, + * to avoid flooding the log with a new message every time we get a few more + * microdescriptors */ +static int notice_bootstrap_percent = 0; + +/** How many problems have we had getting to the next bootstrapping phase? + * These include failure to establish a connection to a Tor relay, + * failures to finish the TLS handshake, failures to validate the + * consensus document, etc. */ +static int bootstrap_problems = 0; + +/** We only tell the controller once we've hit a threshold of problems + * for the current phase. */ +#define BOOTSTRAP_PROBLEM_THRESHOLD 10 + +/** When our bootstrapping progress level changes, but our bootstrapping + * status has not advanced, we only log at NOTICE when we have made at least + * this much progress. + */ +#define BOOTSTRAP_PCT_INCREMENT 5 + +/** Do the actual logging and notifications for + * control_event_bootstrap(). Doesn't change any state beyond that. + */ +static void +control_event_bootstrap_core(int loglevel, bootstrap_status_t status, + int progress) +{ + char buf[BOOTSTRAP_MSG_LEN]; + const char *tag, *summary; + + bootstrap_status_to_string(status, &tag, &summary); + /* Locally reset status if there's incremental progress */ + if (progress) + status = progress; + + tor_log(loglevel, LD_CONTROL, + "Bootstrapped %d%%: %s", status, summary); + tor_snprintf(buf, sizeof(buf), + "BOOTSTRAP PROGRESS=%d TAG=%s SUMMARY=\"%s\"", + status, tag, summary); + tor_snprintf(last_sent_bootstrap_message, + sizeof(last_sent_bootstrap_message), + "NOTICE %s", buf); + control_event_client_status(LOG_NOTICE, "%s", buf); +} + +/** Called when Tor has made progress at bootstrapping its directory + * information and initial circuits. + * + * status is the new status, that is, what task we will be doing + * next. progress is zero if we just started this task, else it + * represents progress on the task. + */ +void +control_event_bootstrap(bootstrap_status_t status, int progress) +{ + int loglevel = LOG_NOTICE; + + if (bootstrap_percent == BOOTSTRAP_STATUS_DONE) + return; /* already bootstrapped; nothing to be done here. */ + + /* special case for handshaking status, since our TLS handshaking code + * can't distinguish what the connection is going to be for. */ + if (status == BOOTSTRAP_STATUS_HANDSHAKE) { + if (bootstrap_percent < BOOTSTRAP_STATUS_CONN_OR) { + status = BOOTSTRAP_STATUS_HANDSHAKE_DIR; + } else { + status = BOOTSTRAP_STATUS_HANDSHAKE_OR; + } + } + + if (status <= bootstrap_percent) { + /* If there's no new progress, return early. */ + if (!progress || progress <= bootstrap_percent) + return; + /* Log at INFO if not enough progress happened. */ + if (progress < notice_bootstrap_percent + BOOTSTRAP_PCT_INCREMENT) + loglevel = LOG_INFO; + } + + control_event_bootstrap_core(loglevel, status, progress); + + if (status > bootstrap_percent) { + bootstrap_phase = status; /* new milestone reached */ + bootstrap_percent = status; + } + if (progress > bootstrap_percent) { + /* incremental progress within a milestone */ + bootstrap_percent = progress; + bootstrap_problems = 0; /* Progress! Reset our problem counter. */ + } + if (loglevel == LOG_NOTICE && + bootstrap_percent > notice_bootstrap_percent) { + /* Remember that we gave a notice at this level. */ + notice_bootstrap_percent = bootstrap_percent; + } +} + +/** Flag whether we've opened an OR_CONN yet */ +static int bootstrap_first_orconn = 0; + +/** Like bootstrap_phase, but for (possibly deferred) directory progress */ +static int bootstrap_dir_phase = BOOTSTRAP_STATUS_UNDEF; + +/** Like bootstrap_problems, but for (possibly deferred) directory progress */ +static int bootstrap_dir_progress = BOOTSTRAP_STATUS_UNDEF; + +/** Defer directory info bootstrap events until we have successfully + * completed our first connection to a router. */ +void +control_event_boot_dir(bootstrap_status_t status, int progress) +{ + if (status > bootstrap_dir_progress) { + bootstrap_dir_progress = status; + bootstrap_dir_phase = status; + } + if (progress && progress >= bootstrap_dir_progress) { + bootstrap_dir_progress = progress; + } + + /* Don't report unless we have successfully opened at least one OR_CONN */ + if (!bootstrap_first_orconn) + return; + + control_event_bootstrap(status, progress); +} + +/** Set a flag to allow reporting of directory bootstrap progress. + * (Code that reports completion of an OR_CONN calls this.) Also, + * report directory progress so far. */ +void +control_event_boot_first_orconn(void) +{ + bootstrap_first_orconn = 1; + control_event_bootstrap(bootstrap_dir_phase, bootstrap_dir_progress); +} + +/** Called when Tor has failed to make bootstrapping progress in a way + * that indicates a problem. warn gives a human-readable hint + * as to why, and reason provides a controller-facing short + * tag. conn is the connection that caused this problem and + * can be NULL if a connection cannot be easily identified. + */ +void +control_event_bootstrap_problem(const char *warn, const char *reason, + const connection_t *conn, int dowarn) +{ + int status = bootstrap_percent; + const char *tag = "", *summary = ""; + char buf[BOOTSTRAP_MSG_LEN]; + const char *recommendation = "ignore"; + int severity; + char *or_id = NULL, *hostaddr = NULL; + or_connection_t *or_conn = NULL; + + /* bootstrap_percent must not be in "undefined" state here. */ + tor_assert(status >= 0); + + if (bootstrap_percent == 100) + return; /* already bootstrapped; nothing to be done here. */ + + bootstrap_problems++; + + if (bootstrap_problems >= BOOTSTRAP_PROBLEM_THRESHOLD) + dowarn = 1; + + /* Don't warn about our bootstrapping status if we are hibernating or + * shutting down. */ + if (we_are_hibernating()) + dowarn = 0; + + tor_assert(bootstrap_status_to_string(bootstrap_phase, &tag, &summary) == 0); + + severity = dowarn ? LOG_WARN : LOG_INFO; + + if (dowarn) + recommendation = "warn"; + + if (conn && conn->type == CONN_TYPE_OR) { + /* XXX TO_OR_CONN can't deal with const */ + or_conn = TO_OR_CONN((connection_t *)conn); + or_id = tor_strdup(hex_str(or_conn->identity_digest, DIGEST_LEN)); + } else { + or_id = tor_strdup("?"); + } + + if (conn) + tor_asprintf(&hostaddr, "%s:%d", conn->address, (int)conn->port); + else + hostaddr = tor_strdup("?"); + + log_fn(severity, + LD_CONTROL, "Problem bootstrapping. Stuck at %d%%: %s. (%s; %s; " + "count %d; recommendation %s; host %s at %s)", + status, summary, warn, reason, + bootstrap_problems, recommendation, + or_id, hostaddr); + + connection_or_report_broken_states(severity, LD_HANDSHAKE); + + tor_snprintf(buf, sizeof(buf), + "BOOTSTRAP PROGRESS=%d TAG=%s SUMMARY=\"%s\" WARNING=\"%s\" REASON=%s " + "COUNT=%d RECOMMENDATION=%s HOSTID=\"%s\" HOSTADDR=\"%s\"", + bootstrap_percent, tag, summary, warn, reason, bootstrap_problems, + recommendation, + or_id, hostaddr); + + tor_snprintf(last_sent_bootstrap_message, + sizeof(last_sent_bootstrap_message), + "WARN %s", buf); + control_event_client_status(LOG_WARN, "%s", buf); + + tor_free(hostaddr); + tor_free(or_id); +} + +/** Called when Tor has failed to make bootstrapping progress in a way + * that indicates a problem. warn gives a hint as to why, and + * reason provides an "or_conn_end_reason" tag. or_conn + * is the connection that caused this problem. + */ +MOCK_IMPL(void, +control_event_bootstrap_prob_or, (const char *warn, int reason, + or_connection_t *or_conn)) +{ + int dowarn = 0; + + if (or_conn->have_noted_bootstrap_problem) + return; + + or_conn->have_noted_bootstrap_problem = 1; + + if (reason == END_OR_CONN_REASON_NO_ROUTE) + dowarn = 1; + + /* If we are using bridges and all our OR connections are now + closed, it means that we totally failed to connect to our + bridges. Throw a warning. */ + if (get_options()->UseBridges && !any_other_active_or_conns(or_conn)) + dowarn = 1; + + control_event_bootstrap_problem(warn, + orconn_end_reason_to_control_string(reason), + TO_CONN(or_conn), dowarn); +} + +/** Return a copy of the last sent bootstrap message. */ +char * +control_event_boot_last_msg(void) +{ + return tor_strdup(last_sent_bootstrap_message); +} + +/** Reset bootstrap tracking state. */ +void +control_event_bootstrap_reset(void) +{ + bootstrap_percent = BOOTSTRAP_STATUS_UNDEF; + bootstrap_phase = BOOTSTRAP_STATUS_UNDEF; + notice_bootstrap_percent = 0; + bootstrap_problems = 0; + bootstrap_first_orconn = 0; + bootstrap_dir_progress = BOOTSTRAP_STATUS_UNDEF; + bootstrap_dir_phase = BOOTSTRAP_STATUS_UNDEF; + memset(last_sent_bootstrap_message, 0, sizeof(last_sent_bootstrap_message)); +} From 7685f8ad35f8662e8af149ae4e530677646f88a0 Mon Sep 17 00:00:00 2001 From: Taylor Yu Date: Mon, 19 Nov 2018 19:43:23 -0600 Subject: [PATCH 0220/2557] Use table lookup for bootstrap_status_to_string It also no longer distinguishes the case of internal-only paths, which was often wrong anyway. Closes ticket 27402. --- changes/ticket27402 | 10 ++ src/feature/control/control_bootstrap.c | 129 ++++++++---------------- 2 files changed, 52 insertions(+), 87 deletions(-) create mode 100644 changes/ticket27402 diff --git a/changes/ticket27402 b/changes/ticket27402 new file mode 100644 index 0000000000..b79fb56760 --- /dev/null +++ b/changes/ticket27402 @@ -0,0 +1,10 @@ + o Minor feature (bootstrap): + - When reporting bootstrap progress, stop distinguishing between + situations where it seems that only internal paths are available + and situations where it seems that external paths are available. + Previously, tor would often erroneously report that it had only + internal paths. Closes ticket 27402. + + o Code simplification and refactoring: + - Split out bootstrap progress reporting from control.c into a + separate file. Part of ticket 27402. diff --git a/src/feature/control/control_bootstrap.c b/src/feature/control/control_bootstrap.c index 58b342c219..7299757f91 100644 --- a/src/feature/control/control_bootstrap.c +++ b/src/feature/control/control_bootstrap.c @@ -16,7 +16,6 @@ #include "core/or/reasons.h" #include "feature/control/control.h" #include "feature/hibernate/hibernate.h" -#include "feature/nodelist/nodelist.h" #include "lib/malloc/malloc.h" /** A sufficiently large size to record the last bootstrap phase string. */ @@ -26,99 +25,55 @@ * of this so we can respond to getinfo status/bootstrap-phase queries. */ static char last_sent_bootstrap_message[BOOTSTRAP_MSG_LEN]; +/** Table to convert bootstrap statuses to strings. */ +static const struct { + bootstrap_status_t status; + const char *tag; + const char *summary; +} boot_to_str_tab[] = { + { BOOTSTRAP_STATUS_UNDEF, "undef", "Undefined" }, + { BOOTSTRAP_STATUS_STARTING, "starting", "Starting" }, + { BOOTSTRAP_STATUS_CONN_DIR, "conn_dir", "Connecting to directory server" }, + { BOOTSTRAP_STATUS_HANDSHAKE, "status_handshake", "Finishing handshake" }, + { BOOTSTRAP_STATUS_HANDSHAKE_DIR, "handshake_dir", + "Finishing handshake with directory server" }, + { BOOTSTRAP_STATUS_ONEHOP_CREATE, "onehop_create", + "Establishing an encrypted directory connection" }, + { BOOTSTRAP_STATUS_REQUESTING_STATUS, "requesting_status", + "Asking for networkstatus consensus" }, + { BOOTSTRAP_STATUS_LOADING_STATUS, "loading_status", + "Loading networkstatus consensus" }, + { BOOTSTRAP_STATUS_LOADING_KEYS, "loading_keys", + "Loading authority key certs" }, + { BOOTSTRAP_STATUS_REQUESTING_DESCRIPTORS, "requesting_descriptors", + "Asking for relay descriptors" }, + { BOOTSTRAP_STATUS_LOADING_DESCRIPTORS, "loading_descriptors", + "Loading relay descriptors" }, + { BOOTSTRAP_STATUS_CONN_OR, "conn_or", "Connecting to the Tor network" }, + { BOOTSTRAP_STATUS_HANDSHAKE_OR, "handshake_or", + "Finishing handshake with first hop" }, + { BOOTSTRAP_STATUS_CIRCUIT_CREATE, "circuit_create", + "Establishing a Tor circuit" }, + { BOOTSTRAP_STATUS_DONE, "done", "Done" }, +}; +#define N_BOOT_TO_STR (sizeof(boot_to_str_tab)/sizeof(boot_to_str_tab[0])) + /** Convert the name of a bootstrapping phase s into strings * tag and summary suitable for display by the controller. */ static int bootstrap_status_to_string(bootstrap_status_t s, const char **tag, const char **summary) { - switch (s) { - case BOOTSTRAP_STATUS_UNDEF: - *tag = "undef"; - *summary = "Undefined"; - break; - case BOOTSTRAP_STATUS_STARTING: - *tag = "starting"; - *summary = "Starting"; - break; - case BOOTSTRAP_STATUS_CONN_DIR: - *tag = "conn_dir"; - *summary = "Connecting to directory server"; - break; - case BOOTSTRAP_STATUS_HANDSHAKE: - *tag = "status_handshake"; - *summary = "Finishing handshake"; - break; - case BOOTSTRAP_STATUS_HANDSHAKE_DIR: - *tag = "handshake_dir"; - *summary = "Finishing handshake with directory server"; - break; - case BOOTSTRAP_STATUS_ONEHOP_CREATE: - *tag = "onehop_create"; - *summary = "Establishing an encrypted directory connection"; - break; - case BOOTSTRAP_STATUS_REQUESTING_STATUS: - *tag = "requesting_status"; - *summary = "Asking for networkstatus consensus"; - break; - case BOOTSTRAP_STATUS_LOADING_STATUS: - *tag = "loading_status"; - *summary = "Loading networkstatus consensus"; - break; - case BOOTSTRAP_STATUS_LOADING_KEYS: - *tag = "loading_keys"; - *summary = "Loading authority key certs"; - break; - case BOOTSTRAP_STATUS_REQUESTING_DESCRIPTORS: - *tag = "requesting_descriptors"; - /* XXXX this appears to incorrectly report internal on most loads */ - *summary = router_have_consensus_path() == CONSENSUS_PATH_INTERNAL ? - "Asking for relay descriptors for internal paths" : - "Asking for relay descriptors"; - break; - /* If we're sure there are no exits in the consensus, - * inform the controller by adding "internal" - * to the status summaries. - * (We only check this while loading descriptors, - * so we may not know in the earlier stages.) - * But if there are exits, we can't be sure whether - * we're creating internal or exit paths/circuits. - * XXXX Or should be use different tags or statuses - * for internal and exit/all? */ - case BOOTSTRAP_STATUS_LOADING_DESCRIPTORS: - *tag = "loading_descriptors"; - *summary = router_have_consensus_path() == CONSENSUS_PATH_INTERNAL ? - "Loading relay descriptors for internal paths" : - "Loading relay descriptors"; - break; - case BOOTSTRAP_STATUS_CONN_OR: - *tag = "conn_or"; - *summary = router_have_consensus_path() == CONSENSUS_PATH_INTERNAL ? - "Connecting to the Tor network internally" : - "Connecting to the Tor network"; - break; - case BOOTSTRAP_STATUS_HANDSHAKE_OR: - *tag = "handshake_or"; - *summary = router_have_consensus_path() == CONSENSUS_PATH_INTERNAL ? - "Finishing handshake with first hop of internal circuit" : - "Finishing handshake with first hop"; - break; - case BOOTSTRAP_STATUS_CIRCUIT_CREATE: - *tag = "circuit_create"; - *summary = router_have_consensus_path() == CONSENSUS_PATH_INTERNAL ? - "Establishing an internal Tor circuit" : - "Establishing a Tor circuit"; - break; - case BOOTSTRAP_STATUS_DONE: - *tag = "done"; - *summary = "Done"; - break; - default: -// log_warn(LD_BUG, "Unrecognized bootstrap status code %d", s); - *tag = *summary = "unknown"; - return -1; + for (size_t i = 0; i < N_BOOT_TO_STR; i++) { + if (s == boot_to_str_tab[i].status) { + *tag = boot_to_str_tab[i].tag; + *summary = boot_to_str_tab[i].summary; + return 0; + } } - return 0; + + *tag = *summary = "unknown"; + return -1; } /** What percentage through the bootstrap process are we? We remember From 701eaef980de4f7dbb5c31c4fee9b7e1e266d7a1 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Sat, 1 Dec 2018 11:36:03 -0500 Subject: [PATCH 0221/2557] Move net.inet.ip.random_id code to lib/net/ --- src/app/config/config.c | 21 +-------------------- src/lib/net/socket.c | 29 +++++++++++++++++++++++++++++ src/lib/net/socket.h | 1 + 3 files changed, 31 insertions(+), 20 deletions(-) diff --git a/src/app/config/config.c b/src/app/config/config.c index d907b07136..50f3793d6c 100644 --- a/src/app/config/config.c +++ b/src/app/config/config.c @@ -157,10 +157,6 @@ #include "core/or/connection_st.h" #include "core/or/port_cfg_st.h" -#ifdef __FreeBSD__ -#include -#endif - #ifdef HAVE_SYSTEMD # if defined(__COVERITY__) && !defined(__INCLUDE_LEVEL__) /* Systemd's use of gcc's __INCLUDE_LEVEL__ extension macro appears to confuse @@ -3386,22 +3382,7 @@ options_validate(or_options_t *old_options, or_options_t *options, if (ContactInfo && !string_is_utf8(ContactInfo, strlen(ContactInfo))) REJECT("ContactInfo config option must be UTF-8."); -#ifdef __FreeBSD__ - if (server_mode(options)) { - int random_id_state; - size_t state_size = sizeof(random_id_state); - - if (sysctlbyname("net.inet.ip.random_id", &random_id_state, - &state_size, NULL, 0)) { - log_warn(LD_CONFIG, - "Failed to figure out if IP ids are randomized."); - } else if (random_id_state == 0) { - log_warn(LD_CONFIG, "Looks like IP ids are not randomized. " - "Please consider setting the net.inet.ip.random_id sysctl, " - "so your relay makes it harder to figure out how busy it is."); - } - } -#endif + check_network_configuration(server_mode(options)); /* Special case on first boot if no Log options are given. */ if (!options->Logs && !options->RunAsDaemon && !from_setconf) { diff --git a/src/lib/net/socket.c b/src/lib/net/socket.c index cd7c9685cd..8940e00591 100644 --- a/src/lib/net/socket.c +++ b/src/lib/net/socket.c @@ -31,6 +31,9 @@ #endif #include #include +#ifdef __FreeBSD__ +#include +#endif /** Called before we make any calls to network-related functions. * (Some operating systems require their network libraries to be @@ -60,6 +63,32 @@ network_init(void) return 0; } +/** + * Warn the user if any system network parameters should be changed. + */ +void +check_network_configuration(bool server_mode) +{ +#ifdef __FreeBSD__ + if (server_mode) { + int random_id_state; + size_t state_size = sizeof(random_id_state); + + if (sysctlbyname("net.inet.ip.random_id", &random_id_state, + &state_size, NULL, 0)) { + log_warn(LD_CONFIG, + "Failed to figure out if IP ids are randomized."); + } else if (random_id_state == 0) { + log_warn(LD_CONFIG, "Looks like IP ids are not randomized. " + "Please consider setting the net.inet.ip.random_id sysctl, " + "so your relay makes it harder to figure out how busy it is."); + } + } +#else + (void) server_mode; +#endif +} + /* When set_max_file_sockets() is called, update this with the max file * descriptor value so we can use it to check the limit when opening a new * socket. Default value is what Debian sets as the default hard limit. */ diff --git a/src/lib/net/socket.h b/src/lib/net/socket.h index 2b87441fc6..822b9975e6 100644 --- a/src/lib/net/socket.h +++ b/src/lib/net/socket.h @@ -54,6 +54,7 @@ int tor_addr_from_getsockname(struct tor_addr_t *addr_out, tor_socket_t sock); int set_socket_nonblocking(tor_socket_t socket); int tor_socketpair(int family, int type, int protocol, tor_socket_t fd[2]); int network_init(void); +void check_network_configuration(bool server_mode); int get_max_sockets(void); void set_max_sockets(int); From a2bb172225e1eecc9e93da655a0d316c4b214ff6 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Tue, 11 Sep 2018 19:38:30 +0300 Subject: [PATCH 0222/2557] tor-resolve: generate SOCKS4a request with trunnel --- src/tools/tor-resolve.c | 75 ++++++++++++++++++++++++++++++++++------- 1 file changed, 63 insertions(+), 12 deletions(-) diff --git a/src/tools/tor-resolve.c b/src/tools/tor-resolve.c index 95411d173d..4ba372b66c 100644 --- a/src/tools/tor-resolve.c +++ b/src/tools/tor-resolve.c @@ -15,6 +15,7 @@ #include "lib/string/util_string.h" #include "lib/net/socks5_status.h" +#include "socks5.h" #include #include @@ -49,11 +50,68 @@ static void usage(void) ATTR_NORETURN; +/** + * Set out to a pointer to newly allocated buffer containing + * SOCKS4a RESOLVE request with username and hostname. + * Return number of bytes in the buffer if succeeded or -1 if failed. + */ +static ssize_t +build_socks4a_resolve_request(uint8_t **out, + const char *username, + const char *hostname) +{ + tor_assert(out); + tor_assert(username); + tor_assert(hostname); + + const char *errmsg = NULL; + uint8_t *output = NULL; + socks4_client_request_t *rq = socks4_client_request_new(); + + socks4_client_request_set_version(rq, 4); + socks4_client_request_set_command(rq, CMD_RESOLVE); + socks4_client_request_set_port(rq, 0); + socks4_client_request_set_addr(rq, 0x00000001u); + socks4_client_request_set_username(rq, username); + socks4_client_request_set_socks4a_addr_hostname(rq, hostname); + + errmsg = socks4_client_request_check(rq); + if (errmsg) { + goto cleanup; + } + + ssize_t encoded_len = socks4_client_request_encoded_len(rq); + if (encoded_len <= 0) { + errmsg = "socks4_client_request_encoded_len failed"; + goto cleanup; + } + + output = tor_malloc(encoded_len); + memset(output, 0, encoded_len); + + encoded_len = socks4_client_request_encode(output, encoded_len, rq); + if (encoded_len <= 0) { + errmsg = "socks4_client_request_encode failed"; + goto cleanup; + } + + *out = output; + + cleanup: + socks4_client_request_free(rq); + if (errmsg) { + log_err(LD_NET, "build_socks4a_resolve_request failed: %s", errmsg); + *out = NULL; + tor_free(output); + } + return errmsg ? -1 : encoded_len; +} + /** Set *out to a newly allocated SOCKS4a resolve request with * username and hostname as provided. Return the number * of bytes in the request. */ static ssize_t -build_socks_resolve_request(char **out, +build_socks_resolve_request(uint8_t **out, const char *username, const char *hostname, int reverse, @@ -65,14 +123,7 @@ build_socks_resolve_request(char **out, tor_assert(hostname); if (version == 4) { - len = 8 + strlen(username) + 1 + strlen(hostname) + 1; - *out = tor_malloc(len); - (*out)[0] = 4; /* SOCKS version 4 */ - (*out)[1] = '\xF0'; /* Command: resolve. */ - set_uint16((*out)+2, htons(0)); /* port: 0. */ - set_uint32((*out)+4, htonl(0x00000001u)); /* addr: 0.0.0.1 */ - memcpy((*out)+8, username, strlen(username)+1); - memcpy((*out)+8+strlen(username)+1, hostname, strlen(hostname)+1); + return build_socks4a_resolve_request(out, username, hostname); } else if (version == 5) { int is_ip_address; tor_addr_t addr; @@ -205,7 +256,7 @@ do_resolve(const char *hostname, int s = -1; struct sockaddr_storage ss; socklen_t socklen; - char *req = NULL; + uint8_t *req = NULL; ssize_t len = 0; tor_assert(hostname); @@ -231,7 +282,7 @@ do_resolve(const char *hostname, if (version == 5) { char method_buf[2]; - if (write_all_to_socket(s, "\x05\x01\x00", 3) != 3) { + if (write_all_to_socket(s, (const char *)"\x05\x01\x00", 3) != 3) { log_err(LD_NET, "Error sending SOCKS5 method list."); goto err; } @@ -257,7 +308,7 @@ do_resolve(const char *hostname, tor_assert(!req); goto err; } - if (write_all_to_socket(s, req, len) != len) { + if (write_all_to_socket(s, (const char *)req, len) != len) { log_sock_error("sending SOCKS request", s); tor_free(req); goto err; From 30582b940e22897e68c50d4d0c9c36f82472e9b2 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Tue, 11 Sep 2018 19:39:00 +0300 Subject: [PATCH 0223/2557] tor-resolve: link tor-resolve binary with trunnel lib --- src/tools/include.am | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/tools/include.am b/src/tools/include.am index f7aa7e0d1e..352d4b5328 100644 --- a/src/tools/include.am +++ b/src/tools/include.am @@ -5,9 +5,11 @@ noinst_PROGRAMS+= src/tools/tor-cov-resolve endif src_tools_tor_resolve_SOURCES = src/tools/tor-resolve.c -src_tools_tor_resolve_LDFLAGS = +src_tools_tor_resolve_LDFLAGS = @TOR_LDFLAGS_openssl@ src_tools_tor_resolve_LDADD = \ + src/trunnel/libor-trunnel.a \ $(TOR_UTIL_LIBS) \ + $(TOR_CRYPTO_LIBS) $(TOR_LIBS_CRYPTLIB)\ $(rust_ldadd) \ @TOR_LIB_MATH@ @TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_USERENV@ From 83af6d6149bb589be01a14b1d800614c1beb9f18 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Tue, 11 Sep 2018 21:42:38 +0300 Subject: [PATCH 0224/2557] tor-resolve: Use trunnel code for SOCKS5 request generation --- src/tools/tor-resolve.c | 158 +++++++++++++++++++++++++++++++--------- 1 file changed, 122 insertions(+), 36 deletions(-) diff --git a/src/tools/tor-resolve.c b/src/tools/tor-resolve.c index 4ba372b66c..c255a71123 100644 --- a/src/tools/tor-resolve.c +++ b/src/tools/tor-resolve.c @@ -107,6 +107,123 @@ build_socks4a_resolve_request(uint8_t **out, return errmsg ? -1 : encoded_len; } +#define SOCKS5_ATYPE_HOSTNAME 0x03 +#define SOCKS5_ATYPE_IPV4 0x01 +#define SOCKS5_ATYPE_IPV6 0x04 + +/** + * Set out to pointer to newly allocated buffer containing + * SOCKS5 RESOLVE/RESOLVE_PTR request with given hostname. + * Generate a reverse request if reverse is true. + * Return the number of bytes in the buffer if succeeded or -1 if failed. + */ +static ssize_t +build_socks5_resolve_request(uint8_t **out, + const char *hostname, + int reverse) +{ + const char *errmsg = NULL; + uint8_t *outbuf = NULL; + int is_ip_address; + tor_addr_t addr; + size_t addrlen; + int ipv6; + is_ip_address = tor_addr_parse(&addr, hostname) != -1; + if (!is_ip_address && reverse) { + log_err(LD_GENERAL, "Tried to do a reverse lookup on a non-IP!"); + return -1; + } + ipv6 = reverse && tor_addr_family(&addr) == AF_INET6; + addrlen = reverse ? (ipv6 ? 16 : 4) : 1 + strlen(hostname); + if (addrlen > UINT8_MAX) { + log_err(LD_GENERAL, "Hostname is too long!"); + return -1; + } + + socks5_client_request_t *rq = socks5_client_request_new(); + + socks5_client_request_set_version(rq, 5); + /* RESOLVE_PTR or RESOLVE */ + socks5_client_request_set_command(rq, reverse ? CMD_RESOLVE_PTR : + CMD_RESOLVE); + socks5_client_request_set_reserved(rq, 0); + + uint8_t atype = SOCKS5_ATYPE_HOSTNAME; + if (reverse) + atype = ipv6 ? SOCKS5_ATYPE_IPV6 : SOCKS5_ATYPE_IPV4; + + socks5_client_request_set_atype(rq, atype); + + switch (atype) { + case SOCKS5_ATYPE_IPV4: { + socks5_client_request_set_dest_addr_ipv4(rq, + tor_addr_to_ipv4h(&addr)); + } break; + case SOCKS5_ATYPE_IPV6: { + uint8_t *ipv6_array = + socks5_client_request_getarray_dest_addr_ipv6(rq); + + tor_assert(ipv6_array); + + memcpy(ipv6_array, tor_addr_to_in6_addr8(&addr), 16); + } break; + + case SOCKS5_ATYPE_HOSTNAME: { + domainname_t *dn = domainname_new(); + domainname_set_len(dn, addrlen - 1); + domainname_setlen_name(dn, addrlen - 1); + char *dn_buf = domainname_getarray_name(dn); + memcpy(dn_buf, hostname, addrlen - 1); + + errmsg = domainname_check(dn); + + if (errmsg) { + domainname_free(dn); + goto cleanup; + } else { + socks5_client_request_set_dest_addr_domainname(rq, dn); + } + } break; + default: + tor_assert_unreached(); + break; + } + + socks5_client_request_set_dest_port(rq, 0); + + errmsg = socks5_client_request_check(rq); + if (errmsg) { + goto cleanup; + } + + ssize_t encoded_len = socks5_client_request_encoded_len(rq); + if (encoded_len < 0) { + errmsg = "Cannot predict encoded length"; + goto cleanup; + } + + outbuf = tor_malloc(encoded_len); + memset(outbuf, 0, encoded_len); + + encoded_len = socks5_client_request_encode(outbuf, encoded_len, rq); + if (encoded_len < 0) { + errmsg = "encoding failed"; + goto cleanup; + } + + *out = outbuf; + + cleanup: + socks5_client_request_free(rq); + if (errmsg) { + tor_free(outbuf); + log_err(LD_NET, "build_socks5_resolve_request failed with error: %s", + errmsg); + } + + return errmsg ? -1 : encoded_len; +} + /** Set *out to a newly allocated SOCKS4a resolve request with * username and hostname as provided. Return the number * of bytes in the request. */ @@ -117,51 +234,20 @@ build_socks_resolve_request(uint8_t **out, int reverse, int version) { - size_t len = 0; tor_assert(out); tor_assert(username); tor_assert(hostname); + tor_assert(version == 4 || version == 5); + if (version == 4) { return build_socks4a_resolve_request(out, username, hostname); } else if (version == 5) { - int is_ip_address; - tor_addr_t addr; - size_t addrlen; - int ipv6; - is_ip_address = tor_addr_parse(&addr, hostname) != -1; - if (!is_ip_address && reverse) { - log_err(LD_GENERAL, "Tried to do a reverse lookup on a non-IP!"); - return -1; - } - ipv6 = reverse && tor_addr_family(&addr) == AF_INET6; - addrlen = reverse ? (ipv6 ? 16 : 4) : 1 + strlen(hostname); - if (addrlen > UINT8_MAX) { - log_err(LD_GENERAL, "Hostname is too long!"); - return -1; - } - len = 6 + addrlen; - *out = tor_malloc(len); - (*out)[0] = 5; /* SOCKS version 5 */ - (*out)[1] = reverse ? '\xF1' : '\xF0'; /* RESOLVE_PTR or RESOLVE */ - (*out)[2] = 0; /* reserved. */ - if (reverse) { - (*out)[3] = ipv6 ? 4 : 1; - if (ipv6) - memcpy((*out)+4, tor_addr_to_in6_addr8(&addr), 16); - else - set_uint32((*out)+4, tor_addr_to_ipv4n(&addr)); - } else { - (*out)[3] = 3; - (*out)[4] = (char)(uint8_t)(addrlen - 1); - memcpy((*out)+5, hostname, addrlen - 1); - } - set_uint16((*out)+4+addrlen, 0); /* port */ - } else { - tor_assert(0); + return build_socks5_resolve_request(out, hostname, reverse); } - return len; + tor_assert_unreached(); + return -1; } static void From d49baa77b5586ecb473a822deb60dbe70c6769c8 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Wed, 12 Sep 2018 11:18:43 +0300 Subject: [PATCH 0225/2557] Allow socks4_server_reply version to be 0 (for tor-resolve) --- src/trunnel/socks5.c | 13 ++++++------- src/trunnel/socks5.trunnel | 2 +- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/trunnel/socks5.c b/src/trunnel/socks5.c index 9e5f6fcfed..057a52b042 100644 --- a/src/trunnel/socks5.c +++ b/src/trunnel/socks5.c @@ -1694,7 +1694,6 @@ socks4_server_reply_new(void) socks4_server_reply_t *val = trunnel_calloc(1, sizeof(socks4_server_reply_t)); if (NULL == val) return NULL; - val->version = 4; return val; } @@ -1724,7 +1723,7 @@ socks4_server_reply_get_version(const socks4_server_reply_t *inp) int socks4_server_reply_set_version(socks4_server_reply_t *inp, uint8_t val) { - if (! ((val == 4))) { + if (! ((val == 0 || val == 4))) { TRUNNEL_SET_ERROR_CODE(inp); return -1; } @@ -1771,7 +1770,7 @@ socks4_server_reply_check(const socks4_server_reply_t *obj) return "Object was NULL"; if (obj->trunnel_error_code_) return "A set function failed on this object"; - if (! (obj->version == 4)) + if (! (obj->version == 0 || obj->version == 4)) return "Integer out of bounds"; return NULL; } @@ -1785,7 +1784,7 @@ socks4_server_reply_encoded_len(const socks4_server_reply_t *obj) return -1; - /* Length of u8 version IN [4] */ + /* Length of u8 version IN [0, 4] */ result += 1; /* Length of u8 status */ @@ -1823,7 +1822,7 @@ socks4_server_reply_encode(uint8_t *output, const size_t avail, const socks4_ser trunnel_assert(encoded_len >= 0); #endif - /* Encode u8 version IN [4] */ + /* Encode u8 version IN [0, 4] */ trunnel_assert(written <= avail); if (avail - written < 1) goto truncated; @@ -1886,11 +1885,11 @@ socks4_server_reply_parse_into(socks4_server_reply_t *obj, const uint8_t *input, ssize_t result = 0; (void)result; - /* Parse u8 version IN [4] */ + /* Parse u8 version IN [0, 4] */ CHECK_REMAINING(1, truncated); obj->version = (trunnel_get_uint8(ptr)); remaining -= 1; ptr += 1; - if (! (obj->version == 4)) + if (! (obj->version == 0 || obj->version == 4)) goto fail; /* Parse u8 status */ diff --git a/src/trunnel/socks5.trunnel b/src/trunnel/socks5.trunnel index b86ec03b9d..17fa2ed996 100644 --- a/src/trunnel/socks5.trunnel +++ b/src/trunnel/socks5.trunnel @@ -86,7 +86,7 @@ struct socks4_client_request { } struct socks4_server_reply { - u8 version IN [4]; + u8 version IN [0,4]; u8 status; u16 port; u32 addr; From 1051969a1d7f642c9062afefd760577d0eda35f5 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Wed, 12 Sep 2018 11:30:01 +0300 Subject: [PATCH 0226/2557] tor-resolve: parse SOCKS4a reply --- src/tools/tor-resolve.c | 39 ++++++++++++++++++++++++++++----------- 1 file changed, 28 insertions(+), 11 deletions(-) diff --git a/src/tools/tor-resolve.c b/src/tools/tor-resolve.c index c255a71123..659d511793 100644 --- a/src/tools/tor-resolve.c +++ b/src/tools/tor-resolve.c @@ -271,34 +271,50 @@ parse_socks4a_resolve_response(const char *hostname, const char *response, size_t len, tor_addr_t *addr_out) { + int result = 0; uint8_t status; tor_assert(response); tor_assert(addr_out); - if (len < RESPONSE_LEN_4) { + socks4_server_reply_t *reply; + + ssize_t parsed = socks4_server_reply_parse(&reply, + (const uint8_t *)response, + len); + + if (parsed == -1) { + log_warn(LD_PROTOCOL, "Failed parsing SOCKS4a response"); + result = -1; goto cleanup; + } + + if (parsed == -2) { log_warn(LD_PROTOCOL,"Truncated socks response."); - return -1; + result = -1; goto cleanup; } - if (((uint8_t)response[0])!=0) { /* version: 0 */ + + if (socks4_server_reply_get_version(reply) != 0) { /* version: 0 */ log_warn(LD_PROTOCOL,"Nonzero version in socks response: bad format."); - return -1; + result = -1; goto cleanup; } - status = (uint8_t)response[1]; - if (get_uint16(response+2)!=0) { /* port: 0 */ + if (socks4_server_reply_get_port(reply) != 0) { /* port: 0 */ log_warn(LD_PROTOCOL,"Nonzero port in socks response: bad format."); - return -1; + result = -1; goto cleanup; } + status = socks4_server_reply_get_status(reply); if (status != 90) { log_warn(LD_NET,"Got status response '%d': socks request failed.", status); if (!strcasecmpend(hostname, ".onion")) { onion_warning(hostname); - return -1; + result = -1; goto cleanup; } - return -1; + result = -1; goto cleanup; } - tor_addr_from_ipv4n(addr_out, get_uint32(response+4)); - return 0; + tor_addr_from_ipv4h(addr_out, socks4_server_reply_get_addr(reply)); + + cleanup: + socks4_server_reply_free(reply); + return result; } /* It would be nice to let someone know what SOCKS5 issue a user may have */ @@ -418,6 +434,7 @@ do_resolve(const char *hostname, log_err(LD_NET, "Error reading SOCKS5 response."); goto err; } + if (reply_buf[0] != 5) { log_err(LD_NET, "Bad SOCKS5 reply version."); goto err; From 8b9d6581f6d00ff1c5739174e6f2ea47b7a393a7 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Wed, 12 Sep 2018 13:12:51 +0300 Subject: [PATCH 0227/2557] tor-resolve: Rework SOCKS5 method negotiation client part with trunnel --- src/tools/tor-resolve.c | 50 ++++++++++++++++++++++++++++++++++------- 1 file changed, 42 insertions(+), 8 deletions(-) diff --git a/src/tools/tor-resolve.c b/src/tools/tor-resolve.c index 659d511793..7aa3871b6d 100644 --- a/src/tools/tor-resolve.c +++ b/src/tools/tor-resolve.c @@ -383,23 +383,57 @@ do_resolve(const char *hostname, } if (version == 5) { - char method_buf[2]; - if (write_all_to_socket(s, (const char *)"\x05\x01\x00", 3) != 3) { + socks5_client_version_t *v = socks5_client_version_new(); + + socks5_client_version_set_version(v, 5); + socks5_client_version_set_n_methods(v, 1); + socks5_client_version_setlen_methods(v, 1); + socks5_client_version_set_methods(v, 0, 0x00); + + tor_assert(!socks5_client_version_check(v)); + ssize_t encoded_len = socks5_client_version_encoded_len(v); + tor_assert(encoded_len > 0); + + uint8_t *buf = tor_malloc(encoded_len); + encoded_len = socks5_client_version_encode(buf, encoded_len, v); + tor_assert(encoded_len > 0); + + socks5_client_version_free(v); + + if (write_all_to_socket(s, (const char *)buf, + encoded_len) != encoded_len) { log_err(LD_NET, "Error sending SOCKS5 method list."); + tor_free(buf); + goto err; } - if (read_all_from_socket(s, method_buf, 2) != 2) { + + tor_free(buf); + + uint8_t method_buf[2]; + + if (read_all_from_socket(s, (char *)method_buf, 2) != 2) { log_err(LD_NET, "Error reading SOCKS5 methods."); goto err; } - if (method_buf[0] != '\x05') { - log_err(LD_NET, "Unrecognized socks version: %u", - (unsigned)method_buf[0]); + + socks5_server_method_t *m; + ssize_t parsed = socks5_server_method_parse(&m, method_buf, + sizeof(method_buf)); + + if (parsed < 2) { + log_err(LD_NET, "Failed to parse SOCKS5 method selection " + "message"); goto err; } - if (method_buf[1] != '\x00') { + + uint8_t method = socks5_server_method_get_method(m); + + socks5_server_method_free(m); + + if (method != 0x00) { log_err(LD_NET, "Unrecognized socks authentication method: %u", - (unsigned)method_buf[1]); + (unsigned)method); goto err; } } From 39e158db365fe3c033c3f8f99a2de1bfed3b707a Mon Sep 17 00:00:00 2001 From: rl1987 Date: Wed, 12 Sep 2018 15:03:08 +0300 Subject: [PATCH 0228/2557] tor-resolve: Rework SOCKS5 response parsing with trunnel --- src/tools/tor-resolve.c | 78 +++++++++++++++++++++++------------------ 1 file changed, 44 insertions(+), 34 deletions(-) diff --git a/src/tools/tor-resolve.c b/src/tools/tor-resolve.c index 7aa3871b6d..ba83f3e3b4 100644 --- a/src/tools/tor-resolve.c +++ b/src/tools/tor-resolve.c @@ -15,7 +15,7 @@ #include "lib/string/util_string.h" #include "lib/net/socks5_status.h" -#include "socks5.h" +#include "trunnel/socks5.h" #include #include @@ -463,56 +463,66 @@ do_resolve(const char *hostname, goto err; } } else { - char reply_buf[16]; - if (read_all_from_socket(s, reply_buf, 4) != 4) { - log_err(LD_NET, "Error reading SOCKS5 response."); + uint8_t reply_buf[512]; + + len = read_all_from_socket(s, (char *)reply_buf, + sizeof(reply_buf)); + + socks5_server_reply_t *reply; + + ssize_t parsed = socks5_server_reply_parse(&reply, + reply_buf, + len); + if (parsed == -1) { + log_err(LD_NET, "Failed parsing SOCKS5 response"); goto err; } - if (reply_buf[0] != 5) { - log_err(LD_NET, "Bad SOCKS5 reply version."); + if (parsed == -2) { + log_err(LD_NET, "Truncated SOCKS5 response"); goto err; } + /* Give a user some useful feedback about SOCKS5 errors */ - if (reply_buf[1] != 0) { + uint8_t reply_field = socks5_server_reply_get_reply(reply); + if (reply_field != 0) { log_warn(LD_NET,"Got SOCKS5 status response '%u': %s", - (unsigned)reply_buf[1], - socks5_reason_to_string(reply_buf[1])); - if (reply_buf[1] == 4 && !strcasecmpend(hostname, ".onion")) { + (unsigned)reply_field, + socks5_reason_to_string(reply_field)); + if (reply_field == 4 && !strcasecmpend(hostname, ".onion")) { onion_warning(hostname); } + + socks5_server_reply_free(reply); goto err; } - if (reply_buf[3] == 1) { + + uint8_t atype = socks5_server_reply_get_atype(reply); + + if (atype == SOCKS5_ATYPE_IPV4) { /* IPv4 address */ - if (read_all_from_socket(s, reply_buf, 4) != 4) { - log_err(LD_NET, "Error reading address in socks5 response."); - goto err; - } - tor_addr_from_ipv4n(result_addr, get_uint32(reply_buf)); - } else if (reply_buf[3] == 4) { + tor_addr_from_ipv4h(result_addr, + socks5_server_reply_get_bind_addr_ipv4(reply)); + } else if (atype == SOCKS5_ATYPE_IPV6) { /* IPv6 address */ - if (read_all_from_socket(s, reply_buf, 16) != 16) { - log_err(LD_NET, "Error reading address in socks5 response."); - goto err; - } - tor_addr_from_ipv6_bytes(result_addr, reply_buf); - } else if (reply_buf[3] == 3) { + tor_addr_from_ipv6_bytes(result_addr, + (const char *)socks5_server_reply_getarray_bind_addr_ipv6(reply)); + } else if (atype == SOCKS5_ATYPE_HOSTNAME) { /* Domain name */ - size_t result_len; - if (read_all_from_socket(s, reply_buf, 1) != 1) { - log_err(LD_NET, "Error reading address_length in socks5 response."); - goto err; - } - result_len = *(uint8_t*)(reply_buf); + domainname_t *dn = + socks5_server_reply_get_bind_addr_domainname(reply); + + size_t result_len = (size_t)domainname_get_len(dn); + *result_hostname = tor_malloc(result_len+1); - if (read_all_from_socket(s, *result_hostname, result_len) - != (int) result_len) { - log_err(LD_NET, "Error reading hostname in socks5 response."); - goto err; - } + + strncpy(*result_hostname, (char *)domainname_getstr_name(dn), + result_len); + (*result_hostname)[result_len] = '\0'; } + + socks5_server_reply_free(reply); } tor_close_socket(s); From 353d2a091d245146576d33a703fe3674c29e0654 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Thu, 13 Sep 2018 09:57:43 +0300 Subject: [PATCH 0229/2557] Fix coverage build --- src/tools/include.am | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/tools/include.am b/src/tools/include.am index 352d4b5328..72dfe6017c 100644 --- a/src/tools/include.am +++ b/src/tools/include.am @@ -17,8 +17,11 @@ if COVERAGE_ENABLED src_tools_tor_cov_resolve_SOURCES = src/tools/tor-resolve.c src_tools_tor_cov_resolve_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_tools_tor_cov_resolve_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) +src_tools_tor_cov_resolve_LDFLAGS = @TOR_LDFLAGS_openssl@ src_tools_tor_cov_resolve_LDADD = \ + src/trunnel/libor-trunnel.a \ $(TOR_UTIL_TESTING_LIBS) \ + $(TOR_CRYPTO_TESTING_LIBS) $(TOR_LIBS_CRYPTLIB) \ @TOR_LIB_MATH@ @TOR_LIB_WS32@ endif From 009205dabe4ba7aba43dffead4087ca94ef9a8e3 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Thu, 13 Sep 2018 10:18:06 +0300 Subject: [PATCH 0230/2557] Add changes file --- changes/ticket27620 | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 changes/ticket27620 diff --git a/changes/ticket27620 b/changes/ticket27620 new file mode 100644 index 0000000000..6c491696d0 --- /dev/null +++ b/changes/ticket27620 @@ -0,0 +1,3 @@ + o Code simplification and refactoring: + - Rework SOCKS wire format handling to rely on trunnel-generated + parsing/generation code. Resolves ticket 27620. From 822cb93cab59e9735e2efda70bc88c47cc92c498 Mon Sep 17 00:00:00 2001 From: Neel Chauhan Date: Wed, 26 Sep 2018 19:14:33 -0400 Subject: [PATCH 0231/2557] Add new option ClientAutoIPv6ORPort to switch between IPv4 and IPv6 OR ports --- changes/ticket27490 | 6 ++++++ doc/tor.1.txt | 6 ++++++ src/app/config/config.c | 1 + src/app/config/or_options_st.h | 3 +++ src/core/mainloop/connection.c | 5 +++++ src/core/or/policies.c | 15 ++++++++++++++- src/feature/client/bridges.c | 3 ++- 7 files changed, 37 insertions(+), 2 deletions(-) create mode 100644 changes/ticket27490 diff --git a/changes/ticket27490 b/changes/ticket27490 new file mode 100644 index 0000000000..523477dfea --- /dev/null +++ b/changes/ticket27490 @@ -0,0 +1,6 @@ + o Minor features (ipv6): + - We add an option ClientAutoIPv6ORPort which makes clients randomly + prefer a node's IPv4 or IPv6 ORPort. The random preference is set + every time a node is loaded from a new consensus or bridge config. + Closes ticket 27490. Patch by Neel Chauhan. + diff --git a/doc/tor.1.txt b/doc/tor.1.txt index 406372433f..bd4dbbcbdb 100644 --- a/doc/tor.1.txt +++ b/doc/tor.1.txt @@ -1748,6 +1748,12 @@ The following options are useful only for clients (that is, if other clients prefer IPv4. Other things may influence the choice. This option breaks a tie to the favor of IPv6. (Default: auto) +[[ClientAutoIPv6ORPort]] **ClientAutoIPv6ORPort** **0**|**1**:: + If this option is set to 1, Tor clients randomly prefer a node's IPv4 or + IPv6 ORPort. The random preference is set every time a node is loaded + from a new consensus or bridge config. When this option is set to 1, + **ClientPreferIPv6ORPort** is ignored. (Default: 0) + [[PathsNeededToBuildCircuits]] **PathsNeededToBuildCircuits** __NUM__:: Tor clients don't build circuits for user traffic until they know about enough of the network so that they could potentially construct diff --git a/src/app/config/config.c b/src/app/config/config.c index 01b48e3c5f..6a510c56da 100644 --- a/src/app/config/config.c +++ b/src/app/config/config.c @@ -332,6 +332,7 @@ static config_var_t option_vars_[] = { V(ClientOnly, BOOL, "0"), V(ClientPreferIPv6ORPort, AUTOBOOL, "auto"), V(ClientPreferIPv6DirPort, AUTOBOOL, "auto"), + V(ClientAutoIPv6ORPort, BOOL, "0"), V(ClientRejectInternalAddresses, BOOL, "1"), V(ClientTransportPlugin, LINELIST, NULL), V(ClientUseIPv6, BOOL, "0"), diff --git a/src/app/config/or_options_st.h b/src/app/config/or_options_st.h index 3524b99b53..ff3d30d7ec 100644 --- a/src/app/config/or_options_st.h +++ b/src/app/config/or_options_st.h @@ -666,6 +666,9 @@ struct or_options_t { * accessing this value directly. */ int ClientPreferIPv6DirPort; + /** If true, prefer an IPv4 or IPv6 OR port at random. */ + int ClientAutoIPv6ORPort; + /** The length of time that we think a consensus should be fresh. */ int V3AuthVotingInterval; /** The length of time we think it will take to distribute votes. */ diff --git a/src/core/mainloop/connection.c b/src/core/mainloop/connection.c index 4231bec014..9f8169082c 100644 --- a/src/core/mainloop/connection.c +++ b/src/core/mainloop/connection.c @@ -2069,6 +2069,11 @@ connection_connect_log_client_use_ip_version(const connection_t *conn) return; } + if (fascist_firewall_use_ipv6(options)) { + log_info(LD_NET, "Our outgoing connection is using IPv%d.", + tor_addr_family(&real_addr) == AF_INET6 ? 6 : 4); + } + /* Check if we couldn't satisfy an address family preference */ if ((!pref_ipv6 && tor_addr_family(&real_addr) == AF_INET6) || (pref_ipv6 && tor_addr_family(&real_addr) == AF_INET)) { diff --git a/src/core/or/policies.c b/src/core/or/policies.c index 3443a17107..e51a49cf60 100644 --- a/src/core/or/policies.c +++ b/src/core/or/policies.c @@ -28,6 +28,7 @@ #include "feature/nodelist/routerparse.h" #include "feature/stats/geoip.h" #include "ht.h" +#include "lib/crypt_ops/crypto_rand.h" #include "lib/encoding/confline.h" #include "core/or/addr_policy_st.h" @@ -487,6 +488,15 @@ fascist_firewall_prefer_ipv6_impl(const or_options_t *options) return -1; } +/* Choose whether we prefer IPv4 or IPv6 by randomly choosing an address + * family. Return 0 for IPv4, and 1 for IPv6. */ +static int +fascist_firewall_rand_prefer_ipv6_addr(void) +{ + /* TODO: Check for failures, and infer our preference based on this. */ + return crypto_rand_int(2); +} + /** Do we prefer to connect to IPv6 ORPorts? * Use node_ipv6_or_preferred() whenever possible: it supports bridge client * per-node IPv6 preferences. @@ -501,7 +511,10 @@ fascist_firewall_prefer_ipv6_orport(const or_options_t *options) } /* We can use both IPv4 and IPv6 - which do we prefer? */ - if (options->ClientPreferIPv6ORPort == 1) { + if (options->ClientAutoIPv6ORPort == 1) { + /* If ClientAutoIPv6ORPort is 1, we prefer IPv4 or IPv6 at random. */ + return fascist_firewall_rand_prefer_ipv6_addr(); + } else if (options->ClientPreferIPv6ORPort == 1) { return 1; } diff --git a/src/feature/client/bridges.c b/src/feature/client/bridges.c index e8afb5a924..e3ed288415 100644 --- a/src/feature/client/bridges.c +++ b/src/feature/client/bridges.c @@ -844,7 +844,8 @@ rewrite_node_address_for_bridge(const bridge_info_t *bridge, node_t *node) } } - if (options->ClientPreferIPv6ORPort == -1) { + if (options->ClientPreferIPv6ORPort == -1 || + options->ClientAutoIPv6ORPort == 0) { /* Mark which address to use based on which bridge_t we got. */ node->ipv6_preferred = (tor_addr_family(&bridge->addr) == AF_INET6 && !tor_addr_is_null(&node->ri->ipv6_addr)); From 81f2828d67f0853c028617511f3846c02f0d6628 Mon Sep 17 00:00:00 2001 From: Neel Chauhan Date: Wed, 26 Sep 2018 19:19:11 -0400 Subject: [PATCH 0232/2557] In fascist_firewall_use_ipv6(), say we can use IPv6 if ClientAutoIPv6ORPort is 1 --- src/core/or/policies.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/core/or/policies.c b/src/core/or/policies.c index e51a49cf60..6da369bf36 100644 --- a/src/core/or/policies.c +++ b/src/core/or/policies.c @@ -461,7 +461,8 @@ fascist_firewall_use_ipv6(const or_options_t *options) * ClientPreferIPv6DirPort is deprecated, but check it anyway. */ return (options->ClientUseIPv6 == 1 || options->ClientUseIPv4 == 0 || options->ClientPreferIPv6ORPort == 1 || - options->ClientPreferIPv6DirPort == 1 || options->UseBridges == 1); + options->ClientPreferIPv6DirPort == 1 || options->UseBridges == 1 || + options->ClientAutoIPv6ORPort == 1); } /** Do we prefer to connect to IPv6, ignoring ClientPreferIPv6ORPort and From ad031b64ce355ac3af15adb716a4322858dce4a8 Mon Sep 17 00:00:00 2001 From: Neel Chauhan Date: Tue, 16 Oct 2018 20:14:27 -0400 Subject: [PATCH 0233/2557] Add regression test for ClientAutoIPv6ORPort --- src/core/or/policies.c | 4 ++-- src/core/or/policies.h | 1 + src/test/test_policy.c | 50 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 53 insertions(+), 2 deletions(-) diff --git a/src/core/or/policies.c b/src/core/or/policies.c index 6da369bf36..c630c2fbdc 100644 --- a/src/core/or/policies.c +++ b/src/core/or/policies.c @@ -491,8 +491,8 @@ fascist_firewall_prefer_ipv6_impl(const or_options_t *options) /* Choose whether we prefer IPv4 or IPv6 by randomly choosing an address * family. Return 0 for IPv4, and 1 for IPv6. */ -static int -fascist_firewall_rand_prefer_ipv6_addr(void) +MOCK_IMPL(int, +fascist_firewall_rand_prefer_ipv6_addr, (void)) { /* TODO: Check for failures, and infer our preference based on this. */ return crypto_rand_int(2); diff --git a/src/core/or/policies.h b/src/core/or/policies.h index 7da3ba031f..0c64ecf378 100644 --- a/src/core/or/policies.h +++ b/src/core/or/policies.h @@ -70,6 +70,7 @@ typedef struct short_policy_t { int firewall_is_fascist_or(void); int firewall_is_fascist_dir(void); int fascist_firewall_use_ipv6(const or_options_t *options); +MOCK_DECL(int, fascist_firewall_rand_prefer_ipv6_addr, (void)); int fascist_firewall_prefer_ipv6_orport(const or_options_t *options); int fascist_firewall_prefer_ipv6_dirport(const or_options_t *options); diff --git a/src/test/test_policy.c b/src/test/test_policy.c index 6a07e5b1f8..cc9023d6a6 100644 --- a/src/test/test_policy.c +++ b/src/test/test_policy.c @@ -2023,6 +2023,20 @@ test_policies_fascist_firewall_allows_address(void *arg) expect_ap); \ STMT_END +/** Mock the preferred address function to return zero (prefer IPv4). */ +static int +mock_fascist_firewall_rand_prefer_ipv6_addr_use_ipv4(void) +{ + return 0; +} + +/** Mock the preferred address function to return one (prefer IPv6). */ +static int +mock_fascist_firewall_rand_prefer_ipv6_addr_use_ipv6(void) +{ + return 1; +} + /** Run unit tests for fascist_firewall_choose_address */ static void test_policies_fascist_firewall_choose_address(void *arg) @@ -2421,6 +2435,42 @@ test_policies_fascist_firewall_choose_address(void *arg) CHECK_CHOSEN_ADDR_RN(fake_rs, fake_node, FIREWALL_DIR_CONNECTION, 1, 1, ipv4_dir_ap); + /* Test ClientAutoIPv6ORPort and pretend we prefer IPv4. */ + memset(&mock_options, 0, sizeof(or_options_t)); + mock_options.ClientAutoIPv6ORPort = 1; + mock_options.ClientUseIPv4 = 1; + mock_options.ClientUseIPv6 = 1; + MOCK(fascist_firewall_rand_prefer_ipv6_addr, + mock_fascist_firewall_rand_prefer_ipv6_addr_use_ipv4); + /* Simulate the initialisation of fake_node.ipv6_preferred */ + fake_node.ipv6_preferred = fascist_firewall_prefer_ipv6_orport( + &mock_options); + + CHECK_CHOSEN_ADDR_RN(fake_rs, fake_node, FIREWALL_OR_CONNECTION, 0, 1, + ipv4_or_ap); + CHECK_CHOSEN_ADDR_RN(fake_rs, fake_node, FIREWALL_OR_CONNECTION, 1, 1, + ipv4_or_ap); + + UNMOCK(fascist_firewall_rand_prefer_ipv6_addr); + + /* Test ClientAutoIPv6ORPort and pretend we prefer IPv6. */ + memset(&mock_options, 0, sizeof(or_options_t)); + mock_options.ClientAutoIPv6ORPort = 1; + mock_options.ClientUseIPv4 = 1; + mock_options.ClientUseIPv6 = 1; + MOCK(fascist_firewall_rand_prefer_ipv6_addr, + mock_fascist_firewall_rand_prefer_ipv6_addr_use_ipv6); + /* Simulate the initialisation of fake_node.ipv6_preferred */ + fake_node.ipv6_preferred = fascist_firewall_prefer_ipv6_orport( + &mock_options); + + CHECK_CHOSEN_ADDR_RN(fake_rs, fake_node, FIREWALL_OR_CONNECTION, 0, 1, + ipv6_or_ap); + CHECK_CHOSEN_ADDR_RN(fake_rs, fake_node, FIREWALL_OR_CONNECTION, 1, 1, + ipv6_or_ap); + + UNMOCK(fascist_firewall_rand_prefer_ipv6_addr); + done: UNMOCK(get_options); } From 0015d0084262ddc60098b15d6cdf30ea66d59a53 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Sat, 1 Dec 2018 20:46:06 -0500 Subject: [PATCH 0234/2557] Use tor_strdup() in place of malloc+strncpy+terminate. --- src/tools/tor-resolve.c | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/src/tools/tor-resolve.c b/src/tools/tor-resolve.c index ba83f3e3b4..803ed26b3b 100644 --- a/src/tools/tor-resolve.c +++ b/src/tools/tor-resolve.c @@ -512,14 +512,7 @@ do_resolve(const char *hostname, domainname_t *dn = socks5_server_reply_get_bind_addr_domainname(reply); - size_t result_len = (size_t)domainname_get_len(dn); - - *result_hostname = tor_malloc(result_len+1); - - strncpy(*result_hostname, (char *)domainname_getstr_name(dn), - result_len); - - (*result_hostname)[result_len] = '\0'; + *result_hostname = tor_strdup(domainname_getstr_name(dn)); } socks5_server_reply_free(reply); From 612b21b8eadea9209b62877e5796d8b461916536 Mon Sep 17 00:00:00 2001 From: teor Date: Mon, 3 Dec 2018 10:19:34 +1000 Subject: [PATCH 0235/2557] comment: replace cached-routers with cached-descriptors cached-routers has been gone for a long time --- src/feature/nodelist/routerlist.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/feature/nodelist/routerlist.c b/src/feature/nodelist/routerlist.c index a8c9003e68..b4d56459df 100644 --- a/src/feature/nodelist/routerlist.c +++ b/src/feature/nodelist/routerlist.c @@ -160,7 +160,7 @@ static time_t last_descriptor_download_attempted = 0; * * From time to time, we replace "cached-descriptors" with a new file * containing only the live, non-superseded descriptors, and clear - * cached-routers.new. + * cached-descriptors.new. * * On startup, we read both files. */ From 90ce8d0a5896348bf0acf284e6e7fdef5b64e813 Mon Sep 17 00:00:00 2001 From: teor Date: Mon, 3 Dec 2018 10:22:49 +1000 Subject: [PATCH 0236/2557] doc: remove cached-routers from the man page cached-routers was removed from the code a long time ago --- doc/tor.1.txt | 4 ---- 1 file changed, 4 deletions(-) diff --git a/doc/tor.1.txt b/doc/tor.1.txt index b147ad68aa..6c4746aa52 100644 --- a/doc/tor.1.txt +++ b/doc/tor.1.txt @@ -3251,10 +3251,6 @@ __CacheDirectory__**/cached-microdescs** and **cached-microdescs.new**:: router. The ".new" file is an append-only journal; when it gets too large, all entries are merged into a new cached-microdescs file. -__CacheDirectory__**/cached-routers** and **cached-routers.new**:: - Obsolete versions of cached-descriptors and cached-descriptors.new. When - Tor can't find the newer files, it looks here instead. - __DataDirectory__**/state**:: A set of persistent key-value mappings. These are documented in the file. These include: From 9369152aae9527cc3764cac8688f258b11bd503d Mon Sep 17 00:00:00 2001 From: rl1987 Date: Mon, 3 Dec 2018 14:28:32 +0200 Subject: [PATCH 0237/2557] Check that new listener connection is actually listening --- src/core/mainloop/connection.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/core/mainloop/connection.c b/src/core/mainloop/connection.c index eaad45c81a..66a8b5147f 100644 --- a/src/core/mainloop/connection.c +++ b/src/core/mainloop/connection.c @@ -1460,6 +1460,19 @@ connection_listener_new(const struct sockaddr *listensockaddr, tor_socket_strerror(tor_socket_errno(s))); goto err; } + +#ifndef __APPLE__ + int value; + socklen_t len = sizeof(value); + + if (!getsockopt(s, SOL_SOCKET, SO_ACCEPTCONN, &value, &len)) { + if (value == 0) { + log_err(LD_NET, "Could not listen on %s - " + "getsockopt(.,SO_ACCEPTCONN,.) yields 0.", address); + goto err; + } + } +#endif /* __APPLE__ */ #endif /* defined(HAVE_SYS_UN_H) */ } else { log_err(LD_BUG, "Got unexpected address family %d.", From 25f3b8244582c49db71c618c3859d7e7a89cd762 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Mon, 3 Dec 2018 14:40:37 +0200 Subject: [PATCH 0238/2557] More logging for #28229 --- src/core/mainloop/connection.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/core/mainloop/connection.c b/src/core/mainloop/connection.c index 66a8b5147f..9d5bad7d8d 100644 --- a/src/core/mainloop/connection.c +++ b/src/core/mainloop/connection.c @@ -2907,6 +2907,10 @@ retry_all_listeners(smartlist_t *new_conns, int close_all_noncontrol) retval = -1; #ifdef ENABLE_LISTENER_REBIND + if (smartlist_len(replacements)) + log_debug(LD_NET, "%d replacements - starting rebinding loop.", + smartlist_len(replacements)); + SMARTLIST_FOREACH_BEGIN(replacements, listener_replacement_t *, r) { int addr_in_use = 0; int skip = 0; @@ -2918,8 +2922,11 @@ retry_all_listeners(smartlist_t *new_conns, int close_all_noncontrol) connection_listener_new_for_port(r->new_port, &skip, &addr_in_use); connection_t *old_conn = r->old_conn; - if (skip) + if (skip) { + log_debug(LD_NET, "Skipping creating new listener for %s:%d", + old_conn->address, old_conn->port); continue; + } connection_close_immediate(old_conn); connection_mark_for_close(old_conn); From db9ab3754a0a45fb421211ea9b815b4449999c66 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Mon, 3 Dec 2018 14:49:33 +0200 Subject: [PATCH 0239/2557] Print error message we get from socket.connect_ex when it fails --- src/test/test_rebind.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/test/test_rebind.py b/src/test/test_rebind.py index 6f759b717e..7d0ccaf628 100644 --- a/src/test/test_rebind.py +++ b/src/test/test_rebind.py @@ -19,9 +19,10 @@ def fail(msg): def try_connecting_to_socksport(): socks_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - if socks_socket.connect_ex(('127.0.0.1', socks_port)): + e = socks_socket.connect_ex(('127.0.0.1', socks_port)) + if e: tor_process.terminate() - fail('Cannot connect to SOCKSPort') + fail('Cannot connect to SOCKSPort: error ' + os.strerror(e)) socks_socket.close() def wait_for_log(s): From 894d207f844cf5fa5816432abfe8d1c02763cd59 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Mon, 3 Dec 2018 15:19:33 +0200 Subject: [PATCH 0240/2557] manpage: Mention that adding new HS is unsupported if Sandbox is enabled --- changes/doc28560 | 3 +++ doc/tor.1.txt | 5 ++++- 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 changes/doc28560 diff --git a/changes/doc28560 b/changes/doc28560 new file mode 100644 index 0000000000..c3356bda0a --- /dev/null +++ b/changes/doc28560 @@ -0,0 +1,3 @@ + o Documentation (hidden service manpage, sandbox): + - Mention that you cannot add new Onion Service if Tor is already + running with Sandbox enabled. Closes ticket 28560. diff --git a/doc/tor.1.txt b/doc/tor.1.txt index 8e6ec7f1a2..dbfb6358c4 100644 --- a/doc/tor.1.txt +++ b/doc/tor.1.txt @@ -610,7 +610,8 @@ GENERAL OPTIONS Logs ServerDNSResolvConfFile Tor must remain in client or server mode (some changes to ClientOnly and - ORPort are not allowed). + ORPort are not allowed). Launching new Onion Services through Control + Port is not supported with current syscall sandboxing implementation. ClientOnionAuthDir and any files in it won't reload on HUP signal. (Default: 0) @@ -2821,6 +2822,8 @@ The following options are used to configure a hidden service. Store data files for a hidden service in DIRECTORY. Every hidden service must have a separate directory. You may use this option multiple times to specify multiple services. If DIRECTORY does not exist, Tor will create it. + Please note that you cannot add new Onion Service to already running Tor + instance if **Sandbox** is enabled. (Note: in current versions of Tor, if DIRECTORY is a relative path, it will be relative to the current working directory of Tor instance, not to its DataDirectory. Do not From 32213fa9ad8a306e1f3bade1c95b1ad95a136bd9 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 3 Dec 2018 12:18:45 -0500 Subject: [PATCH 0241/2557] Keep list of dirauth flags in sync between dirvote.c and fuzz_vrs.c Suggested by Teor on PR --- src/feature/dirauth/dirvote.c | 20 ++++++++++++++++++-- src/feature/dirauth/dirvote.h | 3 +++ src/test/fuzz/fuzz_vrs.c | 8 ++++++-- 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/src/feature/dirauth/dirvote.c b/src/feature/dirauth/dirvote.c index aa4242f678..c6a2193087 100644 --- a/src/feature/dirauth/dirvote.c +++ b/src/feature/dirauth/dirvote.c @@ -4372,6 +4372,23 @@ clear_status_flags_on_sybil(routerstatus_t *rs) * forget to add it to this clause. */ } +/** Space-separated list of all the flags that we will always vote on. */ +const char DIRVOTE_UNIVERSAL_FLAGS[] = + "Authority " + "Exit " + "Fast " + "Guard " + "HSDir " + "Stable " + "StaleDesc " + "V2Dir " + "Valid"; +/** Space-separated list of all flags that we may or may not vote on, + * depending on our configuration. */ +const char DIRVOTE_OPTIONAL_FLAGS[] = + "BadExit " + "Running"; + /** Return a new networkstatus_t* containing our current opinion. (For v3 * authorities) */ networkstatus_t * @@ -4620,8 +4637,7 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key, v3_out->known_flags = smartlist_new(); smartlist_split_string(v3_out->known_flags, - "Authority Exit Fast Guard Stable V2Dir Valid HSDir " - "StaleDesc", + DIRVOTE_UNIVERSAL_FLAGS, 0, SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); if (vote_on_reachability) smartlist_add_strdup(v3_out->known_flags, "Running"); diff --git a/src/feature/dirauth/dirvote.h b/src/feature/dirauth/dirvote.h index a21e9f3455..f99cc2d2bf 100644 --- a/src/feature/dirauth/dirvote.h +++ b/src/feature/dirauth/dirvote.h @@ -92,6 +92,9 @@ /** Maximum size of a line in a vote. */ #define MAX_BW_FILE_HEADERS_LINE_LEN 1024 +extern const char DIRVOTE_UNIVERSAL_FLAGS[]; +extern const char DIRVOTE_OPTIONAL_FLAGS[]; + /* * Public API. Used outside of the dirauth subsystem. * diff --git a/src/test/fuzz/fuzz_vrs.c b/src/test/fuzz/fuzz_vrs.c index 3c6d205a3f..f0d90d7cc6 100644 --- a/src/test/fuzz/fuzz_vrs.c +++ b/src/test/fuzz/fuzz_vrs.c @@ -3,6 +3,7 @@ #define NS_PARSE_PRIVATE #define NETWORKSTATUS_PRIVATE #include "core/or/or.h" +#include "feature/dirauth/dirvote.h" #include "feature/dirparse/ns_parse.h" #include "feature/dirparse/unparseable.h" #include "lib/memarea/memarea.h" @@ -35,9 +36,12 @@ fuzz_init(void) dummy_vote = tor_malloc_zero(sizeof(*dummy_vote)); dummy_vote->known_flags = smartlist_new(); smartlist_split_string(dummy_vote->known_flags, - "Authority BadExit Exit Fast Guard HSDir " - "NoEdConsensus Running Stable V2Dir Valid", + DIRVOTE_UNIVERSAL_FLAGS, " ", 0, 0); + smartlist_split_string(dummy_vote->known_flags, + DIRVOTE_OPTIONAL_FLAGS, + " ", 0, 0); + smartlist_sort_strings(dummy_vote->known_flags); return 0; } From 417a324a8577b0c61185c9d06eb72fbc483d984b Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 3 Dec 2018 12:34:29 -0500 Subject: [PATCH 0242/2557] Make input argument const in set_routerstatus_from_routerinfo. --- src/feature/dirauth/voteflags.c | 4 ++-- src/feature/dirauth/voteflags.h | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/feature/dirauth/voteflags.c b/src/feature/dirauth/voteflags.c index 5adf21ad11..aab322d96f 100644 --- a/src/feature/dirauth/voteflags.c +++ b/src/feature/dirauth/voteflags.c @@ -95,7 +95,7 @@ real_uptime(const routerinfo_t *router, time_t now) */ static int dirserv_thinks_router_is_unreliable(time_t now, - routerinfo_t *router, + const routerinfo_t *router, int need_uptime, int need_capacity) { if (need_uptime) { @@ -541,7 +541,7 @@ dirserv_set_router_is_running(routerinfo_t *router, time_t now) void set_routerstatus_from_routerinfo(routerstatus_t *rs, node_t *node, - routerinfo_t *ri, + const routerinfo_t *ri, time_t now, int listbadexits) { diff --git a/src/feature/dirauth/voteflags.h b/src/feature/dirauth/voteflags.h index 743a666cc8..8dce9fbb04 100644 --- a/src/feature/dirauth/voteflags.h +++ b/src/feature/dirauth/voteflags.h @@ -19,7 +19,8 @@ int running_long_enough_to_decide_unreachable(void); void set_routerstatus_from_routerinfo(routerstatus_t *rs, node_t *node, - routerinfo_t *ri, time_t now, + const routerinfo_t *ri, + time_t now, int listbadexits); void dirserv_compute_performance_thresholds(digestmap_t *omit_as_sybil); From 92af8e511394685f7a39224ec1f73ce79fcf2c42 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 3 Dec 2018 13:22:23 -0500 Subject: [PATCH 0243/2557] Add a framework for testing set_routerstatus_from_routerinfo(). Additionally, use it to test that is_staledesc is set correctly. Eventually we'll want to test all the other flags, but I'm aiming for only adding coverage on the changed code here. --- src/test/include.am | 1 + src/test/test.c | 3 +- src/test/test.h | 1 + src/test/test_voting_flags.c | 192 +++++++++++++++++++++++++++++++++++ 4 files changed, 196 insertions(+), 1 deletion(-) create mode 100644 src/test/test_voting_flags.c diff --git a/src/test/include.am b/src/test/include.am index e5eae56e25..d0f71fa666 100644 --- a/src/test/include.am +++ b/src/test/include.am @@ -177,6 +177,7 @@ src_test_test_SOURCES += \ src/test/test_util.c \ src/test/test_util_format.c \ src/test/test_util_process.c \ + src/test/test_voting_flags.c \ src/test/test_voting_schedule.c \ src/test/test_x509.c \ src/test/test_helpers.c \ diff --git a/src/test/test.c b/src/test/test.c index 17b736d305..bbb00da788 100644 --- a/src/test/test.c +++ b/src/test/test.c @@ -866,7 +866,8 @@ struct testgroup_t testgroups[] = { { "crypto/pem/", pem_tests }, { "dir/", dir_tests }, { "dir/md/", microdesc_tests }, - { "dir/voting-schedule/", voting_schedule_tests }, + { "dir/voting/flags/", voting_flags_tests }, + { "dir/voting/schedule/", voting_schedule_tests }, { "dir_handle_get/", dir_handle_get_tests }, { "dns/", dns_tests }, { "dos/", dos_tests }, diff --git a/src/test/test.h b/src/test/test.h index 092356f0fb..1d8f79717f 100644 --- a/src/test/test.h +++ b/src/test/test.h @@ -265,6 +265,7 @@ extern struct testcase_t tortls_tests[]; extern struct testcase_t util_format_tests[]; extern struct testcase_t util_process_tests[]; extern struct testcase_t util_tests[]; +extern struct testcase_t voting_flags_tests[]; extern struct testcase_t voting_schedule_tests[]; extern struct testcase_t x509_tests[]; diff --git a/src/test/test_voting_flags.c b/src/test/test_voting_flags.c new file mode 100644 index 0000000000..0c4cedb516 --- /dev/null +++ b/src/test/test_voting_flags.c @@ -0,0 +1,192 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "orconfig.h" + +#define VOTEFLAGS_PRIVATE + +#include "core/or/or.h" + +#include "feature/dirauth/voteflags.h" +#include "feature/nodelist/node_st.h" +#include "feature/nodelist/routerstatus_st.h" +#include "feature/nodelist/routerinfo_st.h" + +#include "app/config/config.h" + +#include "test/test.h" + +typedef struct { + time_t now; + routerinfo_t ri; + node_t node; + + routerstatus_t expected; +} flag_vote_test_cfg_t; + +static void +setup_cfg(flag_vote_test_cfg_t *c) +{ + memset(c, 0, sizeof(*c)); + + c->now = approx_time(); + + c->ri.nickname = (char *) "testing100"; + strlcpy(c->expected.nickname, "testing100", sizeof(c->expected.nickname)); + + memset(c->ri.cache_info.identity_digest, 0xff, DIGEST_LEN); + memset(c->ri.cache_info.signed_descriptor_digest, 0xee, DIGEST256_LEN); + + c->ri.cache_info.published_on = c->now - 100; + c->expected.published_on = c->now - 100; + + c->ri.addr = 0x7f010105; + c->expected.addr = 0x7f010105; + c->ri.or_port = 9090; + c->expected.or_port = 9090; + + tor_addr_make_null(&c->ri.ipv6_addr, AF_INET6); + tor_addr_make_null(&c->expected.ipv6_addr, AF_INET6); + + // By default we have no loaded information about stability or speed, + // so we'll default to voting "yeah sure." on these two. + c->expected.is_fast = 1; + c->expected.is_stable = 1; +} + +static bool +check_result(flag_vote_test_cfg_t *c) +{ + bool result = false; + routerstatus_t rs; + memset(&rs, 0, sizeof(rs)); + set_routerstatus_from_routerinfo(&rs, &c->node, &c->ri, c->now, 0); + + tt_i64_op(rs.published_on, OP_EQ, c->expected.published_on); + tt_str_op(rs.nickname, OP_EQ, c->expected.nickname); + + // identity_digest and descriptor_digest are not set here. + + tt_uint_op(rs.addr, OP_EQ, c->expected.addr); + tt_uint_op(rs.or_port, OP_EQ, c->expected.or_port); + tt_uint_op(rs.dir_port, OP_EQ, c->expected.dir_port); + + tt_assert(tor_addr_eq(&rs.ipv6_addr, &c->expected.ipv6_addr)); + tt_uint_op(rs.ipv6_orport, OP_EQ, c->expected.ipv6_orport); + +#define FLAG(flagname) \ + tt_uint_op(rs.flagname, OP_EQ, c->expected.flagname) + + FLAG(is_authority); + FLAG(is_exit); + FLAG(is_stable); + FLAG(is_fast); + FLAG(is_flagged_running); + FLAG(is_named); + FLAG(is_unnamed); + FLAG(is_valid); + FLAG(is_possible_guard); + FLAG(is_bad_exit); + FLAG(is_hs_dir); + FLAG(is_v2_dir); + FLAG(is_staledesc); + FLAG(has_bandwidth); + FLAG(has_exitsummary); + FLAG(bw_is_unmeasured); + + result = true; + + done: + return result; +} + +static void +test_voting_flags_minimal(void *arg) +{ + flag_vote_test_cfg_t *cfg = arg; + check_result(cfg); +} + +static void +test_voting_flags_ipv6(void *arg) +{ + flag_vote_test_cfg_t *cfg = arg; + + tt_assert(tor_addr_parse(&cfg->ri.ipv6_addr, "f00::b42") == AF_INET6); + cfg->ri.ipv6_orport = 9091; + // no change in expected results, since we aren't set up with ipv6 + // connectivity. + if (!check_result(cfg)) + goto done; + + get_options_mutable()->AuthDirHasIPv6Connectivity = 1; + // no change in expected results, since last_reachable6 won't be set. + if (!check_result(cfg)) + goto done; + + cfg->node.last_reachable6 = cfg->now - 10; + // now that lastreachable6 is set, we expect to see the result. + tt_assert(tor_addr_parse(&cfg->expected.ipv6_addr, "f00::b42") == AF_INET6); + cfg->expected.ipv6_orport = 9091; + if (!check_result(cfg)) + goto done; + done: + ; +} + +static void +test_voting_flags_staledesc(void *arg) +{ + flag_vote_test_cfg_t *cfg = arg; + time_t now = cfg->now; + + cfg->ri.cache_info.published_on = now - DESC_IS_STALE_INTERVAL + 10; + cfg->expected.published_on = now - DESC_IS_STALE_INTERVAL + 10; + // no change in expectations for is_staledesc + if (!check_result(cfg)) + goto done; + + cfg->ri.cache_info.published_on = now - DESC_IS_STALE_INTERVAL - 10; + cfg->expected.published_on = now - DESC_IS_STALE_INTERVAL - 10; + cfg->expected.is_staledesc = 1; + if (!check_result(cfg)) + goto done; + + done: + ; +} + +static void * +setup_voting_flags_test(const struct testcase_t *testcase) +{ + (void)testcase; + flag_vote_test_cfg_t *cfg = tor_malloc_zero(sizeof(*cfg)); + setup_cfg(cfg); + return cfg; +} + +static int +teardown_voting_flags_test(const struct testcase_t *testcase, void *arg) +{ + (void)testcase; + flag_vote_test_cfg_t *cfg = arg; + tor_free(cfg); + return 1; +} + +static const struct testcase_setup_t voting_flags_setup = { + .setup_fn = setup_voting_flags_test, + .cleanup_fn = teardown_voting_flags_test, +}; + +#define T(name,flags) \ + { #name, test_voting_flags_##name, (flags), &voting_flags_setup, NULL } + +struct testcase_t voting_flags_tests[] = { + T(minimal, 0), + T(ipv6, TT_FORK), + // TODO: Add more of these tests. + T(staledesc, TT_FORK), + END_OF_TESTCASES +}; + From 31a6d9f49997cfb1266a55b742a15706ae16db5e Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 3 Dec 2018 13:40:08 -0500 Subject: [PATCH 0244/2557] Add tests for parsing each routerstatus flag. --- src/test/test_dir.c | 75 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/src/test/test_dir.c b/src/test/test_dir.c index 26ba269abd..5cdbd877ce 100644 --- a/src/test/test_dir.c +++ b/src/test/test_dir.c @@ -6052,6 +6052,80 @@ test_dir_find_dl_min_delay(void* data) mock_options = NULL; } +static void +test_dir_matching_flags(void *arg) +{ + (void) arg; + routerstatus_t *rs_noflags = NULL; + routerstatus_t *rs = NULL; + char *s = NULL; + + smartlist_t *tokens = smartlist_new(); + memarea_t *area = memarea_new(); + + int expected_val_when_unused = 0; + + const char *ex_noflags = + "r example hereiswhereyouridentitygoes 2015-08-30 12:00:00 " + "192.168.0.1 9001 0\n" + "m thisoneislongerbecauseitisa256bitmddigest33\n" + "s\n"; + const char *cp = ex_noflags; + rs_noflags = routerstatus_parse_entry_from_string( + area, &cp, + cp + strlen(cp), + tokens, NULL, NULL, + MAX_SUPPORTED_CONSENSUS_METHOD, FLAV_MICRODESC); + tt_assert(rs_noflags); + +#define FLAG(string, field) STMT_BEGIN { \ + tor_asprintf(&s,\ + "r example hereiswhereyouridentitygoes 2015-08-30 12:00:00 " \ + "192.168.0.1 9001 0\n" \ + "m thisoneislongerbecauseitisa256bitmddigest33\n" \ + "s %s\n", string); \ + cp = s; \ + rs = routerstatus_parse_entry_from_string( \ + area, &cp, \ + cp + strlen(cp), \ + tokens, NULL, NULL, \ + MAX_SUPPORTED_CONSENSUS_METHOD, FLAV_MICRODESC); \ + /* the field should usually be 0 when no flags are listed */ \ + tt_int_op(rs_noflags->field, OP_EQ, expected_val_when_unused); \ + /* the field should be 1 when this flags islisted */ \ + tt_int_op(rs->field, OP_EQ, 1); \ + tor_free(s); \ + routerstatus_free(rs); \ +} STMT_END + + FLAG("Authority", is_authority); + FLAG("BadExit", is_bad_exit); + FLAG("Exit", is_exit); + FLAG("Fast", is_fast); + FLAG("Guard", is_possible_guard); + FLAG("HSDir", is_hs_dir); + FLAG("Stable", is_stable); + FLAG("StaleDesc", is_staledesc); + FLAG("V2Dir", is_v2_dir); + + // These flags are assumed to be set whether they're declared or not. + expected_val_when_unused = 1; + FLAG("Running", is_flagged_running); + FLAG("Valid", is_valid); + expected_val_when_unused = 0; + + // These flags are no longer used, but still parsed. + FLAG("Named", is_named); + FLAG("Unnamed", is_unnamed); + + done: + tor_free(s); + routerstatus_free(rs); + routerstatus_free(rs_noflags); + memarea_drop_all(area); + smartlist_free(tokens); +} + static void test_dir_assumed_flags(void *arg) { @@ -6377,6 +6451,7 @@ struct testcase_t dir_tests[] = { DIR_ARG(find_dl_min_delay, TT_FORK, "cfr"), DIR_ARG(find_dl_min_delay, TT_FORK, "car"), DIR(assumed_flags, 0), + DIR(matching_flags, 0), DIR(networkstatus_compute_bw_weights_v10, 0), DIR(platform_str, 0), DIR(networkstatus_consensus_has_ipv6, TT_FORK), From b25b8150c282c28d3fa23330891fda7adbbfe584 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 4 Dec 2018 11:56:05 -0500 Subject: [PATCH 0245/2557] Remember in our state file how long we've spent since user activity Rather than initializing the "Dormant" status to "off" and the "last activity" count to "now", initialize them based on our state file: stay dormant if we were dormant, or remember the amount of time we've spent inactive. --- src/app/config/or_state_st.h | 7 ++++++ src/app/config/statefile.c | 8 +++++++ src/core/mainloop/mainloop.c | 7 +----- src/core/mainloop/netstatus.c | 41 +++++++++++++++++++++++++++++++++++ src/core/mainloop/netstatus.h | 3 +++ 5 files changed, 60 insertions(+), 6 deletions(-) diff --git a/src/app/config/or_state_st.h b/src/app/config/or_state_st.h index d95df6236b..00968d3731 100644 --- a/src/app/config/or_state_st.h +++ b/src/app/config/or_state_st.h @@ -87,6 +87,13 @@ struct or_state_t { /** When did we last rotate our onion key? "0" for 'no idea'. */ time_t LastRotatedOnionKey; + + /** Number of minutes since the last user-initiated request (as defined by + * the dormant net-status system.) Set to zero if we are dormant. */ + int MinutesSinceUserActivity; + /** True if we were dormant when we last wrote the file; false if we + * weren't. "auto" on initial startup. */ + int Dormant; }; #endif diff --git a/src/app/config/statefile.c b/src/app/config/statefile.c index 4ba7be1519..97b96f1149 100644 --- a/src/app/config/statefile.c +++ b/src/app/config/statefile.c @@ -34,6 +34,7 @@ #include "app/config/config.h" #include "app/config/confparse.h" #include "core/mainloop/mainloop.h" +#include "core/mainloop/netstatus.h" #include "core/mainloop/connection.h" #include "feature/control/control.h" #include "feature/client/entrynodes.h" @@ -132,6 +133,9 @@ static config_var_t state_vars_[] = { VAR("CircuitBuildTimeBin", LINELIST_S, BuildtimeHistogram, NULL), VAR("BuildtimeHistogram", LINELIST_V, BuildtimeHistogram, NULL), + V(MinutesSinceUserActivity, UINT, NULL), + V(Dormant, AUTOBOOL, "auto"), + END_OF_CONFIG_VARS }; @@ -309,6 +313,8 @@ or_state_set(or_state_t *new_state) get_circuit_build_times_mutable(),global_state) < 0) { ret = -1; } + netstatus_load_from_state(global_state, time(NULL)); + return ret; } @@ -500,6 +506,8 @@ or_state_save(time_t now) entry_guards_update_state(global_state); rep_hist_update_state(global_state); circuit_build_times_update_state(get_circuit_build_times(), global_state); + netstatus_flush_to_state(global_state, now); + if (accounting_is_enabled(get_options())) accounting_run_housekeeping(now); diff --git a/src/core/mainloop/mainloop.c b/src/core/mainloop/mainloop.c index 2e2ae876d4..cd955e0ca0 100644 --- a/src/core/mainloop/mainloop.c +++ b/src/core/mainloop/mainloop.c @@ -1627,6 +1627,7 @@ schedule_rescan_periodic_events,(void)) void rescan_periodic_events(const or_options_t *options) { + puts("RESCAN"); tor_assert(options); /* Avoid scanning the event list if we haven't initialized it yet. This is @@ -2818,12 +2819,6 @@ initialize_mainloop_events(void) int do_main_loop(void) { - /* For now, starting Tor always counts as user activity. Later, we might - * have an option to control this. - */ - reset_user_activity(approx_time()); - set_network_participation(true); - /* initialize the periodic events first, so that code that depends on the * events being present does not assert. */ diff --git a/src/core/mainloop/netstatus.c b/src/core/mainloop/netstatus.c index ed7c952dcd..59fd8f8037 100644 --- a/src/core/mainloop/netstatus.c +++ b/src/core/mainloop/netstatus.c @@ -10,6 +10,8 @@ #include "app/config/config.h" #include "feature/hibernate/hibernate.h" +#include "app/config/or_state_st.h" + /** Return true iff our network is in some sense disabled or shutting down: * either we're hibernating, entering hibernation, or the network is turned * off with DisableNetwork. */ @@ -31,6 +33,10 @@ net_is_completely_disabled(void) /** * The time at which we've last seen "user activity" -- that is, any activity * that should keep us as a participant on the network. + * + * This is not actually the true time. We will adjust this forward if + * our clock jumps, or if Tor is shut down for a while, so that the time + * since our last activity remains as it was before the jump or shutdown. */ static time_t last_user_activity_seen = 0; @@ -99,3 +105,38 @@ is_participating_on_network(void) { return participating_on_network; } + +/** + * Update 'state' with the last time at which we were active on the network. + **/ +void +netstatus_flush_to_state(or_state_t *state, time_t now) +{ + state->Dormant = ! participating_on_network; + if (participating_on_network) { + time_t sec_since_activity = MAX(0, now - last_user_activity_seen); + state->MinutesSinceUserActivity = (int)(sec_since_activity / 60); + } else { + state->MinutesSinceUserActivity = 0; + } +} + +/** + * Update our current view of network participation from an or_state_t object. + **/ +void +netstatus_load_from_state(const or_state_t *state, time_t now) +{ + time_t last_activity; + if (state->Dormant == -1) { // Initial setup. + last_activity = now; + participating_on_network = true; + } else if (state->Dormant) { + last_activity = 0; + participating_on_network = false; + } else { + last_activity = now - 60 * state->MinutesSinceUserActivity; + participating_on_network = true; + } + reset_user_activity(last_activity); +} diff --git a/src/core/mainloop/netstatus.h b/src/core/mainloop/netstatus.h index 58c994fd14..4b008e4cfa 100644 --- a/src/core/mainloop/netstatus.h +++ b/src/core/mainloop/netstatus.h @@ -17,4 +17,7 @@ time_t get_last_user_activity_time(void); void set_network_participation(bool participation); bool is_participating_on_network(void); +void netstatus_flush_to_state(or_state_t *state, time_t now); +void netstatus_load_from_state(const or_state_t *state, time_t now); + #endif From 4f558843151f0c49c077d3d9a1cd8a187904a024 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 4 Dec 2018 12:07:23 -0500 Subject: [PATCH 0246/2557] Add an option to start tor in dormant mode for the first time. --- doc/tor.1.txt | 11 ++++++++++- src/app/config/config.c | 1 + src/app/config/or_options_st.h | 4 ++++ src/core/mainloop/netstatus.c | 10 ++++++++-- 4 files changed, 23 insertions(+), 3 deletions(-) diff --git a/doc/tor.1.txt b/doc/tor.1.txt index 8e6ec7f1a2..82c9e34de4 100644 --- a/doc/tor.1.txt +++ b/doc/tor.1.txt @@ -1803,11 +1803,20 @@ The following options are useful only for clients (that is, if Does not affect servers or onion services. Must be at least 10 minutes. (Default: 24 hours) -[[DormantTimeoutDisabledByIdleStreams]] **DormantTimeoutDisabledByIdleStreams **0**|**1**:: +[[DormantTimeoutDisabledByIdleStreams]] **DormantTimeoutDisabledByIdleStreams** **0**|**1**:: If true, then any open client stream (even one not reading or writing) counts as client activity for the purpose of DormantClientTimeout. If false, then only network activity counts. (Default: 1) +[[DormantOnFirstStartup]] **DormantOnFirstStartup** **0**|**1**:: + If true, then the first time Tor starts up with a fresh DataDirectory, + it starts in dormant mode, and takes no actions until the user has made + a request. (This mode is recommended if installing a Tor client for a + user who might not actually use it.) If false, Tor bootstraps the first + time it is started, whether it sees a user request or not. + + + After the first time Tor starts, it begins in dormant mode if it was + dormant before, and not otherwise. (Default: 0) SERVER OPTIONS -------------- diff --git a/src/app/config/config.c b/src/app/config/config.c index d40e362b32..dcefa3d6a4 100644 --- a/src/app/config/config.c +++ b/src/app/config/config.c @@ -392,6 +392,7 @@ static config_var_t option_vars_[] = { OBSOLETE("DNSListenAddress"), V(DormantClientTimeout, INTERVAL, "24 hours"), V(DormantTimeoutDisabledByIdleStreams, BOOL, "1"), + V(DormantOnFirstStartup, BOOL, "0"), /* DoS circuit creation options. */ V(DoSCircuitCreationEnabled, AUTOBOOL, "auto"), V(DoSCircuitCreationMinConnections, UINT, "0"), diff --git a/src/app/config/or_options_st.h b/src/app/config/or_options_st.h index 9065248a9c..c2bc1079a5 100644 --- a/src/app/config/or_options_st.h +++ b/src/app/config/or_options_st.h @@ -1085,6 +1085,10 @@ struct or_options_t { * from becoming dormant. **/ int DormantTimeoutDisabledByIdleStreams; + + /** Boolean: true if Tor should be dormant the first time it starts with + * a datadirectory; false otherwise. */ + int DormantOnFirstStartup; }; #endif diff --git a/src/core/mainloop/netstatus.c b/src/core/mainloop/netstatus.c index 59fd8f8037..2426baae34 100644 --- a/src/core/mainloop/netstatus.c +++ b/src/core/mainloop/netstatus.c @@ -129,8 +129,14 @@ netstatus_load_from_state(const or_state_t *state, time_t now) { time_t last_activity; if (state->Dormant == -1) { // Initial setup. - last_activity = now; - participating_on_network = true; + if (get_options()->DormantOnFirstStartup) { + last_activity = 0; + participating_on_network = false; + } else { + // Start up as active, treat activity as happening now. + last_activity = now; + participating_on_network = true; + } } else if (state->Dormant) { last_activity = 0; participating_on_network = false; From 1f6d7bc4afb5dd00c84c21c9323c2e2109f2378e Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 16 Nov 2018 11:51:58 -0500 Subject: [PATCH 0247/2557] Update ReleasingTor.md to reflect current practice See #28479. --- doc/HACKING/ReleasingTor.md | 109 ++++++++++++++++++++++-------------- 1 file changed, 68 insertions(+), 41 deletions(-) diff --git a/doc/HACKING/ReleasingTor.md b/doc/HACKING/ReleasingTor.md index b5444afa96..3073cfb108 100644 --- a/doc/HACKING/ReleasingTor.md +++ b/doc/HACKING/ReleasingTor.md @@ -20,30 +20,29 @@ new Tor release: === I. Make sure it works -1. Use it for a while, as a client, as a relay, as a hidden service, - and as a directory authority. See if it has any obvious bugs, and - resolve those. +1. Make sure that CI passes: have a look at Travis, Appveyor, and + Jenkins. Make sure you're looking at the right branches. - As applicable, merge the `maint-X` branch into the `release-X` branch. - But you've been doing that all along, right? + If there are any unexplained failures, try to fix them or figure them + out. -2. Are all of the jenkins builders happy? See jenkins.torproject.org. +2. Verify that there are no big outstanding issues. You might find such + issues -- - What about the bsd buildbots? - See http://buildbot.pixelminers.net/builders/ + * On Trac - What about Coverity Scan? + * On coverity scan - What about clang scan-build? + * On OSS-Fuzz - Does `make distcheck` complain? +3. Run checks that aren't covered above, including: - How about `make test-stem` and `make test-network` and - `make test-network-all`? + * clang scan-build. (See the script in ./scripts/test/scan_build.sh) - - Are all those tests still happy with --enable-expensive-hardening ? + * make test-network and make test-network-all (with + --enable-expensive-hardening) - Any memory leaks? + * Running Tor yourself and making sure that it actually works for you. === II. Write a changelog @@ -55,11 +54,14 @@ new Tor release: of them and reordering to focus on what users and funders would find interesting and understandable. - To do this, first run `./scripts/maint/lintChanges.py changes/*` and - fix as many warnings as you can. Then run `./scripts/maint/sortChanges.py - changes/* > changelog.in` to combine headings and sort the entries. - After that, it's time to hand-edit and fix the issues that lintChanges - can't find: + To do this, run + `./scripts/maint/sortChanges.py changes/* > changelog.in` + to combine headings and sort the entries. Copy the changelog.in file + into the ChangeLog. Run 'format_changelog.py' (see below) to clean + up the line breaks. + + After that, it's time to hand-edit and fix the issues that + lintChanges can't find: 1. Within each section, sort by "version it's a bugfix on", else by numerical ticket order. @@ -68,8 +70,6 @@ new Tor release: Make stuff very terse - Make sure each section name ends with a colon - Describe the user-visible problem right away Mention relevant config options by name. If they're rare or unusual, @@ -81,6 +81,8 @@ new Tor release: "Relays", not "servers" or "nodes" or "Tor relays". + "Onion services", not "hidden services". + "Stop FOOing", not "Fix a bug where we would FOO". Try not to let any given section be longer than about a page. Break up @@ -106,6 +108,8 @@ new Tor release: changelog: instead, look up the corrected versions that were merged into ChangeLog in the master branch, and use those. + Add "backport from X.Y.Z" in the section header for these entries. + 2. Compose a short release blurb to highlight the user-facing changes. Insert said release blurb into the ChangeLog stanza. If it's a stable release, add it to the ReleaseNotes file too. If we're adding @@ -142,33 +146,50 @@ new Tor release: master, merge it with "-s ours" to avoid a needless version bump. 2. Make distcheck, put the tarball up in somewhere (how about your - homedir on your homedir on people.torproject.org?) , and tell `#tor` - about it. Wait a while to see if anybody has problems building it. - (Though jenkins is usually pretty good about catching these things.) + homedir on your homedir on people.torproject.org?) , and tell `#tor-dev` + about it. + + If you want, wait until at least one person has built it + successfully. (We used to say "wait for others to test it", but our + CI has successfully caught these kinds of errors for the last several + years.) + + +3. Make sure that the new version is recommended in the latest consensus. + (Otherwise, users will get confused when it complains to them + about its status.) + + If it is not, you'll need to poke Roger, Weasel, and Sebastian again: see + item 0.1 at the start of this document. === IV. Commit, upload, announce 1. Sign the tarball, then sign and push the git tag: gpg -ba - git tag -u tor-0.3.x.y-status - git push origin tag tor-0.3.x.y-status + git tag -s tor-0.4.x.y- + git push origin tag tor-0.4.x.y- - (You must do this before you update the website: it relies on finding - the version by tag.) + (You must do this before you update the website: the website scripts + rely on finding the version by tag.) + + (If your default PGP key is not the one you want to sign with, then say + "-u " instead of "-s".) 2. scp the tarball and its sig to the dist website, i.e. - `/srv/dist-master.torproject.org/htdocs/` on dist-master. When you want - it to go live, you run "static-update-component dist.torproject.org" - on dist-master. + `/srv/dist-master.torproject.org/htdocs/` on dist-master. Run + "static-update-component dist.torproject.org" on dist-master. In the webwml.git repository, `include/versions.wmi` and `Makefile` - to note the new version. + to note the new version. Push these changes to master. (NOTE: Due to #17805, there can only be one stable version listed at once. Nonetheless, do not call your version "alpha" if it is stable, or people will get confused.) + (NOTE: It will take a while for the website update scripts to update + the website.) + 3. Email the packagers (cc'ing tor-team) that a new tarball is up. The current list of packagers is: @@ -186,29 +207,35 @@ new Tor release: Also, email tor-packagers@lists.torproject.org. + Mention where to download the tarball (https://dist.torproject.org). + + Include a link to the changelog. + + 4. Add the version number to Trac. To do this, go to Trac, log in, select "Admin" near the top of the screen, then select "Versions" from the menu on the left. At the right, there will be an "Add version" box. By convention, we enter the version in the form "Tor: - 0.2.2.23-alpha" (or whatever the version is), and we select the date as + 0.4.0.1-alpha" (or whatever the version is), and we select the date as the date in the ChangeLog. -5. Double-check: did the version get recommended in the consensus yet? Is - the website updated? If not, don't announce until they have the - up-to-date versions, or people will get confused. +5. Wait for the download page to be updated. (If you don't do this before you + announce, people will be confused.) 6. Mail the release blurb and ChangeLog to tor-talk (development release) or tor-announce (stable). Post the changelog on the blog as well. You can generate a - blog-formatted version of the changelog with the -B option to - format-changelog. + blog-formatted version of the changelog with + `./scripts/maint/format_changelog.py --B` When you post, include an estimate of when the next TorBrowser releases will come out that include this Tor release. This will usually track https://wiki.mozilla.org/RapidRelease/Calendar , but it can vary. + For templates to use when announcing, see: + https://trac.torproject.org/projects/tor/wiki/org/teams/NetworkTeam/AnnouncementTemplates === V. Aftermath and cleanup @@ -216,7 +243,7 @@ new Tor release: `maint-x.y.z` branch to "newversion-dev", and do a `merge -s ours` merge to avoid taking that change into master. -2. Forward-port the ChangeLog (and ReleaseNotes if appropriate). +2. Forward-port the ChangeLog (and ReleaseNotes if appropriate) to the + master branch. 3. Keep an eye on the blog post, to moderate comments and answer questions. - From c01507a5fe73d0e49902a8abaa1f73112ae1fefa Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 5 Dec 2018 08:14:21 -0500 Subject: [PATCH 0248/2557] remember why we are doing getsockopt() --- src/core/mainloop/connection.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/core/mainloop/connection.c b/src/core/mainloop/connection.c index 9d5bad7d8d..55a1dd361e 100644 --- a/src/core/mainloop/connection.c +++ b/src/core/mainloop/connection.c @@ -1462,6 +1462,7 @@ connection_listener_new(const struct sockaddr *listensockaddr, } #ifndef __APPLE__ + /* This code was introduced to help debug #28229. */ int value; socklen_t len = sizeof(value); From a2f81b644b2a4679c634744d3830cdb6397f0814 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 3 Dec 2018 14:06:53 -0500 Subject: [PATCH 0249/2557] Write tests for mark_my_descriptor_dirty_if_too_old() --- src/feature/nodelist/networkstatus.c | 4 +- src/feature/nodelist/networkstatus.h | 5 +- src/feature/relay/router.c | 4 +- src/feature/relay/router.h | 4 ++ src/test/test_router.c | 98 ++++++++++++++++++++++++++++ 5 files changed, 109 insertions(+), 6 deletions(-) diff --git a/src/feature/nodelist/networkstatus.c b/src/feature/nodelist/networkstatus.c index f1def9afb1..a730f7d834 100644 --- a/src/feature/nodelist/networkstatus.c +++ b/src/feature/nodelist/networkstatus.c @@ -724,8 +724,8 @@ networkstatus_vote_find_mutable_entry(networkstatus_t *ns, const char *digest) /** Return the entry in ns for the identity digest digest, or * NULL if none was found. */ -const routerstatus_t * -networkstatus_vote_find_entry(networkstatus_t *ns, const char *digest) +MOCK_IMPL(const routerstatus_t *, +networkstatus_vote_find_entry,(networkstatus_t *ns, const char *digest)) { return networkstatus_vote_find_mutable_entry(ns, digest); } diff --git a/src/feature/nodelist/networkstatus.h b/src/feature/nodelist/networkstatus.h index 8802de2d65..3fd151cf37 100644 --- a/src/feature/nodelist/networkstatus.h +++ b/src/feature/nodelist/networkstatus.h @@ -40,8 +40,9 @@ int compare_digest_to_routerstatus_entry(const void *_key, const void **_member); int compare_digest_to_vote_routerstatus_entry(const void *_key, const void **_member); -const routerstatus_t *networkstatus_vote_find_entry(networkstatus_t *ns, - const char *digest); +MOCK_DECL(const routerstatus_t *,networkstatus_vote_find_entry,( + networkstatus_t *ns, + const char *digest)); routerstatus_t *networkstatus_vote_find_mutable_entry(networkstatus_t *ns, const char *digest); int networkstatus_vote_find_entry_idx(networkstatus_t *ns, diff --git a/src/feature/relay/router.c b/src/feature/relay/router.c index 2d4ab9b0a0..ef433db8bc 100644 --- a/src/feature/relay/router.c +++ b/src/feature/relay/router.c @@ -1471,9 +1471,9 @@ static extrainfo_t *desc_extrainfo = NULL; static const char *desc_gen_reason = "uninitialized reason"; /** Since when has our descriptor been "clean"? 0 if we need to regenerate it * now. */ -static time_t desc_clean_since = 0; +STATIC time_t desc_clean_since = 0; /** Why did we mark the descriptor dirty? */ -static const char *desc_dirty_reason = "Tor just started"; +STATIC const char *desc_dirty_reason = "Tor just started"; /** Boolean: do we need to regenerate the above? */ static int desc_needs_upload = 0; diff --git a/src/feature/relay/router.h b/src/feature/relay/router.h index 4575172afb..217511df9f 100644 --- a/src/feature/relay/router.h +++ b/src/feature/relay/router.h @@ -117,6 +117,10 @@ void router_free_all(void); /* Used only by router.c and test.c */ STATIC void get_platform_str(char *platform, size_t len); STATIC int router_write_fingerprint(int hashed); +#ifdef TOR_UNIT_TESTS +extern time_t desc_clean_since; +extern const char *desc_dirty_reason; +#endif #endif #endif /* !defined(TOR_ROUTER_H) */ diff --git a/src/test/test_router.c b/src/test/test_router.c index 921ec42904..18740dcb84 100644 --- a/src/test/test_router.c +++ b/src/test/test_router.c @@ -7,12 +7,17 @@ * \brief Unittests for code in router.c **/ +#define ROUTER_PRIVATE + #include "core/or/or.h" #include "app/config/config.h" #include "core/mainloop/mainloop.h" #include "feature/hibernate/hibernate.h" +#include "feature/nodelist/networkstatus.h" +#include "feature/nodelist/networkstatus_st.h" #include "feature/nodelist/routerinfo_st.h" #include "feature/nodelist/routerlist.h" +#include "feature/nodelist/routerstatus_st.h" #include "feature/relay/router.h" #include "feature/stats/rephist.h" #include "lib/crypt_ops/crypto_curve25519.h" @@ -231,11 +236,104 @@ test_router_check_descriptor_bandwidth_changed(void *arg) UNMOCK(we_are_hibernating); } +static networkstatus_t *mock_ns = NULL; +static networkstatus_t * +mock_networkstatus_get_live_consensus(time_t now) +{ + (void)now; + return mock_ns; +} + +static routerstatus_t *mock_rs = NULL; +static const routerstatus_t * +mock_networkstatus_vote_find_entry(networkstatus_t *ns, const char *digest) +{ + (void)ns; + (void)digest; + return mock_rs; +} + +static void +test_router_mark_if_too_old(void *arg) +{ + (void)arg; + time_t now = approx_time(); + MOCK(networkstatus_get_live_consensus, + mock_networkstatus_get_live_consensus); + MOCK(networkstatus_vote_find_entry, mock_networkstatus_vote_find_entry); + + routerstatus_t rs; + networkstatus_t ns; + memset(&rs, 0, sizeof(rs)); + memset(&ns, 0, sizeof(ns)); + mock_ns = &ns; + mock_ns->valid_after = now-3600; + mock_rs = &rs; + mock_rs->published_on = now - 10; + + // no reason to mark this time. + desc_clean_since = now-10; + desc_dirty_reason = NULL; + mark_my_descriptor_dirty_if_too_old(now); + tt_i64_op(desc_clean_since, OP_EQ, now-10); + + // Doesn't appear in consensus? Still don't mark it. + mock_ns = NULL; + mark_my_descriptor_dirty_if_too_old(now); + tt_i64_op(desc_clean_since, OP_EQ, now-10); + mock_ns = &ns; + + // No new descriptor in a long time? Mark it. + desc_clean_since = now - 3600 * 96; + mark_my_descriptor_dirty_if_too_old(now); + tt_i64_op(desc_clean_since, OP_EQ, 0); + tt_str_op(desc_dirty_reason, OP_EQ, "time for new descriptor"); + + // Version in consensus published a long time ago? We won't mark it + // if it's been clean for only a short time. + desc_clean_since = now - 10; + desc_dirty_reason = NULL; + mock_rs->published_on = now - 3600 * 96; + mark_my_descriptor_dirty_if_too_old(now); + tt_i64_op(desc_clean_since, OP_EQ, now - 10); + + // ... but if it's been clean a while, we mark. + desc_clean_since = now - 2 * 3600; + mark_my_descriptor_dirty_if_too_old(now); + tt_i64_op(desc_clean_since, OP_EQ, 0); + tt_str_op(desc_dirty_reason, OP_EQ, + "version listed in consensus is quite old"); + + // same deal if we're marked stale. + desc_clean_since = now - 2 * 3600; + desc_dirty_reason = NULL; + mock_rs->published_on = now - 10; + mock_rs->is_staledesc = 1; + mark_my_descriptor_dirty_if_too_old(now); + tt_i64_op(desc_clean_since, OP_EQ, 0); + tt_str_op(desc_dirty_reason, OP_EQ, + "listed as stale in consensus"); + + // same deal if we're absent from the consensus. + desc_clean_since = now - 2 * 3600; + desc_dirty_reason = NULL; + mock_rs = NULL; + mark_my_descriptor_dirty_if_too_old(now); + tt_i64_op(desc_clean_since, OP_EQ, 0); + tt_str_op(desc_dirty_reason, OP_EQ, + "not listed in consensus"); + + done: + UNMOCK(networkstatus_get_live_consensus); + UNMOCK(networkstatus_vote_find_entry); +} + #define ROUTER_TEST(name, flags) \ { #name, test_router_ ## name, flags, NULL, NULL } struct testcase_t router_tests[] = { ROUTER_TEST(check_descriptor_bandwidth_changed, TT_FORK), ROUTER_TEST(dump_router_to_string_no_bridge_distribution_method, TT_FORK), + ROUTER_TEST(mark_if_too_old, TT_FORK), END_OF_TESTCASES }; From c31346ffb444065820407670ecf64b925e3775fc Mon Sep 17 00:00:00 2001 From: rl1987 Date: Thu, 6 Dec 2018 15:26:34 +0200 Subject: [PATCH 0250/2557] Print Python version during each Travis CI job --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index b5713d6933..2ea529e252 100644 --- a/.travis.yml +++ b/.travis.yml @@ -193,6 +193,9 @@ install: - if [[ "$RUST_OPTIONS" != "" ]]; then rustup --version; fi - if [[ "$RUST_OPTIONS" != "" ]]; then rustc --version; fi - if [[ "$RUST_OPTIONS" != "" ]]; then cargo --version; fi + ## Get python version + - python --version + ## run stem tests if they are enabled. - if [[ "$TEST_STEM" != "" ]]; then pushd stem; python -c "from stem import stem; print(stem.__version__);"; git log -1; popd; fi script: From ecaecaddd89ce0d89e5f90effbf066065ccccd10 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Thu, 6 Dec 2018 15:31:33 +0200 Subject: [PATCH 0251/2557] Add changes file --- changes/ticket28551 | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 changes/ticket28551 diff --git a/changes/ticket28551 b/changes/ticket28551 new file mode 100644 index 0000000000..46ba9d713b --- /dev/null +++ b/changes/ticket28551 @@ -0,0 +1,3 @@ + o Minor features (Continuous Integration): + - Log Python version during each Travis CI job. Resolves issue + 28551. From d9f36d3e929e5acebbf483e908d0c90aeabf6558 Mon Sep 17 00:00:00 2001 From: teor Date: Fri, 7 Dec 2018 09:21:17 +1000 Subject: [PATCH 0252/2557] Fallbacks: minor script comment changes --- scripts/maint/updateFallbackDirs.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/scripts/maint/updateFallbackDirs.py b/scripts/maint/updateFallbackDirs.py index 0ea3992d8f..cf923bfa32 100755 --- a/scripts/maint/updateFallbackDirs.py +++ b/scripts/maint/updateFallbackDirs.py @@ -18,8 +18,8 @@ # Optionally uses ipaddress (python 3 builtin) or py2-ipaddress (package) # for netblock analysis. # -# Then read the logs to make sure the fallbacks aren't dominated by a single -# netblock or port. +# After running this script, read the logs to make sure the fallbacks aren't +# dominated by a single netblock or port. # Script by weasel, April 2015 # Portions by gsathya & karsten, 2013 @@ -39,8 +39,6 @@ import urllib import urllib2 import hashlib import dateutil.parser -# bson_lazy provides bson -#from bson import json_util import copy import re @@ -1400,7 +1398,7 @@ class CandidateList(dict): each line's key/value pairs are placed in a dictonary, (of string -> string key/value pairs), and these dictionaries are placed in an array. - comments start with # and are ignored """ + comments start with # and are ignored. """ file_data = file_obj['data'] file_name = file_obj['name'] relaylist = [] From 766fd6cf7666cecc25a94e6cfe46b8b27188ff5d Mon Sep 17 00:00:00 2001 From: teor Date: Fri, 7 Dec 2018 09:29:39 +1000 Subject: [PATCH 0253/2557] Fallbacks: use a 24 hour consensus expiry tolerance Tor clients will use a consensus that expired up to 24 hours ago. Clients on 0.3.5.5-alpha? and earlier won't select guards from an expired consensus, but they can still bootstrap if they have existing guards. Update the fallback expiry tolerance to match tor's checks. Part of 28768, follow-up on 24661. --- changes/ticket28768 | 4 ++++ scripts/maint/updateFallbackDirs.py | 29 +++++++++++++++++------------ 2 files changed, 21 insertions(+), 12 deletions(-) create mode 100644 changes/ticket28768 diff --git a/changes/ticket28768 b/changes/ticket28768 new file mode 100644 index 0000000000..ce991c8a42 --- /dev/null +++ b/changes/ticket28768 @@ -0,0 +1,4 @@ + o Minor features (fallback directory mirrors): + - Accept fallbacks that deliver reasonably live consensuses. + (Consensuses that expired less than 24 hours ago.) + Closes ticket 28768. diff --git a/scripts/maint/updateFallbackDirs.py b/scripts/maint/updateFallbackDirs.py index cf923bfa32..41c4da675e 100755 --- a/scripts/maint/updateFallbackDirs.py +++ b/scripts/maint/updateFallbackDirs.py @@ -98,19 +98,24 @@ MUST_BE_RUNNING_NOW = (PERFORM_IPV4_DIRPORT_CHECKS # Clients have been using microdesc consensuses by default for a while now DOWNLOAD_MICRODESC_CONSENSUS = True -# If a relay delivers an expired consensus, if it expired less than this many -# seconds ago, we still allow the relay. This should never be less than -90, -# as all directory mirrors should have downloaded a consensus 90 minutes -# before it expires. It should never be more than 24 hours, because clients -# reject consensuses that are older than REASONABLY_LIVE_TIME. -# For the consensus expiry check to be accurate, the machine running this -# script needs an accurate clock. +# If a relay delivers an invalid consensus, if it expired less than this many +# seconds ago, accept the relay as a fallback. For the consensus expiry check +# to be accurate, the machine running this script needs an accurate clock. # -# Relays on 0.3.0 and later return a 404 when they are about to serve an -# expired consensus. This makes them fail the download check. -# We use a tolerance of 0, so that 0.2.x series relays also fail the download -# check if they serve an expired consensus. -CONSENSUS_EXPIRY_TOLERANCE = 0 +# Relays on 0.3.0 and later return a 404 when they are about to serve a +# consensus that expired more than 24 hours ago. 0.2.9 and earlier relays +# will serve consensuses that are very old. +# +# A 404 makes relays fail the download check. We use a tolerance of 24 hours, +# so that 0.2.9 relays also fail the download check if they serve a consensus +# that is not reasonably live. +# +# CONSENSUS_EXPIRY_TOLERANCE should never be more than 24 hours, because +# clients reject consensuses that are older than REASONABLY_LIVE_TIME. Clients +# on 0.3.5.5-alpha? and earlier also won't select guards from consensuses that +# have expired, but can bootstrap if they already have guards in their state +# file. +CONSENSUS_EXPIRY_TOLERANCE = 24*60*60 # Output fallback name, flags, bandwidth, and ContactInfo in a C comment? OUTPUT_COMMENTS = True if OUTPUT_CANDIDATES else False From c3fe405e217d1551b3a58f2469f05650dd9d7579 Mon Sep 17 00:00:00 2001 From: teor Date: Fri, 7 Dec 2018 09:41:28 +1000 Subject: [PATCH 0254/2557] Fallbacks: use a 24 hour future consensus tolerance Tor clients on 0.3.5.6-rc? and later will use a consensus that will become valid up to 24 hours in the future. Clients on 0.3.5.5-alpha? and earlier won't accept future consensuses. Update the fallback expiry tolerance to match tor's checks. Part of 28768, follow-up on 28591. --- changes/ticket28768 | 4 ++-- scripts/maint/updateFallbackDirs.py | 21 ++++++++++++++++++--- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/changes/ticket28768 b/changes/ticket28768 index ce991c8a42..27d90febc8 100644 --- a/changes/ticket28768 +++ b/changes/ticket28768 @@ -1,4 +1,4 @@ o Minor features (fallback directory mirrors): - Accept fallbacks that deliver reasonably live consensuses. - (Consensuses that expired less than 24 hours ago.) - Closes ticket 28768. + (Consensuses that will become valid less than 24 hours in the future, + or that expired less than 24 hours ago.) Closes ticket 28768. diff --git a/scripts/maint/updateFallbackDirs.py b/scripts/maint/updateFallbackDirs.py index 41c4da675e..142d468273 100755 --- a/scripts/maint/updateFallbackDirs.py +++ b/scripts/maint/updateFallbackDirs.py @@ -98,14 +98,19 @@ MUST_BE_RUNNING_NOW = (PERFORM_IPV4_DIRPORT_CHECKS # Clients have been using microdesc consensuses by default for a while now DOWNLOAD_MICRODESC_CONSENSUS = True -# If a relay delivers an invalid consensus, if it expired less than this many -# seconds ago, accept the relay as a fallback. For the consensus expiry check -# to be accurate, the machine running this script needs an accurate clock. +# If a relay delivers an invalid consensus, if it will become valid less than +# this many seconds in the future, or expired less than this many seconds ago, +# accept the relay as a fallback. For the consensus expiry check to be +# accurate, the machine running this script needs an accurate clock. # # Relays on 0.3.0 and later return a 404 when they are about to serve a # consensus that expired more than 24 hours ago. 0.2.9 and earlier relays # will serve consensuses that are very old. # +# Relays on 0.3.5.6-rc? and later return a 404 when they are about to serve a +# consensus that will become valid more than 24 hours in the future. Older +# relays don't serve future consensuses. +# # A 404 makes relays fail the download check. We use a tolerance of 24 hours, # so that 0.2.9 relays also fail the download check if they serve a consensus # that is not reasonably live. @@ -1127,6 +1132,7 @@ class Candidate(object): ).run()[0] end = datetime.datetime.utcnow() time_since_expiry = (end - consensus.valid_until).total_seconds() + time_until_valid = (consensus.valid_after - end).total_seconds() except Exception, stem_error: end = datetime.datetime.utcnow() log_excluded('Unable to retrieve a consensus from %s: %s', nickname, @@ -1151,6 +1157,15 @@ class Candidate(object): status += ', invalid' level = logging.WARNING download_failed = True + elif (time_until_valid > 0): + status = 'future consensus, valid in %ds'%(int(time_until_valid)) + if time_until_valid <= CONSENSUS_EXPIRY_TOLERANCE: + status += ', tolerating up to %ds'%(CONSENSUS_EXPIRY_TOLERANCE) + level = logging.INFO + else: + status += ', invalid' + level = logging.WARNING + download_failed = True else: status = 'ok' level = logging.DEBUG From 7f3a7d9a2713ecddaf0cd8e08e054de4c2870792 Mon Sep 17 00:00:00 2001 From: teor Date: Fri, 7 Dec 2018 15:53:13 +1000 Subject: [PATCH 0255/2557] Fallbacks: Rename CONSENSUS_EXPIRY_TOLERANCE to REASONABLY_LIVE_TIME Cleanup after 28768. --- scripts/maint/updateFallbackDirs.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/scripts/maint/updateFallbackDirs.py b/scripts/maint/updateFallbackDirs.py index 142d468273..914d121345 100755 --- a/scripts/maint/updateFallbackDirs.py +++ b/scripts/maint/updateFallbackDirs.py @@ -115,12 +115,12 @@ DOWNLOAD_MICRODESC_CONSENSUS = True # so that 0.2.9 relays also fail the download check if they serve a consensus # that is not reasonably live. # -# CONSENSUS_EXPIRY_TOLERANCE should never be more than 24 hours, because -# clients reject consensuses that are older than REASONABLY_LIVE_TIME. Clients -# on 0.3.5.5-alpha? and earlier also won't select guards from consensuses that -# have expired, but can bootstrap if they already have guards in their state -# file. -CONSENSUS_EXPIRY_TOLERANCE = 24*60*60 +# REASONABLY_LIVE_TIME should never be more than Tor's REASONABLY_LIVE_TIME, +# (24 hours), because clients reject consensuses that are older than that. +# Clients on 0.3.5.5-alpha? and earlier also won't select guards from +# consensuses that have expired, but can bootstrap if they already have guards +# in their state file. +REASONABLY_LIVE_TIME = 24*60*60 # Output fallback name, flags, bandwidth, and ContactInfo in a C comment? OUTPUT_COMMENTS = True if OUTPUT_CANDIDATES else False @@ -1150,8 +1150,8 @@ class Candidate(object): download_failed = True elif (time_since_expiry > 0): status = 'outdated consensus, expired %ds ago'%(int(time_since_expiry)) - if time_since_expiry <= CONSENSUS_EXPIRY_TOLERANCE: - status += ', tolerating up to %ds'%(CONSENSUS_EXPIRY_TOLERANCE) + if time_since_expiry <= REASONABLY_LIVE_TIME: + status += ', tolerating up to %ds'%(REASONABLY_LIVE_TIME) level = logging.INFO else: status += ', invalid' @@ -1159,8 +1159,8 @@ class Candidate(object): download_failed = True elif (time_until_valid > 0): status = 'future consensus, valid in %ds'%(int(time_until_valid)) - if time_until_valid <= CONSENSUS_EXPIRY_TOLERANCE: - status += ', tolerating up to %ds'%(CONSENSUS_EXPIRY_TOLERANCE) + if time_until_valid <= REASONABLY_LIVE_TIME: + status += ', tolerating up to %ds'%(REASONABLY_LIVE_TIME) level = logging.INFO else: status += ', invalid' From 75b5cc047d3e257701de6dfab9f80d358025fb95 Mon Sep 17 00:00:00 2001 From: teor Date: Fri, 7 Dec 2018 11:16:15 +1000 Subject: [PATCH 0256/2557] Fallbacks: refactor is_in_whitelist() to take an exact match boolean No behaviour change. Preparation for 24838. --- scripts/maint/fallback.whitelist | 23 +-- scripts/maint/updateFallbackDirs.py | 220 ++++++++++++++++++++-------- 2 files changed, 167 insertions(+), 76 deletions(-) diff --git a/scripts/maint/fallback.whitelist b/scripts/maint/fallback.whitelist index 79551948c6..97291d73be 100644 --- a/scripts/maint/fallback.whitelist +++ b/scripts/maint/fallback.whitelist @@ -1,34 +1,21 @@ # updateFallbackDirs.py directory mirror whitelist -# -# Format: -# IPv4:DirPort orport= id= [ ipv6=: ] -# or use: -# scripts/maint/generateFallbackDirLine.py fingerprint ... -# + # All attributes must match for the directory mirror to be included. # If the fallback has an ipv6 key, the whitelist line must also have # it, and vice versa, otherwise they don't match. -# (The blacklist overrides the whitelist.) # To replace this list with the hard-coded fallback list (for testing), use -# a command similar to: +# "updateFallbackDirs.py check_existing", or a command similar to: # cat src/app/config/fallback_dirs.inc | grep \" | grep -v weight | \ # tr -d '\n' | \ # sed 's/"" / /g' | sed 's/""/"/g' | tr \" '\n' | grep -v '^$' \ # > scripts/maint/fallback.whitelist -# -# When testing before a release, exclusions due to changed details will result -# in a warning, unless the IPv4 address or port change happened recently. -# Then it is only logged at info level, as part of the eligibility check. -# Exclusions due to stability also are only shown at info level. -# -# Add the number of selected, slow, and excluded relays, and compare that to -# the number of hard-coded relays. If it's less, use info-level logs to find -# out why each of the missing relays was excluded. # If a relay operator wants their relay to be a FallbackDir, # enter the following information here: -# : orport= id= [ ipv6=: ] +# : orport= id= ( ipv6=[]: )? +# or use: +# scripts/maint/generateFallbackDirLine.py fingerprint ... # https://lists.torproject.org/pipermail/tor-relays/2015-December/008362.html # https://trac.torproject.org/projects/tor/ticket/22321#comment:22 diff --git a/scripts/maint/updateFallbackDirs.py b/scripts/maint/updateFallbackDirs.py index 914d121345..b334352152 100755 --- a/scripts/maint/updateFallbackDirs.py +++ b/scripts/maint/updateFallbackDirs.py @@ -920,61 +920,155 @@ class Candidate(object): return False return True - def is_in_whitelist(self, relaylist): - """ A fallback matches if each key in the whitelist line matches: + def id_matches(self, id, exact=False): + """ Does this fallback's id match id? + exact is ignored. """ + return self._fpr == id + + def ipv4_addr_matches(self, ipv4_addr, exact=False): + """ Does this fallback's IPv4 address match ipv4_addr? + exact is ignored. """ + return self.dirip == ipv4_addr + + def ipv4_dirport_matches(self, ipv4_dirport, exact=False): + """ Does this fallback's IPv4 dirport match ipv4_dirport? + If exact is False, always return True. """ + if exact: + return self.dirport == int(ipv4_dirport) + else: + return True + + def ipv4_and_dirport_matches(self, ipv4_addr, ipv4_dirport, exact=False): + """ Does this fallback's IPv4 address match ipv4_addr? + If exact is True, also check ipv4_dirport. """ + ipv4_match = self.ipv4_addr_matches(ipv4_addr, exact=exact) + if exact: + return ipv4_match and self.ipv4_dirport_matches(ipv4_dirport, + exact=exact) + else: + return ipv4_match + + def ipv4_orport_matches(self, ipv4_orport, exact=False): + """ Does this fallback's IPv4 orport match ipv4_orport? + If exact is False, always return True. """ + if exact: + return self.orport == int(ipv4_orport) + else: + return True + + def ipv4_and_orport_matches(self, ipv4_addr, ipv4_orport, exact=False): + """ Does this fallback's IPv4 address match ipv4_addr? + If exact is True, also check ipv4_orport. """ + ipv4_match = self.ipv4_addr_matches(ipv4_addr, exact=exact) + if exact: + return ipv4_match and self.ipv4_orport_matches(ipv4_orport, + exact=exact) + else: + return ipv4_match + + def ipv6_addr_matches(self, ipv6_addr, exact=False): + """ Does this fallback's IPv6 address match ipv6_addr? + Both addresses must be present to match. + exact is ignored. """ + if self.has_ipv6() and ipv6_addr is not None: + # Check that we have a bracketed IPv6 address without a port + assert(ipv6_addr.startswith('[') and ipv6_addr.endswith(']')) + return self.ipv6addr == ipv6_addr + else: + return False + + def ipv6_orport_matches(self, ipv6_orport, exact=False): + """ Does this fallback's IPv6 orport match ipv6_orport? + Both ports must be present to match. + If exact is False, always return True. """ + if exact: + return (self.has_ipv6() and ipv6_orport is not None and + self.ipv6orport == int(ipv6_orport)) + else: + return True + + def ipv6_and_orport_matches(self, ipv6_addr, ipv6_orport, exact=False): + """ Does this fallback's IPv6 address match ipv6_addr? + If exact is True, also check ipv6_orport. """ + ipv6_match = self.ipv6_addr_matches(ipv6_addr, exact=exact) + if exact: + return ipv6_match and self.ipv6_orport_matches(ipv6_orport, + exact=exact) + else: + return ipv6_match + + def entry_matches_exact(self, entry): + """ Is entry an exact match for this fallback? + A fallback is an exact match for entry if each key in entry matches: ipv4 dirport orport id - ipv6 address and port (if present) + ipv6 address and port (if present in the fallback or the whitelist) If the fallback has an ipv6 key, the whitelist line must also have - it, and vice versa, otherwise they don't match. """ - ipv6 = None - if self.has_ipv6(): - ipv6 = '%s:%d'%(self.ipv6addr, self.ipv6orport) + it, otherwise they don't match. + + Logs a warning-level message if the fallback would be an exact match, + but one of the id, ipv4, ipv4 orport, ipv4 dirport, or ipv6 orport + have changed. """ + if not self.id_matches(entry['id'], exact=True): + # can't log here unless we match an IP and port, because every relay's + # fingerprint is compared to every entry's fingerprint + if self.ipv4_and_orport_matches(entry['ipv4'], + entry['orport'], + exact=True): + logging.warning('%s excluded: has OR %s:%d changed fingerprint to ' + + '%s?', entry['id'], self.dirip, self.orport, + self._fpr) + if self.ipv6_and_orport_matches(entry.get('ipv6_addr'), + entry.get('ipv6_orport'), + exact=True): + logging.warning('%s excluded: has OR %s changed fingerprint to ' + + '%s?', entry['id'], entry['ipv6'], self._fpr) + return False + if not self.ipv4_addr_matches(entry['ipv4'], exact=True): + logging.warning('%s excluded: has it changed IPv4 from %s to %s?', + self._fpr, entry['ipv4'], self.dirip) + return False + if not self.ipv4_dirport_matches(entry['dirport'], exact=True): + logging.warning('%s excluded: has it changed DirPort from %s:%d to ' + + '%s:%d?', self._fpr, self.dirip, int(entry['dirport']), + self.dirip, self.dirport) + return False + if not self.ipv4_orport_matches(entry['orport'], exact=True): + logging.warning('%s excluded: has it changed ORPort from %s:%d to ' + + '%s:%d?', self._fpr, self.dirip, int(entry['orport']), + self.dirip, self.orport) + return False + if entry.has_key('ipv6') and self.has_ipv6(): + # if both entry and fallback have an ipv6 address, compare them + if not self.ipv6_and_orport_matches(entry['ipv6_addr'], + entry['ipv6_orport'], + exact=True): + logging.warning('%s excluded: has it changed IPv6 ORPort from %s ' + + 'to %s:%d?', self._fpr, entry['ipv6'], + self.ipv6addr, self.ipv6orport) + return False + # if the fallback has an IPv6 address but the whitelist entry + # doesn't, or vice versa, the whitelist entry doesn't match + elif entry.has_key('ipv6') and not self.has_ipv6(): + logging.warning('%s excluded: has it lost its former IPv6 address %s?', + self._fpr, entry['ipv6']) + return False + elif not entry.has_key('ipv6') and self.has_ipv6(): + logging.warning('%s excluded: has it gained an IPv6 address %s:%d?', + self._fpr, self.ipv6addr, self.ipv6orport) + return False + return True + + def is_in_whitelist(self, relaylist, exact=False): + """ If exact is True (existing fallback list), check if this fallback is + an exact match for any whitelist entry, using entry_matches_exact(). + """ for entry in relaylist: - if entry['id'] != self._fpr: - # can't log here unless we match an IP and port, because every relay's - # fingerprint is compared to every entry's fingerprint - if entry['ipv4'] == self.dirip and int(entry['orport']) == self.orport: - logging.warning('%s excluded: has OR %s:%d changed fingerprint to ' + - '%s?', entry['id'], self.dirip, self.orport, - self._fpr) - if self.has_ipv6() and entry.has_key('ipv6') and entry['ipv6'] == ipv6: - logging.warning('%s excluded: has OR %s changed fingerprint to ' + - '%s?', entry['id'], ipv6, self._fpr) - continue - if entry['ipv4'] != self.dirip: - logging.warning('%s excluded: has it changed IPv4 from %s to %s?', - self._fpr, entry['ipv4'], self.dirip) - continue - if int(entry['dirport']) != self.dirport: - logging.warning('%s excluded: has it changed DirPort from %s:%d to ' + - '%s:%d?', self._fpr, self.dirip, int(entry['dirport']), - self.dirip, self.dirport) - continue - if int(entry['orport']) != self.orport: - logging.warning('%s excluded: has it changed ORPort from %s:%d to ' + - '%s:%d?', self._fpr, self.dirip, int(entry['orport']), - self.dirip, self.orport) - continue - if entry.has_key('ipv6') and self.has_ipv6(): - # if both entry and fallback have an ipv6 address, compare them - if entry['ipv6'] != ipv6: - logging.warning('%s excluded: has it changed IPv6 ORPort from %s ' + - 'to %s?', self._fpr, entry['ipv6'], ipv6) - continue - # if the fallback has an IPv6 address but the whitelist entry - # doesn't, or vice versa, the whitelist entry doesn't match - elif entry.has_key('ipv6') and not self.has_ipv6(): - logging.warning('%s excluded: has it lost its former IPv6 address %s?', - self._fpr, entry['ipv6']) - continue - elif not entry.has_key('ipv6') and self.has_ipv6(): - logging.warning('%s excluded: has it gained an IPv6 address %s?', - self._fpr, ipv6) - continue - return True + if exact: + if self.entry_matches_exact(entry): + return True return False def cw_to_bw_factor(self): @@ -1458,18 +1552,28 @@ class CandidateList(dict): relay_entry['dirport'] = ipv4_maybe_dirport_split[1] elif kvl == 2: relay_entry[key_value_split[0]] = key_value_split[1] + # split ipv6 addresses and orports + if key_value_split[0] == 'ipv6': + ipv6_orport_split = key_value_split[1].rsplit(':', 1) + ipv6l = len(ipv6_orport_split) + if ipv6l != 2: + print '#error Bad %s IPv6 item: %s, format is [ipv6]:orport.'%( + file_name, item) + relay_entry['ipv6_addr'] = ipv6_orport_split[0] + relay_entry['ipv6_orport'] = ipv6_orport_split[1] relaylist.append(relay_entry) return relaylist - # apply the fallback whitelist - def apply_filter_lists(self, whitelist_obj): + def apply_filter_lists(self, whitelist_obj, exact=False): + """ Apply the fallback whitelist_obj to this fallback list, + passing exact to is_in_whitelist(). """ excluded_count = 0 logging.debug('Applying whitelist') # parse the whitelist whitelist = self.load_relaylist(whitelist_obj) filtered_fallbacks = [] for f in self.fallbacks: - in_whitelist = f.is_in_whitelist(whitelist) + in_whitelist = f.is_in_whitelist(whitelist, exact=exact) if in_whitelist: # include filtered_fallbacks.append(f) @@ -2082,14 +2186,14 @@ def process_existing(): logging.getLogger('stem').setLevel(logging.INFO) whitelist = {'data': parse_fallback_file(FALLBACK_FILE_NAME), 'name': FALLBACK_FILE_NAME} - list_fallbacks(whitelist) + list_fallbacks(whitelist, exact=True) def process_default(): logging.basicConfig(level=logging.WARNING) logging.getLogger('stem').setLevel(logging.WARNING) whitelist = {'data': read_from_file(WHITELIST_FILE_NAME, MAX_LIST_FILE_SIZE), 'name': WHITELIST_FILE_NAME} - list_fallbacks(whitelist) + list_fallbacks(whitelist, exact=True) ## Main Function def main(): @@ -2110,10 +2214,10 @@ def log_excluded(msg, *args): else: logging.info(msg, *args) -def list_fallbacks(whitelist): +def list_fallbacks(whitelist, exact=False): """ Fetches required onionoo documents and evaluates the - fallback directory criteria for each of the relays """ - + fallback directory criteria for each of the relays, + passing exact to apply_filter_lists(). """ print "/* type=fallback */" print ("/* version={} */" .format(cleanse_c_multiline_comment(FALLBACK_FORMAT_VERSION))) @@ -2153,7 +2257,7 @@ def list_fallbacks(whitelist): # warning that the details have changed from those in the whitelist. # instead, there will be an info-level log during the eligibility check. initial_count = len(candidates.fallbacks) - excluded_count = candidates.apply_filter_lists(whitelist) + excluded_count = candidates.apply_filter_lists(whitelist, exact=exact) print candidates.summarise_filters(initial_count, excluded_count) eligible_count = len(candidates.fallbacks) From 6bc5c06dc25630c5a5f97f1da2af29f88683d243 Mon Sep 17 00:00:00 2001 From: teor Date: Fri, 7 Dec 2018 13:23:57 +1000 Subject: [PATCH 0257/2557] Fallbacks: accept relays that are a fuzzy match to the whitelist If a relay matches at least one fingerprint, IPv4 address, or IPv6 address in the fallback whitelist, it can become a fallback. This reduces the work required to keep the list up to date. Closes ticket 28768. --- changes/ticket24838 | 6 ++++++ scripts/maint/fallback.whitelist | 26 +++++++++++++------------ scripts/maint/updateFallbackDirs.py | 30 +++++++++++++++++++++++++++-- 3 files changed, 48 insertions(+), 14 deletions(-) create mode 100644 changes/ticket24838 diff --git a/changes/ticket24838 b/changes/ticket24838 new file mode 100644 index 0000000000..d068e31b91 --- /dev/null +++ b/changes/ticket24838 @@ -0,0 +1,6 @@ + o Minor features (fallback directory mirrors): + - Accept relays that are a fuzzy match to a fallback whitelist entry. + If a relay matches at least one fingerprint, IPv4 address, or IPv6 + address in the fallback whitelist, it can become a fallback. This + reduces the work required to keep the list up to date. + Closes ticket 24838. diff --git a/scripts/maint/fallback.whitelist b/scripts/maint/fallback.whitelist index 97291d73be..eb7fd92d4b 100644 --- a/scripts/maint/fallback.whitelist +++ b/scripts/maint/fallback.whitelist @@ -1,16 +1,18 @@ # updateFallbackDirs.py directory mirror whitelist - -# All attributes must match for the directory mirror to be included. -# If the fallback has an ipv6 key, the whitelist line must also have -# it, and vice versa, otherwise they don't match. - -# To replace this list with the hard-coded fallback list (for testing), use -# "updateFallbackDirs.py check_existing", or a command similar to: -# cat src/app/config/fallback_dirs.inc | grep \" | grep -v weight | \ -# tr -d '\n' | \ -# sed 's/"" / /g' | sed 's/""/"/g' | tr \" '\n' | grep -v '^$' \ -# > scripts/maint/fallback.whitelist - +# +# At least one of these keys must match for a directory mirror to be included +# in the fallback list: +# id +# ipv4 +# ipv6 +# The ports and nickname are ignored. Missing or extra ipv6 addresses +# are ignored. +# +# The latest relay details from Onionoo are included in the generated list. +# +# To check the hard-coded fallback list (for testing), use: +# $ updateFallbackDirs.py check_existing +# # If a relay operator wants their relay to be a FallbackDir, # enter the following information here: # : orport= id= ( ipv6=[]: )? diff --git a/scripts/maint/updateFallbackDirs.py b/scripts/maint/updateFallbackDirs.py index b334352152..14372d0e83 100755 --- a/scripts/maint/updateFallbackDirs.py +++ b/scripts/maint/updateFallbackDirs.py @@ -1061,14 +1061,40 @@ class Candidate(object): return False return True + def entry_matches_fuzzy(self, entry): + """ Is entry a fuzzy match for this fallback? + A fallback is a fuzzy match for entry if at least one of these keys + in entry matches: + id + ipv4 + ipv6 (if present in both the fallback and whitelist) + The ports and nickname are ignored. Missing or extra ipv6 addresses + are ignored. + + Doesn't log any warning messages. """ + if self.id_matches(entry['id'], exact=False): + return True + if self.ipv4_addr_matches(entry['ipv4'], exact=False): + return True + if entry.has_key('ipv6') and self.has_ipv6(): + # if both entry and fallback have an ipv6 address, compare them + if self.ipv6_addr_matches(entry['ipv6_addr'], exact=False): + return True + return False + def is_in_whitelist(self, relaylist, exact=False): """ If exact is True (existing fallback list), check if this fallback is an exact match for any whitelist entry, using entry_matches_exact(). - """ + + If exact is False (new fallback whitelist), check if this fallback is + a fuzzy match for any whitelist entry, using entry_matches_fuzzy(). """ for entry in relaylist: if exact: if self.entry_matches_exact(entry): return True + else: + if self.entry_matches_fuzzy(entry): + return True return False def cw_to_bw_factor(self): @@ -2193,7 +2219,7 @@ def process_default(): logging.getLogger('stem').setLevel(logging.WARNING) whitelist = {'data': read_from_file(WHITELIST_FILE_NAME, MAX_LIST_FILE_SIZE), 'name': WHITELIST_FILE_NAME} - list_fallbacks(whitelist, exact=True) + list_fallbacks(whitelist, exact=False) ## Main Function def main(): From d9b9c1fa76dacacd502141aeca08c5a9722d0898 Mon Sep 17 00:00:00 2001 From: teor Date: Fri, 7 Dec 2018 17:03:46 +1000 Subject: [PATCH 0258/2557] Changes file for 24805 --- changes/ticket24805 | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 changes/ticket24805 diff --git a/changes/ticket24805 b/changes/ticket24805 new file mode 100644 index 0000000000..4ba6f6ecd4 --- /dev/null +++ b/changes/ticket24805 @@ -0,0 +1,3 @@ + o Minor features (fallback directory list): + - Update the fallback whitelist based on operator opt-ins and opt-outs. + Closes ticket 24805, patch by Phoul. From da264f7c766b332f596a92766f7625c4a17abf70 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Sat, 8 Dec 2018 15:15:20 +0200 Subject: [PATCH 0259/2557] Let's not double-quote OUTPUTARG after all --- scripts/test/scan-build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/test/scan-build.sh b/scripts/test/scan-build.sh index 9a63383804..26e05ff101 100755 --- a/scripts/test/scan-build.sh +++ b/scripts/test/scan-build.sh @@ -72,7 +72,7 @@ scan-build \ # shellcheck disable=SC2086 scan-build \ - $CHECKERS "$OUTPUTARG" \ + $CHECKERS $OUTPUTARG \ make -j5 -k CHECKERS="\ From f0a8664677b8e4a3503172d6e7564da33496be8f Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 6 Dec 2018 14:13:29 -0500 Subject: [PATCH 0260/2557] Add code to parse K=V lines into config_line_t format. Closes ticket 28755 --- src/lib/encoding/.may_include | 1 + src/lib/encoding/include.am | 2 + src/lib/encoding/kvline.c | 239 ++++++++++++++++++++++++++++++++++ src/lib/encoding/kvline.h | 24 ++++ src/test/test_config.c | 78 +++++++++++ 5 files changed, 344 insertions(+) create mode 100644 src/lib/encoding/kvline.c create mode 100644 src/lib/encoding/kvline.h diff --git a/src/lib/encoding/.may_include b/src/lib/encoding/.may_include index 7c2ef36929..c9bf4b1786 100644 --- a/src/lib/encoding/.may_include +++ b/src/lib/encoding/.may_include @@ -1,5 +1,6 @@ orconfig.h lib/cc/*.h +lib/container/*.h lib/ctime/*.h lib/encoding/*.h lib/intmath/*.h diff --git a/src/lib/encoding/include.am b/src/lib/encoding/include.am index 2d2aa3988a..83e9211b6f 100644 --- a/src/lib/encoding/include.am +++ b/src/lib/encoding/include.am @@ -9,6 +9,7 @@ src_lib_libtor_encoding_a_SOURCES = \ src/lib/encoding/confline.c \ src/lib/encoding/cstring.c \ src/lib/encoding/keyval.c \ + src/lib/encoding/kvline.c \ src/lib/encoding/pem.c \ src/lib/encoding/time_fmt.c @@ -22,5 +23,6 @@ noinst_HEADERS += \ src/lib/encoding/confline.h \ src/lib/encoding/cstring.h \ src/lib/encoding/keyval.h \ + src/lib/encoding/kvline.h \ src/lib/encoding/pem.h \ src/lib/encoding/time_fmt.h diff --git a/src/lib/encoding/kvline.c b/src/lib/encoding/kvline.c new file mode 100644 index 0000000000..11ff4f0f96 --- /dev/null +++ b/src/lib/encoding/kvline.c @@ -0,0 +1,239 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file kvline.c + * + * \brief Manipulating lines of key-value pairs. + **/ + +#include "orconfig.h" + +#include "lib/container/smartlist.h" +#include "lib/encoding/confline.h" +#include "lib/encoding/cstring.h" +#include "lib/encoding/kvline.h" +#include "lib/malloc/malloc.h" +#include "lib/string/compat_ctype.h" +#include "lib/string/printf.h" +#include "lib/string/util_string.h" +#include "lib/log/escape.h" +#include "lib/log/util_bug.h" + +#include +#include +#include + +/** Return true iff we need to quote and escape the string s to encode + * it. */ +static bool +needs_escape(const char *s, bool as_keyless_val) +{ + if (as_keyless_val && *s == 0) + return true; + + for (; *s; ++s) { + if (*s >= 127 || TOR_ISSPACE(*s) || ! TOR_ISPRINT(*s) || + *s == '\'' || *s == '\"') { + return true; + } + } + return false; +} + +/** + * Return true iff the key in line is not set. + **/ +static bool +line_has_no_key(const config_line_t *line) +{ + return line->key == NULL || strlen(line->key) == 0; +} + +/** + * Return true iff the all the lines in line can be encoded + * using flags. + **/ +static bool +kvline_can_encode_lines(const config_line_t *line, unsigned flags) +{ + for ( ; line; line = line->next) { + const bool keyless = line_has_no_key(line); + if (keyless) { + if (! (flags & KV_OMIT_KEYS)) { + /* If KV_OMIT_KEYS is not set, we can't encode a line with no key. */ + return false; + } + if (strchr(line->value, '=') && !( flags & KV_QUOTED)) { + /* We can't have a keyless value with = without quoting it. */ + return false; + } + } + + if (needs_escape(line->value, keyless) && ! (flags & KV_QUOTED)) { + /* If KV_QUOTED is false, we can't encode a value that needs quotes. */ + return false; + } + if (line->key && strlen(line->key) && + (needs_escape(line->key, false) || strchr(line->key, '='))) { + /* We can't handle keys that need quoting. */ + return false; + } + } + return true; +} + +/** + * Encode a linked list of lines in line as a series of 'Key=Value' + * pairs, using the provided flags to encode it. Return a newly + * allocated string on success, or NULL on failure. + * + * If KV_QUOTED is set in flags, then all values that contain + * spaces or unusual characters are escaped and quoted. Otherwise, such + * values are not allowed. + * + * If KV_OMIT_KEYS is set in flags, then pairs with empty keys are + * allowed, and are encoded as 'Value'. Otherwise, such pairs are not + * allowed. + */ +char * +kvline_encode(const config_line_t *line, + unsigned flags) +{ + if (!kvline_can_encode_lines(line, flags)) + return NULL; + + smartlist_t *elements = smartlist_new(); + + for (; line; line = line->next) { + + const char *k = ""; + const char *eq = "="; + const char *v = ""; + const bool keyless = line_has_no_key(line); + bool esc = needs_escape(line->value, keyless); + char *tmp = NULL; + + if (! keyless) { + k = line->key; + } else { + eq = ""; + if (strchr(line->value, '=')) { + esc = true; + } + } + + if (esc) { + tmp = esc_for_log(line->value); + v = tmp; + } else { + v = line->value; + } + + smartlist_add_asprintf(elements, "%s%s%s", k, eq, v); + tor_free(tmp); + } + + char *result = smartlist_join_strings(elements, " ", 0, NULL); + + SMARTLIST_FOREACH(elements, char *, cp, tor_free(cp)); + smartlist_free(elements); + + return result; +} + +/** + * Decode a line containing a series of space-separated 'Key=Value' + * pairs, using the provided flags to decode it. Return a newly + * allocated list of pairs on success, or NULL on failure. + * + * If KV_QUOTED is set in flags, then (double-)quoted values are + * allowed. Otherwise, such values are not allowed. + * + * If KV_OMIT_KEYS is set in flags, then values without keys are + * allowed. Otherwise, such values are not allowed. + */ +config_line_t * +kvline_parse(const char *line, unsigned flags) +{ + const char *cp = line, *cplast = NULL; + bool omit_keys = (flags & KV_OMIT_KEYS) != 0; + bool quoted = (flags & KV_QUOTED) != 0; + + config_line_t *result = NULL; + config_line_t **next_line = &result; + + char *key = NULL; + char *val = NULL; + + while (*cp) { + key = val = NULL; + { + size_t idx = strspn(cp, " \t\r\v\n"); + cp += idx; + } + if (BUG(cp == cplast)) { + /* If we didn't parse anything, this code is broken. */ + goto err; // LCOV_EXCL_LINE + } + cplast = cp; + if (! *cp) + break; /* End of string; we're done. */ + + /* Possible formats are K=V, K="V", V, and "V", depending on flags. */ + + /* Find the key. */ + if (*cp != '\"') { + size_t idx = strcspn(cp, " \t\r\v\n="); + + if (cp[idx] == '=') { + key = tor_memdup_nulterm(cp, idx); + cp += idx + 1; + } else { + if (!omit_keys) + goto err; + } + } + + if (*cp == '\"') { + /* The type is "V". */ + if (!quoted) + goto err; + size_t len=0; + cp = unescape_string(cp, &val, &len); + if (cp == NULL || len != strlen(val)) { + // The string contains a NUL or is badly coded. + goto err; + } + } else { + size_t idx = strcspn(cp, " \t\r\v\n"); + val = tor_memdup_nulterm(cp, idx); + cp += idx; + } + + if (key && strlen(key) == 0) { + /* We don't allow empty keys. */ + goto err; + } + + *next_line = tor_malloc_zero(sizeof(config_line_t)); + (*next_line)->key = key ? key : tor_strdup(""); + (*next_line)->value = val; + next_line = &(*next_line)->next; + key = val = NULL; + } + + if (!kvline_can_encode_lines(result, flags)) { + goto err; + } + return result; + + err: + tor_free(key); + tor_free(val); + config_free_lines(result); + return NULL; +} diff --git a/src/lib/encoding/kvline.h b/src/lib/encoding/kvline.h new file mode 100644 index 0000000000..3272cc1754 --- /dev/null +++ b/src/lib/encoding/kvline.h @@ -0,0 +1,24 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file kvline.h + * + * \brief Header for kvline.c + **/ + +#ifndef TOR_KVLINE_H +#define TOR_KVLINE_H + +struct config_line_t; + +#define KV_QUOTED (1u<<0) +#define KV_OMIT_KEYS (1u<<1) + +struct config_line_t *kvline_parse(const char *line, unsigned flags); +char *kvline_encode(const struct config_line_t *line, unsigned flags); + +#endif /* !defined(TOR_KVLINE_H) */ diff --git a/src/test/test_config.c b/src/test/test_config.c index dae4d83766..5140c3c1a8 100644 --- a/src/test/test_config.c +++ b/src/test/test_config.c @@ -54,6 +54,7 @@ #include "lib/meminfo/meminfo.h" #include "lib/net/gethostname.h" #include "lib/encoding/confline.h" +#include "lib/encoding/kvline.h" #ifdef HAVE_UNISTD_H #include @@ -5813,6 +5814,82 @@ test_config_extended_fmt(void *arg) config_free_lines(lines); } +static void +test_config_kvline_parse(void *arg) +{ + (void)arg; + + config_line_t *lines = NULL; + char *enc = NULL; + + lines = kvline_parse("A=B CD=EF", 0); + tt_assert(lines); + tt_str_op(lines->key, OP_EQ, "A"); + tt_str_op(lines->value, OP_EQ, "B"); + tt_str_op(lines->next->key, OP_EQ, "CD"); + tt_str_op(lines->next->value, OP_EQ, "EF"); + enc = kvline_encode(lines, 0); + tt_str_op(enc, OP_EQ, "A=B CD=EF"); + tor_free(enc); + enc = kvline_encode(lines, KV_QUOTED|KV_OMIT_KEYS); + tt_str_op(enc, OP_EQ, "A=B CD=EF"); + tor_free(enc); + config_free_lines(lines); + + lines = kvline_parse("AB CDE=F", 0); + tt_assert(! lines); + + lines = kvline_parse("AB CDE=F", KV_OMIT_KEYS); + tt_assert(lines); + tt_str_op(lines->key, OP_EQ, ""); + tt_str_op(lines->value, OP_EQ, "AB"); + tt_str_op(lines->next->key, OP_EQ, "CDE"); + tt_str_op(lines->next->value, OP_EQ, "F"); + tt_assert(lines); + enc = kvline_encode(lines, 0); + tt_assert(!enc); + enc = kvline_encode(lines, KV_QUOTED|KV_OMIT_KEYS); + tt_str_op(enc, OP_EQ, "AB CDE=F"); + tor_free(enc); + config_free_lines(lines); + + lines = kvline_parse("AB=C CDE=\"F G\"", 0); + tt_assert(!lines); + + lines = kvline_parse("AB=C CDE=\"F G\" \"GHI\" ", KV_QUOTED|KV_OMIT_KEYS); + tt_assert(lines); + tt_str_op(lines->key, OP_EQ, "AB"); + tt_str_op(lines->value, OP_EQ, "C"); + tt_str_op(lines->next->key, OP_EQ, "CDE"); + tt_str_op(lines->next->value, OP_EQ, "F G"); + tt_str_op(lines->next->next->key, OP_EQ, ""); + tt_str_op(lines->next->next->value, OP_EQ, "GHI"); + enc = kvline_encode(lines, 0); + tt_assert(!enc); + enc = kvline_encode(lines, KV_QUOTED|KV_OMIT_KEYS); + tt_str_op(enc, OP_EQ, "AB=C CDE=\"F G\" GHI"); + tor_free(enc); + config_free_lines(lines); + + lines = kvline_parse("A\"B=C CDE=\"F\" \"GHI\" ", KV_QUOTED|KV_OMIT_KEYS); + tt_assert(! lines); + + lines = kvline_parse("AB=", KV_QUOTED); + tt_assert(lines); + tt_str_op(lines->key, OP_EQ, "AB"); + tt_str_op(lines->value, OP_EQ, ""); + config_free_lines(lines); + + lines = kvline_parse("AB=", 0); + tt_assert(lines); + tt_str_op(lines->key, OP_EQ, "AB"); + tt_str_op(lines->value, OP_EQ, ""); + + done: + config_free_lines(lines); + tor_free(enc); +} + #define CONFIG_TEST(name, flags) \ { #name, test_config_ ## name, flags, NULL, NULL } @@ -5864,5 +5941,6 @@ struct testcase_t config_tests[] = { CONFIG_TEST(include_opened_file_list, 0), CONFIG_TEST(compute_max_mem_in_queues, 0), CONFIG_TEST(extended_fmt, 0), + CONFIG_TEST(kvline_parse, 0), END_OF_TESTCASES }; From 845e8dbe5904b4b2d3eb2db9e2681ea2f4d98008 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 10 Dec 2018 10:00:26 -0500 Subject: [PATCH 0261/2557] Fuzzing module for various string operations, currently focusing on encoding and decoding. There are bunches of places where we don't want to invest in a full fuzzer, but we would like to make sure that some string operation can handle all its possible inputs. This fuzzer uses the first byte of its input to decide what to do with the rest of the input. Right now, all the possibilities are decoding a string, and seeing whether it is decodeable. If it is, we try to re-encode it and do the whole thing again, to make sure we get the same result. This turned up a lot of bugs in the key-value parser, and I think it will help in other cases too. Closes ticket 28808. --- scripts/codegen/fuzzing_include_am.py | 1 + src/test/fuzz/fuzz_strops.c | 247 ++++++++++++++++++++++++++ src/test/fuzz/include.am | 29 +++ 3 files changed, 277 insertions(+) create mode 100644 src/test/fuzz/fuzz_strops.c diff --git a/scripts/codegen/fuzzing_include_am.py b/scripts/codegen/fuzzing_include_am.py index 3c948d87cf..5b51158843 100755 --- a/scripts/codegen/fuzzing_include_am.py +++ b/scripts/codegen/fuzzing_include_am.py @@ -13,6 +13,7 @@ FUZZERS = """ iptsv2 microdesc socks + strops vrs """ diff --git a/src/test/fuzz/fuzz_strops.c b/src/test/fuzz/fuzz_strops.c new file mode 100644 index 0000000000..5da590acfa --- /dev/null +++ b/src/test/fuzz/fuzz_strops.c @@ -0,0 +1,247 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file fuzz_strops.c + * \brief Fuzzers for various string encoding/decoding operations + **/ + +#include "orconfig.h" + +#include "lib/cc/torint.h" +#include "lib/ctime/di_ops.h" +#include "lib/encoding/binascii.h" +#include "lib/encoding/cstring.h" +#include "lib/encoding/kvline.h" +#include "lib/encoding/confline.h" +#include "lib/malloc/malloc.h" +#include "lib/log/escape.h" +#include "lib/log/util_bug.h" +#include "lib/intmath/muldiv.h" + +#include "test/fuzz/fuzzing.h" + +#include +#include + +int +fuzz_init(void) +{ + return 0; +} + +int +fuzz_cleanup(void) +{ + return 0; +} + +typedef struct chunk_t { + uint8_t *buf; + size_t len; +} chunk_t; + +#define chunk_free(ch) \ + FREE_AND_NULL(chunk_t, chunk_free_, (ch)) + +static chunk_t * +chunk_new(size_t len) +{ + chunk_t *ch = tor_malloc(sizeof(chunk_t)); + ch->buf = tor_malloc(len); + ch->len = len; + return ch; +} +static void +chunk_free_(chunk_t *ch) +{ + if (!ch) + return; + tor_free(ch->buf); + tor_free(ch); +} +static bool +chunk_eq(const chunk_t *a, const chunk_t *b) +{ + return a->len == b->len && fast_memeq(a->buf, b->buf, a->len); +} + +static chunk_t * +b16_dec(const chunk_t *inp) +{ + chunk_t *ch = chunk_new(CEIL_DIV(inp->len, 2)); + int r = base16_decode((char *)ch->buf, ch->len, (char *)inp->buf, inp->len); + if (r >= 0) { + ch->len = r; + } else { + chunk_free(ch); + } + return ch; +} +static chunk_t * +b16_enc(const chunk_t *inp) +{ + chunk_t *ch = chunk_new(inp->len * 2 + 1); + base16_encode((char *)ch->buf, ch->len, (char*)inp->buf, inp->len); + return ch; +} + +#if 0 +static chunk_t * +b32_dec(const chunk_t *inp) +{ + chunk_t *ch = chunk_new(inp->len);//XXXX + int r = base32_decode((char *)ch->buf, ch->len, (char *)inp->buf, inp->len); + if (r >= 0) { + ch->len = r; // XXXX we need some way to get the actual length of + // XXXX the output here. + } else { + chunk_free(ch); + } + return ch; +} +static chunk_t * +b32_enc(const chunk_t *inp) +{ + chunk_t *ch = chunk_new(base32_encoded_size(inp->len)); + base32_encode((char *)ch->buf, ch->len, (char*)inp->buf, inp->len); + ch->len = strlen((char *) ch->buf); + return ch; +} +#endif + +static chunk_t * +b64_dec(const chunk_t *inp) +{ + chunk_t *ch = chunk_new(inp->len);//XXXX This could be shorter. + int r = base64_decode((char *)ch->buf, ch->len, (char *)inp->buf, inp->len); + if (r >= 0) { + ch->len = r; + } else { + chunk_free(ch); + } + return ch; +} +static chunk_t * +b64_enc(const chunk_t *inp) +{ + chunk_t *ch = chunk_new(BASE64_BUFSIZE(inp->len)); + base64_encode((char *)ch->buf, ch->len, (char *)inp->buf, inp->len, 0); + ch->len = strlen((char *) ch->buf); + return ch; +} + +static chunk_t * +c_dec(const chunk_t *inp) +{ + char *s = tor_memdup_nulterm(inp->buf, inp->len); + chunk_t *ch = tor_malloc(sizeof(chunk_t)); + char *r = NULL; + (void) unescape_string(s, &r, &ch->len); + tor_free(s); + ch->buf = (uint8_t*) r; + if (!ch->buf) { + tor_free(ch); + } + return ch; +} +static chunk_t * +c_enc(const chunk_t *inp) +{ + char *s = tor_memdup_nulterm(inp->buf, inp->len); + chunk_t *ch = tor_malloc(sizeof(chunk_t)); + ch->buf = (uint8_t*)esc_for_log(s); + tor_free(s); + ch->len = strlen((char*)ch->buf); + return ch; +} + +static int kv_flags = 0; +static config_line_t * +kv_dec(const chunk_t *inp) +{ + char *s = tor_memdup_nulterm(inp->buf, inp->len); + config_line_t *res = kvline_parse(s, kv_flags); + tor_free(s); + return res; +} +static chunk_t * +kv_enc(const config_line_t *inp) +{ + char *s = kvline_encode(inp, kv_flags); + if (!s) + return NULL; + chunk_t *res = tor_malloc(sizeof(chunk_t)); + res->buf = (uint8_t*)s; + res->len = strlen(s); + return res; +} + +/* Given an encoder function, a decoder function, and a function to free + * the decoded object, check whether any string that successfully decoded + * will then survive an encode-decode-encode round-trip unchanged. + */ +#define ENCODE_ROUNDTRIP(E,D,FREE) \ + STMT_BEGIN { \ + bool err = false; \ + a = D(&inp); \ + if (!a) \ + return 0; \ + b = E(a); \ + tor_assert(b); \ + c = D(b); \ + tor_assert(c); \ + d = E(c); \ + tor_assert(d); \ + if (!chunk_eq(b,d)) { \ + printf("Unequal chunks: %s\n", \ + hex_str((char*)b->buf, b->len)); \ + printf(" vs %s\n", \ + hex_str((char*)d->buf, d->len)); \ + err = true; \ + } \ + FREE(a); \ + chunk_free(b); \ + FREE(c); \ + chunk_free(d); \ + tor_assert(!err); \ + } STMT_END + +int +fuzz_main(const uint8_t *stdin_buf, size_t data_size) +{ + if (!data_size) + return 0; + + chunk_t inp = { (uint8_t*)stdin_buf, data_size }; + chunk_t *b=NULL,*d=NULL; + void *a=NULL,*c=NULL; + + switch (stdin_buf[0]) { + case 0: + ENCODE_ROUNDTRIP(b16_enc, b16_dec, chunk_free_); + break; + case 1: + /* + XXXX see notes above about our base-32 functions. + ENCODE_ROUNDTRIP(b32_enc, b32_dec, chunk_free_); + */ + break; + case 2: + ENCODE_ROUNDTRIP(b64_enc, b64_dec, chunk_free_); + break; + case 3: + ENCODE_ROUNDTRIP(c_enc, c_dec, chunk_free_); + break; + case 5: + kv_flags = KV_QUOTED|KV_OMIT_KEYS; + ENCODE_ROUNDTRIP(kv_enc, kv_dec, config_free_lines_); + break; + case 6: + kv_flags = 0; + ENCODE_ROUNDTRIP(kv_enc, kv_dec, config_free_lines_); + break; + } + + return 0; +} diff --git a/src/test/fuzz/include.am b/src/test/fuzz/include.am index 27eeced8c5..d0711f05d6 100644 --- a/src/test/fuzz/include.am +++ b/src/test/fuzz/include.am @@ -152,6 +152,16 @@ src_test_fuzz_fuzz_socks_LDFLAGS = $(FUZZING_LDFLAG) src_test_fuzz_fuzz_socks_LDADD = $(FUZZING_LIBS) endif +if UNITTESTS_ENABLED +src_test_fuzz_fuzz_strops_SOURCES = \ + src/test/fuzz/fuzzing_common.c \ + src/test/fuzz/fuzz_strops.c +src_test_fuzz_fuzz_strops_CPPFLAGS = $(FUZZING_CPPFLAGS) +src_test_fuzz_fuzz_strops_CFLAGS = $(FUZZING_CFLAGS) +src_test_fuzz_fuzz_strops_LDFLAGS = $(FUZZING_LDFLAG) +src_test_fuzz_fuzz_strops_LDADD = $(FUZZING_LIBS) +endif + if UNITTESTS_ENABLED src_test_fuzz_fuzz_vrs_SOURCES = \ src/test/fuzz/fuzzing_common.c \ @@ -176,6 +186,7 @@ FUZZERS = \ src/test/fuzz/fuzz-iptsv2 \ src/test/fuzz/fuzz-microdesc \ src/test/fuzz/fuzz-socks \ + src/test/fuzz/fuzz-strops \ src/test/fuzz/fuzz-vrs endif @@ -290,6 +301,15 @@ src_test_fuzz_lf_fuzz_socks_LDFLAGS = $(LIBFUZZER_LDFLAG) src_test_fuzz_lf_fuzz_socks_LDADD = $(LIBFUZZER_LIBS) endif +if UNITTESTS_ENABLED +src_test_fuzz_lf_fuzz_strops_SOURCES = \ + $(src_test_fuzz_fuzz_strops_SOURCES) +src_test_fuzz_lf_fuzz_strops_CPPFLAGS = $(LIBFUZZER_CPPFLAGS) +src_test_fuzz_lf_fuzz_strops_CFLAGS = $(LIBFUZZER_CFLAGS) +src_test_fuzz_lf_fuzz_strops_LDFLAGS = $(LIBFUZZER_LDFLAG) +src_test_fuzz_lf_fuzz_strops_LDADD = $(LIBFUZZER_LIBS) +endif + if UNITTESTS_ENABLED src_test_fuzz_lf_fuzz_vrs_SOURCES = \ $(src_test_fuzz_fuzz_vrs_SOURCES) @@ -312,6 +332,7 @@ LIBFUZZER_FUZZERS = \ src/test/fuzz/lf-fuzz-iptsv2 \ src/test/fuzz/lf-fuzz-microdesc \ src/test/fuzz/lf-fuzz-socks \ + src/test/fuzz/lf-fuzz-strops \ src/test/fuzz/lf-fuzz-vrs else @@ -405,6 +426,13 @@ src_test_fuzz_liboss_fuzz_socks_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS) src_test_fuzz_liboss_fuzz_socks_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS) endif +if UNITTESTS_ENABLED +src_test_fuzz_liboss_fuzz_strops_a_SOURCES = \ + $(src_test_fuzz_fuzz_strops_SOURCES) +src_test_fuzz_liboss_fuzz_strops_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS) +src_test_fuzz_liboss_fuzz_strops_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS) +endif + if UNITTESTS_ENABLED src_test_fuzz_liboss_fuzz_vrs_a_SOURCES = \ $(src_test_fuzz_fuzz_vrs_SOURCES) @@ -425,6 +453,7 @@ OSS_FUZZ_FUZZERS = \ src/test/fuzz/liboss-fuzz-iptsv2.a \ src/test/fuzz/liboss-fuzz-microdesc.a \ src/test/fuzz/liboss-fuzz-socks.a \ + src/test/fuzz/liboss-fuzz-strops.a \ src/test/fuzz/liboss-fuzz-vrs.a else From 5200df85570dae7dede1473a169398e83ff64a37 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 10 Dec 2018 17:44:53 -0500 Subject: [PATCH 0262/2557] Copy the nss-related changes into fuzzing_include_am.py. --- scripts/codegen/fuzzing_include_am.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/codegen/fuzzing_include_am.py b/scripts/codegen/fuzzing_include_am.py index 5b51158843..a944584453 100755 --- a/scripts/codegen/fuzzing_include_am.py +++ b/scripts/codegen/fuzzing_include_am.py @@ -24,12 +24,12 @@ FUZZING_CPPFLAGS = \ FUZZING_CFLAGS = \ $(AM_CFLAGS) $(TEST_CFLAGS) FUZZING_LDFLAG = \ - @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ @TOR_LDFLAGS_libevent@ + @TOR_LDFLAGS_zlib@ $(TOR_LDFLAGS_CRYPTLIB) @TOR_LDFLAGS_libevent@ FUZZING_LIBS = \ $(TOR_INTERNAL_TESTING_LIBS) \ $(rust_ldadd) \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ \ - @TOR_LIBEVENT_LIBS@ @TOR_OPENSSL_LIBS@ \ + @TOR_LIBEVENT_LIBS@ $(TOR_LIBS_CRYPTLIB) \ @TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@ @CURVE25519_LIBS@ \ @TOR_SYSTEMD_LIBS@ \ @TOR_LZMA_LIBS@ \ From 53855d72b71cca7c61124bbdfac0a93b1386cb43 Mon Sep 17 00:00:00 2001 From: David Goulet Date: Wed, 12 Dec 2018 13:23:46 -0500 Subject: [PATCH 0263/2557] man: Add that "GETINFO address" won't work with "Sandbox 1" Patch by "wagon". Closes #28538 Signed-off-by: David Goulet --- doc/tor.1.txt | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/doc/tor.1.txt b/doc/tor.1.txt index c886e37613..5b73e25e16 100644 --- a/doc/tor.1.txt +++ b/doc/tor.1.txt @@ -598,21 +598,26 @@ GENERAL OPTIONS Otherwise the sandbox will be disabled. The option is currently an experimental feature. It only works on Linux-based operating systems, and only when Tor has been built with the libseccomp library. This option - can not be changed while tor is running. + can not be changed while tor is running. + + - When the Sandbox is 1, the following options can not be changed when tor + When the **Sandbox** is 1, the following options can not be changed when tor is running: - Address - ConnLimit - CookieAuthFile - DirPortFrontPage - ExtORPortCookieAuthFile - Logs - ServerDNSResolvConfFile - Tor must remain in client or server mode (some changes to ClientOnly and - ORPort are not allowed). Launching new Onion Services through Control - Port is not supported with current syscall sandboxing implementation. - ClientOnionAuthDir and any files in it won't reload on HUP signal. + **Address**, + **ConnLimit**, + **CookieAuthFile**, + **DirPortFrontPage**, + **ExtORPortCookieAuthFile**, + **Logs**, + **ServerDNSResolvConfFile**, + **ClientOnionAuthDir** (and any files in it won't reload on HUP signal). + + + Launching new Onion Services through the control port is not supported + with current syscall sandboxing implementation. + + + Tor must remain in client or server mode (some changes to **ClientOnly** + and **ORPort** are not allowed). Currently, if **Sandbox** is 1, + **ControlPort** command "GETINFO address" will not work. + + (Default: 0) [[Socks4Proxy]] **Socks4Proxy** __host__[:__port__]:: From c037bf58173db56766381a7c1cd5973789f0fd0f Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 4 Dec 2018 12:11:42 -0500 Subject: [PATCH 0264/2557] changes file for ticket26864 --- changes/ticket28624 | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 changes/ticket28624 diff --git a/changes/ticket28624 b/changes/ticket28624 new file mode 100644 index 0000000000..353f962be9 --- /dev/null +++ b/changes/ticket28624 @@ -0,0 +1,5 @@ + o Minor features (battery management, client, dormant mode): + - The client's memory of whether it is "dormant", and how long it has + spend idle, persists across invocations. Implements ticket 28624. + - There is a DormantOnFirstStartup option that integrators can use if + they expect that in many cases, Tor will be installed but not used. From b5c04173c88369d0c4cdaf8a34c2474dc25c79fa Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 4 Dec 2018 12:21:14 -0500 Subject: [PATCH 0265/2557] Change interaction between dormant mode and clock jumps. When the clock jumps, and we have a record of last user activity, adjust that record. This way if I'm inactive for 10 minutes and then the laptop is sleeping for an hour, I'll still count as having been inactive for 10 minutes. Previously, we treat every jump as if it were activity, which is ridiculous, and would prevent a Tor instance with a jumpy clock from ever going dormant. --- src/core/mainloop/mainloop.c | 21 +++++++++++---------- src/core/mainloop/netstatus.c | 12 ++++++++++++ src/core/mainloop/netstatus.h | 1 + src/lib/intmath/cmp.h | 3 +++ 4 files changed, 27 insertions(+), 10 deletions(-) diff --git a/src/core/mainloop/mainloop.c b/src/core/mainloop/mainloop.c index cd955e0ca0..6b9c03798e 100644 --- a/src/core/mainloop/mainloop.c +++ b/src/core/mainloop/mainloop.c @@ -1627,7 +1627,6 @@ schedule_rescan_periodic_events,(void)) void rescan_periodic_events(const or_options_t *options) { - puts("RESCAN"); tor_assert(options); /* Avoid scanning the event list if we haven't initialized it yet. This is @@ -2687,6 +2686,17 @@ update_current_time(time_t now) memcpy(&last_updated, ¤t_second_last_changed, sizeof(last_updated)); monotime_coarse_get(¤t_second_last_changed); + /** How much clock jumping means that we should adjust our idea of when + * to go dormant? */ +#define NUM_JUMPED_SECONDS_BEFORE_NETSTATUS_UPDATE 20 + + /* Don't go dormant early or late just because we jumped in time. */ + if (ABS(seconds_elapsed) >= NUM_JUMPED_SECONDS_BEFORE_NETSTATUS_UPDATE) { + if (is_participating_on_network()) { + netstatus_note_clock_jumped(seconds_elapsed); + } + } + /** How much clock jumping do we tolerate? */ #define NUM_JUMPED_SECONDS_BEFORE_WARN 100 @@ -2697,10 +2707,6 @@ update_current_time(time_t now) // moving back in time is always a bad sign. circuit_note_clock_jumped(seconds_elapsed, false); - /* Don't go dormant just because we jumped in time. */ - if (is_participating_on_network()) { - reset_user_activity(now); - } } else if (seconds_elapsed >= NUM_JUMPED_SECONDS_BEFORE_WARN) { /* Compare the monotonic clock to the result of time(). */ const int32_t monotime_msec_passed = @@ -2722,11 +2728,6 @@ update_current_time(time_t now) if (clock_jumped || seconds_elapsed >= NUM_IDLE_SECONDS_BEFORE_WARN) { circuit_note_clock_jumped(seconds_elapsed, ! clock_jumped); } - - /* Don't go dormant just because we jumped in time. */ - if (is_participating_on_network()) { - reset_user_activity(now); - } } else if (seconds_elapsed > 0) { stats_n_seconds_working += seconds_elapsed; } diff --git a/src/core/mainloop/netstatus.c b/src/core/mainloop/netstatus.c index 2426baae34..d1989cb839 100644 --- a/src/core/mainloop/netstatus.c +++ b/src/core/mainloop/netstatus.c @@ -146,3 +146,15 @@ netstatus_load_from_state(const or_state_t *state, time_t now) } reset_user_activity(last_activity); } + +/** + * Adjust the time at which the user was last active by seconds_diff + * in response to a clock jump. + */ +void +netstatus_note_clock_jumped(time_t seconds_diff) +{ + time_t last_active = get_last_user_activity_time(); + if (last_active) + reset_user_activity(last_active + seconds_diff); +} diff --git a/src/core/mainloop/netstatus.h b/src/core/mainloop/netstatus.h index 4b008e4cfa..9a0fa410fd 100644 --- a/src/core/mainloop/netstatus.h +++ b/src/core/mainloop/netstatus.h @@ -19,5 +19,6 @@ bool is_participating_on_network(void); void netstatus_flush_to_state(or_state_t *state, time_t now); void netstatus_load_from_state(const or_state_t *state, time_t now); +void netstatus_note_clock_jumped(time_t seconds_diff); #endif diff --git a/src/lib/intmath/cmp.h b/src/lib/intmath/cmp.h index 16952bee3e..11b6fdf98e 100644 --- a/src/lib/intmath/cmp.h +++ b/src/lib/intmath/cmp.h @@ -36,4 +36,7 @@ ((v) > (max)) ? (max) : \ (v) ) +/** Give the absolute value of x, independent of its type. */ +#define ABS(x) ( ((x)<0) ? -(x) : (x) ) + #endif /* !defined(TOR_INTMATH_CMP_H) */ From e3b7fd2a8129f0d2a7879976e57496b6fd4a7d00 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 4 Dec 2018 17:52:06 -0500 Subject: [PATCH 0266/2557] Unit tests for back-end functions for persistent dormant state --- src/test/test_mainloop.c | 76 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/src/test/test_mainloop.c b/src/test/test_mainloop.c index 8dfd5f619a..d797417912 100644 --- a/src/test/test_mainloop.c +++ b/src/test/test_mainloop.c @@ -8,6 +8,7 @@ #define CONFIG_PRIVATE #define MAINLOOP_PRIVATE +#define STATEFILE_PRIVATE #include "test/test.h" #include "test/log_test_helpers.h" @@ -20,6 +21,8 @@ #include "feature/hs/hs_service.h" #include "app/config/config.h" +#include "app/config/statefile.h" +#include "app/config/or_state_st.h" static const uint64_t BILLION = 1000000000; @@ -190,6 +193,13 @@ test_mainloop_user_activity(void *arg) tt_int_op(true, OP_EQ, is_participating_on_network()); tt_int_op(schedule_rescan_called, OP_EQ, 1); + // We _will_ adjust if the clock jumps though. + netstatus_note_clock_jumped(500); + tt_i64_op(get_last_user_activity_time(), OP_EQ, start+525); + + netstatus_note_clock_jumped(-400); + tt_i64_op(get_last_user_activity_time(), OP_EQ, start+125); + done: UNMOCK(schedule_rescan_periodic_events); } @@ -273,6 +283,70 @@ test_mainloop_check_participation(void *arg) UNMOCK(connection_get_by_type_nonlinked); } +static void +test_mainloop_dormant_load_state(void *arg) +{ + (void)arg; + or_state_t *state = or_state_new(); + const time_t start = 1543956575; + + reset_user_activity(0); + set_network_participation(false); + + // When we construct a new state, it starts out in "auto" mode. + tt_int_op(state->Dormant, OP_EQ, -1); + + // Initializing from "auto" makes us start out (by default) non-Dormant, + // with activity right now. + netstatus_load_from_state(state, start); + tt_assert(is_participating_on_network()); + tt_i64_op(get_last_user_activity_time(), OP_EQ, start); + + // Initializing from dormant clears the last user activity time, and + // makes us dormant. + state->Dormant = 1; + netstatus_load_from_state(state, start); + tt_assert(! is_participating_on_network()); + tt_i64_op(get_last_user_activity_time(), OP_EQ, 0); + + // Initializing from non-dormant sets the last user activity time, and + // makes us non-dormant. + state->Dormant = 0; + state->MinutesSinceUserActivity = 123; + netstatus_load_from_state(state, start); + tt_assert(is_participating_on_network()); + tt_i64_op(get_last_user_activity_time(), OP_EQ, start - 123*60); + + done: + or_state_free(state); +} + +static void +test_mainloop_dormant_save_state(void *arg) +{ + (void)arg; + or_state_t *state = or_state_new(); + const time_t start = 1543956575; + + // Can we save a non-dormant state correctly? + reset_user_activity(start - 1000); + set_network_participation(true); + netstatus_flush_to_state(state, start); + + tt_int_op(state->Dormant, OP_EQ, 0); + tt_int_op(state->MinutesSinceUserActivity, OP_EQ, 1000 / 60); + + // Can we save a dormant state correctly? + set_network_participation(false); + netstatus_flush_to_state(state, start); + + tt_int_op(state->Dormant, OP_EQ, 1); + tt_int_op(state->MinutesSinceUserActivity, OP_EQ, 0); + + done: + or_state_free(state); +} + #define MAINLOOP_TEST(name) \ { #name, test_mainloop_## name , TT_FORK, NULL, NULL } @@ -281,5 +355,7 @@ struct testcase_t mainloop_tests[] = { MAINLOOP_TEST(update_time_jumps), MAINLOOP_TEST(user_activity), MAINLOOP_TEST(check_participation), + MAINLOOP_TEST(dormant_load_state), + MAINLOOP_TEST(dormant_save_state), END_OF_TESTCASES }; From 4bc3983f64ab2a60a79127f47573836373ddf587 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 13 Dec 2018 19:35:02 -0500 Subject: [PATCH 0267/2557] Add a DROPOWNERSHIP controller command to undo TAKEOWNERSHIP. Closes ticket 28843. --- changes/ticket28843 | 3 +++ src/feature/control/control.c | 23 +++++++++++++++++++++++ 2 files changed, 26 insertions(+) create mode 100644 changes/ticket28843 diff --git a/changes/ticket28843 b/changes/ticket28843 new file mode 100644 index 0000000000..00905293c4 --- /dev/null +++ b/changes/ticket28843 @@ -0,0 +1,3 @@ + o Minor features (controller): + - Add a DROPOWNERSHIP command to undo the effects of TAKEOWNERSHIP. + Implements ticket 28843. diff --git a/src/feature/control/control.c b/src/feature/control/control.c index c2da7fe48b..50f43c0f44 100644 --- a/src/feature/control/control.c +++ b/src/feature/control/control.c @@ -1741,6 +1741,26 @@ handle_control_takeownership(control_connection_t *conn, uint32_t len, return 0; } +/** Called when we get a DROPOWNERSHIP command. Mark this connection + * as a non-owning connection, so that we will not exit if the connection + * closes. */ +static int +handle_control_dropownership(control_connection_t *conn, uint32_t len, + const char *body) +{ + (void)len; + (void)body; + + conn->is_owning_control_connection = 0; + + log_info(LD_CONTROL, "Control connection %d has dropped ownership of this " + "Tor instance.", + (int)(conn->base_.s)); + + send_control_done(conn); + return 0; +} + /** Return true iff addr is unusable as a mapaddress target because of * containing funny characters. */ static int @@ -5548,6 +5568,9 @@ connection_control_process_inbuf(control_connection_t *conn) } else if (!strcasecmp(conn->incoming_cmd, "TAKEOWNERSHIP")) { if (handle_control_takeownership(conn, cmd_data_len, args)) return -1; + } else if (!strcasecmp(conn->incoming_cmd, "DROPOWNERSHIP")) { + if (handle_control_dropownership(conn, cmd_data_len, args)) + return -1; } else if (!strcasecmp(conn->incoming_cmd, "MAPADDRESS")) { if (handle_control_mapaddress(conn, cmd_data_len, args)) return -1; From 325348b360fc8a51d1344d918c1bb20017f1e863 Mon Sep 17 00:00:00 2001 From: Rob Jansen Date: Wed, 12 Dec 2018 15:27:45 -0500 Subject: [PATCH 0268/2557] allow any value for HearbeatPeriod in testing Tor networks --- src/app/config/config.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/app/config/config.c b/src/app/config/config.c index dcefa3d6a4..071c33d530 100644 --- a/src/app/config/config.c +++ b/src/app/config/config.c @@ -3897,7 +3897,8 @@ options_validate(or_options_t *old_options, or_options_t *options, } if (options->HeartbeatPeriod && - options->HeartbeatPeriod < MIN_HEARTBEAT_PERIOD) { + options->HeartbeatPeriod < MIN_HEARTBEAT_PERIOD && + !options->TestingTorNetwork) { log_warn(LD_CONFIG, "HeartbeatPeriod option is too short; " "raising to %d seconds.", MIN_HEARTBEAT_PERIOD); options->HeartbeatPeriod = MIN_HEARTBEAT_PERIOD; From 06046c726fb09ec745d2a565abfef28097856a19 Mon Sep 17 00:00:00 2001 From: Matt Traudt Date: Fri, 14 Dec 2018 09:24:19 -0500 Subject: [PATCH 0269/2557] Add changes file for <30min HeartbeatPeriod patch --- changes/ticket28840 | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 changes/ticket28840 diff --git a/changes/ticket28840 b/changes/ticket28840 new file mode 100644 index 0000000000..05d3fbb94c --- /dev/null +++ b/changes/ticket28840 @@ -0,0 +1,3 @@ + o Minor features (testing): + - Allow HeartbeatPeriod of less than 30 minutes in testing Tor networks. + Closes ticket 28840, patch by robgjansen From cf7342ab6f653c2dc49134024d668b06bac2c96c Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 14 Dec 2018 13:45:22 -0500 Subject: [PATCH 0270/2557] Add a benchmark for parsing a microdescriptor --- src/test/bench.c | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/src/test/bench.c b/src/test/bench.c index ff8707d41c..f8680c3ab6 100644 --- a/src/test/bench.c +++ b/src/test/bench.c @@ -39,6 +39,9 @@ #include "lib/crypt_ops/digestset.h" #include "lib/crypt_ops/crypto_init.h" +#include "feature/dirparse/microdesc_parse.h" +#include "feature/nodelist/microdesc.h" + #if defined(HAVE_CLOCK_GETTIME) && defined(CLOCK_PROCESS_CPUTIME_ID) static uint64_t nanostart; static inline uint64_t @@ -639,6 +642,41 @@ bench_ecdh_p224(void) } #endif +static void +bench_md_parse(void) +{ + uint64_t start, end; + const int N = 100000; + // selected arbitrarily + const char md_text[] = + "@last-listed 2018-12-14 18:14:14\n" + "onion-key\n" + "-----BEGIN RSA PUBLIC KEY-----\n" + "MIGJAoGBAMHkZeXNDX/49JqM2BVLmh1Fnb5iMVnatvZZTLJyedqDLkbXZ1WKP5oh\n" + "7ec14dj/k3ntpwHD4s2o3Lb6nfagWbug4+F/rNJ7JuFru/PSyOvDyHGNAuegOXph\n" + "3gTGjdDpv/yPoiadGebbVe8E7n6hO+XxM2W/4dqheKimF0/s9B7HAgMBAAE=\n" + "-----END RSA PUBLIC KEY-----\n" + "ntor-onion-key QgF/EjqlNG1wRHLIop/nCekEH+ETGZSgYOhu26eiTF4=\n" + "family $00E9A86E7733240E60D8435A7BBD634A23894098 " + "$329BD7545DEEEBBDC8C4285F243916F248972102 " + "$69E06EBB2573A4F89330BDF8BC869794A3E10E4D " + "$DCA2A3FAE50B3729DAA15BC95FB21AF03389818B\n" + "p accept 53,80,443,5222-5223,25565\n" + "id ed25519 BzffzY99z6Q8KltcFlUTLWjNTBU7yKK+uQhyi1Ivb3A\n"; + + reset_perftime(); + start = perftime(); + for (int i = 0; i < N; ++i) { + smartlist_t *s = microdescs_parse_from_string(md_text, NULL, 1, + SAVED_IN_CACHE, NULL); + SMARTLIST_FOREACH(s, microdesc_t *, md, microdesc_free(md)); + smartlist_free(s); + } + + end = perftime(); + printf("Microdesc parse: %f nsec\n", NANOCOUNT(start, end, N)); +} + typedef void (*bench_fn)(void); typedef struct benchmark_t { @@ -666,6 +704,8 @@ static struct benchmark_t benchmarks[] = { ENT(ecdh_p256), ENT(ecdh_p224), #endif + + ENT(md_parse), {NULL,NULL,0} }; From 3c35c0d441cc25f750524056113970a376d8432c Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 14 Dec 2018 13:07:55 -0500 Subject: [PATCH 0271/2557] Add a function to provide an upper bound on base64 decoded length --- src/lib/encoding/binascii.c | 12 ++++++++++++ src/lib/encoding/binascii.h | 1 + src/test/test_util_format.c | 4 +++- 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/src/lib/encoding/binascii.c b/src/lib/encoding/binascii.c index e9140d9432..067db075ad 100644 --- a/src/lib/encoding/binascii.c +++ b/src/lib/encoding/binascii.c @@ -179,6 +179,18 @@ base64_encode_size(size_t srclen, int flags) return enclen; } +/** Return an upper bound on the number of bytes that might be needed to hold + * the data from decoding the base64 string srclen. This is only an + * upper bound, since some part of the base64 string might be padding or + * space. */ +size_t +base64_decode_maxsize(size_t srclen) +{ + tor_assert(srclen < INT_MAX / 3); + + return CEIL_DIV(srclen * 3, 4); +} + /** Internal table mapping 6 bit values to the Base64 alphabet. */ static const char base64_encode_table[64] = { 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', diff --git a/src/lib/encoding/binascii.h b/src/lib/encoding/binascii.h index 23cbaa7070..c71ba65dfb 100644 --- a/src/lib/encoding/binascii.h +++ b/src/lib/encoding/binascii.h @@ -42,6 +42,7 @@ const char *hex_str(const char *from, size_t fromlen); #define BASE64_ENCODE_MULTILINE 1 size_t base64_encode_size(size_t srclen, int flags); +size_t base64_decode_maxsize(size_t srclen); int base64_encode(char *dest, size_t destlen, const char *src, size_t srclen, int flags); int base64_decode(char *dest, size_t destlen, const char *src, size_t srclen); diff --git a/src/test/test_util_format.c b/src/test/test_util_format.c index 85d8a8e62e..fd57125b86 100644 --- a/src/test/test_util_format.c +++ b/src/test/test_util_format.c @@ -392,10 +392,13 @@ test_util_format_encoded_size(void *arg) base64_encode(outbuf, sizeof(outbuf), (char *)inbuf, i, 0); tt_int_op(strlen(outbuf), OP_EQ, base64_encode_size(i, 0)); + tt_int_op(i, OP_LE, base64_decode_maxsize(strlen(outbuf))); + base64_encode(outbuf, sizeof(outbuf), (char *)inbuf, i, BASE64_ENCODE_MULTILINE); tt_int_op(strlen(outbuf), OP_EQ, base64_encode_size(i, BASE64_ENCODE_MULTILINE)); + tt_int_op(i, OP_LE, base64_decode_maxsize(strlen(outbuf))); } done: @@ -417,4 +420,3 @@ struct testcase_t util_format_tests[] = { { "encoded_size", test_util_format_encoded_size, 0, NULL, NULL }, END_OF_TESTCASES }; - From 6dc90d290daa29b4ff2c7692be3a2ed64f25dfc1 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 14 Dec 2018 13:11:58 -0500 Subject: [PATCH 0272/2557] Use 25% less RAM for base64-encoded directory objects We were allocating N bytes to decode an N-byte base64 encoding, when 3N/4 would have been enough. --- src/feature/dirparse/parsecommon.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/feature/dirparse/parsecommon.c b/src/feature/dirparse/parsecommon.c index e00af0eea2..91b775533b 100644 --- a/src/feature/dirparse/parsecommon.c +++ b/src/feature/dirparse/parsecommon.c @@ -393,8 +393,9 @@ get_next_token(memarea_t *area, RET_ERR("Couldn't parse private key."); } else { /* If it's something else, try to base64-decode it */ int r; - tok->object_body = ALLOC(next-*s); /* really, this is too much RAM. */ - r = base64_decode(tok->object_body, next-*s, *s, next-*s); + size_t maxsize = base64_decode_maxsize(next-*s); + tok->object_body = ALLOC(maxsize); + r = base64_decode(tok->object_body, maxsize, *s, next-*s); if (r<0) RET_ERR("Malformed object: bad base64-encoded data"); tok->object_size = r; From 9dc53bc68f5e038c9531e3b12a58026d4007f652 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 14 Dec 2018 14:48:12 -0500 Subject: [PATCH 0273/2557] Remove a needless memset() in get_token_arguments() I believe we originally added this for "just in case" safety, but it isn't actually needed -- we never copy uninitialized stack here. What's more, this one memset is showing up on our startup profiles, so we ought to remove it. Closes ticket 28852. --- changes/ticket28852 | 4 ++++ src/feature/dirparse/parsecommon.c | 1 - 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 changes/ticket28852 diff --git a/changes/ticket28852 b/changes/ticket28852 new file mode 100644 index 0000000000..a58cc3ba0e --- /dev/null +++ b/changes/ticket28852 @@ -0,0 +1,4 @@ + o Minor features (performance): + - Remove a needless memset() call from get_token_arguments, + thereby speeding up the tokenization of directory objects by about + 20%. Closes ticket 28852. diff --git a/src/feature/dirparse/parsecommon.c b/src/feature/dirparse/parsecommon.c index e00af0eea2..5280f2ed2f 100644 --- a/src/feature/dirparse/parsecommon.c +++ b/src/feature/dirparse/parsecommon.c @@ -169,7 +169,6 @@ get_token_arguments(memarea_t *area, directory_token_t *tok, char *cp = mem; int j = 0; char *args[MAX_ARGS]; - memset(args, 0, sizeof(args)); while (*cp) { if (j == MAX_ARGS) return -1; From 3dd1f064a7d5708585f88beffaf3897ba6555208 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 14 Dec 2018 16:07:10 -0500 Subject: [PATCH 0274/2557] Rewrite the core of parse_short_policy() to be faster. The old implementation did some funky out-of-order lexing, and tended to parse every port twice if the %d-%d pattern didn't match. Closes ticket 28853. --- changes/ticket28853 | 3 ++ src/core/or/policies.c | 80 ++++++++++++++++++++++-------------------- 2 files changed, 45 insertions(+), 38 deletions(-) create mode 100644 changes/ticket28853 diff --git a/changes/ticket28853 b/changes/ticket28853 new file mode 100644 index 0000000000..e76f6bd8c9 --- /dev/null +++ b/changes/ticket28853 @@ -0,0 +1,3 @@ + o Minor features (performance): + - Replace parse_short_policy() with a faster implementation, to improve + microdescriptor parsing time. Closes ticket 28853. diff --git a/src/core/or/policies.c b/src/core/or/policies.c index 123fc8566e..bffdb1fddd 100644 --- a/src/core/or/policies.c +++ b/src/core/or/policies.c @@ -2720,7 +2720,7 @@ parse_short_policy(const char *summary) int is_accept; int n_entries; short_policy_entry_t entries[MAX_EXITPOLICY_SUMMARY_LEN]; /* overkill */ - const char *next; + char *next; if (!strcmpstart(summary, "accept ")) { is_accept = 1; @@ -2735,57 +2735,56 @@ parse_short_policy(const char *summary) n_entries = 0; for ( ; *summary; summary = next) { - const char *comma = strchr(summary, ','); - unsigned low, high; - char dummy; - char ent_buf[32]; - size_t len; - - next = comma ? comma+1 : strchr(summary, '\0'); - len = comma ? (size_t)(comma - summary) : strlen(summary); - if (n_entries == MAX_EXITPOLICY_SUMMARY_LEN) { log_fn(LOG_PROTOCOL_WARN, LD_DIR, "Impossibly long policy summary %s", escaped(orig_summary)); return NULL; } - if (! TOR_ISDIGIT(*summary) || len > (sizeof(ent_buf)-1)) { - /* unrecognized entry format. skip it. */ - continue; - } - if (len < 1) { - /* empty; skip it. */ - /* XXX This happens to be unreachable, since if len==0, then *summary is - * ',' or '\0', and the TOR_ISDIGIT test above would have failed. */ - continue; + unsigned low, high; + int ok; + low = (unsigned) tor_parse_ulong(summary, 10, 1, 65535, &ok, &next); + if (!ok) { + if (! TOR_ISDIGIT(*summary) || *summary == ',') { + /* Unrecognized format: skip it. */ + goto skip_ent; + } else { + goto bad_ent; + } } - memcpy(ent_buf, summary, len); - ent_buf[len] = '\0'; + switch (*next) { + case ',': + ++next; + /* fall through */ + case '\0': + high = low; + break; + case '-': + high = (unsigned) tor_parse_ulong(next+1, 10, low, 65535, &ok, &next); + if (!ok) + goto bad_ent; - if (tor_sscanf(ent_buf, "%u-%u%c", &low, &high, &dummy) == 2) { - if (low<1 || low>65535 || high<1 || high>65535 || low>high) { - log_fn(LOG_PROTOCOL_WARN, LD_DIR, - "Found bad entry in policy summary %s", escaped(orig_summary)); - return NULL; - } - } else if (tor_sscanf(ent_buf, "%u%c", &low, &dummy) == 1) { - if (low<1 || low>65535) { - log_fn(LOG_PROTOCOL_WARN, LD_DIR, - "Found bad entry in policy summary %s", escaped(orig_summary)); - return NULL; - } - high = low; - } else { - log_fn(LOG_PROTOCOL_WARN, LD_DIR,"Found bad entry in policy summary %s", - escaped(orig_summary)); - return NULL; + if (*next == ',') + ++next; + else if (*next != '\0') + goto bad_ent; + + break; + default: + goto bad_ent; } entries[n_entries].min_port = low; entries[n_entries].max_port = high; n_entries++; + + continue; + skip_ent: + next = strchr(next, ','); + if (!next) + break; + ++next; } if (n_entries == 0) { @@ -2806,6 +2805,11 @@ parse_short_policy(const char *summary) result->n_entries = n_entries; memcpy(result->entries, entries, sizeof(short_policy_entry_t)*n_entries); return result; + + bad_ent: + log_fn(LOG_PROTOCOL_WARN, LD_DIR,"Found bad entry in policy summary %s", + escaped(orig_summary)); + return NULL; } /** Write policy back out into a string. */ From 3bec371d04b291ec0351f01eeafdc842ea2a51d6 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Sun, 16 Dec 2018 10:05:35 +0200 Subject: [PATCH 0275/2557] Refrain from hardcoding address length and type in netinfo.trunnel --- src/trunnel/netinfo.c | 30 ++++++------------------------ src/trunnel/netinfo.trunnel | 6 +++--- 2 files changed, 9 insertions(+), 27 deletions(-) diff --git a/src/trunnel/netinfo.c b/src/trunnel/netinfo.c index 170e9bf034..de389eb13f 100644 --- a/src/trunnel/netinfo.c +++ b/src/trunnel/netinfo.c @@ -34,8 +34,6 @@ netinfo_addr_new(void) netinfo_addr_t *val = trunnel_calloc(1, sizeof(netinfo_addr_t)); if (NULL == val) return NULL; - val->addr_type = NETINFO_ADDR_TYPE_IPV4; - val->len = 4; return val; } @@ -65,10 +63,6 @@ netinfo_addr_get_addr_type(const netinfo_addr_t *inp) int netinfo_addr_set_addr_type(netinfo_addr_t *inp, uint8_t val) { - if (! ((val == NETINFO_ADDR_TYPE_IPV4 || val == NETINFO_ADDR_TYPE_IPV6))) { - TRUNNEL_SET_ERROR_CODE(inp); - return -1; - } inp->addr_type = val; return 0; } @@ -80,10 +74,6 @@ netinfo_addr_get_len(const netinfo_addr_t *inp) int netinfo_addr_set_len(netinfo_addr_t *inp, uint8_t val) { - if (! ((val == 4 || val == 16))) { - TRUNNEL_SET_ERROR_CODE(inp); - return -1; - } inp->len = val; return 0; } @@ -141,10 +131,6 @@ netinfo_addr_check(const netinfo_addr_t *obj) return "Object was NULL"; if (obj->trunnel_error_code_) return "A set function failed on this object"; - if (! (obj->addr_type == NETINFO_ADDR_TYPE_IPV4 || obj->addr_type == NETINFO_ADDR_TYPE_IPV6)) - return "Integer out of bounds"; - if (! (obj->len == 4 || obj->len == 16)) - return "Integer out of bounds"; switch (obj->addr_type) { case NETINFO_ADDR_TYPE_IPV4: @@ -169,10 +155,10 @@ netinfo_addr_encoded_len(const netinfo_addr_t *obj) return -1; - /* Length of u8 addr_type IN [NETINFO_ADDR_TYPE_IPV4, NETINFO_ADDR_TYPE_IPV6] */ + /* Length of u8 addr_type */ result += 1; - /* Length of u8 len IN [4, 16] */ + /* Length of u8 len */ result += 1; switch (obj->addr_type) { @@ -219,14 +205,14 @@ netinfo_addr_encode(uint8_t *output, const size_t avail, const netinfo_addr_t *o trunnel_assert(encoded_len >= 0); #endif - /* Encode u8 addr_type IN [NETINFO_ADDR_TYPE_IPV4, NETINFO_ADDR_TYPE_IPV6] */ + /* Encode u8 addr_type */ trunnel_assert(written <= avail); if (avail - written < 1) goto truncated; trunnel_set_uint8(ptr, (obj->addr_type)); written += 1; ptr += 1; - /* Encode u8 len IN [4, 16] */ + /* Encode u8 len */ trunnel_assert(written <= avail); if (avail - written < 1) goto truncated; @@ -296,19 +282,15 @@ netinfo_addr_parse_into(netinfo_addr_t *obj, const uint8_t *input, const size_t ssize_t result = 0; (void)result; - /* Parse u8 addr_type IN [NETINFO_ADDR_TYPE_IPV4, NETINFO_ADDR_TYPE_IPV6] */ + /* Parse u8 addr_type */ CHECK_REMAINING(1, truncated); obj->addr_type = (trunnel_get_uint8(ptr)); remaining -= 1; ptr += 1; - if (! (obj->addr_type == NETINFO_ADDR_TYPE_IPV4 || obj->addr_type == NETINFO_ADDR_TYPE_IPV6)) - goto fail; - /* Parse u8 len IN [4, 16] */ + /* Parse u8 len */ CHECK_REMAINING(1, truncated); obj->len = (trunnel_get_uint8(ptr)); remaining -= 1; ptr += 1; - if (! (obj->len == 4 || obj->len == 16)) - goto fail; /* Parse union addr[addr_type] */ switch (obj->addr_type) { diff --git a/src/trunnel/netinfo.trunnel b/src/trunnel/netinfo.trunnel index 21afcd136e..83c3a9e40f 100644 --- a/src/trunnel/netinfo.trunnel +++ b/src/trunnel/netinfo.trunnel @@ -5,14 +5,14 @@ const NETINFO_ADDR_TYPE_IPV4 = 4; const NETINFO_ADDR_TYPE_IPV6 = 6; struct netinfo_addr { - u8 addr_type IN [NETINFO_ADDR_TYPE_IPV4, NETINFO_ADDR_TYPE_IPV6]; - u8 len IN [4, 16]; + u8 addr_type; + u8 len; union addr[addr_type] { NETINFO_ADDR_TYPE_IPV4: u32 ipv4; NETINFO_ADDR_TYPE_IPV6: u8 ipv6[16]; default: fail; }; - + } struct netinfo_cell { From 5b2acbec0e50a1858c43d0cafe9c4696152cde27 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Sun, 16 Dec 2018 10:19:37 +0200 Subject: [PATCH 0276/2557] Refrain from closing connection if found one unrecognized address in NETINFO cell --- src/core/or/channeltls.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/core/or/channeltls.c b/src/core/or/channeltls.c index 392e78506e..8f407d5e15 100644 --- a/src/core/or/channeltls.c +++ b/src/core/or/channeltls.c @@ -1809,10 +1809,8 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan) if (tor_addr_from_netinfo_addr(&addr, netinfo_addr) == -1) { log_fn(LOG_PROTOCOL_WARN, LD_OR, - "Bad address in netinfo cell; closing connection."); - connection_or_close_for_error(chan->conn, 0); - netinfo_cell_free(netinfo_cell); - return; + "Bad address in netinfo cell; Skipping."); + continue; } /* A relay can connect from anywhere and be canonical, so * long as it tells you from where it came. This may sound a bit From a0fad3981e9d072188c6ba19f25bdd78962330d8 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 17 Dec 2018 09:03:04 -0500 Subject: [PATCH 0277/2557] Replace use of strcmp_len() with new mem_eq_token(). The strcmp_len() function was somewhat misconceived, since we're only using it to test whether a length+extent string is equal to a NUL-terminated string or not. By simplifying it and making it inlined, we should be able to make it a little faster. (It *does* show up in profiles.) Closes ticket 28856. --- src/feature/dirparse/parsecommon.c | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/src/feature/dirparse/parsecommon.c b/src/feature/dirparse/parsecommon.c index e00af0eea2..a69957e74a 100644 --- a/src/feature/dirparse/parsecommon.c +++ b/src/feature/dirparse/parsecommon.c @@ -15,6 +15,7 @@ #include "lib/string/printf.h" #include "lib/memarea/memarea.h" #include "lib/crypt_ops/crypto_rsa.h" +#include "lib/ctime/di_ops.h" #include @@ -251,6 +252,16 @@ token_check_object(memarea_t *area, const char *kwd, return tok; } +/** Return true iff the memlen-byte chunk of memory at + * memlen is the same length as token, and their + * contents are equal. */ +static bool +mem_eq_token(const void *mem, size_t memlen, const char *token) +{ + size_t len = strlen(token); + return memlen == len && fast_memeq(mem, token, len); +} + /** Helper function: read the next token from *s, advance *s to the end of the * token, and return the parsed token. Parse *s according to the list * of tokens in table. @@ -290,7 +301,7 @@ get_next_token(memarea_t *area, next = find_whitespace_eos(*s, eol); - if (!strcmp_len(*s, "opt", next-*s)) { + if (mem_eq_token(*s, next-*s, "opt")) { /* Skip past an "opt" at the start of the line. */ *s = eat_whitespace_eos_no_nl(next, eol); next = find_whitespace_eos(*s, eol); @@ -301,7 +312,7 @@ get_next_token(memarea_t *area, /* Search the table for the appropriate entry. (I tried a binary search * instead, but it wasn't any faster.) */ for (i = 0; table[i].t ; ++i) { - if (!strcmp_len(*s, table[i].t, next-*s)) { + if (mem_eq_token(*s, next-*s, table[i].t)) { /* We've found the keyword. */ kwd = table[i].t; tok->tp = table[i].v; @@ -354,7 +365,7 @@ get_next_token(memarea_t *area, obstart = *s; /* Set obstart to start of object spec */ if (eol - *s <= 16 || memchr(*s+11,'\0',eol-*s-16) || /* no short lines, */ - strcmp_len(eol-5, "-----", 5) || /* nuls or invalid endings */ + !mem_eq_token(eol-5, 5, "-----") || /* nuls or invalid endings */ (eol-*s) > MAX_UNPARSED_OBJECT_SIZE) { /* name too long */ RET_ERR("Malformed object: bad begin line"); } @@ -373,8 +384,8 @@ get_next_token(memarea_t *area, eol = eos; /* Validate the ending tag, which should be 9 + NAME + 5 + eol */ if ((size_t)(eol-next) != 9+obname_len+5 || - strcmp_len(next+9, tok->object_type, obname_len) || - strcmp_len(eol-5, "-----", 5)) { + !mem_eq_token(next+9, obname_len, tok->object_type) || + !mem_eq_token(eol-5, 5, "-----")) { tor_snprintf(ebuf, sizeof(ebuf), "Malformed object: mismatched end tag %s", tok->object_type); ebuf[sizeof(ebuf)-1] = '\0'; From 29254812a3428da4ffeaede5d211364d942f94ab Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 17 Dec 2018 09:04:25 -0500 Subject: [PATCH 0278/2557] Remove strcmp_len(): it is now unused (See 28856.) --- src/lib/string/util_string.c | 15 --------------- src/lib/string/util_string.h | 1 - src/test/test_util.c | 9 --------- 3 files changed, 25 deletions(-) diff --git a/src/lib/string/util_string.c b/src/lib/string/util_string.c index e76e73046f..36e19d029c 100644 --- a/src/lib/string/util_string.c +++ b/src/lib/string/util_string.c @@ -212,21 +212,6 @@ strcmpstart(const char *s1, const char *s2) return strncmp(s1, s2, n); } -/** Compare the s1_len-byte string s1 with s2, - * without depending on a terminating nul in s1. Sorting order is first by - * length, then lexically; return values are as for strcmp. - */ -int -strcmp_len(const char *s1, const char *s2, size_t s1_len) -{ - size_t s2_len = strlen(s2); - if (s1_len < s2_len) - return -1; - if (s1_len > s2_len) - return 1; - return fast_memcmp(s1, s2, s2_len); -} - /** Compares the first strlen(s2) characters of s1 with s2. Returns as for * strcasecmp. */ diff --git a/src/lib/string/util_string.h b/src/lib/string/util_string.h index 99467a27c3..6541afa4cb 100644 --- a/src/lib/string/util_string.h +++ b/src/lib/string/util_string.h @@ -33,7 +33,6 @@ int tor_strisnonupper(const char *s); int tor_strisspace(const char *s); int strcmp_opt(const char *s1, const char *s2); int strcmpstart(const char *s1, const char *s2); -int strcmp_len(const char *s1, const char *s2, size_t len); int strcasecmpstart(const char *s1, const char *s2); int strcmpend(const char *s1, const char *s2); int strcasecmpend(const char *s1, const char *s2); diff --git a/src/test/test_util.c b/src/test/test_util.c index 2b4d64e42e..7ac1a1ec42 100644 --- a/src/test/test_util.c +++ b/src/test/test_util.c @@ -2153,15 +2153,6 @@ test_util_strmisc(void *arg) tt_int_op(strcmp_opt(NULL, "foo"), OP_LT, 0); tt_int_op(strcmp_opt("foo", NULL), OP_GT, 0); - /* Test strcmp_len */ - tt_int_op(strcmp_len("foo", "bar", 3), OP_GT, 0); - tt_int_op(strcmp_len("foo", "bar", 2), OP_LT, 0); - tt_int_op(strcmp_len("foo2", "foo1", 4), OP_GT, 0); - tt_int_op(strcmp_len("foo2", "foo1", 3), OP_LT, 0); /* Really stop at len */ - tt_int_op(strcmp_len("foo2", "foo", 3), OP_EQ, 0); /* Really stop at len */ - tt_int_op(strcmp_len("blah", "", 4), OP_GT, 0); - tt_int_op(strcmp_len("blah", "", 0), OP_EQ, 0); - done: tor_free(cp_tmp); } From 82fb40c8dc2f21753298559e79d898add80bf6b7 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 17 Dec 2018 09:26:57 -0500 Subject: [PATCH 0279/2557] Fix dead-assignment warnings in test_config.c Found by scan-build. --- src/test/test_config.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/test/test_config.c b/src/test/test_config.c index 5140c3c1a8..67a43d669e 100644 --- a/src/test/test_config.c +++ b/src/test/test_config.c @@ -5783,6 +5783,7 @@ test_config_extended_fmt(void *arg) tt_str_op(lp->value, OP_EQ, "is back here"); tt_int_op(lp->command, OP_EQ, CONFIG_LINE_NORMAL); lp = lp->next; + tt_assert(!lp); config_free_lines(lines); /* Try with the "extended" flag enabled. */ @@ -5809,6 +5810,7 @@ test_config_extended_fmt(void *arg) tt_str_op(lp->value, OP_EQ, ""); tt_int_op(lp->command, OP_EQ, CONFIG_LINE_CLEAR); lp = lp->next; + tt_assert(!lp); done: config_free_lines(lines); From f50558ce8ce33339b86ac642d92a27430e066d62 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 17 Dec 2018 09:27:40 -0500 Subject: [PATCH 0280/2557] Fix dead-assignment warning in test_shared_random.c --- src/test/test_shared_random.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test/test_shared_random.c b/src/test/test_shared_random.c index 433661f128..16e28afafe 100644 --- a/src/test/test_shared_random.c +++ b/src/test/test_shared_random.c @@ -309,6 +309,7 @@ test_get_start_time_of_current_run(void *arg) retval = parse_rfc1123_time("Mon, 19 Apr 2015 23:00:00 UTC", &mock_consensus.valid_after); + tt_int_op(retval, OP_EQ, 0); retval = parse_rfc1123_time("Mon, 20 Apr 2015 00:08:00 UTC", ¤t_time); From d58a597a55dba2ac468cbdef5172ecda1dc2344f Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 17 Dec 2018 09:28:08 -0500 Subject: [PATCH 0281/2557] Fix dead assignment warning in test_hs_service.c --- src/test/test_hs_service.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/test_hs_service.c b/src/test/test_hs_service.c index ee2d71aa75..79cbd2e5d4 100644 --- a/src/test/test_hs_service.c +++ b/src/test/test_hs_service.c @@ -1396,7 +1396,6 @@ static void test_build_update_descriptors(void *arg) { int ret; - time_t now = time(NULL); node_t *node; hs_service_t *service; hs_service_intro_point_t *ip_cur, *ip_next; @@ -1422,7 +1421,8 @@ test_build_update_descriptors(void *arg) voting_schedule_recalculate_timing(get_options(), mock_ns.valid_after); update_approx_time(mock_ns.valid_after+1); - now = mock_ns.valid_after+1; + + time_t now = mock_ns.valid_after+1; /* Create a service without a current descriptor to trigger a build. */ service = helper_create_service(); From ce3d501040fb994bb3c7dd82b21e02648621d3f1 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 17 Dec 2018 09:28:24 -0500 Subject: [PATCH 0282/2557] Fix null-pointer-deref warning from scan-build in test_hs_service.c --- src/test/test_hs_service.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/test/test_hs_service.c b/src/test/test_hs_service.c index 79cbd2e5d4..b2aafc1cd6 100644 --- a/src/test/test_hs_service.c +++ b/src/test/test_hs_service.c @@ -1654,6 +1654,9 @@ test_build_descriptors(void *arg) service->desc_current = NULL; build_all_descriptors(now); + tt_assert(service->desc_current); + tt_assert(service->desc_current->desc); + hs_desc_superencrypted_data_t *superencrypted; superencrypted = &service->desc_current->desc->superencrypted_data; tt_int_op(smartlist_len(superencrypted->clients), OP_EQ, 16); From 35509978dd4985901431abe895d1443e75afc00a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Thu, 22 Nov 2018 04:25:11 +0100 Subject: [PATCH 0283/2557] Add new Process subsystem. This patch adds a new Process subsystem for running external programs in the background of Tor. The design is focused around a new type named `process_t` which have an API that allows the developer to easily write code that interacts with the given child process. These interactions includes: - Easy API for writing output to the child process's standard input handle. - Receive callbacks whenever the child has output on either its standard output or standard error handles. - Receive callback when the child process terminates. We also support two different "protocols" for handling output from the child process. The default protocol is the "line" protocol where the process output callbacks will be invoked only when there is complete lines (either "\r\n" or "\n" terminated). We also support the "raw" protocol where the read callbacks will get whatever the operating system delivered to us in a single read operation. This patch does not include any operating system backends, but the Unix and Windows backends will be included in separate commits. See: https://bugs.torproject.org/28179 --- src/app/main/main.c | 6 + src/lib/process/.may_include | 5 +- src/lib/process/include.am | 2 + src/lib/process/process.c | 644 +++++++++++++++++++++++++++++++++++ src/lib/process/process.h | 127 +++++++ src/test/include.am | 1 + src/test/test.c | 1 + src/test/test.h | 1 + src/test/test_process.c | 558 ++++++++++++++++++++++++++++++ 9 files changed, 1343 insertions(+), 2 deletions(-) create mode 100644 src/lib/process/process.c create mode 100644 src/lib/process/process.h create mode 100644 src/test/test_process.c diff --git a/src/app/main/main.c b/src/app/main/main.c index 03b3a95d03..15b6b48ff2 100644 --- a/src/app/main/main.c +++ b/src/app/main/main.c @@ -73,6 +73,7 @@ #include "lib/geoip/geoip.h" #include "lib/process/waitpid.h" +#include "lib/process/process.h" #include "lib/meminfo/meminfo.h" #include "lib/osinfo/uname.h" @@ -557,6 +558,10 @@ tor_init(int argc, char *argv[]) rend_cache_init(); addressmap_init(); /* Init the client dns cache. Do it always, since it's * cheap. */ + + /* Initialize Process subsystem. */ + process_init(); + /* Initialize the HS subsystem. */ hs_init(); @@ -784,6 +789,7 @@ tor_free_all(int postfork) circuitmux_ewma_free_all(); accounting_free_all(); protover_summary_cache_free_all(); + process_free_all(); if (!postfork) { config_free_all(); diff --git a/src/lib/process/.may_include b/src/lib/process/.may_include index a2d57a52f3..b1c50c24a6 100644 --- a/src/lib/process/.may_include +++ b/src/lib/process/.may_include @@ -4,8 +4,9 @@ lib/cc/*.h lib/container/*.h lib/ctime/*.h lib/err/*.h -lib/intmath/*.h +lib/evloop/*.h lib/fs/*.h +lib/intmath/*.h lib/log/*.h lib/malloc/*.h lib/net/*.h @@ -15,4 +16,4 @@ lib/subsys/*.h lib/testsupport/*.h lib/thread/*.h -ht.h \ No newline at end of file +ht.h diff --git a/src/lib/process/include.am b/src/lib/process/include.am index 2aa30cc3c1..1d213d1c00 100644 --- a/src/lib/process/include.am +++ b/src/lib/process/include.am @@ -9,6 +9,7 @@ src_lib_libtor_process_a_SOURCES = \ src/lib/process/daemon.c \ src/lib/process/env.c \ src/lib/process/pidfile.c \ + src/lib/process/process.c \ src/lib/process/restrict.c \ src/lib/process/setuid.c \ src/lib/process/subprocess.c \ @@ -24,6 +25,7 @@ noinst_HEADERS += \ src/lib/process/daemon.h \ src/lib/process/env.h \ src/lib/process/pidfile.h \ + src/lib/process/process.h \ src/lib/process/restrict.h \ src/lib/process/setuid.h \ src/lib/process/subprocess.h \ diff --git a/src/lib/process/process.c b/src/lib/process/process.c new file mode 100644 index 0000000000..d3967a52da --- /dev/null +++ b/src/lib/process/process.c @@ -0,0 +1,644 @@ +/* Copyright (c) 2003, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file process.c + * \brief Module for working with other processes. + **/ + +#define PROCESS_PRIVATE +#include "lib/container/buffers.h" +#include "lib/net/buffers_net.h" +#include "lib/container/smartlist.h" +#include "lib/log/log.h" +#include "lib/log/util_bug.h" +#include "lib/process/process.h" +#include "lib/process/env.h" + +#ifdef HAVE_STDDEF_H +#include +#endif + +/** A list of all process_t instances currently allocated. */ +static smartlist_t *processes; + +/** Structure to represent a child process. */ +struct process_t { + /** Process status. */ + process_status_t status; + + /** Which protocol is the process using? */ + process_protocol_t protocol; + + /** Which function to call when we have data ready from stdout? */ + process_read_callback_t stdout_read_callback; + + /** Which function to call when we have data ready from stderr? */ + process_read_callback_t stderr_read_callback; + + /** Which function call when our process terminated? */ + process_exit_callback_t exit_callback; + + /** Our exit code when the process have terminated. */ + process_exit_code_t exit_code; + + /** Name of the command we want to execute (for example: /bin/ls). */ + char *command; + + /** The arguments used for the new process. */ + smartlist_t *arguments; + + /** The environment used for the new process. */ + smartlist_t *environment; + + /** Buffer to store data from stdout when it is read. */ + buf_t *stdout_buffer; + + /** Buffer to store data from stderr when it is read. */ + buf_t *stderr_buffer; + + /** Buffer to store data to stdin before it is written. */ + buf_t *stdin_buffer; + + /** Do we need to store some custom data with the process? */ + void *data; +}; + +/** Convert a given process status in status to its string + * representation. */ +const char * +process_status_to_string(process_status_t status) +{ + switch (status) { + case PROCESS_STATUS_NOT_RUNNING: + return "not running"; + case PROCESS_STATUS_RUNNING: + return "running"; + case PROCESS_STATUS_ERROR: + return "error"; + } + + /* LCOV_EXCL_START */ + tor_assert_unreached(); + return NULL; + /* LCOV_EXCL_STOP */ +} + +/** Convert a given process protocol in protocol to its string + * representation. */ +const char * +process_protocol_to_string(process_protocol_t protocol) +{ + switch (protocol) { + case PROCESS_PROTOCOL_LINE: + return "Line"; + case PROCESS_PROTOCOL_RAW: + return "Raw"; + } + + /* LCOV_EXCL_START */ + tor_assert_unreached(); + return NULL; + /* LCOV_EXCL_STOP */ +} + +/** Initialize the Process subsystem. This function initializes the Process + * subsystem's global state. For cleaning up, process_free_all() should + * be called. */ +void +process_init(void) +{ + processes = smartlist_new(); +} + +/** Free up all resources that is handled by the Process subsystem. Note that + * this call does not terminate already running processes. */ +void +process_free_all(void) +{ + SMARTLIST_FOREACH(processes, process_t *, x, process_free(x)); + smartlist_free(processes); +} + +/** Get a list of all processes. This function returns a smartlist of + * process_t containing all the currently allocated processes. */ +const smartlist_t * +process_get_all_processes(void) +{ + return processes; +} + +/** Allocate and initialize a new process. This function returns a newly + * allocated and initialized process data, which can be used to configure and + * later run a subprocess of Tor. Use the various process_set_*() + * methods to configure it and run the process using process_exec(). Use + * command to specify the path to the command to run. You can either + * specify an absolute path to the command or relative where Tor will use the + * underlying operating system's functionality for finding the command to run. + * */ +process_t * +process_new(const char *command) +{ + tor_assert(command); + + process_t *process; + process = tor_malloc_zero(sizeof(process_t)); + + /* Set our command. */ + process->command = tor_strdup(command); + + /* By default we are not running. */ + process->status = PROCESS_STATUS_NOT_RUNNING; + + /* Prepare process environment. */ + process->arguments = smartlist_new(); + process->environment = smartlist_new(); + + /* Prepare the buffers. */ + process->stdout_buffer = buf_new(); + process->stderr_buffer = buf_new(); + process->stdin_buffer = buf_new(); + + smartlist_add(processes, process); + + return process; +} + +/** Deallocate the given process in process. */ +void +process_free_(process_t *process) +{ + if (! process) + return; + + /* Cleanup parameters. */ + tor_free(process->command); + + /* Cleanup arguments and environment. */ + SMARTLIST_FOREACH(process->arguments, char *, x, tor_free(x)); + smartlist_free(process->arguments); + + SMARTLIST_FOREACH(process->environment, char *, x, tor_free(x)); + smartlist_free(process->environment); + + /* Cleanup the buffers. */ + buf_free(process->stdout_buffer); + buf_free(process->stderr_buffer); + buf_free(process->stdin_buffer); + + smartlist_remove(processes, process); + + tor_free(process); +} + +/** Execute the given process. This function executes the given process as a + * subprocess of Tor. Returns PROCESS_STATUS_RUNNING upon success. */ +process_status_t +process_exec(process_t *process) +{ + tor_assert(process); + + process_status_t status = PROCESS_STATUS_NOT_RUNNING; + + log_info(LD_PROCESS, "Starting new process: %s", process->command); + + /* Update our state. */ + process_set_status(process, status); + + if (status != PROCESS_STATUS_RUNNING) { + log_warn(LD_PROCESS, "Failed to start process: %s", + process_get_command(process)); + } + + return status; +} + +/** Set the callback function for output from the child process's standard out + * handle. This function sets the callback function which is called every time + * the child process have written output to its standard out file handle. + * + * Use process_set_protocol(process, PROCESS_PROTOCOL_LINE) if you want + * the callback to only contain complete "\n" or "\r\n" terminated lines. */ +void +process_set_stdout_read_callback(process_t *process, + process_read_callback_t callback) +{ + tor_assert(process); + process->stdout_read_callback = callback; +} + +/** Set the callback function for output from the child process's standard + * error handle. This function sets the callback function which is called + * every time the child process have written output to its standard error file + * handle. + * + * Use process_set_protocol(process, PROCESS_PROTOCOL_LINE) if you want + * the callback to only contain complete "\n" or "\r\n" terminated lines. */ +void +process_set_stderr_read_callback(process_t *process, + process_read_callback_t callback) +{ + tor_assert(process); + process->stderr_read_callback = callback; +} + +/** Set the callback function for process exit notification. The + * callback function will be called every time your child process have + * terminated. */ +void +process_set_exit_callback(process_t *process, + process_exit_callback_t callback) +{ + tor_assert(process); + process->exit_callback = callback; +} + +/** Get the current command of the given process. */ +const char * +process_get_command(const process_t *process) +{ + tor_assert(process); + return process->command; +} + +void +process_set_protocol(process_t *process, process_protocol_t protocol) +{ + tor_assert(process); + process->protocol = protocol; +} + +/** Get the currently used protocol of the given process. */ +process_protocol_t +process_get_protocol(const process_t *process) +{ + tor_assert(process); + return process->protocol; +} + +/** Set opague pointer to data. This function allows you to store a pointer to + * your own data in the given process. Use process_get_data() in the + * various callback functions to retrieve the data again. + * + * Note that the given process does NOT take ownership of the data and you are + * responsible for freeing up any resources allocated by the given data. + * */ +void +process_set_data(process_t *process, void *data) +{ + tor_assert(process); + process->data = data; +} + +/** Get the opaque pointer to callback data from the given process. This + * function allows you get the data you stored with process_set_data() + * in the different callback functions. */ +void * +process_get_data(const process_t *process) +{ + tor_assert(process); + return process->data; +} + +/** Set the status of a given process. */ +void +process_set_status(process_t *process, process_status_t status) +{ + tor_assert(process); + process->status = status; +} + +/** Get the status of the given process. */ +process_status_t +process_get_status(const process_t *process) +{ + tor_assert(process); + return process->status; +} + +/** Append an argument to the list of arguments in the given process. */ +void +process_append_argument(process_t *process, const char *argument) +{ + tor_assert(process); + tor_assert(argument); + + smartlist_add(process->arguments, tor_strdup(argument)); +} + +/** Returns a list of arguments (excluding the command itself) from the + * given process. */ +const smartlist_t * +process_get_arguments(const process_t *process) +{ + tor_assert(process); + return process->arguments; +} + +/** Returns a newly allocated Unix style argument vector. Use tor_free() + * to deallocate it after use. */ +char ** +process_get_argv(const process_t *process) +{ + tor_assert(process); + + /** Generate a Unix style process argument vector from our process's + * arguments smartlist_t. */ + char **argv = NULL; + + char *filename = process->command; + const smartlist_t *arguments = process->arguments; + const size_t size = smartlist_len(arguments); + + /* Make space for the process filename as argv[0] and a trailing NULL. */ + argv = tor_malloc_zero(sizeof(char *) * (size + 2)); + + /* Set our filename as first argument. */ + argv[0] = filename; + + /* Put in the rest of the values from arguments. */ + SMARTLIST_FOREACH_BEGIN(arguments, char *, arg_val) { + tor_assert(arg_val != NULL); + + argv[arg_val_sl_idx + 1] = arg_val; + } SMARTLIST_FOREACH_END(arg_val); + + return argv; +} + +/** Set the given key/value pair as environment variable in the + * given process. */ +void +process_set_environment(process_t *process, + const char *key, + const char *value) +{ + tor_assert(process); + tor_assert(key); + tor_assert(value); + + smartlist_add_asprintf(process->environment, "%s=%s", key, value); +} + +/** Returns a newly allocated process_environment_t containing the + * environment variables for the given process. */ +process_environment_t * +process_get_environment(const process_t *process) +{ + tor_assert(process); + return process_environment_make(process->environment); +} + +/** Write size bytes of data to the given process's standard + * input. */ +void +process_write(process_t *process, + const uint8_t *data, size_t size) +{ + tor_assert(process); + tor_assert(data); + + buf_add(process->stdin_buffer, (char *)data, size); + process_write_stdin(process, process->stdin_buffer); +} + +/** As tor_vsnprintf(), but write the data to the given process's standard + * input. */ +void +process_vprintf(process_t *process, + const char *format, va_list args) +{ + tor_assert(process); + tor_assert(format); + + int size; + char *data; + + size = tor_vasprintf(&data, format, args); + process_write(process, (uint8_t *)data, size); + tor_free(data); +} + +/** As tor_snprintf(), but write the data to the given process's standard + * input. */ +void +process_printf(process_t *process, + const char *format, ...) +{ + tor_assert(process); + tor_assert(format); + + va_list ap; + va_start(ap, format); + process_vprintf(process, format, ap); + va_end(ap); +} + +/** This function is called by the Process backend when a given process have + * data that is ready to be read from the child process's standard output + * handle. */ +void +process_notify_event_stdout(process_t *process) +{ + tor_assert(process); + + int ret; + ret = process_read_stdout(process, process->stdout_buffer); + + if (ret > 0) + process_read_data(process, + process->stdout_buffer, + process->stdout_read_callback); +} + +/** This function is called by the Process backend when a given process have + * data that is ready to be read from the child process's standard error + * handle. */ +void +process_notify_event_stderr(process_t *process) +{ + tor_assert(process); + + int ret; + ret = process_read_stderr(process, process->stderr_buffer); + + if (ret > 0) + process_read_data(process, + process->stderr_buffer, + process->stderr_read_callback); +} + +/** This function is called by the Process backend when a given process is + * allowed to begin writing data to the standard input of the child process. */ +void +process_notify_event_stdin(process_t *process) +{ + tor_assert(process); + + process_write_stdin(process, process->stdin_buffer); +} + +/** This function is called by the Process backend when a given process have + * terminated. The exit status code is passed in exit_code. We mark the + * process as no longer running and calls the exit_callback with + * information about the process termination. */ +void +process_notify_event_exit(process_t *process, process_exit_code_t exit_code) +{ + tor_assert(process); + + log_debug(LD_PROCESS, + "Process terminated with exit code: %"PRIu64, exit_code); + + /* Update our state. */ + process_set_status(process, PROCESS_STATUS_NOT_RUNNING); + process->exit_code = exit_code; + + /* Call our exit callback, if it exists. */ + if (process->exit_callback) { + process->exit_callback(process, exit_code); + } +} + +/** This function is called whenever the Process backend have notified us that + * there is data to be read from its standard out handle. Returns the number of + * bytes that have been put into the given buffer. */ +MOCK_IMPL(STATIC int, process_read_stdout, (process_t *process, buf_t *buffer)) +{ + tor_assert(process); + tor_assert(buffer); + + return 0; +} + +/** This function is called whenever the Process backend have notified us that + * there is data to be read from its standard error handle. Returns the number + * of bytes that have been put into the given buffer. */ +MOCK_IMPL(STATIC int, process_read_stderr, (process_t *process, buf_t *buffer)) +{ + tor_assert(process); + tor_assert(buffer); + + return 0; +} + +/** This function calls the backend function for the given process whenever + * there is data to be written to the backends' file handles. */ +MOCK_IMPL(STATIC void, process_write_stdin, + (process_t *process, buf_t *buffer)) +{ + tor_assert(process); + tor_assert(buffer); +} + +/** This function calls the protocol handlers based on the value of + * process_get_protocol(process). Currently we call + * process_read_buffer() for PROCESS_PROTOCOL_RAW and + * process_read_lines() for PROCESS_PROTOCOL_LINE. */ +STATIC void +process_read_data(process_t *process, + buf_t *buffer, + process_read_callback_t callback) +{ + tor_assert(process); + tor_assert(buffer); + + switch (process_get_protocol(process)) { + case PROCESS_PROTOCOL_RAW: + process_read_buffer(process, buffer, callback); + break; + case PROCESS_PROTOCOL_LINE: + process_read_lines(process, buffer, callback); + break; + default: + /* LCOV_EXCL_START */ + tor_assert_unreached(); + return; + /* LCOV_EXCL_STOP */ + } +} + +/** This function takes the content of the given buffer and passes it to + * the given callback function, but ensures that an additional zero byte + * is added to the end of the data such that the given callback implementation + * can threat the content as a ASCIIZ string. */ +STATIC void +process_read_buffer(process_t *process, + buf_t *buffer, + process_read_callback_t callback) +{ + tor_assert(process); + tor_assert(buffer); + + const size_t size = buf_datalen(buffer); + + /* We allocate an extra byte for the zero byte in the end. */ + char *data = tor_malloc_zero(size + 1); + + buf_get_bytes(buffer, data, size); + log_debug(LD_PROCESS, "Read data from process"); + + if (callback) + callback(process, data, size); + + tor_free(data); +} + +/** This function tries to extract complete lines from the given buffer + * and calls the given callback function whenever it has a complete + * line. Before calling callback we remove the trailing "\n" or "\r\n" + * from the line. If we are unable to extract a complete line we leave the data + * in the buffer for next call. */ +STATIC void +process_read_lines(process_t *process, + buf_t *buffer, + process_read_callback_t callback) +{ + tor_assert(process); + tor_assert(buffer); + + const size_t size = buf_datalen(buffer) + 1; + size_t line_size = 0; + char *data = tor_malloc_zero(size); + int ret; + + while (true) { + line_size = size; + ret = buf_get_line(buffer, data, &line_size); + + /* A complete line should always be smaller than the size of our + * buffer. */ + tor_assert(ret != -1); + + /* Remove \n from the end of the line. */ + if (data[line_size - 1] == '\n') { + data[line_size - 1] = '\0'; + --line_size; + } + + /* Remove \r from the end of the line. */ + if (data[line_size - 1] == '\r') { + data[line_size - 1] = '\0'; + --line_size; + } + + if (ret == 1) { + log_debug(LD_PROCESS, "Read line from process: \"%s\"", data); + + if (callback) + callback(process, data, line_size); + + /* We have read a whole line, let's see if there is more lines to read. + * */ + continue; + } + + /* No complete line for us to read. We are done for now. */ + tor_assert_nonfatal(ret == 0); + break; + } + + tor_free(data); +} diff --git a/src/lib/process/process.h b/src/lib/process/process.h new file mode 100644 index 0000000000..cf20a5d80b --- /dev/null +++ b/src/lib/process/process.h @@ -0,0 +1,127 @@ +/* Copyright (c) 2003-2004, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file process.h + * \brief Header for process.c + **/ + +#ifndef TOR_PROCESS_H +#define TOR_PROCESS_H + +#include "orconfig.h" +#include "lib/malloc/malloc.h" +#include "lib/string/printf.h" + +/** Maximum number of bytes to write to a process' stdin. */ +#define PROCESS_MAX_WRITE (1024) + +/** Maximum number of bytes to read from a process' stdout/stderr. */ +#define PROCESS_MAX_READ (1024) + +typedef enum { + /** The process is not running. */ + PROCESS_STATUS_NOT_RUNNING, + + /** The process is running. */ + PROCESS_STATUS_RUNNING, + + /** The process is in an erroneous state. */ + PROCESS_STATUS_ERROR +} process_status_t; + +const char *process_status_to_string(process_status_t status); + +typedef enum { + /** Pass complete \n-terminated lines to the + * callback (with the \n or \r\n removed). */ + PROCESS_PROTOCOL_LINE, + + /** Pass the raw response from read() to the callback. */ + PROCESS_PROTOCOL_RAW +} process_protocol_t; + +const char *process_protocol_to_string(process_protocol_t protocol); + +struct process_t; +typedef struct process_t process_t; + +typedef uint64_t process_exit_code_t; + +typedef void (*process_read_callback_t)(process_t *, + char *, + size_t); +typedef void (*process_exit_callback_t)(process_t *, + process_exit_code_t); + +void process_init(void); +void process_free_all(void); +const smartlist_t *process_get_all_processes(void); + +process_t *process_new(const char *command); +void process_free_(process_t *process); +#define process_free(s) FREE_AND_NULL(process_t, process_free_, (s)) + +process_status_t process_exec(process_t *process); + +void process_set_stdout_read_callback(process_t *, + process_read_callback_t); +void process_set_stderr_read_callback(process_t *, + process_read_callback_t); +void process_set_exit_callback(process_t *, + process_exit_callback_t); + +const char *process_get_command(const process_t *process); + +void process_append_argument(process_t *process, const char *argument); +const smartlist_t *process_get_arguments(const process_t *process); +char **process_get_argv(const process_t *process); + +void process_set_environment(process_t *process, + const char *key, + const char *value); + +struct process_environment_t; +struct process_environment_t *process_get_environment(const process_t *); + +void process_set_protocol(process_t *process, process_protocol_t protocol); +process_protocol_t process_get_protocol(const process_t *process); + +void process_set_data(process_t *process, void *data); +void *process_get_data(const process_t *process); + +void process_set_status(process_t *process, process_status_t status); +process_status_t process_get_status(const process_t *process); + +void process_write(process_t *process, + const uint8_t *data, size_t size); +void process_vprintf(process_t *process, + const char *format, va_list args) CHECK_PRINTF(2, 0); +void process_printf(process_t *process, + const char *format, ...) CHECK_PRINTF(2, 3); + +void process_notify_event_stdout(process_t *process); +void process_notify_event_stderr(process_t *process); +void process_notify_event_stdin(process_t *process); +void process_notify_event_exit(process_t *process, + process_exit_code_t); + +#ifdef PROCESS_PRIVATE +MOCK_DECL(STATIC int, process_read_stdout, (process_t *, buf_t *)); +MOCK_DECL(STATIC int, process_read_stderr, (process_t *, buf_t *)); +MOCK_DECL(STATIC void, process_write_stdin, (process_t *, buf_t *)); + +STATIC void process_read_data(process_t *process, + buf_t *buffer, + process_read_callback_t callback); +STATIC void process_read_buffer(process_t *process, + buf_t *buffer, + process_read_callback_t callback); +STATIC void process_read_lines(process_t *process, + buf_t *buffer, + process_read_callback_t callback); +#endif /* defined(PROCESS_PRIVATE). */ + +#endif /* defined(TOR_PROCESS_H). */ diff --git a/src/test/include.am b/src/test/include.am index e5eae56e25..482897c7a5 100644 --- a/src/test/include.am +++ b/src/test/include.am @@ -153,6 +153,7 @@ src_test_test_SOURCES += \ src/test/test_pem.c \ src/test/test_periodic_event.c \ src/test/test_policy.c \ + src/test/test_process.c \ src/test/test_procmon.c \ src/test/test_proto_http.c \ src/test/test_proto_misc.c \ diff --git a/src/test/test.c b/src/test/test.c index 17b736d305..b13c3ffbe2 100644 --- a/src/test/test.c +++ b/src/test/test.c @@ -898,6 +898,7 @@ struct testgroup_t testgroups[] = { { "periodic-event/" , periodic_event_tests }, { "policy/" , policy_tests }, { "procmon/", procmon_tests }, + { "process/", process_tests }, { "proto/http/", proto_http_tests }, { "proto/misc/", proto_misc_tests }, { "protover/", protover_tests }, diff --git a/src/test/test.h b/src/test/test.h index 092356f0fb..7c17389776 100644 --- a/src/test/test.h +++ b/src/test/test.h @@ -241,6 +241,7 @@ extern struct testcase_t pem_tests[]; extern struct testcase_t periodic_event_tests[]; extern struct testcase_t policy_tests[]; extern struct testcase_t procmon_tests[]; +extern struct testcase_t process_tests[]; extern struct testcase_t proto_http_tests[]; extern struct testcase_t proto_misc_tests[]; extern struct testcase_t protover_tests[]; diff --git a/src/test/test_process.c b/src/test/test_process.c new file mode 100644 index 0000000000..2adbde7ad2 --- /dev/null +++ b/src/test/test_process.c @@ -0,0 +1,558 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file test_process.c + * \brief Test cases for the Process API. + */ + +#include "orconfig.h" +#include "core/or/or.h" +#include "test/test.h" + +#define PROCESS_PRIVATE +#include "lib/process/process.h" + +static const char *stdout_read_buffer; +static const char *stderr_read_buffer; + +struct process_data_t { + smartlist_t *stdout_data; + smartlist_t *stderr_data; + smartlist_t *stdin_data; + process_exit_code_t exit_code; +}; + +typedef struct process_data_t process_data_t; + +static process_data_t * +process_data_new(void) +{ + process_data_t *process_data = tor_malloc_zero(sizeof(process_data_t)); + process_data->stdout_data = smartlist_new(); + process_data->stderr_data = smartlist_new(); + process_data->stdin_data = smartlist_new(); + return process_data; +} + +static void +process_data_free(process_data_t *process_data) +{ + if (process_data == NULL) + return; + + SMARTLIST_FOREACH(process_data->stdout_data, char *, x, tor_free(x)); + SMARTLIST_FOREACH(process_data->stderr_data, char *, x, tor_free(x)); + SMARTLIST_FOREACH(process_data->stdin_data, char *, x, tor_free(x)); + + smartlist_free(process_data->stdout_data); + smartlist_free(process_data->stderr_data); + smartlist_free(process_data->stdin_data); + tor_free(process_data); +} + +static int +process_mocked_read_stdout(process_t *process, buf_t *buffer) +{ + (void)process; + + if (stdout_read_buffer != NULL) { + buf_add_string(buffer, stdout_read_buffer); + stdout_read_buffer = NULL; + } + + return (int)buf_datalen(buffer); +} + +static int +process_mocked_read_stderr(process_t *process, buf_t *buffer) +{ + (void)process; + + if (stderr_read_buffer != NULL) { + buf_add_string(buffer, stderr_read_buffer); + stderr_read_buffer = NULL; + } + + return (int)buf_datalen(buffer); +} + +static void +process_mocked_write_stdin(process_t *process, buf_t *buffer) +{ + const size_t size = buf_datalen(buffer); + + if (size == 0) + return; + + char *data = tor_malloc_zero(size + 1); + process_data_t *process_data = process_get_data(process); + + buf_get_bytes(buffer, data, size); + smartlist_add(process_data->stdin_data, data); +} + +static void +process_stdout_callback(process_t *process, char *data, size_t size) +{ + tt_ptr_op(process, OP_NE, NULL); + tt_ptr_op(data, OP_NE, NULL); + tt_int_op(strlen(data), OP_EQ, size); + + process_data_t *process_data = process_get_data(process); + smartlist_add(process_data->stdout_data, tor_strdup(data)); + + done: + return; +} + +static void +process_stderr_callback(process_t *process, char *data, size_t size) +{ + tt_ptr_op(process, OP_NE, NULL); + tt_ptr_op(data, OP_NE, NULL); + tt_int_op(strlen(data), OP_EQ, size); + + process_data_t *process_data = process_get_data(process); + smartlist_add(process_data->stderr_data, tor_strdup(data)); + + done: + return; +} + +static void +process_exit_callback(process_t *process, process_exit_code_t exit_code) +{ + tt_ptr_op(process, OP_NE, NULL); + + process_data_t *process_data = process_get_data(process); + process_data->exit_code = exit_code; + + done: + return; +} + +static void +test_default_values(void *arg) +{ + (void)arg; + process_init(); + + process_t *process = process_new("/path/to/nothing"); + + /* We are not running by default. */ + tt_int_op(PROCESS_STATUS_NOT_RUNNING, OP_EQ, process_get_status(process)); + + /* We use the line protocol by default. */ + tt_int_op(PROCESS_PROTOCOL_LINE, OP_EQ, process_get_protocol(process)); + + /* We don't set any custom data by default. */ + tt_ptr_op(NULL, OP_EQ, process_get_data(process)); + + /* Our command was given to the process_t's constructor in process_new(). */ + tt_str_op("/path/to/nothing", OP_EQ, process_get_command(process)); + + /* Make sure we are listed in the list of proccesses. */ + tt_assert(smartlist_contains(process_get_all_processes(), + process)); + + /* Our arguments should be empty. */ + tt_int_op(0, OP_EQ, + smartlist_len(process_get_arguments(process))); + + done: + process_free(process); + process_free_all(); +} + +static void +test_stringified_types(void *arg) +{ + (void)arg; + + /* process_protocol_t values. */ + tt_str_op("Raw", OP_EQ, process_protocol_to_string(PROCESS_PROTOCOL_RAW)); + tt_str_op("Line", OP_EQ, process_protocol_to_string(PROCESS_PROTOCOL_LINE)); + + /* process_status_t values. */ + tt_str_op("not running", OP_EQ, + process_status_to_string(PROCESS_STATUS_NOT_RUNNING)); + tt_str_op("running", OP_EQ, + process_status_to_string(PROCESS_STATUS_RUNNING)); + tt_str_op("error", OP_EQ, + process_status_to_string(PROCESS_STATUS_ERROR)); + + done: + return; +} + +static void +test_line_protocol_simple(void *arg) +{ + (void)arg; + process_init(); + + process_data_t *process_data = process_data_new(); + + process_t *process = process_new(""); + process_set_data(process, process_data); + + process_set_stdout_read_callback(process, process_stdout_callback); + process_set_stderr_read_callback(process, process_stderr_callback); + + MOCK(process_read_stdout, process_mocked_read_stdout); + MOCK(process_read_stderr, process_mocked_read_stderr); + + /* Make sure we are running with the line protocol. */ + tt_int_op(PROCESS_PROTOCOL_LINE, OP_EQ, process_get_protocol(process)); + + tt_int_op(0, OP_EQ, smartlist_len(process_data->stdout_data)); + tt_int_op(0, OP_EQ, smartlist_len(process_data->stderr_data)); + + stdout_read_buffer = "Hello stdout\n"; + process_notify_event_stdout(process); + tt_ptr_op(NULL, OP_EQ, stdout_read_buffer); + + stderr_read_buffer = "Hello stderr\r\n"; + process_notify_event_stderr(process); + tt_ptr_op(NULL, OP_EQ, stderr_read_buffer); + + /* Data should be ready. */ + tt_int_op(1, OP_EQ, smartlist_len(process_data->stdout_data)); + tt_int_op(1, OP_EQ, smartlist_len(process_data->stderr_data)); + + /* Check if the data is correct. */ + tt_str_op(smartlist_get(process_data->stdout_data, 0), OP_EQ, + "Hello stdout"); + tt_str_op(smartlist_get(process_data->stderr_data, 0), OP_EQ, + "Hello stderr"); + + done: + process_data_free(process_data); + process_free(process); + process_free_all(); + + UNMOCK(process_read_stdout); + UNMOCK(process_read_stderr); +} + +static void +test_line_protocol_multi(void *arg) +{ + (void)arg; + process_init(); + + process_data_t *process_data = process_data_new(); + + process_t *process = process_new(""); + process_set_data(process, process_data); + process_set_stdout_read_callback(process, process_stdout_callback); + process_set_stderr_read_callback(process, process_stderr_callback); + + MOCK(process_read_stdout, process_mocked_read_stdout); + MOCK(process_read_stderr, process_mocked_read_stderr); + + /* Make sure we are running with the line protocol. */ + tt_int_op(PROCESS_PROTOCOL_LINE, OP_EQ, process_get_protocol(process)); + + tt_int_op(0, OP_EQ, smartlist_len(process_data->stdout_data)); + tt_int_op(0, OP_EQ, smartlist_len(process_data->stderr_data)); + + stdout_read_buffer = "Hello stdout\r\nOnion Onion Onion\nA B C D\r\n\r\n"; + process_notify_event_stdout(process); + tt_ptr_op(NULL, OP_EQ, stdout_read_buffer); + + stderr_read_buffer = "Hello stderr\nFoo bar baz\nOnion Onion Onion\n"; + process_notify_event_stderr(process); + tt_ptr_op(NULL, OP_EQ, stderr_read_buffer); + + /* Data should be ready. */ + tt_int_op(4, OP_EQ, smartlist_len(process_data->stdout_data)); + tt_int_op(3, OP_EQ, smartlist_len(process_data->stderr_data)); + + /* Check if the data is correct. */ + tt_str_op(smartlist_get(process_data->stdout_data, 0), OP_EQ, + "Hello stdout"); + tt_str_op(smartlist_get(process_data->stdout_data, 1), OP_EQ, + "Onion Onion Onion"); + tt_str_op(smartlist_get(process_data->stdout_data, 2), OP_EQ, + "A B C D"); + tt_str_op(smartlist_get(process_data->stdout_data, 3), OP_EQ, + ""); + + tt_str_op(smartlist_get(process_data->stderr_data, 0), OP_EQ, + "Hello stderr"); + tt_str_op(smartlist_get(process_data->stderr_data, 1), OP_EQ, + "Foo bar baz"); + tt_str_op(smartlist_get(process_data->stderr_data, 2), OP_EQ, + "Onion Onion Onion"); + + done: + process_data_free(process_data); + process_free(process); + process_free_all(); + + UNMOCK(process_read_stdout); + UNMOCK(process_read_stderr); +} + +static void +test_line_protocol_partial(void *arg) +{ + (void)arg; + process_init(); + + process_data_t *process_data = process_data_new(); + + process_t *process = process_new(""); + process_set_data(process, process_data); + process_set_stdout_read_callback(process, process_stdout_callback); + process_set_stderr_read_callback(process, process_stderr_callback); + + MOCK(process_read_stdout, process_mocked_read_stdout); + MOCK(process_read_stderr, process_mocked_read_stderr); + + /* Make sure we are running with the line protocol. */ + tt_int_op(PROCESS_PROTOCOL_LINE, OP_EQ, process_get_protocol(process)); + + tt_int_op(0, OP_EQ, smartlist_len(process_data->stdout_data)); + tt_int_op(0, OP_EQ, smartlist_len(process_data->stderr_data)); + + stdout_read_buffer = "Hello stdout this is a partial line ..."; + process_notify_event_stdout(process); + tt_ptr_op(NULL, OP_EQ, stdout_read_buffer); + + stderr_read_buffer = "Hello stderr this is a partial line ..."; + process_notify_event_stderr(process); + tt_ptr_op(NULL, OP_EQ, stderr_read_buffer); + + /* Data should NOT be ready. */ + tt_int_op(0, OP_EQ, smartlist_len(process_data->stdout_data)); + tt_int_op(0, OP_EQ, smartlist_len(process_data->stderr_data)); + + stdout_read_buffer = " the end\nAnother partial string goes here ..."; + process_notify_event_stdout(process); + tt_ptr_op(NULL, OP_EQ, stdout_read_buffer); + + stderr_read_buffer = " the end\nAnother partial string goes here ..."; + process_notify_event_stderr(process); + tt_ptr_op(NULL, OP_EQ, stderr_read_buffer); + + /* Some data should be ready. */ + tt_int_op(1, OP_EQ, smartlist_len(process_data->stdout_data)); + tt_int_op(1, OP_EQ, smartlist_len(process_data->stderr_data)); + + stdout_read_buffer = " the end\nFoo bar baz\n"; + process_notify_event_stdout(process); + tt_ptr_op(NULL, OP_EQ, stdout_read_buffer); + + stderr_read_buffer = " the end\nFoo bar baz\n"; + process_notify_event_stderr(process); + tt_ptr_op(NULL, OP_EQ, stderr_read_buffer); + + /* Some data should be ready. */ + tt_int_op(3, OP_EQ, smartlist_len(process_data->stdout_data)); + tt_int_op(3, OP_EQ, smartlist_len(process_data->stderr_data)); + + /* Check if the data is correct. */ + tt_str_op(smartlist_get(process_data->stdout_data, 0), OP_EQ, + "Hello stdout this is a partial line ... the end"); + tt_str_op(smartlist_get(process_data->stdout_data, 1), OP_EQ, + "Another partial string goes here ... the end"); + tt_str_op(smartlist_get(process_data->stdout_data, 2), OP_EQ, + "Foo bar baz"); + + tt_str_op(smartlist_get(process_data->stderr_data, 0), OP_EQ, + "Hello stderr this is a partial line ... the end"); + tt_str_op(smartlist_get(process_data->stderr_data, 1), OP_EQ, + "Another partial string goes here ... the end"); + tt_str_op(smartlist_get(process_data->stderr_data, 2), OP_EQ, + "Foo bar baz"); + + done: + process_data_free(process_data); + process_free(process); + process_free_all(); + + UNMOCK(process_read_stdout); + UNMOCK(process_read_stderr); +} + +static void +test_raw_protocol_simple(void *arg) +{ + (void)arg; + process_init(); + + process_data_t *process_data = process_data_new(); + + process_t *process = process_new(""); + process_set_data(process, process_data); + process_set_protocol(process, PROCESS_PROTOCOL_RAW); + + process_set_stdout_read_callback(process, process_stdout_callback); + process_set_stderr_read_callback(process, process_stderr_callback); + + MOCK(process_read_stdout, process_mocked_read_stdout); + MOCK(process_read_stderr, process_mocked_read_stderr); + + /* Make sure we are running with the raw protocol. */ + tt_int_op(PROCESS_PROTOCOL_RAW, OP_EQ, process_get_protocol(process)); + + tt_int_op(0, OP_EQ, smartlist_len(process_data->stdout_data)); + tt_int_op(0, OP_EQ, smartlist_len(process_data->stderr_data)); + + stdout_read_buffer = "Hello stdout\n"; + process_notify_event_stdout(process); + tt_ptr_op(NULL, OP_EQ, stdout_read_buffer); + + stderr_read_buffer = "Hello stderr\n"; + process_notify_event_stderr(process); + tt_ptr_op(NULL, OP_EQ, stderr_read_buffer); + + /* Data should be ready. */ + tt_int_op(1, OP_EQ, smartlist_len(process_data->stdout_data)); + tt_int_op(1, OP_EQ, smartlist_len(process_data->stderr_data)); + + stdout_read_buffer = "Hello, again, stdout\nThis contains multiple lines"; + process_notify_event_stdout(process); + tt_ptr_op(NULL, OP_EQ, stdout_read_buffer); + + stderr_read_buffer = "Hello, again, stderr\nThis contains multiple lines"; + process_notify_event_stderr(process); + tt_ptr_op(NULL, OP_EQ, stderr_read_buffer); + + /* Data should be ready. */ + tt_int_op(2, OP_EQ, smartlist_len(process_data->stdout_data)); + tt_int_op(2, OP_EQ, smartlist_len(process_data->stderr_data)); + + /* Check if the data is correct. */ + tt_str_op(smartlist_get(process_data->stdout_data, 0), OP_EQ, + "Hello stdout\n"); + tt_str_op(smartlist_get(process_data->stdout_data, 1), OP_EQ, + "Hello, again, stdout\nThis contains multiple lines"); + + tt_str_op(smartlist_get(process_data->stderr_data, 0), OP_EQ, + "Hello stderr\n"); + tt_str_op(smartlist_get(process_data->stderr_data, 1), OP_EQ, + "Hello, again, stderr\nThis contains multiple lines"); + + done: + process_data_free(process_data); + process_free(process); + process_free_all(); + + UNMOCK(process_read_stdout); + UNMOCK(process_read_stderr); +} + +static void +test_write_simple(void *arg) +{ + (void)arg; + + process_init(); + + process_data_t *process_data = process_data_new(); + + process_t *process = process_new(""); + process_set_data(process, process_data); + + MOCK(process_write_stdin, process_mocked_write_stdin); + + process_write(process, (uint8_t *)"Hello world\n", 12); + process_notify_event_stdin(process); + tt_int_op(1, OP_EQ, smartlist_len(process_data->stdin_data)); + + process_printf(process, "Hello %s !\n", "moon"); + process_notify_event_stdin(process); + tt_int_op(2, OP_EQ, smartlist_len(process_data->stdin_data)); + + done: + process_data_free(process_data); + process_free(process); + process_free_all(); + + UNMOCK(process_write_stdin); +} + +static void +test_exit_simple(void *arg) +{ + (void)arg; + + process_init(); + + process_data_t *process_data = process_data_new(); + + process_t *process = process_new(""); + process_set_data(process, process_data); + process_set_exit_callback(process, process_exit_callback); + + /* Our default is 0. */ + tt_int_op(0, OP_EQ, process_data->exit_code); + + /* Fake that we are a running process. */ + process_set_status(process, PROCESS_STATUS_RUNNING); + tt_int_op(process_get_status(process), OP_EQ, PROCESS_STATUS_RUNNING); + + /* Fake an exit. */ + process_notify_event_exit(process, 1337); + + /* Check if our state changed and if our callback fired. */ + tt_int_op(process_get_status(process), OP_EQ, PROCESS_STATUS_NOT_RUNNING); + tt_int_op(1337, OP_EQ, process_data->exit_code); + + done: + process_set_data(process, process_data); + process_data_free(process_data); + process_free(process); + process_free_all(); +} + +static void +test_argv_simple(void *arg) +{ + (void)arg; + process_init(); + + process_t *process = process_new("/bin/cat"); + char **argv = NULL; + + /* Setup some arguments. */ + process_append_argument(process, "foo"); + process_append_argument(process, "bar"); + process_append_argument(process, "baz"); + + /* Check the number of elements. */ + tt_int_op(3, OP_EQ, + smartlist_len(process_get_arguments(process))); + + /* Let's try to convert it into a Unix style char **argv. */ + argv = process_get_argv(process); + + /* Check our values. */ + tt_str_op(argv[0], OP_EQ, "/bin/cat"); + tt_str_op(argv[1], OP_EQ, "foo"); + tt_str_op(argv[2], OP_EQ, "bar"); + tt_str_op(argv[3], OP_EQ, "baz"); + tt_ptr_op(argv[4], OP_EQ, NULL); + + done: + tor_free(argv); + process_free(process); + process_free_all(); +} + +struct testcase_t process_tests[] = { + { "default_values", test_default_values, TT_FORK, NULL, NULL }, + { "stringified_types", test_stringified_types, TT_FORK, NULL, NULL }, + { "line_protocol_simple", test_line_protocol_simple, TT_FORK, NULL, NULL }, + { "line_protocol_multi", test_line_protocol_multi, TT_FORK, NULL, NULL }, + { "line_protocol_partial", test_line_protocol_partial, TT_FORK, NULL, NULL }, + { "raw_protocol_simple", test_raw_protocol_simple, TT_FORK, NULL, NULL }, + { "write_simple", test_write_simple, TT_FORK, NULL, NULL }, + { "exit_simple", test_exit_simple, TT_FORK, NULL, NULL }, + { "argv_simple", test_argv_simple, TT_FORK, NULL, NULL }, + END_OF_TESTCASES +}; From 2e957027e28449d4c3254cc404d154f4bce41bfc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Thu, 22 Nov 2018 04:43:27 +0100 Subject: [PATCH 0284/2557] Add Unix backend for the Process subsystem. This patch adds the Unix backend for the Process subsystem. The Unix backend attaches file descriptors from the child process's standard in, out and error to Tor's libevent based main loop using traditional Unix pipes. We use the already available `waitpid` module to get events whenever the child process terminates. See: https://bugs.torproject.org/28179 --- src/lib/process/include.am | 2 + src/lib/process/process.c | 43 +++ src/lib/process/process.h | 5 + src/lib/process/process_unix.c | 611 +++++++++++++++++++++++++++++++++ src/lib/process/process_unix.h | 64 ++++ src/test/test_process.c | 21 ++ 6 files changed, 746 insertions(+) create mode 100644 src/lib/process/process_unix.c create mode 100644 src/lib/process/process_unix.h diff --git a/src/lib/process/include.am b/src/lib/process/include.am index 1d213d1c00..f068032681 100644 --- a/src/lib/process/include.am +++ b/src/lib/process/include.am @@ -10,6 +10,7 @@ src_lib_libtor_process_a_SOURCES = \ src/lib/process/env.c \ src/lib/process/pidfile.c \ src/lib/process/process.c \ + src/lib/process/process_unix.c \ src/lib/process/restrict.c \ src/lib/process/setuid.c \ src/lib/process/subprocess.c \ @@ -26,6 +27,7 @@ noinst_HEADERS += \ src/lib/process/env.h \ src/lib/process/pidfile.h \ src/lib/process/process.h \ + src/lib/process/process_unix.h \ src/lib/process/restrict.h \ src/lib/process/setuid.h \ src/lib/process/subprocess.h \ diff --git a/src/lib/process/process.c b/src/lib/process/process.c index d3967a52da..8d6a9d3fa0 100644 --- a/src/lib/process/process.c +++ b/src/lib/process/process.c @@ -15,6 +15,7 @@ #include "lib/log/log.h" #include "lib/log/util_bug.h" #include "lib/process/process.h" +#include "lib/process/process_unix.h" #include "lib/process/env.h" #ifdef HAVE_STDDEF_H @@ -64,6 +65,11 @@ struct process_t { /** Do we need to store some custom data with the process? */ void *data; + +#ifndef _WIN32 + /** Our Unix process handle. */ + process_unix_t *unix_process; +#endif }; /** Convert a given process status in status to its string @@ -161,6 +167,11 @@ process_new(const char *command) process->stderr_buffer = buf_new(); process->stdin_buffer = buf_new(); +#ifndef _WIN32 + /* Prepare our Unix process handle. */ + process->unix_process = process_unix_new(); +#endif + smartlist_add(processes, process); return process; @@ -188,6 +199,11 @@ process_free_(process_t *process) buf_free(process->stderr_buffer); buf_free(process->stdin_buffer); +#ifndef _WIN32 + /* Cleanup our Unix process handle. */ + process_unix_free(process->unix_process); +#endif + smartlist_remove(processes, process); tor_free(process); @@ -204,6 +220,10 @@ process_exec(process_t *process) log_info(LD_PROCESS, "Starting new process: %s", process->command); +#ifndef _WIN32 + status = process_unix_exec(process); +#endif + /* Update our state. */ process_set_status(process, status); @@ -391,6 +411,17 @@ process_get_environment(const process_t *process) return process_environment_make(process->environment); } +#ifndef _WIN32 +/** Get the internal handle for the Unix backend. */ +process_unix_t * +process_get_unix_process(const process_t *process) +{ + tor_assert(process); + tor_assert(process->unix_process); + return process->unix_process; +} +#endif + /** Write size bytes of data to the given process's standard * input. */ void @@ -510,7 +541,11 @@ MOCK_IMPL(STATIC int, process_read_stdout, (process_t *process, buf_t *buffer)) tor_assert(process); tor_assert(buffer); +#ifndef _WIN32 + return process_unix_read_stdout(process, buffer); +#else return 0; +#endif } /** This function is called whenever the Process backend have notified us that @@ -521,7 +556,11 @@ MOCK_IMPL(STATIC int, process_read_stderr, (process_t *process, buf_t *buffer)) tor_assert(process); tor_assert(buffer); +#ifndef _WIN32 + return process_unix_read_stderr(process, buffer); +#else return 0; +#endif } /** This function calls the backend function for the given process whenever @@ -531,6 +570,10 @@ MOCK_IMPL(STATIC void, process_write_stdin, { tor_assert(process); tor_assert(buffer); + +#ifndef _WIN32 + process_unix_write(process, buffer); +#endif } /** This function calls the protocol handlers based on the value of diff --git a/src/lib/process/process.h b/src/lib/process/process.h index cf20a5d80b..b17b8dac7c 100644 --- a/src/lib/process/process.h +++ b/src/lib/process/process.h @@ -95,6 +95,11 @@ void *process_get_data(const process_t *process); void process_set_status(process_t *process, process_status_t status); process_status_t process_get_status(const process_t *process); +#ifndef _WIN32 +struct process_unix_t; +struct process_unix_t *process_get_unix_process(const process_t *process); +#endif + void process_write(process_t *process, const uint8_t *data, size_t size); void process_vprintf(process_t *process, diff --git a/src/lib/process/process_unix.c b/src/lib/process/process_unix.c new file mode 100644 index 0000000000..c3691f1851 --- /dev/null +++ b/src/lib/process/process_unix.c @@ -0,0 +1,611 @@ +/* Copyright (c) 2003, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file process_unix.c + * \brief Module for working with Unix processes. + **/ + +#define PROCESS_UNIX_PRIVATE +#include "lib/intmath/cmp.h" +#include "lib/container/buffers.h" +#include "lib/net/buffers_net.h" +#include "lib/container/smartlist.h" +#include "lib/evloop/compat_libevent.h" +#include "lib/log/log.h" +#include "lib/log/util_bug.h" +#include "lib/process/process.h" +#include "lib/process/process_unix.h" +#include "lib/process/waitpid.h" +#include "lib/process/env.h" + +#include + +#ifdef HAVE_STRING_H +#include +#endif + +#ifdef HAVE_ERRNO_H +#include +#endif + +#ifdef HAVE_UNISTD_H +#include +#endif + +#ifdef HAVE_FCNTL_H +#include +#endif + +#if defined(HAVE_SYS_PRCTL_H) && defined(__linux__) +#include +#endif + +#if HAVE_SIGNAL_H +#include +#endif + +#ifndef _WIN32 + +/** Maximum number of file descriptors, if we cannot get it via sysconf() */ +#define DEFAULT_MAX_FD 256 + +/** Internal state for Unix handles. */ +struct process_unix_handle_t { + /** Unix File Descriptor. */ + int fd; + + /** Have we reached end of file? */ + bool reached_eof; + + /** Event structure for libevent. */ + struct event *event; + + /** Are we writing? */ + bool is_writing; +}; + +/** Internal state for our Unix process. */ +struct process_unix_t { + /** Standard in handle. */ + process_unix_handle_t stdin_handle; + + /** Standard out handle. */ + process_unix_handle_t stdout_handle; + + /** Standard error handle. */ + process_unix_handle_t stderr_handle; + + /** The process identifier of our process. */ + pid_t pid; + + /** Waitpid Callback structure. */ + waitpid_callback_t *waitpid; +}; + +/** Returns a newly allocated process_unix_t. */ +process_unix_t * +process_unix_new(void) +{ + process_unix_t *unix_process; + unix_process = tor_malloc_zero(sizeof(process_unix_t)); + + unix_process->stdin_handle.fd = -1; + unix_process->stderr_handle.fd = -1; + unix_process->stdout_handle.fd = -1; + + return unix_process; +} + +/** Deallocates the given unix_process. */ +void +process_unix_free_(process_unix_t *unix_process) +{ + if (! unix_process) + return; + + /* Clean up our waitpid callback. */ + clear_waitpid_callback(unix_process->waitpid); + + /* FIXME(ahf): Refactor waitpid code? */ + unix_process->waitpid = NULL; + + /* Cleanup our events. */ + if (! unix_process->stdout_handle.reached_eof) + process_unix_stop_reading(&unix_process->stdout_handle); + + if (! unix_process->stderr_handle.reached_eof) + process_unix_stop_reading(&unix_process->stderr_handle); + + process_unix_stop_writing(&unix_process->stdin_handle); + + tor_event_free(unix_process->stdout_handle.event); + tor_event_free(unix_process->stderr_handle.event); + tor_event_free(unix_process->stdin_handle.event); + + tor_free(unix_process); +} + +/** Executes the given process as a child process of Tor. This function is + * responsible for setting up the child process and run it. This includes + * setting up pipes for interprocess communication, initialize the waitpid + * callbacks, and finally run fork() followed by execve(). Returns + * PROCESS_STATUS_RUNNING upon success. */ +process_status_t +process_unix_exec(process_t *process) +{ + static int max_fd = -1; + + process_unix_t *unix_process; + pid_t pid; + int stdin_pipe[2]; + int stdout_pipe[2]; + int stderr_pipe[2]; + int retval, fd; + + unix_process = process_get_unix_process(process); + + /* Create standard in pipe. */ + retval = pipe(stdin_pipe); + + if (-1 == retval) { + log_warn(LD_PROCESS, + "Unable to create pipe for stdin " + "communication with process: %s", + strerror(errno)); + + return PROCESS_STATUS_ERROR; + } + + /* Create standard out pipe. */ + retval = pipe(stdout_pipe); + + if (-1 == retval) { + log_warn(LD_PROCESS, + "Unable to create pipe for stdout " + "communication with process: %s", + strerror(errno)); + + /** Cleanup standard in pipe. */ + close(stdin_pipe[0]); + close(stdin_pipe[1]); + + return PROCESS_STATUS_ERROR; + } + + /* Create standard error pipe. */ + retval = pipe(stderr_pipe); + + if (-1 == retval) { + log_warn(LD_PROCESS, + "Unable to create pipe for stderr " + "communication with process: %s", + strerror(errno)); + + /** Cleanup standard in pipe. */ + close(stdin_pipe[0]); + close(stdin_pipe[1]); + + /** Cleanup standard out pipe. */ + close(stdin_pipe[0]); + close(stdin_pipe[1]); + + return PROCESS_STATUS_ERROR; + } + +#ifdef _SC_OPEN_MAX + if (-1 == max_fd) { + max_fd = (int)sysconf(_SC_OPEN_MAX); + + if (max_fd == -1) { + max_fd = DEFAULT_MAX_FD; + log_warn(LD_PROCESS, + "Cannot find maximum file descriptor, assuming: %d", max_fd); + } + } +#else /* !(defined(_SC_OPEN_MAX)) */ + max_fd = DEFAULT_MAX_FD; +#endif /* defined(_SC_OPEN_MAX) */ + + pid = fork(); + + if (0 == pid) { + /* This code is running in the child process context. */ + +#if defined(HAVE_SYS_PRCTL_H) && defined(__linux__) + /* Attempt to have the kernel issue a SIGTERM if the parent + * goes away. Certain attributes of the binary being execve()ed + * will clear this during the execve() call, but it's better + * than nothing. + */ + prctl(PR_SET_PDEATHSIG, SIGTERM); +#endif /* defined(HAVE_SYS_PRCTL_H) && defined(__linux__) */ + + /* Link process stdout to the write end of the pipe. */ + retval = dup2(stdout_pipe[1], STDOUT_FILENO); + if (-1 == retval) + goto error; + + /* Link process stderr to the write end of the pipe. */ + retval = dup2(stderr_pipe[1], STDERR_FILENO); + if (-1 == retval) + goto error; + + /* Link process stdin to the read end of the pipe */ + retval = dup2(stdin_pipe[0], STDIN_FILENO); + if (-1 == retval) + goto error; + + /* Close our pipes now after they have been dup2()'ed. */ + close(stderr_pipe[0]); + close(stderr_pipe[1]); + close(stdout_pipe[0]); + close(stdout_pipe[1]); + close(stdin_pipe[0]); + close(stdin_pipe[1]); + + /* Close all other fds, including the read end of the pipe. XXX: We should + * now be doing enough FD_CLOEXEC setting to make this needless. + */ + for (fd = STDERR_FILENO + 1; fd < max_fd; fd++) + close(fd); + + /* Create the argv value for our new process. */ + char **argv = process_get_argv(process); + + /* Create the env value for our new process. */ + process_environment_t *env = process_get_environment(process); + + /* Call the requested program. */ + retval = execve(argv[0], argv, env->unixoid_environment_block); + + /* If we made it here it is because execve failed :-( */ + if (-1 == retval) + fprintf(stderr, "Call to execve() failed: %s", strerror(errno)); + + tor_free(argv); + process_environment_free(env); + + tor_assert_unreached(); + + error: + /* LCOV_EXCL_START */ + fprintf(stderr, "Error from child process: %s", strerror(errno)); + _exit(1); + /* LCOV_EXCL_STOP */ + } + + /* We are in the parent process. */ + if (-1 == pid) { + log_warn(LD_PROCESS, + "Failed to create child process: %s", strerror(errno)); + + /** Cleanup standard in pipe. */ + close(stdin_pipe[0]); + close(stdin_pipe[1]); + + /** Cleanup standard out pipe. */ + close(stdin_pipe[0]); + close(stdin_pipe[1]); + + /** Cleanup standard error pipe. */ + close(stderr_pipe[0]); + close(stderr_pipe[1]); + + return PROCESS_STATUS_ERROR; + } + + /* Register our PID. */ + unix_process->pid = pid; + + /* Setup waitpid callbacks. */ + unix_process->waitpid = set_waitpid_callback(pid, + process_unix_waitpid_callback, + process); + + /* Handle standard out. */ + unix_process->stdout_handle.fd = stdout_pipe[0]; + retval = close(stdout_pipe[1]); + + if (-1 == retval) { + log_warn(LD_PROCESS, "Failed to close write end of standard out pipe: %s", + strerror(errno)); + } + + /* Handle standard error. */ + unix_process->stderr_handle.fd = stderr_pipe[0]; + retval = close(stderr_pipe[1]); + + if (-1 == retval) { + log_warn(LD_PROCESS, + "Failed to close write end of standard error pipe: %s", + strerror(errno)); + } + + /* Handle standard in. */ + unix_process->stdin_handle.fd = stdin_pipe[1]; + retval = close(stdin_pipe[0]); + + if (-1 == retval) { + log_warn(LD_PROCESS, "Failed to close read end of standard in pipe: %s", + strerror(errno)); + } + + /* Setup our handles. */ + process_unix_setup_handle(process, + &unix_process->stdout_handle, + EV_READ|EV_PERSIST, + stdout_read_callback); + + process_unix_setup_handle(process, + &unix_process->stderr_handle, + EV_READ|EV_PERSIST, + stderr_read_callback); + + process_unix_setup_handle(process, + &unix_process->stdin_handle, + EV_WRITE|EV_PERSIST, + stdin_write_callback); + + /* Start reading from standard out and standard error. */ + process_unix_start_reading(&unix_process->stdout_handle); + process_unix_start_reading(&unix_process->stderr_handle); + + return PROCESS_STATUS_RUNNING; +} + +/** Write the given buffer as input to the given process's + * standard input. Returns the number of bytes written. */ +int +process_unix_write(process_t *process, buf_t *buffer) +{ + tor_assert(process); + tor_assert(buffer); + + process_unix_t *unix_process = process_get_unix_process(process); + + size_t buffer_flush_len = buf_datalen(buffer); + const size_t max_to_write = MIN(PROCESS_MAX_WRITE, buffer_flush_len); + + /* If we have data to write (when buffer_flush_len > 0) and we are not + * currently getting file descriptor events from the kernel, we tell the + * kernel to start notifying us about when we can write to our file + * descriptor and return. */ + if (buffer_flush_len > 0 && ! unix_process->stdin_handle.is_writing) { + process_unix_start_writing(&unix_process->stdin_handle); + return 0; + } + + /* We don't have any data to write, but the kernel is currently notifying us + * about whether we are able to write or not. Tell the kernel to stop + * notifying us until we have data to write. */ + if (buffer_flush_len == 0 && unix_process->stdin_handle.is_writing) { + process_unix_stop_writing(&unix_process->stdin_handle); + return 0; + } + + /* We have data to write and the kernel have told us to write it. */ + return buf_flush_to_pipe(buffer, + process_get_unix_process(process)->stdin_handle.fd, + max_to_write, &buffer_flush_len); +} + +/** Read data from the given process's standard output and put it into + * buffer. Returns the number of bytes read. */ +int +process_unix_read_stdout(process_t *process, buf_t *buffer) +{ + tor_assert(process); + tor_assert(buffer); + + process_unix_t *unix_process = process_get_unix_process(process); + + return process_unix_read_handle(process, + &unix_process->stdout_handle, + buffer); +} + +/** Read data from the given process's standard error and put it into + * buffer. Returns the number of bytes read. */ +int +process_unix_read_stderr(process_t *process, buf_t *buffer) +{ + tor_assert(process); + tor_assert(buffer); + + process_unix_t *unix_process = process_get_unix_process(process); + + return process_unix_read_handle(process, + &unix_process->stderr_handle, + buffer); +} + +/** This function is called whenever libevent thinks we have data that could be + * read from the child process's standard output. We notify the Process + * subsystem, which is then responsible for calling back to us for doing the + * actual reading of the data. */ +STATIC void +stdout_read_callback(evutil_socket_t fd, short event, void *data) +{ + (void)fd; + (void)event; + + process_t *process = data; + tor_assert(process); + + process_notify_event_stdout(process); +} + +/** This function is called whenever libevent thinks we have data that could be + * read from the child process's standard error. We notify the Process + * subsystem, which is then responsible for calling back to us for doing the + * actual reading of the data. */ +STATIC void +stderr_read_callback(evutil_socket_t fd, short event, void *data) +{ + (void)fd; + (void)event; + + process_t *process = data; + tor_assert(process); + + process_notify_event_stderr(process); +} + +/** This function is called whenever libevent thinks we have data that could be + * written the child process's standard input. We notify the Process subsystem, + * which is then responsible for calling back to us for doing the actual write + * of the data. */ +STATIC void +stdin_write_callback(evutil_socket_t fd, short event, void *data) +{ + (void)fd; + (void)event; + + process_t *process = data; + tor_assert(process); + + process_notify_event_stdin(process); +} + +/** This function tells libevent that we are interested in receiving read + * events from the given handle. */ +STATIC void +process_unix_start_reading(process_unix_handle_t *handle) +{ + tor_assert(handle); + + if (event_add(handle->event, NULL)) + log_warn(LD_PROCESS, + "Unable to add libevent event for handle."); +} + +/** This function tells libevent that we are no longer interested in receiving + * read events from the given handle. */ +STATIC void +process_unix_stop_reading(process_unix_handle_t *handle) +{ + tor_assert(handle); + + if (handle->event == NULL) + return; + + if (event_del(handle->event)) + log_warn(LD_PROCESS, + "Unable to delete libevent event for handle."); +} + +/** This function tells libevent that we are interested in receiving write + * events from the given handle. */ +STATIC void +process_unix_start_writing(process_unix_handle_t *handle) +{ + tor_assert(handle); + + if (event_add(handle->event, NULL)) + log_warn(LD_PROCESS, + "Unable to add libevent event for handle."); + + handle->is_writing = true; +} + +/** This function tells libevent that we are no longer interested in receiving + * write events from the given handle. */ +STATIC void +process_unix_stop_writing(process_unix_handle_t *handle) +{ + tor_assert(handle); + + if (handle->event == NULL) + return; + + if (event_del(handle->event)) + log_warn(LD_PROCESS, + "Unable to delete libevent event for handle."); + + handle->is_writing = false; +} + +/** This function is called when the waitpid system have detected that our + * process have terminated. We disable the waitpid system and notify the + * Process subsystem that we have terminated. */ +STATIC void +process_unix_waitpid_callback(int status, void *data) +{ + tor_assert(data); + + process_t *process = data; + process_unix_t *unix_process = process_get_unix_process(process); + + /* Notify our process. */ + process_notify_event_exit(process, status); + + /* Remove our waitpid callback. */ + clear_waitpid_callback(unix_process->waitpid); + unix_process->waitpid = NULL; +} + +/** This function sets the file descriptor in the handle as non-blocking + * and configures the libevent event structure based on the given flags + * to ensure that callback is called whenever we have events on the + * given handle. */ +STATIC void +process_unix_setup_handle(process_t *process, + process_unix_handle_t *handle, + short flags, + event_callback_fn callback) +{ + tor_assert(process); + tor_assert(handle); + tor_assert(callback); + + /* Put our file descriptor into non-blocking mode. */ + if (fcntl(handle->fd, F_SETFL, O_NONBLOCK) < 0) { + log_warn(LD_PROCESS, "Unable mark Unix handle as non-blocking: %s", + strerror(errno)); + } + + /* Setup libevent event. */ + handle->event = tor_event_new(tor_libevent_get_base(), + handle->fd, + flags, + callback, + process); +} + +/** This function reads data from the given handle and puts it into + * buffer. Returns the number of bytes read this way. */ +STATIC int +process_unix_read_handle(process_t *process, + process_unix_handle_t *handle, + buf_t *buffer) +{ + tor_assert(process); + tor_assert(handle); + tor_assert(buffer); + + int ret = 0; + int eof = 0; + int error = 0; + + ret = buf_read_from_pipe(buffer, + handle->fd, + PROCESS_MAX_READ, + &eof, + &error); + + if (error) + log_warn(LD_PROCESS, + "Unable to read data: %s", strerror(error)); + + if (eof) { + handle->reached_eof = true; + process_unix_stop_reading(handle); + } + + return ret; +} + +#endif /* defined(_WIN32). */ diff --git a/src/lib/process/process_unix.h b/src/lib/process/process_unix.h new file mode 100644 index 0000000000..5fc23bcf07 --- /dev/null +++ b/src/lib/process/process_unix.h @@ -0,0 +1,64 @@ +/* Copyright (c) 2003-2004, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file process_unix.h + * \brief Header for process_unix.c + **/ + +#ifndef TOR_PROCESS_UNIX_H +#define TOR_PROCESS_UNIX_H + +#ifndef _WIN32 + +#include "orconfig.h" +#include "lib/malloc/malloc.h" + +#include + +struct process_t; + +struct process_unix_t; +typedef struct process_unix_t process_unix_t; + +process_unix_t *process_unix_new(void); +void process_unix_free_(process_unix_t *unix_process); +#define process_unix_free(s) \ + FREE_AND_NULL(process_unix_t, process_unix_free_, (s)) + +process_status_t process_unix_exec(struct process_t *process); + +int process_unix_write(struct process_t *process, buf_t *buffer); +int process_unix_read_stdout(struct process_t *process, buf_t *buffer); +int process_unix_read_stderr(struct process_t *process, buf_t *buffer); + +#ifdef PROCESS_UNIX_PRIVATE +struct process_unix_handle_t; +typedef struct process_unix_handle_t process_unix_handle_t; + +STATIC void stdout_read_callback(evutil_socket_t fd, short event, void *data); +STATIC void stderr_read_callback(evutil_socket_t fd, short event, void *data); +STATIC void stdin_write_callback(evutil_socket_t fd, short event, void *data); + +STATIC void process_unix_start_reading(process_unix_handle_t *); +STATIC void process_unix_stop_reading(process_unix_handle_t *); + +STATIC void process_unix_start_writing(process_unix_handle_t *); +STATIC void process_unix_stop_writing(process_unix_handle_t *); + +STATIC void process_unix_waitpid_callback(int status, void *data); + +STATIC void process_unix_setup_handle(process_t *process, + process_unix_handle_t *handle, + short flags, + event_callback_fn callback); +STATIC int process_unix_read_handle(process_t *, + process_unix_handle_t *, + buf_t *); +#endif /* defined(PROCESS_UNIX_PRIVATE). */ + +#endif /* defined(_WIN32). */ + +#endif /* defined(TOR_PROCESS_UNIX_H). */ diff --git a/src/test/test_process.c b/src/test/test_process.c index 2adbde7ad2..816695ccab 100644 --- a/src/test/test_process.c +++ b/src/test/test_process.c @@ -12,6 +12,8 @@ #define PROCESS_PRIVATE #include "lib/process/process.h" +#define PROCESS_UNIX_PRIVATE +#include "lib/process/process_unix.h" static const char *stdout_read_buffer; static const char *stderr_read_buffer; @@ -544,6 +546,24 @@ test_argv_simple(void *arg) process_free_all(); } +static void +test_unix(void *arg) +{ + (void)arg; +#ifndef _WIN32 + process_init(); + + process_t *process = process_new(); + + /* On Unix all processes should have a Unix process handle. */ + tt_ptr_op(NULL, OP_NE, process_get_unix_process(process)); + + done: + process_free(process); + process_free_all(); +#endif +} + struct testcase_t process_tests[] = { { "default_values", test_default_values, TT_FORK, NULL, NULL }, { "stringified_types", test_stringified_types, TT_FORK, NULL, NULL }, @@ -554,5 +574,6 @@ struct testcase_t process_tests[] = { { "write_simple", test_write_simple, TT_FORK, NULL, NULL }, { "exit_simple", test_exit_simple, TT_FORK, NULL, NULL }, { "argv_simple", test_argv_simple, TT_FORK, NULL, NULL }, + { "unix", test_unix, TT_FORK, NULL, NULL }, END_OF_TESTCASES }; From bb784cf4f36256a8276ba60641d3ff766b9cd9df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Thu, 22 Nov 2018 04:49:23 +0100 Subject: [PATCH 0285/2557] Add Windows backend for the Process subsystem. This patch adds support for Microsoft Windows in the Process subsystem. Libevent does not support mixing different types of handles (sockets, named pipes, etc.) on Windows in its core event loop code. This have historically meant that Tor have avoided attaching any non-networking handles to the event loop. This patch uses a slightly different approach to roughly support the same features for the Process subsystem as we do with the Unix backend. In this patch we use Windows Extended I/O functions (ReadFileEx() and WriteFileEx()) which executes asynchronously in the background and executes a completion routine when the scheduled read or write operation have completed. This is much different from the Unix backend where the operating system signals to us whenever a file descriptor is "ready" to either being read from or written to. To make the Windows operating system execute the completion routines of ReadFileEx() and WriteFileEx() we must get the Tor process into what Microsoft calls an "alertable" state. To do this we execute SleepEx() with a zero millisecond sleep time from a main loop timer that ticks once a second. This moves the process into the "alertable" state and when we return from the zero millisecond timeout all the outstanding I/O completion routines will be called and we can schedule the next reads and writes. The timer loop is also responsible for detecting whether our child processes have terminated since the last timer tick. See: https://bugs.torproject.org/28179 --- src/lib/process/include.am | 2 + src/lib/process/process.c | 35 +- src/lib/process/process.h | 3 + src/lib/process/process_win32.c | 857 ++++++++++++++++++++++++++++++++ src/lib/process/process_win32.h | 90 ++++ src/test/test_process.c | 23 +- 6 files changed, 1007 insertions(+), 3 deletions(-) create mode 100644 src/lib/process/process_win32.c create mode 100644 src/lib/process/process_win32.h diff --git a/src/lib/process/include.am b/src/lib/process/include.am index f068032681..aa73356146 100644 --- a/src/lib/process/include.am +++ b/src/lib/process/include.am @@ -11,6 +11,7 @@ src_lib_libtor_process_a_SOURCES = \ src/lib/process/pidfile.c \ src/lib/process/process.c \ src/lib/process/process_unix.c \ + src/lib/process/process_win32.c \ src/lib/process/restrict.c \ src/lib/process/setuid.c \ src/lib/process/subprocess.c \ @@ -28,6 +29,7 @@ noinst_HEADERS += \ src/lib/process/pidfile.h \ src/lib/process/process.h \ src/lib/process/process_unix.h \ + src/lib/process/process_win32.h \ src/lib/process/restrict.h \ src/lib/process/setuid.h \ src/lib/process/subprocess.h \ diff --git a/src/lib/process/process.c b/src/lib/process/process.c index 8d6a9d3fa0..d4237b2b1f 100644 --- a/src/lib/process/process.c +++ b/src/lib/process/process.c @@ -16,6 +16,7 @@ #include "lib/log/util_bug.h" #include "lib/process/process.h" #include "lib/process/process_unix.h" +#include "lib/process/process_win32.h" #include "lib/process/env.h" #ifdef HAVE_STDDEF_H @@ -69,6 +70,9 @@ struct process_t { #ifndef _WIN32 /** Our Unix process handle. */ process_unix_t *unix_process; +#else + /** Our Win32 process handle. */ + process_win32_t *win32_process; #endif }; @@ -117,6 +121,10 @@ void process_init(void) { processes = smartlist_new(); + +#ifdef _WIN32 + process_win32_init(); +#endif } /** Free up all resources that is handled by the Process subsystem. Note that @@ -124,6 +132,10 @@ process_init(void) void process_free_all(void) { +#ifdef _WIN32 + process_win32_deinit(); +#endif + SMARTLIST_FOREACH(processes, process_t *, x, process_free(x)); smartlist_free(processes); } @@ -170,6 +182,9 @@ process_new(const char *command) #ifndef _WIN32 /* Prepare our Unix process handle. */ process->unix_process = process_unix_new(); +#else + /* Prepare our Win32 process handle. */ + process->win32_process = process_win32_new(); #endif smartlist_add(processes, process); @@ -202,6 +217,9 @@ process_free_(process_t *process) #ifndef _WIN32 /* Cleanup our Unix process handle. */ process_unix_free(process->unix_process); +#else + /* Cleanup our Win32 process handle. */ + process_win32_free(process->win32_process); #endif smartlist_remove(processes, process); @@ -222,6 +240,8 @@ process_exec(process_t *process) #ifndef _WIN32 status = process_unix_exec(process); +#else + status = process_win32_exec(process); #endif /* Update our state. */ @@ -420,6 +440,15 @@ process_get_unix_process(const process_t *process) tor_assert(process->unix_process); return process->unix_process; } +#else +/** Get the internal handle for Windows backend. */ +process_win32_t * +process_get_win32_process(const process_t *process) +{ + tor_assert(process); + tor_assert(process->win32_process); + return process->win32_process; +} #endif /** Write size bytes of data to the given process's standard @@ -544,7 +573,7 @@ MOCK_IMPL(STATIC int, process_read_stdout, (process_t *process, buf_t *buffer)) #ifndef _WIN32 return process_unix_read_stdout(process, buffer); #else - return 0; + return process_win32_read_stdout(process, buffer); #endif } @@ -559,7 +588,7 @@ MOCK_IMPL(STATIC int, process_read_stderr, (process_t *process, buf_t *buffer)) #ifndef _WIN32 return process_unix_read_stderr(process, buffer); #else - return 0; + return process_win32_read_stderr(process, buffer); #endif } @@ -573,6 +602,8 @@ MOCK_IMPL(STATIC void, process_write_stdin, #ifndef _WIN32 process_unix_write(process, buffer); +#else + process_win32_write(process, buffer); #endif } diff --git a/src/lib/process/process.h b/src/lib/process/process.h index b17b8dac7c..f759c71939 100644 --- a/src/lib/process/process.h +++ b/src/lib/process/process.h @@ -98,6 +98,9 @@ process_status_t process_get_status(const process_t *process); #ifndef _WIN32 struct process_unix_t; struct process_unix_t *process_get_unix_process(const process_t *process); +#else +struct process_win32_t; +struct process_win32_t *process_get_win32_process(const process_t *process); #endif void process_write(process_t *process, diff --git a/src/lib/process/process_win32.c b/src/lib/process/process_win32.c new file mode 100644 index 0000000000..a019e0b4f3 --- /dev/null +++ b/src/lib/process/process_win32.c @@ -0,0 +1,857 @@ +/* Copyright (c) 2003, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file process_win32.c + * \brief Module for working with Windows processes. + **/ + +#define PROCESS_WIN32_PRIVATE +#include "lib/intmath/cmp.h" +#include "lib/container/buffers.h" +#include "lib/net/buffers_net.h" +#include "lib/container/smartlist.h" +#include "lib/log/log.h" +#include "lib/log/util_bug.h" +#include "lib/log/win32err.h" +#include "lib/process/process.h" +#include "lib/process/process_win32.h" +#include "lib/process/subprocess.h" +#include "lib/process/env.h" + +#ifdef HAVE_SYS_TIME_H +#include +#endif + +#ifdef _WIN32 + +/** The size of our intermediate buffers. */ +#define BUFFER_SIZE (1024) + +/** Timer that ticks once a second and calls the process_win32_timer_callback() + * function. */ +static periodic_timer_t *periodic_timer; + +/** Structure to represent the state around the pipe HANDLE. + * + * This structure is used to store state about a given HANDLE, including + * whether we have reached end of file, its intermediate buffers, and how much + * data that is available in the intermediate buffer. */ +struct process_win32_handle_t { + /** Standard out pipe handle. */ + HANDLE pipe; + + /** True iff we have reached EOF from the pipe. */ + bool reached_eof; + + /** How much data is available in buffer. */ + size_t data_available; + + /** Intermediate buffer for ReadFileEx() and WriteFileEx(). */ + char buffer[BUFFER_SIZE]; + + /** Overlapped structure for ReadFileEx() and WriteFileEx(). */ + OVERLAPPED overlapped; + + /** Are we waiting for another I/O operation to complete? */ + bool busy; +}; + +/** Structure to represent the Windows specific implementation details of this + * Process backend. + * + * This structure is attached to process_t (see process.h) and is + * reachable from process_t via the process_get_win32_process() + * method. */ +struct process_win32_t { + /** Standard in state. */ + process_win32_handle_t stdin_handle; + + /** Standard out state. */ + process_win32_handle_t stdout_handle; + + /** Standard error state. */ + process_win32_handle_t stderr_handle; + + /** Process Information. */ + PROCESS_INFORMATION process_information; +}; + +/** Create a new process_win32_t. + * + * This function constructs a new process_win32_t and initializes the + * default values. */ +process_win32_t * +process_win32_new(void) +{ + process_win32_t *win32_process; + win32_process = tor_malloc_zero(sizeof(process_win32_t)); + + win32_process->stdin_handle.pipe = INVALID_HANDLE_VALUE; + win32_process->stdout_handle.pipe = INVALID_HANDLE_VALUE; + win32_process->stderr_handle.pipe = INVALID_HANDLE_VALUE; + + return win32_process; +} + +/** Free a given process_win32_t. + * + * This function deinitializes and frees up the resources allocated for the + * given process_win32_t. */ +void +process_win32_free_(process_win32_t *win32_process) +{ + if (! win32_process) + return; + + /* Cleanup our handles. */ + process_win32_cleanup_handle(&win32_process->stdin_handle); + process_win32_cleanup_handle(&win32_process->stdout_handle); + process_win32_cleanup_handle(&win32_process->stderr_handle); + + tor_free(win32_process); +} + +/** Initialize the Windows backend of the Process subsystem. */ +void +process_win32_init(void) +{ + /* We don't start the periodic timer here because it makes no sense to have + * the timer running until we have some processes that benefits from the + * timer timer ticks. */ +} + +/** Deinitialize the Windows backend of the Process subsystem. */ +void +process_win32_deinit(void) +{ + /* Stop our timer, but only if it's running. */ + if (process_win32_timer_running()) + process_win32_timer_stop(); +} + +/** Execute the given process. This function is responsible for setting up + * named pipes for I/O between the child process and the Tor process. Returns + * PROCESS_STATUS_RUNNING upon success. */ +process_status_t +process_win32_exec(process_t *process) +{ + tor_assert(process); + + process_win32_t *win32_process = process_get_win32_process(process); + + HANDLE stdout_pipe_read = NULL; + HANDLE stdout_pipe_write = NULL; + HANDLE stderr_pipe_read = NULL; + HANDLE stderr_pipe_write = NULL; + HANDLE stdin_pipe_read = NULL; + HANDLE stdin_pipe_write = NULL; + BOOL ret = FALSE; + const char *filename = process_get_command(process); + + /* Not much we can do if we haven't been told what to start. */ + if (BUG(filename == NULL)) + return PROCESS_STATUS_ERROR; + + /* Setup our security attributes. */ + SECURITY_ATTRIBUTES security_attributes; + security_attributes.nLength = sizeof(security_attributes); + security_attributes.bInheritHandle = TRUE; + /* FIXME: should we set explicit security attributes? + * (See Ticket #2046, comment 5) */ + security_attributes.lpSecurityDescriptor = NULL; + + /* Create our standard out pipe. */ + if (! process_win32_create_pipe(&stdout_pipe_read, + &stdout_pipe_write, + &security_attributes, + PROCESS_WIN32_PIPE_TYPE_READER)) { + return PROCESS_STATUS_ERROR; + } + + /* Create our standard error pipe. */ + if (! process_win32_create_pipe(&stderr_pipe_read, + &stderr_pipe_write, + &security_attributes, + PROCESS_WIN32_PIPE_TYPE_READER)) { + return PROCESS_STATUS_ERROR; + } + + /* Create out standard in pipe. */ + if (! process_win32_create_pipe(&stdin_pipe_read, + &stdin_pipe_write, + &security_attributes, + PROCESS_WIN32_PIPE_TYPE_WRITER)) { + return PROCESS_STATUS_ERROR; + } + + /* Configure startup info for our child process. */ + STARTUPINFOA startup_info; + + memset(&startup_info, 0, sizeof(startup_info)); + startup_info.cb = sizeof(startup_info); + startup_info.hStdError = stderr_pipe_write; + startup_info.hStdOutput = stdout_pipe_write; + startup_info.hStdInput = stdin_pipe_read; + startup_info.dwFlags |= STARTF_USESTDHANDLES; + + /* Create the env value for our new process. */ + process_environment_t *env = process_get_environment(process); + + /* Create the argv value for our new process. */ + char **argv = process_get_argv(process); + + /* Windows expects argv to be a whitespace delimited string, so join argv up + */ + char *joined_argv = tor_join_win_cmdline((const char **)argv); + + /* Create the child process */ + ret = CreateProcessA(filename, + joined_argv, + NULL, + NULL, + TRUE, + CREATE_NO_WINDOW, + env->windows_environment_block[0] == '\0' ? + NULL : env->windows_environment_block, + NULL, + &startup_info, + &win32_process->process_information); + + tor_free(argv); + tor_free(joined_argv); + process_environment_free(env); + + if (! ret) { + log_warn(LD_PROCESS, "CreateProcessA() failed: %s", + format_win32_error(GetLastError())); + + /* Cleanup our handles. */ + CloseHandle(stdout_pipe_read); + CloseHandle(stdout_pipe_write); + CloseHandle(stderr_pipe_read); + CloseHandle(stderr_pipe_write); + CloseHandle(stdin_pipe_read); + CloseHandle(stdin_pipe_write); + + return PROCESS_STATUS_ERROR; + } + + /* TODO: Should we close hProcess and hThread in + * process_handle->process_information? */ + win32_process->stdout_handle.pipe = stdout_pipe_read; + win32_process->stderr_handle.pipe = stderr_pipe_read; + win32_process->stdin_handle.pipe = stdin_pipe_write; + + /* Used by the callback functions from ReadFileEx() and WriteFileEx() such + * that we can figure out which process_t that was responsible for the event. + * + * Warning, here be dragons: + * + * MSDN says that the hEvent member of the overlapped structure is unused + * for ReadFileEx() and WriteFileEx, which allows us to store a pointer to + * our process state there. + */ + win32_process->stdout_handle.overlapped.hEvent = (HANDLE)process; + win32_process->stderr_handle.overlapped.hEvent = (HANDLE)process; + win32_process->stdin_handle.overlapped.hEvent = (HANDLE)process; + + /* Start our timer if it is not already running. */ + if (! process_win32_timer_running()) + process_win32_timer_start(); + + /* We use Windows Extended I/O functions, so our completion callbacks are + * called automatically for us when there is data to read. Because of this + * we start the read of standard out and error right away. */ + process_notify_event_stdout(process); + process_notify_event_stderr(process); + + return PROCESS_STATUS_RUNNING; +} + +/** Schedule an async write of the data found in buffer for the given + * process. This function runs an async write operation of the content of + * buffer, if we are not already waiting for a pending I/O request. Returns the + * number of bytes that Windows will hopefully write for us in the background. + * */ +int +process_win32_write(struct process_t *process, buf_t *buffer) +{ + tor_assert(process); + tor_assert(buffer); + + process_win32_t *win32_process = process_get_win32_process(process); + BOOL ret = FALSE; + const size_t buffer_size = buf_datalen(buffer); + + /* Windows is still writing our buffer. */ + if (win32_process->stdin_handle.busy) + return 0; + + /* Nothing for us to do right now. */ + if (buffer_size == 0) + return 0; + + /* We have reached end of file already? */ + if (BUG(win32_process->stdin_handle.reached_eof)) + return 0; + + /* Figure out how much data we should read. */ + const size_t write_size = MIN(buffer_size, + sizeof(win32_process->stdin_handle.buffer)); + + /* Read data from the process_t buffer into our intermediate buffer. */ + buf_get_bytes(buffer, win32_process->stdin_handle.buffer, write_size); + + /* Schedule our write. */ + ret = WriteFileEx(win32_process->stdin_handle.pipe, + win32_process->stdin_handle.buffer, + write_size, + &win32_process->stdin_handle.overlapped, + process_win32_stdin_write_done); + + if (! ret) { + log_warn(LD_PROCESS, "WriteFileEx() failed: %s", + format_win32_error(GetLastError())); + return 0; + } + + /* This cast should be safe since our buffer can maximum be BUFFER_SIZE + * large. */ + return (int)write_size; +} + +/** This function is called from the Process subsystem whenever the Windows + * backend says it has data ready. This function also ensures that we are + * starting a new background read from the standard output of the child process + * and asks Windows to call process_win32_stdout_read_done() when that + * operation is finished. Returns the number of bytes moved into buffer. + * */ +int +process_win32_read_stdout(struct process_t *process, buf_t *buffer) +{ + tor_assert(process); + tor_assert(buffer); + + process_win32_t *win32_process = process_get_win32_process(process); + + return process_win32_read_from_handle(&win32_process->stdout_handle, + buffer, + process_win32_stdout_read_done); +} + +/** This function is called from the Process subsystem whenever the Windows + * backend says it has data ready. This function also ensures that we are + * starting a new background read from the standard error of the child process + * and asks Windows to call process_win32_stderr_read_done() when that + * operation is finished. Returns the number of bytes moved into buffer. + * */ +int +process_win32_read_stderr(struct process_t *process, buf_t *buffer) +{ + tor_assert(process); + tor_assert(buffer); + + process_win32_t *win32_process = process_get_win32_process(process); + + return process_win32_read_from_handle(&win32_process->stderr_handle, + buffer, + process_win32_stderr_read_done); +} + +/** This function is responsible for moving the Tor process into what Microsoft + * calls an "alertable" state. Once the process is in an alertable state the + * Windows kernel will notify us when our background I/O requests have finished + * and the callbacks will be executed. */ +void +process_win32_trigger_completion_callbacks(void) +{ + DWORD ret; + + /* The call to SleepEx(dwMilliseconds, dwAlertable) makes the process sleep + * for dwMilliseconds and if dwAlertable is set to TRUE it will also cause + * the process to enter alertable state, where the Windows kernel will notify + * us about completed I/O requests from ReadFileEx() and WriteFileEX(), which + * will cause our completion callbacks to be executed. + * + * This function returns 0 if the time interval expired or WAIT_IO_COMPLETION + * if one or more I/O callbacks were executed. */ + ret = SleepEx(0, TRUE); + + /* Warn us if the function returned something we did not anticipate. */ + if (ret != 0 && ret != WAIT_IO_COMPLETION) { + log_warn(LD_PROCESS, "SleepEx() returned %lu", ret); + } +} + +/** Start the periodic timer which is reponsible for checking whether processes + * are still alive and to make sure that the Tor process is periodically being + * moved into an alertable state. */ +STATIC void +process_win32_timer_start(void) +{ + /* Make sure we never start our timer if it's already running. */ + if (BUG(process_win32_timer_running())) + return; + + /* Wake up once a second. */ + static const struct timeval interval = {1, 0}; + + log_info(LD_PROCESS, "Starting Windows Process I/O timer"); + periodic_timer = periodic_timer_new(tor_libevent_get_base(), + &interval, + process_win32_timer_callback, + NULL); +} + +/** Stops the periodic timer. */ +STATIC void +process_win32_timer_stop(void) +{ + if (BUG(periodic_timer == NULL)) + return; + + log_info(LD_PROCESS, "Stopping Windows Process I/O timer"); + periodic_timer_free(periodic_timer); +} + +/** Returns true iff the periodic timer is running. */ +STATIC bool +process_win32_timer_running(void) +{ + return periodic_timer != NULL; +} + +/** This function is called whenever the periodic_timer ticks. The function is + * responsible for moving the Tor process into an alertable state once a second + * and checking for whether our child processes have terminated since the last + * tick. */ +STATIC void +process_win32_timer_callback(periodic_timer_t *timer, void *data) +{ + tor_assert(timer == periodic_timer); + tor_assert(data == NULL); + + log_debug(LD_PROCESS, "Windows Process I/O timer ticked"); + const smartlist_t *processes = process_get_all_processes(); + + SMARTLIST_FOREACH(processes, process_t *, p, + process_win32_timer_test_process(p)); + + process_win32_trigger_completion_callbacks(); +} + +/** Test whether a given process is still alive. Notify the Process subsystem + * if our process have died. */ +STATIC void +process_win32_timer_test_process(process_t *process) +{ + tor_assert(process); + + /* No need to look at processes that don't claim they are running. */ + if (process_get_status(process) != PROCESS_STATUS_RUNNING) + return; + + process_win32_t *win32_process = process_get_win32_process(process); + BOOL ret = FALSE; + DWORD exit_code = 0; + + /* We start by testing whether our process is still running. */ + ret = GetExitCodeProcess(win32_process->process_information.hProcess, + &exit_code); + + if (! ret) { + log_warn(LD_PROCESS, "GetExitCodeProcess() failed: %s", + format_win32_error(GetLastError())); + return; + } + + /* Notify our process_t that our process have terminated. */ + if (exit_code != STILL_ACTIVE) + process_notify_event_exit(process, exit_code); +} + +/** Create a new overlapped named pipe. This function creates a new connected, + * named, pipe in *read_pipe and *write_pipe if the function is + * succesful. Returns true on sucess, false on failure. */ +STATIC bool +process_win32_create_pipe(HANDLE *read_pipe, + HANDLE *write_pipe, + SECURITY_ATTRIBUTES *attributes, + process_win32_pipe_type_t pipe_type) +{ + tor_assert(read_pipe); + tor_assert(write_pipe); + tor_assert(attributes); + + BOOL ret = FALSE; + + /* Buffer size. */ + const size_t size = 4096; + + /* Our additional read/write modes that depends on which pipe type we are + * creating. */ + DWORD read_mode = 0; + DWORD write_mode = 0; + + /* Generate the unique pipe name. */ + char pipe_name[MAX_PATH]; + static DWORD process_id = 0; + static DWORD counter = 0; + + if (process_id == 0) + process_id = GetCurrentProcessId(); + + tor_snprintf(pipe_name, sizeof(pipe_name), + "\\\\.\\Pipe\\Tor-Process-Pipe-%lu-%lu", + process_id, counter++); + + /* Only one of our handles can be overlapped. */ + switch (pipe_type) { + case PROCESS_WIN32_PIPE_TYPE_READER: + read_mode = FILE_FLAG_OVERLAPPED; + break; + case PROCESS_WIN32_PIPE_TYPE_WRITER: + write_mode = FILE_FLAG_OVERLAPPED; + break; + default: + /* LCOV_EXCL_START */ + tor_assert_nonfatal_unreached_once(); + /* LCOV_EXCL_STOP */ + } + + /* Setup our read and write handles. */ + HANDLE read_handle; + HANDLE write_handle; + + /* Create our named pipe. */ + read_handle = CreateNamedPipeA(pipe_name, + (PIPE_ACCESS_INBOUND|read_mode), + (PIPE_TYPE_BYTE|PIPE_WAIT), + 1, + size, + size, + 1000, + attributes); + + if (read_handle == INVALID_HANDLE_VALUE) { + log_warn(LD_PROCESS, "CreateNamedPipeA() failed: %s", + format_win32_error(GetLastError())); + return false; + } + + /* Create our file in the pipe namespace. */ + write_handle = CreateFileA(pipe_name, + GENERIC_WRITE, + 0, + attributes, + OPEN_EXISTING, + (FILE_ATTRIBUTE_NORMAL|write_mode), + NULL); + + if (write_handle == INVALID_HANDLE_VALUE) { + log_warn(LD_PROCESS, "CreateFileA() failed: %s", + format_win32_error(GetLastError())); + + CloseHandle(read_handle); + + return false; + } + + /* Set the inherit flag for our pipe. */ + switch (pipe_type) { + case PROCESS_WIN32_PIPE_TYPE_READER: + ret = SetHandleInformation(read_handle, HANDLE_FLAG_INHERIT, 0); + break; + case PROCESS_WIN32_PIPE_TYPE_WRITER: + ret = SetHandleInformation(write_handle, HANDLE_FLAG_INHERIT, 0); + break; + default: + /* LCOV_EXCL_START */ + tor_assert_nonfatal_unreached_once(); + /* LCOV_EXCL_STOP */ + } + + if (! ret) { + log_warn(LD_PROCESS, "SetHandleInformation() failed: %s", + format_win32_error(GetLastError())); + + CloseHandle(read_handle); + CloseHandle(write_handle); + + return false; + } + + /* Everything is good. */ + *read_pipe = read_handle; + *write_pipe = write_handle; + + return true; +} + +/** Cleanup a given handle. */ +STATIC void +process_win32_cleanup_handle(process_win32_handle_t *handle) +{ + tor_assert(handle); + +#if 0 + /* FIXME(ahf): My compiler does not set _WIN32_WINNT to a high enough value + * for this code to be available. Should we force it? CancelIoEx() is + * available from Windows 7 and above. If we decide to require this, we need + * to update the checks in all the three I/O completion callbacks to handle + * the ERROR_OPERATION_ABORTED as well as ERROR_BROKEN_PIPE. */ + +#if _WIN32_WINNT >= 0x0600 + /* This code is only supported from Windows 7 and onwards. */ + BOOL ret; + DWORD error_code; + + /* Cancel any pending I/O requests. */ + ret = CancelIoEx(handle->pipe, &handle->overlapped); + + if (! ret) { + error_code = GetLastError(); + + /* There was no pending I/O requests for our handle. */ + if (error_code != ERROR_NOT_FOUND) { + log_warn(LD_PROCESS, "CancelIoEx() failed: %s", + format_win32_error(error_code)); + } + } +#endif +#endif + + /* Close our handle. */ + if (handle->pipe != INVALID_HANDLE_VALUE) { + CloseHandle(handle->pipe); + handle->pipe = INVALID_HANDLE_VALUE; + } +} + +/** This function is called when ReadFileEx() completes its background read + * from the child process's standard output. We notify the Process subsystem if + * there is data available for it to read from us. */ +STATIC VOID WINAPI +process_win32_stdout_read_done(DWORD error_code, + DWORD byte_count, + LPOVERLAPPED overlapped) +{ + tor_assert(overlapped); + tor_assert(overlapped->hEvent); + + /* This happens when we have asked ReadFileEx() to read some data, but we + * then decided to call CloseHandle() on the HANDLE. This can happen if + * someone runs process_free() in the exit_callback of process_t, which means + * we cannot call process_get_win32_process() here. */ + if (error_code == ERROR_BROKEN_PIPE) { + log_debug(LD_PROCESS, "Process reported broken pipe on standard out"); + return; + } + + /* Extract our process_t from the hEvent member of OVERLAPPED. */ + process_t *process = (process_t *)overlapped->hEvent; + process_win32_t *win32_process = process_get_win32_process(process); + + if (process_win32_handle_read_completion(&win32_process->stdout_handle, + error_code, + byte_count)) { + /* Schedule our next read. */ + process_notify_event_stdout(process); + } +} + +/** This function is called when ReadFileEx() completes its background read + * from the child process's standard error. We notify the Process subsystem if + * there is data available for it to read from us. */ +STATIC VOID WINAPI +process_win32_stderr_read_done(DWORD error_code, + DWORD byte_count, + LPOVERLAPPED overlapped) +{ + tor_assert(overlapped); + tor_assert(overlapped->hEvent); + + /* This happens when we have asked ReadFileEx() to read some data, but we + * then decided to call CloseHandle() on the HANDLE. This can happen if + * someone runs process_free() in the exit_callback of process_t, which means + * we cannot call process_get_win32_process() here. */ + if (error_code == ERROR_BROKEN_PIPE) { + log_debug(LD_PROCESS, "Process reported broken pipe on standard error"); + return; + } + + /* Extract our process_t from the hEvent member of OVERLAPPED. */ + process_t *process = (process_t *)overlapped->hEvent; + process_win32_t *win32_process = process_get_win32_process(process); + + if (process_win32_handle_read_completion(&win32_process->stderr_handle, + error_code, + byte_count)) { + /* Schedule our next read. */ + process_notify_event_stderr(process); + } +} + +/** This function is called when WriteFileEx() completes its background write + * to the child process's standard input. We notify the Process subsystem that + * it can write data to us again. */ +STATIC VOID WINAPI +process_win32_stdin_write_done(DWORD error_code, + DWORD byte_count, + LPOVERLAPPED overlapped) +{ + tor_assert(overlapped); + tor_assert(overlapped->hEvent); + + (void)byte_count; + + /* This happens when we have asked WriteFileEx() to write some data, but we + * then decided to call CloseHandle() on the HANDLE. This can happen if + * someone runs process_free() in the exit_callback of process_t, which means + * we cannot call process_get_win32_process() here. */ + if (error_code == ERROR_BROKEN_PIPE) { + log_debug(LD_PROCESS, "Process reported broken pipe on standard input"); + return; + } + + process_t *process = (process_t *)overlapped->hEvent; + process_win32_t *win32_process = process_get_win32_process(process); + + /* Mark our handle as not having any outstanding I/O requests. */ + win32_process->stdin_handle.busy = false; + + /* Check if we have been asked to write to the handle that have been marked + * as having reached EOF. */ + if (BUG(win32_process->stdin_handle.reached_eof)) + return; + + if (error_code == 0) { + /** Our data have been succesfully written. Clear our state and schedule + * the next write. */ + win32_process->stdin_handle.data_available = 0; + memset(win32_process->stdin_handle.buffer, 0, + sizeof(win32_process->stdin_handle.buffer)); + + /* Schedule the next write. */ + process_notify_event_stdin(process); + } else if (error_code == ERROR_HANDLE_EOF) { + /* Our WriteFileEx() call was succesful, but we reached the end of our + * file. We mark our handle as having reached EOF and returns. */ + tor_assert(byte_count == 0); + + win32_process->stdin_handle.reached_eof = true; + } else { + /* An error happened: We warn the user and mark our handle as having + * reached EOF */ + log_warn(LD_PROCESS, + "Error in I/O completion routine from WriteFileEx(): %s", + format_win32_error(error_code)); + win32_process->stdin_handle.reached_eof = true; + } +} + +/** This function reads data from the given handle's internal buffer and + * moves it into the given buffer. Additionally, we start the next + * ReadFileEx() background operation with the given callback as + * completion callback. Returns the number of bytes written to the buffer. */ +STATIC int +process_win32_read_from_handle(process_win32_handle_t *handle, + buf_t *buffer, + LPOVERLAPPED_COMPLETION_ROUTINE callback) +{ + tor_assert(handle); + tor_assert(buffer); + tor_assert(callback); + + BOOL ret = FALSE; + int bytes_available = 0; + + /* We already have a request to read data that isn't complete yet. */ + if (BUG(handle->busy)) + return 0; + + /* Check if we have been asked to read from a handle that have already told + * us that we have reached the end of the file. */ + if (BUG(handle->reached_eof)) + return 0; + + /* This cast should be safe since our buffer can be at maximum up to + * BUFFER_SIZE in size. */ + bytes_available = (int)handle->data_available; + + if (handle->data_available > 0) { + /* Read data from our intermediate buffer into the process_t buffer. */ + buf_add(buffer, handle->buffer, handle->data_available); + + /* Reset our read state. */ + handle->data_available = 0; + memset(handle->buffer, 0, sizeof(handle->buffer)); + } + + /* Ask the Windows kernel to read data from our pipe into our buffer and call + * the callback function when it is done. */ + ret = ReadFileEx(handle->pipe, + handle->buffer, + sizeof(handle->buffer), + &handle->overlapped, + callback); + + if (! ret) { + log_warn(LD_PROCESS, "ReadFileEx() failed: %s", + format_win32_error(GetLastError())); + return bytes_available; + } + + /* We mark our handle as having a pending I/O request. */ + handle->busy = true; + + return bytes_available; +} + +/** This function checks the callback values from ReadFileEx() in + * error_code and byte_count if we have read data. Returns true + * iff our caller should request more data from ReadFileEx(). */ +STATIC bool +process_win32_handle_read_completion(process_win32_handle_t *handle, + DWORD error_code, + DWORD byte_count) +{ + tor_assert(handle); + + /* Mark our handle as not having any outstanding I/O requests. */ + handle->busy = false; + + if (error_code == 0) { + /* Our ReadFileEx() call was succesful and there is data for us. */ + + /* This cast should be safe since byte_count should never be larger than + * BUFFER_SIZE. */ + tor_assert(byte_count <= BUFFER_SIZE); + handle->data_available = (size_t)byte_count; + + /* Tell our caller to schedule the next read. */ + return true; + } else if (error_code == ERROR_HANDLE_EOF) { + /* Our ReadFileEx() call was succesful, but we reached the end of our file. + * We mark our handle as having reached EOF and returns. */ + tor_assert(byte_count == 0); + + handle->reached_eof = true; + } else { + /* An error happened: We warn the user and mark our handle as having + * reached EOF */ + log_warn(LD_PROCESS, + "Error in I/O completion routine from ReadFileEx(): %s", + format_win32_error(error_code)); + + handle->reached_eof = true; + } + + /* Our caller should NOT schedule the next read. */ + return false; +} + +#endif /* ! defined(_WIN32). */ diff --git a/src/lib/process/process_win32.h b/src/lib/process/process_win32.h new file mode 100644 index 0000000000..8c3b80d341 --- /dev/null +++ b/src/lib/process/process_win32.h @@ -0,0 +1,90 @@ +/* Copyright (c) 2003-2004, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file process_win32.h + * \brief Header for process_win32.c + **/ + +#ifndef TOR_PROCESS_WIN32_H +#define TOR_PROCESS_WIN32_H + +#ifdef _WIN32 + +#include "orconfig.h" +#include "lib/malloc/malloc.h" +#include "lib/evloop/compat_libevent.h" + +#include + +struct process_t; + +struct process_win32_t; +typedef struct process_win32_t process_win32_t; + +process_win32_t *process_win32_new(void); +void process_win32_free_(process_win32_t *win32_process); +#define process_win32_free(s) \ + FREE_AND_NULL(process_win32_t, process_win32_free_, (s)) + +void process_win32_init(void); +void process_win32_deinit(void); + +process_status_t process_win32_exec(struct process_t *process); + +int process_win32_write(struct process_t *process, buf_t *buffer); +int process_win32_read_stdout(struct process_t *process, buf_t *buffer); +int process_win32_read_stderr(struct process_t *process, buf_t *buffer); + +void process_win32_trigger_completion_callbacks(void); + +#ifdef PROCESS_WIN32_PRIVATE +/* Timer handling. */ +STATIC void process_win32_timer_start(void); +STATIC void process_win32_timer_stop(void); +STATIC bool process_win32_timer_running(void); +STATIC void process_win32_timer_callback(periodic_timer_t *, void *); +STATIC void process_win32_timer_test_process(process_t *); + +/* I/O pipe handling. */ +struct process_win32_handle_t; +typedef struct process_win32_handle_t process_win32_handle_t; + +typedef enum process_win32_pipe_type_t { + /** This pipe is used for reading. */ + PROCESS_WIN32_PIPE_TYPE_READER, + + /** This pipe is used for writing. */ + PROCESS_WIN32_PIPE_TYPE_WRITER +} process_win32_pipe_type_t; + +STATIC bool process_win32_create_pipe(HANDLE *, + HANDLE *, + SECURITY_ATTRIBUTES *, + process_win32_pipe_type_t); + +STATIC void process_win32_cleanup_handle(process_win32_handle_t *handle); + +STATIC VOID WINAPI process_win32_stdout_read_done(DWORD, + DWORD, + LPOVERLAPPED); +STATIC VOID WINAPI process_win32_stderr_read_done(DWORD, + DWORD, + LPOVERLAPPED); +STATIC VOID WINAPI process_win32_stdin_write_done(DWORD, + DWORD, + LPOVERLAPPED); + +STATIC int process_win32_read_from_handle(process_win32_handle_t *, + buf_t *, + LPOVERLAPPED_COMPLETION_ROUTINE); +STATIC bool process_win32_handle_read_completion(process_win32_handle_t *, + DWORD, + DWORD); +#endif /* defined(PROCESS_WIN32_PRIVATE). */ + +#endif /* ! defined(_WIN32). */ + +#endif /* defined(TOR_PROCESS_WIN32_H). */ diff --git a/src/test/test_process.c b/src/test/test_process.c index 816695ccab..85ee9691a4 100644 --- a/src/test/test_process.c +++ b/src/test/test_process.c @@ -14,6 +14,8 @@ #include "lib/process/process.h" #define PROCESS_UNIX_PRIVATE #include "lib/process/process_unix.h" +#define PROCESS_WIN32_PRIVATE +#include "lib/process/process_win32.h" static const char *stdout_read_buffer; static const char *stderr_read_buffer; @@ -553,7 +555,7 @@ test_unix(void *arg) #ifndef _WIN32 process_init(); - process_t *process = process_new(); + process_t *process = process_new(""); /* On Unix all processes should have a Unix process handle. */ tt_ptr_op(NULL, OP_NE, process_get_unix_process(process)); @@ -564,6 +566,24 @@ test_unix(void *arg) #endif } +static void +test_win32(void *arg) +{ + (void)arg; +#ifdef _WIN32 + process_init(); + + process_t *process = process_new(""); + + /* On Win32 all processes should have a Win32 process handle. */ + tt_ptr_op(NULL, OP_NE, process_get_win32_process(process)); + + done: + process_free(process); + process_free_all(); +#endif +} + struct testcase_t process_tests[] = { { "default_values", test_default_values, TT_FORK, NULL, NULL }, { "stringified_types", test_stringified_types, TT_FORK, NULL, NULL }, @@ -575,5 +595,6 @@ struct testcase_t process_tests[] = { { "exit_simple", test_exit_simple, TT_FORK, NULL, NULL }, { "argv_simple", test_argv_simple, TT_FORK, NULL, NULL }, { "unix", test_unix, TT_FORK, NULL, NULL }, + { "win32", test_win32, TT_FORK, NULL, NULL }, END_OF_TESTCASES }; From 89393a77e5db804784d4f08ef67fd2831799d65b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Thu, 22 Nov 2018 05:22:24 +0100 Subject: [PATCH 0286/2557] Add process_get_pid() to the Process subsystem. This patch adds support for getting the unique process identifier from a given process_t. This patch implements both support for both the Unix and Microsoft Windows backend. See: https://bugs.torproject.org/28179 --- src/lib/process/process.c | 13 +++++++++++++ src/lib/process/process.h | 3 +++ src/lib/process/process_unix.c | 10 ++++++++++ src/lib/process/process_unix.h | 2 ++ src/lib/process/process_win32.c | 10 ++++++++++ src/lib/process/process_win32.h | 2 ++ src/test/test_process.c | 3 +++ 7 files changed, 43 insertions(+) diff --git a/src/lib/process/process.c b/src/lib/process/process.c index d4237b2b1f..ab19378a93 100644 --- a/src/lib/process/process.c +++ b/src/lib/process/process.c @@ -255,6 +255,19 @@ process_exec(process_t *process) return status; } +/** Returns the unique process identifier for the given process. */ +process_pid_t +process_get_pid(process_t *process) +{ + tor_assert(process); + +#ifndef _WIN32 + return process_unix_get_pid(process); +#else + return process_win32_get_pid(process); +#endif +} + /** Set the callback function for output from the child process's standard out * handle. This function sets the callback function which is called every time * the child process have written output to its standard out file handle. diff --git a/src/lib/process/process.h b/src/lib/process/process.h index f759c71939..7fd6cf53d0 100644 --- a/src/lib/process/process.h +++ b/src/lib/process/process.h @@ -49,6 +49,7 @@ struct process_t; typedef struct process_t process_t; typedef uint64_t process_exit_code_t; +typedef uint64_t process_pid_t; typedef void (*process_read_callback_t)(process_t *, char *, @@ -66,6 +67,8 @@ void process_free_(process_t *process); process_status_t process_exec(process_t *process); +process_pid_t process_get_pid(process_t *process); + void process_set_stdout_read_callback(process_t *, process_read_callback_t); void process_set_stderr_read_callback(process_t *, diff --git a/src/lib/process/process_unix.c b/src/lib/process/process_unix.c index c3691f1851..fa03fdbbe4 100644 --- a/src/lib/process/process_unix.c +++ b/src/lib/process/process_unix.c @@ -356,6 +356,16 @@ process_unix_exec(process_t *process) return PROCESS_STATUS_RUNNING; } +/** Returns the unique process identifier for the given process. */ +process_pid_t +process_unix_get_pid(process_t *process) +{ + tor_assert(process); + + process_unix_t *unix_process = process_get_unix_process(process); + return (process_pid_t)unix_process->pid; +} + /** Write the given buffer as input to the given process's * standard input. Returns the number of bytes written. */ int diff --git a/src/lib/process/process_unix.h b/src/lib/process/process_unix.h index 5fc23bcf07..0474746b26 100644 --- a/src/lib/process/process_unix.h +++ b/src/lib/process/process_unix.h @@ -30,6 +30,8 @@ void process_unix_free_(process_unix_t *unix_process); process_status_t process_unix_exec(struct process_t *process); +process_pid_t process_unix_get_pid(struct process_t *process); + int process_unix_write(struct process_t *process, buf_t *buffer); int process_unix_read_stdout(struct process_t *process, buf_t *buffer); int process_unix_read_stderr(struct process_t *process, buf_t *buffer); diff --git a/src/lib/process/process_win32.c b/src/lib/process/process_win32.c index a019e0b4f3..3e97f37801 100644 --- a/src/lib/process/process_win32.c +++ b/src/lib/process/process_win32.c @@ -271,6 +271,16 @@ process_win32_exec(process_t *process) return PROCESS_STATUS_RUNNING; } +/** Returns the unique process identifier for the given process. */ +process_pid_t +process_win32_get_pid(process_t *process) +{ + tor_assert(process); + + process_win32_t *win32_process = process_get_win32_process(process); + return (process_pid_t)win32_process->process_information.dwProcessId; +} + /** Schedule an async write of the data found in buffer for the given * process. This function runs an async write operation of the content of * buffer, if we are not already waiting for a pending I/O request. Returns the diff --git a/src/lib/process/process_win32.h b/src/lib/process/process_win32.h index 8c3b80d341..dbd264104c 100644 --- a/src/lib/process/process_win32.h +++ b/src/lib/process/process_win32.h @@ -34,6 +34,8 @@ void process_win32_deinit(void); process_status_t process_win32_exec(struct process_t *process); +process_pid_t process_win32_get_pid(struct process_t *process); + int process_win32_write(struct process_t *process, buf_t *buffer); int process_win32_read_stdout(struct process_t *process, buf_t *buffer); int process_win32_read_stderr(struct process_t *process, buf_t *buffer); diff --git a/src/test/test_process.c b/src/test/test_process.c index 85ee9691a4..4f86e786cb 100644 --- a/src/test/test_process.c +++ b/src/test/test_process.c @@ -160,6 +160,9 @@ test_default_values(void *arg) tt_assert(smartlist_contains(process_get_all_processes(), process)); + /* Default PID is 0. */ + tt_int_op(0, OP_EQ, process_get_pid(process)); + /* Our arguments should be empty. */ tt_int_op(0, OP_EQ, smartlist_len(process_get_arguments(process))); From e982fb1dae6ff0888ae419246578048470dd65b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Thu, 22 Nov 2018 17:38:40 +0100 Subject: [PATCH 0287/2557] Add documentation for the is_socket and error argument of read_to_chunk(). See: https://bugs.torproject.org/28179 --- src/lib/net/buffers_net.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/lib/net/buffers_net.c b/src/lib/net/buffers_net.c index 1b65819dbb..da7043d5cb 100644 --- a/src/lib/net/buffers_net.c +++ b/src/lib/net/buffers_net.c @@ -33,8 +33,10 @@ /** Read up to at_most bytes from the file descriptor fd into * chunk (which must be on buf). If we get an EOF, set - * *reached_eof to 1. Return -1 on error, 0 on eof or blocking, - * and the number of bytes read otherwise. */ + * *reached_eof to 1. Uses tor_socket_recv() iff is_socket + * is true, otherwise it uses read(). Return -1 on error (and sets + * *error to errno), 0 on eof or blocking, and the number of bytes read + * otherwise. */ static inline int read_to_chunk(buf_t *buf, chunk_t *chunk, tor_socket_t fd, size_t at_most, int *reached_eof, int *error, bool is_socket) From 338137221c8bd89f6d611c0cd3bf7b8a85d02517 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Thu, 22 Nov 2018 18:14:03 +0100 Subject: [PATCH 0288/2557] Make sure we call process_notify_event_exit() as the last thing in different callbacks. This patch makes sure that we call process_notify_event_exit() after we have done any modifications we need to do to the state of a process_t. This allows application developers to call process_free() in the exit_callback of the process. See: https://bugs.torproject.org/28179 --- src/lib/process/process_unix.c | 10 +++++++--- src/lib/process/process_win32.c | 7 +++++-- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/src/lib/process/process_unix.c b/src/lib/process/process_unix.c index fa03fdbbe4..4f46bbd888 100644 --- a/src/lib/process/process_unix.c +++ b/src/lib/process/process_unix.c @@ -549,12 +549,16 @@ process_unix_waitpid_callback(int status, void *data) process_t *process = data; process_unix_t *unix_process = process_get_unix_process(process); - /* Notify our process. */ - process_notify_event_exit(process, status); - /* Remove our waitpid callback. */ clear_waitpid_callback(unix_process->waitpid); unix_process->waitpid = NULL; + + /* Notify our process. */ + process_notify_event_exit(process, status); + + /* Make sure you don't modify the process after we have called + * process_notify_event_exit() on it, to allow users to process_free() it in + * the exit callback. */ } /** This function sets the file descriptor in the handle as non-blocking diff --git a/src/lib/process/process_win32.c b/src/lib/process/process_win32.c index 3e97f37801..7422493deb 100644 --- a/src/lib/process/process_win32.c +++ b/src/lib/process/process_win32.c @@ -445,12 +445,15 @@ process_win32_timer_callback(periodic_timer_t *timer, void *data) tor_assert(data == NULL); log_debug(LD_PROCESS, "Windows Process I/O timer ticked"); + + /* Move the process into an alertable state. */ + process_win32_trigger_completion_callbacks(); + + /* Check if our processes are still alive. */ const smartlist_t *processes = process_get_all_processes(); SMARTLIST_FOREACH(processes, process_t *, p, process_win32_timer_test_process(p)); - - process_win32_trigger_completion_callbacks(); } /** Test whether a given process is still alive. Notify the Process subsystem From 4f611a1df70d2c5e4cb6261f75c1b82c9ed04598 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Fri, 23 Nov 2018 03:44:59 +0100 Subject: [PATCH 0289/2557] Add process_terminate(). This patch adds support for process termination to the Process subsystem. See: https://bugs.torproject.org/28179 --- src/lib/process/process.c | 20 ++++++++++++++++++++ src/lib/process/process.h | 1 + src/lib/process/process_unix.c | 26 ++++++++++++++++++++++++++ src/lib/process/process_unix.h | 1 + src/lib/process/process_win32.c | 22 ++++++++++++++++++++++ src/lib/process/process_win32.h | 1 + 6 files changed, 71 insertions(+) diff --git a/src/lib/process/process.c b/src/lib/process/process.c index ab19378a93..657b319f54 100644 --- a/src/lib/process/process.c +++ b/src/lib/process/process.c @@ -255,6 +255,26 @@ process_exec(process_t *process) return status; } +/** Terminate the given process. Returns true on success, + * otherwise false. */ +bool +process_terminate(process_t *process) +{ + tor_assert(process); + + /* Terminating a non-running process isn't going to work. */ + if (process_get_status(process) != PROCESS_STATUS_RUNNING) + return false; + + log_debug(LD_PROCESS, "Terminating process"); + +#ifndef _WIN32 + return process_unix_terminate(process); +#else + return process_win32_terminate(process); +#endif +} + /** Returns the unique process identifier for the given process. */ process_pid_t process_get_pid(process_t *process) diff --git a/src/lib/process/process.h b/src/lib/process/process.h index 7fd6cf53d0..c6b733a065 100644 --- a/src/lib/process/process.h +++ b/src/lib/process/process.h @@ -66,6 +66,7 @@ void process_free_(process_t *process); #define process_free(s) FREE_AND_NULL(process_t, process_free_, (s)) process_status_t process_exec(process_t *process); +bool process_terminate(process_t *process); process_pid_t process_get_pid(process_t *process); diff --git a/src/lib/process/process_unix.c b/src/lib/process/process_unix.c index 4f46bbd888..4a9aaa2edd 100644 --- a/src/lib/process/process_unix.c +++ b/src/lib/process/process_unix.c @@ -356,6 +356,32 @@ process_unix_exec(process_t *process) return PROCESS_STATUS_RUNNING; } +/** Terminate the given process. Returns true on success, otherwise false. */ +bool +process_unix_terminate(process_t *process) +{ + tor_assert(process); + + process_unix_t *unix_process = process_get_unix_process(process); + + /* All running processes should have a waitpid. */ + if (BUG(unix_process->waitpid == NULL)) + return false; + + /* Send a SIGTERM to our child process. */ + int ret; + + ret = kill(unix_process->pid, SIGTERM); + + if (ret == -1) { + log_warn(LD_PROCESS, "Unable to terminate process: %s", + strerror(errno)); + return false; + } + + return ret == 0; +} + /** Returns the unique process identifier for the given process. */ process_pid_t process_unix_get_pid(process_t *process) diff --git a/src/lib/process/process_unix.h b/src/lib/process/process_unix.h index 0474746b26..e17c59ea81 100644 --- a/src/lib/process/process_unix.h +++ b/src/lib/process/process_unix.h @@ -29,6 +29,7 @@ void process_unix_free_(process_unix_t *unix_process); FREE_AND_NULL(process_unix_t, process_unix_free_, (s)) process_status_t process_unix_exec(struct process_t *process); +bool process_unix_terminate(struct process_t *process); process_pid_t process_unix_get_pid(struct process_t *process); diff --git a/src/lib/process/process_win32.c b/src/lib/process/process_win32.c index 7422493deb..7b18903c70 100644 --- a/src/lib/process/process_win32.c +++ b/src/lib/process/process_win32.c @@ -271,6 +271,28 @@ process_win32_exec(process_t *process) return PROCESS_STATUS_RUNNING; } +/** Terminate the given process. Returns true on success, otherwise false. */ +bool +process_win32_terminate(process_t *process) +{ + tor_assert(process); + + process_win32_t *win32_process = process_get_win32_process(process); + + /* Terminate our process. */ + BOOL ret; + + ret = TerminateProcess(win32_process->process_information.hProcess, 0); + + if (! ret) { + log_warn(LD_PROCESS, "TerminateProcess() failed: %s", + format_win32_error(GetLastError())); + return false; + } + + return true; +} + /** Returns the unique process identifier for the given process. */ process_pid_t process_win32_get_pid(process_t *process) diff --git a/src/lib/process/process_win32.h b/src/lib/process/process_win32.h index dbd264104c..9a42e6c713 100644 --- a/src/lib/process/process_win32.h +++ b/src/lib/process/process_win32.h @@ -33,6 +33,7 @@ void process_win32_init(void); void process_win32_deinit(void); process_status_t process_win32_exec(struct process_t *process); +bool process_win32_terminate(struct process_t *process); process_pid_t process_win32_get_pid(struct process_t *process); From b0d268a82264f28f6b78f734d45a13e8272e70ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Fri, 23 Nov 2018 05:43:34 +0100 Subject: [PATCH 0290/2557] Add process_reset_environment() to the Process subsystem. This patch adds a new function that allows us to reset the environment of a given process_t with a list of key/value pairs. See: https://bugs.torproject.org/28179 --- src/lib/process/process.c | 17 +++++++++++++++++ src/lib/process/process.h | 1 + 2 files changed, 18 insertions(+) diff --git a/src/lib/process/process.c b/src/lib/process/process.c index 657b319f54..915217e132 100644 --- a/src/lib/process/process.c +++ b/src/lib/process/process.c @@ -441,6 +441,23 @@ process_get_argv(const process_t *process) return argv; } +/** This function clears the internal environment and copies over every string + * from env as the new environment. */ +void +process_reset_environment(process_t *process, const smartlist_t *env) +{ + tor_assert(process); + tor_assert(env); + + /* Cleanup old environment. */ + SMARTLIST_FOREACH(process->environment, char *, x, tor_free(x)); + smartlist_free(process->environment); + process->environment = smartlist_new(); + + SMARTLIST_FOREACH(env, char *, x, + smartlist_add(process->environment, tor_strdup(x))); +} + /** Set the given key/value pair as environment variable in the * given process. */ void diff --git a/src/lib/process/process.h b/src/lib/process/process.h index c6b733a065..6092c2da7d 100644 --- a/src/lib/process/process.h +++ b/src/lib/process/process.h @@ -83,6 +83,7 @@ void process_append_argument(process_t *process, const char *argument); const smartlist_t *process_get_arguments(const process_t *process); char **process_get_argv(const process_t *process); +void process_reset_environment(process_t *process, const smartlist_t *env); void process_set_environment(process_t *process, const char *key, const char *value); From ad4cc89c5d1987cbcb231bf054433c7f05b83a95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Thu, 22 Nov 2018 19:00:21 +0100 Subject: [PATCH 0291/2557] Add "PT" log domain. See: https://bugs.torproject.org/28179 --- doc/tor.1.txt | 4 ++-- src/lib/log/log.c | 2 +- src/lib/log/log.h | 4 +++- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/doc/tor.1.txt b/doc/tor.1.txt index 42fe8f1bcc..1984b05d64 100644 --- a/doc/tor.1.txt +++ b/doc/tor.1.txt @@ -669,8 +669,8 @@ GENERAL OPTIONS + The currently recognized domains are: general, crypto, net, config, fs, protocol, mm, http, app, control, circ, rend, bug, dir, dirserv, or, edge, - acct, hist, handshake, heartbeat, channel, sched, guard, consdiff, dos, and - process. + acct, hist, handshake, heartbeat, channel, sched, guard, consdiff, dos, + process, and pt. Domain names are case-insensitive. + + For example, "`Log [handshake]debug [~net,~mm]info notice stdout`" sends diff --git a/src/lib/log/log.c b/src/lib/log/log.c index 861de6e963..2c25ddf1aa 100644 --- a/src/lib/log/log.c +++ b/src/lib/log/log.c @@ -1268,7 +1268,7 @@ static const char *domain_list[] = { "GENERAL", "CRYPTO", "NET", "CONFIG", "FS", "PROTOCOL", "MM", "HTTP", "APP", "CONTROL", "CIRC", "REND", "BUG", "DIR", "DIRSERV", "OR", "EDGE", "ACCT", "HIST", "HANDSHAKE", "HEARTBEAT", "CHANNEL", - "SCHED", "GUARD", "CONSDIFF", "DOS", "PROCESS", NULL + "SCHED", "GUARD", "CONSDIFF", "DOS", "PROCESS", "PT", NULL }; /** Return a bitmask for the log domain for which domain is the name, diff --git a/src/lib/log/log.h b/src/lib/log/log.h index 1cd6087eb8..c8820ee037 100644 --- a/src/lib/log/log.h +++ b/src/lib/log/log.h @@ -109,8 +109,10 @@ #define LD_DOS (1u<<25) /** Processes */ #define LD_PROCESS (1u<<26) +/** Pluggable Transports. */ +#define LD_PT (1u<<27) /** Number of logging domains in the code. */ -#define N_LOGGING_DOMAINS 27 +#define N_LOGGING_DOMAINS 28 /** This log message is not safe to send to a callback-based logger * immediately. Used as a flag, not a log domain. */ From bfb94dd2ca8e04fb1fe8aba9ad48effbb8b70662 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Fri, 23 Nov 2018 05:56:41 +0100 Subject: [PATCH 0292/2557] Use process_t for managed proxies. This patch makes the managed proxy subsystem use the process_t data structure such that we can get events from the PT process while Tor is running and not just when the PT process is being configured. See: https://bugs.torproject.org/28179 --- src/core/mainloop/mainloop.c | 4 - src/feature/client/transports.c | 174 ++++++++++++++++++-------------- src/feature/client/transports.h | 12 ++- src/test/test_pt.c | 66 ++++++------ 4 files changed, 137 insertions(+), 119 deletions(-) diff --git a/src/core/mainloop/mainloop.c b/src/core/mainloop/mainloop.c index 2e2ae876d4..aaaa5009cb 100644 --- a/src/core/mainloop/mainloop.c +++ b/src/core/mainloop/mainloop.c @@ -1853,10 +1853,6 @@ second_elapsed_callback(time_t now, const or_options_t *options) run_connection_housekeeping(i, now); } - /* 11b. check pending unconfigured managed proxies */ - if (!net_is_disabled() && pt_proxies_configuration_pending()) - pt_configure_remaining_proxies(); - /* Run again in a second. */ return 1; } diff --git a/src/feature/client/transports.c b/src/feature/client/transports.c index 85d4d2e080..f0400b713d 100644 --- a/src/feature/client/transports.c +++ b/src/feature/client/transports.c @@ -102,10 +102,11 @@ #include "feature/relay/ext_orport.h" #include "feature/control/control.h" -#include "lib/process/env.h" +#include "lib/process/process.h" #include "lib/process/subprocess.h" +#include "lib/process/env.h" -static process_environment_t * +static smartlist_t * create_managed_proxy_environment(const managed_proxy_t *mp); static inline int proxy_configuration_finished(const managed_proxy_t *mp); @@ -490,8 +491,8 @@ proxy_prepare_for_restart(managed_proxy_t *mp) tor_assert(mp->conf_state == PT_PROTO_COMPLETED); /* destroy the process handle and terminate the process. */ - tor_process_handle_destroy(mp->process_handle, 1); - mp->process_handle = NULL; + process_set_data(mp->process, NULL); + process_terminate(mp->process); /* destroy all its registered transports, since we will no longer use them. */ @@ -520,34 +521,35 @@ proxy_prepare_for_restart(managed_proxy_t *mp) static int launch_managed_proxy(managed_proxy_t *mp) { - int retval; + tor_assert(mp); - process_environment_t *env = create_managed_proxy_environment(mp); + smartlist_t *env = create_managed_proxy_environment(mp); -#ifdef _WIN32 - /* Passing NULL as lpApplicationName makes Windows search for the .exe */ - retval = tor_spawn_background(NULL, - (const char **)mp->argv, - env, - &mp->process_handle); -#else /* !(defined(_WIN32)) */ - retval = tor_spawn_background(mp->argv[0], - (const char **)mp->argv, - env, - &mp->process_handle); -#endif /* defined(_WIN32) */ + /* Configure our process. */ + process_set_data(mp->process, mp); + process_set_stdout_read_callback(mp->process, managed_proxy_stdout_callback); + process_set_stderr_read_callback(mp->process, managed_proxy_stderr_callback); + process_set_exit_callback(mp->process, managed_proxy_exit_callback); + process_set_protocol(mp->process, PROCESS_PROTOCOL_LINE); + process_reset_environment(mp->process, env); - process_environment_free(env); + /* Cleanup our env. */ + SMARTLIST_FOREACH(env, char *, x, tor_free(x)); + smartlist_free(env); - if (retval == PROCESS_STATUS_ERROR) { - log_warn(LD_GENERAL, "Managed proxy at '%s' failed at launch.", + /* Skip the argv[0] as we get that from process_new(argv[0]). */ + for (int i = 1; mp->argv[i] != NULL; ++i) + process_append_argument(mp->process, mp->argv[i]); + + if (process_exec(mp->process) != PROCESS_STATUS_RUNNING) { + log_warn(LD_CONFIG, "Managed proxy at '%s' failed at launch.", mp->argv[0]); return -1; } - log_info(LD_CONFIG, "Managed proxy at '%s' has spawned with PID '%d'.", - mp->argv[0], tor_process_get_pid(mp->process_handle)); - + log_info(LD_CONFIG, + "Managed proxy at '%s' has spawned with PID '%" PRIu64 "'.", + mp->argv[0], process_get_pid(mp->process)); mp->conf_state = PT_PROTO_LAUNCHED; return 0; @@ -615,10 +617,6 @@ pt_configure_remaining_proxies(void) STATIC int configure_proxy(managed_proxy_t *mp) { - int configuration_finished = 0; - smartlist_t *proxy_output = NULL; - enum stream_status stream_status = 0; - /* if we haven't launched the proxy yet, do it now */ if (mp->conf_state == PT_PROTO_INFANT) { if (launch_managed_proxy(mp) < 0) { /* launch fail */ @@ -629,45 +627,8 @@ configure_proxy(managed_proxy_t *mp) } tor_assert(mp->conf_state != PT_PROTO_INFANT); - tor_assert(mp->process_handle); - - proxy_output = - tor_get_lines_from_handle(tor_process_get_stdout_pipe(mp->process_handle), - &stream_status); - if (!proxy_output) { /* failed to get input from proxy */ - if (stream_status != IO_STREAM_EAGAIN) { /* bad stream status! */ - mp->conf_state = PT_PROTO_BROKEN; - log_warn(LD_GENERAL, "The communication stream of managed proxy '%s' " - "is '%s'. Most probably the managed proxy stopped running. " - "This might be a bug of the managed proxy, a bug of Tor, or " - "a misconfiguration. Please enable logging on your managed " - "proxy and check the logs for errors.", - mp->argv[0], stream_status_to_string(stream_status)); - } - - goto done; - } - - /* Handle lines. */ - SMARTLIST_FOREACH_BEGIN(proxy_output, const char *, line) { - handle_proxy_line(line, mp); - if (proxy_configuration_finished(mp)) - goto done; - } SMARTLIST_FOREACH_END(line); - - done: - /* if the proxy finished configuring, exit the loop. */ - if (proxy_configuration_finished(mp)) { - handle_finished_proxy(mp); - configuration_finished = 1; - } - - if (proxy_output) { - SMARTLIST_FOREACH(proxy_output, char *, cp, tor_free(cp)); - smartlist_free(proxy_output); - } - - return configuration_finished; + tor_assert(mp->process); + return mp->conf_state == PT_PROTO_COMPLETED; } /** Register server managed proxy mp transports to state */ @@ -748,8 +709,11 @@ managed_proxy_destroy(managed_proxy_t *mp, /* free the outgoing proxy URI */ tor_free(mp->proxy_uri); - tor_process_handle_destroy(mp->process_handle, also_terminate_process); - mp->process_handle = NULL; + /* do we want to terminate our process if it's still running? */ + if (also_terminate_process && mp->process) + process_terminate(mp->process); + + process_free(mp->process); tor_free(mp); } @@ -1257,7 +1221,7 @@ get_bindaddr_for_server_proxy(const managed_proxy_t *mp) /** Return a newly allocated process_environment_t * for mp's * process. */ -static process_environment_t * +static smartlist_t * create_managed_proxy_environment(const managed_proxy_t *mp) { const or_options_t *options = get_options(); @@ -1272,8 +1236,6 @@ create_managed_proxy_environment(const managed_proxy_t *mp) /* The final environment to be passed to mp. */ smartlist_t *merged_env_vars = get_current_process_environment_variables(); - process_environment_t *env; - { char *state_tmp = get_datadir_fname("pt_state/"); /* XXX temp */ smartlist_add_asprintf(envs, "TOR_PT_STATE_LOCATION=%s", state_tmp); @@ -1366,14 +1328,9 @@ create_managed_proxy_environment(const managed_proxy_t *mp) tor_free_, 1); } SMARTLIST_FOREACH_END(env_var); - env = process_environment_make(merged_env_vars); - smartlist_free(envs); - SMARTLIST_FOREACH(merged_env_vars, void *, x, tor_free(x)); - smartlist_free(merged_env_vars); - - return env; + return merged_env_vars; } /** Create and return a new managed proxy for transport using @@ -1392,6 +1349,7 @@ managed_proxy_create(const smartlist_t *with_transport_list, mp->argv = proxy_argv; mp->transports = smartlist_new(); mp->proxy_uri = get_pt_proxy_uri(); + mp->process = process_new(proxy_argv[0]); mp->transports_to_launch = smartlist_new(); SMARTLIST_FOREACH(with_transport_list, const char *, transport, @@ -1736,3 +1694,63 @@ tor_escape_str_for_pt_args(const char *string, const char *chars_to_escape) return new_string; } + +/** Callback function that is called when our PT process have data on its + * stdout. Our process can be found in process, the data can be found in + * line and the length of our line is given in size. */ +STATIC void +managed_proxy_stdout_callback(process_t *process, char *line, size_t size) +{ + tor_assert(process); + tor_assert(line); + + (void)size; + + managed_proxy_t *mp = process_get_data(process); + + handle_proxy_line(line, mp); + + if (proxy_configuration_finished(mp)) { + handle_finished_proxy(mp); + tor_assert(mp->conf_state == PT_PROTO_COMPLETED); + } +} + +/** Callback function that is called when our PT process have data on its + * stderr. Our process can be found in process, the data can be found in + * line and the length of our line is given in size. */ +STATIC void +managed_proxy_stderr_callback(process_t *process, char *line, size_t size) +{ + tor_assert(process); + tor_assert(line); + + (void)size; + + managed_proxy_t *mp = process_get_data(process); + + log_warn(LD_PT, "Managed proxy at '%s' reported: %s", mp->argv[0], line); +} + +/** Callback function that is called when our PT process terminates. The + * process exit code can be found in exit_code and our process can be + * found in process. */ +STATIC void +managed_proxy_exit_callback(process_t *process, process_exit_code_t exit_code) +{ + tor_assert(process); + + log_warn(LD_PT, + "Pluggable Transport process terminated with status code %" PRIu64, + exit_code); + + /* We detach ourself from the MP (if we are attached) and free ourself. */ + managed_proxy_t *mp = process_get_data(process); + + if (BUG(mp != NULL)) { + mp->process = NULL; + process_set_data(process, NULL); + } + + process_free(process); +} diff --git a/src/feature/client/transports.h b/src/feature/client/transports.h index b80875c95c..59df637d81 100644 --- a/src/feature/client/transports.h +++ b/src/feature/client/transports.h @@ -11,6 +11,8 @@ #ifndef TOR_TRANSPORTS_H #define TOR_TRANSPORTS_H +#include "lib/process/process.h" + /** Represents a pluggable transport used by a bridge. */ typedef struct transport_t { /** SOCKS version: One of PROXY_SOCKS4, PROXY_SOCKS5. */ @@ -81,7 +83,7 @@ enum pt_proto_state { PT_PROTO_FAILED_LAUNCH /* failed while launching */ }; -struct process_handle_t; +struct process_t; /** Structure containing information of a managed proxy. */ typedef struct { @@ -94,8 +96,8 @@ typedef struct { int is_server; /* is it a server proxy? */ - /* A pointer to the process handle of this managed proxy. */ - struct process_handle_t *process_handle; + /* A pointer to the process of this managed proxy. */ + struct process_t *process; /** Boolean: We are re-parsing our config, and we are going to * remove this managed proxy if we don't find it any transport @@ -140,6 +142,10 @@ STATIC char* get_pt_proxy_uri(void); STATIC void free_execve_args(char **arg); +STATIC void managed_proxy_stdout_callback(process_t *, char *, size_t); +STATIC void managed_proxy_stderr_callback(process_t *, char *, size_t); +STATIC void managed_proxy_exit_callback(process_t *, process_exit_code_t); + #endif /* defined(PT_PRIVATE) */ #endif /* !defined(TOR_TRANSPORTS_H) */ diff --git a/src/test/test_pt.c b/src/test/test_pt.c index d0160d1148..2501b867bb 100644 --- a/src/test/test_pt.c +++ b/src/test/test_pt.c @@ -9,6 +9,7 @@ #define STATEFILE_PRIVATE #define CONTROL_PRIVATE #define SUBPROCESS_PRIVATE +#define PROCESS_PRIVATE #include "core/or/or.h" #include "app/config/config.h" #include "app/config/confparse.h" @@ -20,6 +21,7 @@ #include "lib/process/subprocess.h" #include "lib/encoding/confline.h" #include "lib/net/resolve.h" +#include "lib/process/process.h" #include "app/config/or_state_st.h" @@ -151,6 +153,8 @@ test_pt_get_transport_options(void *arg) config_line_t *cl = NULL; (void)arg; + process_init(); + execve_args = tor_malloc(sizeof(char*)*2); execve_args[0] = tor_strdup("cheeseshop"); execve_args[1] = NULL; @@ -190,6 +194,7 @@ test_pt_get_transport_options(void *arg) config_free_lines(cl); managed_proxy_destroy(mp, 0); smartlist_free(transport_list); + process_free_all(); } static void @@ -253,6 +258,8 @@ test_pt_get_extrainfo_string(void *arg) char *s = NULL; (void) arg; + process_init(); + argv1 = tor_malloc_zero(sizeof(char*)*3); argv1[0] = tor_strdup("ewige"); argv1[1] = tor_strdup("Blumenkraft"); @@ -286,43 +293,25 @@ test_pt_get_extrainfo_string(void *arg) smartlist_free(t1); smartlist_free(t2); tor_free(s); + process_free_all(); } -#ifdef _WIN32 -#define STDIN_HANDLE HANDLE* -#else -#define STDIN_HANDLE int -#endif - -static smartlist_t * -tor_get_lines_from_handle_replacement(STDIN_HANDLE handle, - enum stream_status *stream_status_out) +static int +process_read_stdout_replacement(process_t *process, buf_t *buffer) { + (void)process; static int times_called = 0; - smartlist_t *retval_sl = smartlist_new(); - - (void) handle; - (void) stream_status_out; /* Generate some dummy CMETHOD lines the first 5 times. The 6th time, send 'CMETHODS DONE' to finish configuring the proxy. */ if (times_called++ != 5) { - smartlist_add_asprintf(retval_sl, "SMETHOD mock%d 127.0.0.1:555%d", + buf_add_printf(buffer, "SMETHOD mock%d 127.0.0.1:555%d\n", times_called, times_called); } else { - smartlist_add_strdup(retval_sl, "SMETHODS DONE"); + buf_add_string(buffer, "SMETHODS DONE\n"); } - return retval_sl; -} - -/* NOP mock */ -static void -tor_process_handle_destroy_replacement(process_handle_t *process_handle, - int also_terminate_process) -{ - (void) process_handle; - (void) also_terminate_process; + return (int)buf_datalen(buffer); } static or_state_t *dummy_state = NULL; @@ -355,12 +344,11 @@ test_pt_configure_proxy(void *arg) managed_proxy_t *mp = NULL; (void) arg; + process_init(); + dummy_state = tor_malloc_zero(sizeof(or_state_t)); - MOCK(tor_get_lines_from_handle, - tor_get_lines_from_handle_replacement); - MOCK(tor_process_handle_destroy, - tor_process_handle_destroy_replacement); + MOCK(process_read_stdout, process_read_stdout_replacement); MOCK(get_or_state, get_or_state_replacement); MOCK(queue_control_event_string, @@ -372,24 +360,34 @@ test_pt_configure_proxy(void *arg) mp->conf_state = PT_PROTO_ACCEPTING_METHODS; mp->transports = smartlist_new(); mp->transports_to_launch = smartlist_new(); - mp->process_handle = tor_malloc_zero(sizeof(process_handle_t)); mp->argv = tor_malloc_zero(sizeof(char*)*2); mp->argv[0] = tor_strdup(""); mp->is_server = 1; + /* Configure the process. */ + mp->process = process_new(""); + process_set_stdout_read_callback(mp->process, managed_proxy_stdout_callback); + process_set_data(mp->process, mp); + /* Test the return value of configure_proxy() by calling it some times while it is uninitialized and then finally finalizing its configuration. */ for (i = 0 ; i < 5 ; i++) { + /* force a read from our mocked stdout reader. */ + process_notify_event_stdout(mp->process); + /* try to configure our proxy. */ retval = configure_proxy(mp); /* retval should be zero because proxy hasn't finished configuring yet */ tt_int_op(retval, OP_EQ, 0); /* check the number of registered transports */ - tt_assert(smartlist_len(mp->transports) == i+1); + tt_int_op(smartlist_len(mp->transports), OP_EQ, i+1); /* check that the mp is still waiting for transports */ tt_assert(mp->conf_state == PT_PROTO_ACCEPTING_METHODS); } + /* Get the SMETHOD DONE written to the process. */ + process_notify_event_stdout(mp->process); + /* this last configure_proxy() should finalize the proxy configuration. */ retval = configure_proxy(mp); /* retval should be 1 since the proxy finished configuring */ @@ -435,8 +433,7 @@ test_pt_configure_proxy(void *arg) done: or_state_free(dummy_state); - UNMOCK(tor_get_lines_from_handle); - UNMOCK(tor_process_handle_destroy); + UNMOCK(process_read_stdout); UNMOCK(get_or_state); UNMOCK(queue_control_event_string); if (controlevent_msgs) { @@ -449,10 +446,11 @@ test_pt_configure_proxy(void *arg) smartlist_free(mp->transports); } smartlist_free(mp->transports_to_launch); - tor_free(mp->process_handle); + process_free(mp->process); tor_free(mp->argv[0]); tor_free(mp->argv); tor_free(mp); + process_free_all(); } /* Test the get_pt_proxy_uri() function. */ From e3ceaebba25127a1d4a3da16e9128ab52491c080 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Fri, 23 Nov 2018 22:50:55 +0100 Subject: [PATCH 0293/2557] Add support for logging messages from pluggable transports. This patch adds support for the "LOG" protocol message from a pluggable transport. This allows pluggable transport developers to relay log messages from their binary to Tor, which will both emit them as log messages from the Tor process itself, but also pass them on via the control port. See: https://bugs.torproject.org/28180 See: https://bugs.torproject.org/28181 See: https://bugs.torproject.org/28182 --- src/feature/client/transports.c | 42 +++++++++++++++++++++++++++++++++ src/feature/client/transports.h | 1 + src/feature/control/control.c | 11 +++++++++ src/feature/control/control.h | 5 +++- src/test/test_pt.c | 19 +++++++++++++-- 5 files changed, 75 insertions(+), 3 deletions(-) diff --git a/src/feature/client/transports.c b/src/feature/client/transports.c index f0400b713d..dc76e9c245 100644 --- a/src/feature/client/transports.c +++ b/src/feature/client/transports.c @@ -128,6 +128,7 @@ static void parse_method_error(const char *line, int is_server_method); #define PROTO_SMETHODS_DONE "SMETHODS DONE" #define PROTO_PROXY_DONE "PROXY DONE" #define PROTO_PROXY_ERROR "PROXY-ERROR" +#define PROTO_LOG "LOG" /** The first and only supported - at the moment - configuration protocol version. */ @@ -909,6 +910,9 @@ handle_proxy_line(const char *line, managed_proxy_t *mp) parse_proxy_error(line); goto err; + } else if (!strcmpstart(line, PROTO_LOG)) { + parse_log_line(line); + return; } else if (!strcmpstart(line, SPAWN_ERROR_MESSAGE)) { /* managed proxy launch failed: parse error message to learn why. */ int retval, child_state, saved_errno; @@ -1146,6 +1150,44 @@ parse_proxy_error(const char *line) line+strlen(PROTO_PROXY_ERROR)+1); } +/** Parses a LOG line and emit log events accordingly. */ +STATIC void +parse_log_line(const char *line) +{ + smartlist_t *items = smartlist_new(); + + if (strlen(line) < (strlen(PROTO_LOG) + 1)) { + log_warn(LD_PT, "Managed proxy sent us a %s line " + "with missing arguments.", PROTO_LOG); + goto done; + } + + const char *arguments = line + strlen(PROTO_LOG) + 1; + + /* The format is 'LOG '. We accept empty messages. */ + smartlist_split_string(items, arguments, NULL, + SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 2); + + if (smartlist_len(items) < 2) { + log_warn(LD_PT, "Managed proxy sent us a %s line " + "with too few arguments.", PROTO_LOG); + goto done; + } + + const char *transport_name = smartlist_get(items, 0); + const char *message = smartlist_get(items, 1); + + log_info(LD_PT, "Managed proxy transport \"%s\" says: %s", + transport_name, message); + + /* Emit control port event. */ + control_event_transport_log(transport_name, message); + + done: + SMARTLIST_FOREACH(items, char *, s, tor_free(s)); + smartlist_free(items); +} + /** Return a newly allocated string that tor should place in * TOR_PT_SERVER_TRANSPORT_OPTIONS while configuring the server * manged proxy in mp. Return NULL if no such options are found. */ diff --git a/src/feature/client/transports.h b/src/feature/client/transports.h index 59df637d81..fbb720aac6 100644 --- a/src/feature/client/transports.h +++ b/src/feature/client/transports.h @@ -128,6 +128,7 @@ STATIC int parse_version(const char *line, managed_proxy_t *mp); STATIC void parse_env_error(const char *line); STATIC void parse_proxy_error(const char *line); STATIC void handle_proxy_line(const char *line, managed_proxy_t *mp); +STATIC void parse_log_line(const char *line); STATIC char *get_transport_options_for_server_proxy(const managed_proxy_t *mp); STATIC void managed_proxy_destroy(managed_proxy_t *mp, diff --git a/src/feature/control/control.c b/src/feature/control/control.c index 94679dfd22..b6505a85d6 100644 --- a/src/feature/control/control.c +++ b/src/feature/control/control.c @@ -7395,6 +7395,17 @@ control_event_transport_launched(const char *mode, const char *transport_name, mode, transport_name, fmt_addr(addr), port); } +/** A pluggable transport called transport_name has emitted a log + * message found in message. */ +void +control_event_transport_log(const char *transport_name, const char *message) +{ + send_control_event(EVENT_TRANSPORT_LOG, + "650 TRANSPORT_LOG %s %s\r\n", + transport_name, + message); +} + /** Convert rendezvous auth type to string for HS_DESC control events */ const char * diff --git a/src/feature/control/control.h b/src/feature/control/control.h index cd5402d455..eb2b5676ea 100644 --- a/src/feature/control/control.h +++ b/src/feature/control/control.h @@ -205,6 +205,8 @@ void control_event_clients_seen(const char *controller_str); void control_event_transport_launched(const char *mode, const char *transport_name, tor_addr_t *addr, uint16_t port); +void control_event_transport_log(const char *transport_name, + const char *message); const char *rend_auth_type_to_string(rend_auth_type_t auth_type); MOCK_DECL(const char *, node_describe_longname_by_id,(const char *id_digest)); void control_event_hs_descriptor_requested(const char *onion_address, @@ -293,7 +295,8 @@ void control_free_all(void); #define EVENT_HS_DESC 0x0021 #define EVENT_HS_DESC_CONTENT 0x0022 #define EVENT_NETWORK_LIVENESS 0x0023 -#define EVENT_MAX_ 0x0023 +#define EVENT_TRANSPORT_LOG 0x0024 +#define EVENT_MAX_ 0x0024 /* sizeof(control_connection_t.event_mask) in bits, currently a uint64_t */ #define EVENT_CAPACITY_ 0x0040 diff --git a/src/test/test_pt.c b/src/test/test_pt.c index 2501b867bb..c980276b14 100644 --- a/src/test/test_pt.c +++ b/src/test/test_pt.c @@ -304,11 +304,16 @@ process_read_stdout_replacement(process_t *process, buf_t *buffer) /* Generate some dummy CMETHOD lines the first 5 times. The 6th time, send 'CMETHODS DONE' to finish configuring the proxy. */ - if (times_called++ != 5) { + times_called++; + + if (times_called <= 5) { buf_add_printf(buffer, "SMETHOD mock%d 127.0.0.1:555%d\n", times_called, times_called); - } else { + } else if (times_called <= 6) { buf_add_string(buffer, "SMETHODS DONE\n"); + } else if (times_called <= 7) { + buf_add_string(buffer, "LOG mock3 Oh noes, something bad happened. " + "What do we do!?\n"); } return (int)buf_datalen(buffer); @@ -410,6 +415,16 @@ test_pt_configure_proxy(void *arg) tt_str_op(smartlist_get(controlevent_msgs, 4), OP_EQ, "650 TRANSPORT_LAUNCHED server mock5 127.0.0.1 5555\r\n"); + /* Get the log message out. */ + process_notify_event_stdout(mp->process); + + tt_int_op(controlevent_n, OP_EQ, 6); + tt_int_op(controlevent_event, OP_EQ, EVENT_TRANSPORT_LOG); + tt_int_op(smartlist_len(controlevent_msgs), OP_EQ, 6); + tt_str_op(smartlist_get(controlevent_msgs, 5), OP_EQ, + "650 TRANSPORT_LOG mock3 Oh noes, something bad happened. " + "What do we do!?\r\n"); + { /* check that the transport info were saved properly in the tor state */ config_line_t *transport_in_state = NULL; smartlist_t *transport_info_sl = smartlist_new(); From 9b6a10a26f25e0da07e62bd5617b519b0952b40a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Mon, 26 Nov 2018 02:18:04 +0100 Subject: [PATCH 0294/2557] Add slow test for process_t for main loop interaction. This patch adds test cases for process_t which uses Tor's main loop. This allows us to test that the callbacks are actually invoked by the main loop when we expect them. See: https://bugs.torproject.org/28179 --- .gitignore | 2 + src/test/include.am | 2 + src/test/test-process.c | 85 +++++++++ src/test/test.h | 1 + src/test/test_process_slow.c | 330 +++++++++++++++++++++++++++++++++++ src/test/test_slow.c | 1 + 6 files changed, 421 insertions(+) create mode 100644 src/test/test-process.c create mode 100644 src/test/test_process_slow.c diff --git a/.gitignore b/.gitignore index ee2de376a6..df8db1113a 100644 --- a/.gitignore +++ b/.gitignore @@ -240,6 +240,7 @@ uptime-*.json /src/test/test-slow /src/test/test-bt-cl /src/test/test-child +/src/test/test-process /src/test/test-memwipe /src/test/test-ntor-cl /src/test/test-hs-ntor-cl @@ -250,6 +251,7 @@ uptime-*.json /src/test/test-slow.exe /src/test/test-bt-cl.exe /src/test/test-child.exe +/src/test/test-process.exe /src/test/test-ntor-cl.exe /src/test/test-hs-ntor-cl.exe /src/test/test-memwipe.exe diff --git a/src/test/include.am b/src/test/include.am index 482897c7a5..9df2fd481d 100644 --- a/src/test/include.am +++ b/src/test/include.am @@ -66,6 +66,7 @@ noinst_PROGRAMS+= \ src/test/test-slow \ src/test/test-memwipe \ src/test/test-child \ + src/test/test-process \ src/test/test_workqueue \ src/test/test-switch-id \ src/test/test-timers @@ -202,6 +203,7 @@ if UNITTESTS_ENABLED src_test_test_slow_SOURCES += \ src/test/test_slow.c \ src/test/test_crypto_slow.c \ + src/test/test_process_slow.c \ src/test/test_util_slow.c \ src/test/testing_common.c \ src/test/testing_rsakeys.c \ diff --git a/src/test/test-process.c b/src/test/test-process.c new file mode 100644 index 0000000000..ec1b395002 --- /dev/null +++ b/src/test/test-process.c @@ -0,0 +1,85 @@ +/* Copyright (c) 2011-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "orconfig.h" +#include +#ifdef _WIN32 +#define WINDOWS_LEAN_AND_MEAN +#include +#else +#include +#endif /* defined(_WIN32) */ +#include +#include + +#ifdef _WIN32 +#define SLEEP(sec) Sleep((sec)*1000) +#else +#define SLEEP(sec) sleep(sec) +#endif + +/* Trivial test program to test process_t. */ +int +main(int argc, char **argv) +{ + /* Does our process get the right arguments? */ + for (int i = 0; i < argc; ++i) { + fprintf(stdout, "argv[%d] = '%s'\n", i, argv[i]); + fflush(stdout); + } + + /* Make sure our process got our environment variable. */ + fprintf(stdout, "Environment variable TOR_TEST_ENV = '%s'\n", + getenv("TOR_TEST_ENV")); + fflush(stdout); + + /* Test line handling on stdout and stderr. */ + fprintf(stdout, "Output on stdout\nThis is a new line\n"); + fflush(stdout); + + fprintf(stderr, "Output on stderr\nThis is a new line\n"); + fflush(stderr); + + fprintf(stdout, "Partial line on stdout ..."); + fflush(stdout); + + fprintf(stderr, "Partial line on stderr ..."); + fflush(stderr); + + SLEEP(2); + + fprintf(stdout, "end of partial line on stdout\n"); + fflush(stdout); + fprintf(stderr, "end of partial line on stderr\n"); + fflush(stderr); + + /* Echo input from stdin. */ + char buffer[1024]; + + int count = 0; + + while (fgets(buffer, sizeof(buffer), stdin)) { + /* Strip the newline. */ + size_t size = strlen(buffer); + + if (size >= 1 && buffer[size - 1] == '\n') { + buffer[size - 1] = '\0'; + --size; + } + + if (size >= 1 && buffer[size - 1] == '\r') { + buffer[size - 1] = '\0'; + --size; + } + + fprintf(stdout, "Read line from stdin: '%s'\n", buffer); + fflush(stdout); + + if (++count == 3) + break; + } + + fprintf(stdout, "We are done for here, thank you!\n"); + + return 0; +} diff --git a/src/test/test.h b/src/test/test.h index 7c17389776..9f72623968 100644 --- a/src/test/test.h +++ b/src/test/test.h @@ -270,6 +270,7 @@ extern struct testcase_t voting_schedule_tests[]; extern struct testcase_t x509_tests[]; extern struct testcase_t slow_crypto_tests[]; +extern struct testcase_t slow_process_tests[]; extern struct testcase_t slow_util_tests[]; extern struct testgroup_t testgroups[]; diff --git a/src/test/test_process_slow.c b/src/test/test_process_slow.c new file mode 100644 index 0000000000..b4c5a2d5e2 --- /dev/null +++ b/src/test/test_process_slow.c @@ -0,0 +1,330 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file test_process_slow.c + * \brief Slow test cases for the Process API. + */ + +#include "orconfig.h" +#include "core/or/or.h" +#include "core/mainloop/mainloop.h" +#include "lib/evloop/compat_libevent.h" +#include "lib/process/process.h" +#include "lib/process/waitpid.h" +#include "test/test.h" + +#ifndef BUILDDIR +#define BUILDDIR "." +#endif + +#ifdef _WIN32 +#define TEST_PROCESS "test-process.exe" +#else +#define TEST_PROCESS BUILDDIR "/src/test/test-process" +#endif /* defined(_WIN32) */ + +/** Timer that ticks once a second and stop the event loop after 5 ticks. */ +static periodic_timer_t *main_loop_timeout_timer; + +/** How many times have our timer ticked? */ +static int timer_tick_count; + +struct process_data_t { + smartlist_t *stdout_data; + smartlist_t *stderr_data; + smartlist_t *stdin_data; + process_exit_code_t exit_code; +}; + +typedef struct process_data_t process_data_t; + +static process_data_t * +process_data_new(void) +{ + process_data_t *process_data = tor_malloc_zero(sizeof(process_data_t)); + process_data->stdout_data = smartlist_new(); + process_data->stderr_data = smartlist_new(); + process_data->stdin_data = smartlist_new(); + return process_data; +} + +static void +process_data_free(process_data_t *process_data) +{ + if (process_data == NULL) + return; + + SMARTLIST_FOREACH(process_data->stdout_data, char *, x, tor_free(x)); + SMARTLIST_FOREACH(process_data->stderr_data, char *, x, tor_free(x)); + SMARTLIST_FOREACH(process_data->stdin_data, char *, x, tor_free(x)); + + smartlist_free(process_data->stdout_data); + smartlist_free(process_data->stderr_data); + smartlist_free(process_data->stdin_data); + tor_free(process_data); +} + +static void +process_stdout_callback(process_t *process, char *data, size_t size) +{ + tt_ptr_op(process, OP_NE, NULL); + tt_ptr_op(data, OP_NE, NULL); + tt_int_op(strlen(data), OP_EQ, size); + + process_data_t *process_data = process_get_data(process); + smartlist_add(process_data->stdout_data, tor_strdup(data)); + + done: + return; +} + +static void +process_stderr_callback(process_t *process, char *data, size_t size) +{ + tt_ptr_op(process, OP_NE, NULL); + tt_ptr_op(data, OP_NE, NULL); + tt_int_op(strlen(data), OP_EQ, size); + + process_data_t *process_data = process_get_data(process); + smartlist_add(process_data->stderr_data, tor_strdup(data)); + + done: + return; +} + +static void +process_exit_callback(process_t *process, process_exit_code_t exit_code) +{ + tt_ptr_op(process, OP_NE, NULL); + + process_data_t *process_data = process_get_data(process); + process_data->exit_code = exit_code; + + /* Our process died. Let's check the values it returned. */ + tor_shutdown_event_loop_and_exit(0); + + done: + return; +} + +#ifdef _WIN32 +static const char * +get_win32_test_binary_path(void) +{ + static char buffer[MAX_PATH]; + + /* Get the absolute path of our binary: \path\to\test-slow.exe. */ + GetModuleFileNameA(GetModuleHandle(0), buffer, sizeof(buffer)); + + /* Find our process name. */ + char *offset = strstr(buffer, "test-slow.exe"); + tt_ptr_op(offset, OP_NE, NULL); + + /* Change test-slow.exe to test-process.exe. */ + memcpy(offset, TEST_PROCESS, strlen(TEST_PROCESS)); + + return buffer; + done: + return NULL; +} +#endif + +static void +main_loop_timeout_cb(periodic_timer_t *timer, void *data) +{ + /* Sanity check. */ + tt_ptr_op(timer, OP_EQ, main_loop_timeout_timer); + tt_ptr_op(data, OP_EQ, NULL); + + /* Have we been called 10 times we exit. */ + timer_tick_count++; + + tt_int_op(timer_tick_count, OP_LT, 10); + +#ifndef _WIN32 + /* Call waitpid callbacks. */ + notify_pending_waitpid_callbacks(); +#endif + + return; + done: + /* Exit with an error. */ + tor_shutdown_event_loop_and_exit(-1); +} + +static void +run_main_loop(void) +{ + int ret; + + /* Wake up after 1 seconds. */ + static const struct timeval interval = {1, 0}; + + timer_tick_count = 0; + main_loop_timeout_timer = periodic_timer_new(tor_libevent_get_base(), + &interval, + main_loop_timeout_cb, + NULL); + + /* Run our main loop. */ + ret = do_main_loop(); + + /* Clean up our main loop timeout timer. */ + tt_int_op(ret, OP_EQ, 0); + + done: + periodic_timer_free(main_loop_timeout_timer); +} + +static void +test_callbacks(void *arg) +{ + (void)arg; + const char *filename = NULL; + +#ifdef _WIN32 + filename = get_win32_test_binary_path(); +#else + filename = TEST_PROCESS; +#endif + + /* Initialize Process subsystem. */ + process_init(); + + /* Process callback data. */ + process_data_t *process_data = process_data_new(); + + /* Setup our process. */ + process_t *process = process_new(filename); + process_set_data(process, process_data); + process_set_stdout_read_callback(process, process_stdout_callback); + process_set_stderr_read_callback(process, process_stderr_callback); + process_set_exit_callback(process, process_exit_callback); + + /* Set environment variable. */ + process_set_environment(process, "TOR_TEST_ENV", "Hello, from Tor!"); + + /* Add some arguments. */ + process_append_argument(process, "This is the first one"); + process_append_argument(process, "Second one"); + process_append_argument(process, "Third: Foo bar baz"); + + /* Run our process. */ + process_status_t status; + + status = process_exec(process); + tt_int_op(status, OP_EQ, PROCESS_STATUS_RUNNING); + + /* Write some lines to stdin. */ + process_printf(process, "Hi process!\r\n"); + process_printf(process, "Can you read more than one line?\n"); + process_printf(process, "Can you read partial ..."); + process_printf(process, " lines?\r\n"); + + /* Start our main loop. */ + run_main_loop(); + + /* Check if our process is still running? */ + status = process_get_status(process); + tt_int_op(status, OP_EQ, PROCESS_STATUS_NOT_RUNNING); + + /* We returned. Let's see what our event loop said. */ + tt_int_op(smartlist_len(process_data->stdout_data), OP_EQ, 12); + tt_int_op(smartlist_len(process_data->stderr_data), OP_EQ, 3); + tt_int_op(process_data->exit_code, OP_EQ, 0); + + /* Check stdout output. */ + char argv0_expected[256]; + tor_snprintf(argv0_expected, sizeof(argv0_expected), + "argv[0] = '%s'", filename); + + tt_str_op(smartlist_get(process_data->stdout_data, 0), OP_EQ, + argv0_expected); + tt_str_op(smartlist_get(process_data->stdout_data, 1), OP_EQ, + "argv[1] = 'This is the first one'"); + tt_str_op(smartlist_get(process_data->stdout_data, 2), OP_EQ, + "argv[2] = 'Second one'"); + tt_str_op(smartlist_get(process_data->stdout_data, 3), OP_EQ, + "argv[3] = 'Third: Foo bar baz'"); + tt_str_op(smartlist_get(process_data->stdout_data, 4), OP_EQ, + "Environment variable TOR_TEST_ENV = 'Hello, from Tor!'"); + tt_str_op(smartlist_get(process_data->stdout_data, 5), OP_EQ, + "Output on stdout"); + tt_str_op(smartlist_get(process_data->stdout_data, 6), OP_EQ, + "This is a new line"); + tt_str_op(smartlist_get(process_data->stdout_data, 7), OP_EQ, + "Partial line on stdout ...end of partial line on stdout"); + tt_str_op(smartlist_get(process_data->stdout_data, 8), OP_EQ, + "Read line from stdin: 'Hi process!'"); + tt_str_op(smartlist_get(process_data->stdout_data, 9), OP_EQ, + "Read line from stdin: 'Can you read more than one line?'"); + tt_str_op(smartlist_get(process_data->stdout_data, 10), OP_EQ, + "Read line from stdin: 'Can you read partial ... lines?'"); + tt_str_op(smartlist_get(process_data->stdout_data, 11), OP_EQ, + "We are done for here, thank you!"); + + /* Check stderr output. */ + tt_str_op(smartlist_get(process_data->stderr_data, 0), OP_EQ, + "Output on stderr"); + tt_str_op(smartlist_get(process_data->stderr_data, 1), OP_EQ, + "This is a new line"); + tt_str_op(smartlist_get(process_data->stderr_data, 2), OP_EQ, + "Partial line on stderr ...end of partial line on stderr"); + + done: + process_data_free(process_data); + process_free(process); + process_free_all(); +} + +static void +test_callbacks_terminate(void *arg) +{ + (void)arg; + const char *filename = NULL; + +#ifdef _WIN32 + filename = get_win32_test_binary_path(); +#else + filename = TEST_PROCESS; +#endif + + /* Initialize Process subsystem. */ + process_init(); + + /* Process callback data. */ + process_data_t *process_data = process_data_new(); + + /* Setup our process. */ + process_t *process = process_new(filename); + process_set_data(process, process_data); + process_set_exit_callback(process, process_exit_callback); + + /* Run our process. */ + process_status_t status; + + status = process_exec(process); + tt_int_op(status, OP_EQ, PROCESS_STATUS_RUNNING); + + /* Zap our process. */ + process_terminate(process); + + /* Start our main loop. */ + run_main_loop(); + + /* Check if our process is still running? */ + status = process_get_status(process); + tt_int_op(status, OP_EQ, PROCESS_STATUS_NOT_RUNNING); + + done: + process_data_free(process_data); + process_free(process); + process_free_all(); +} + +struct testcase_t slow_process_tests[] = { + { "callbacks", test_callbacks, TT_FORK, NULL, NULL }, + { "callbacks_terminate", test_callbacks_terminate, TT_FORK, NULL, NULL }, + END_OF_TESTCASES +}; diff --git a/src/test/test_slow.c b/src/test/test_slow.c index 0b665363ab..0c66eff930 100644 --- a/src/test/test_slow.c +++ b/src/test/test_slow.c @@ -20,6 +20,7 @@ struct testgroup_t testgroups[] = { { "slow/crypto/", slow_crypto_tests }, + { "slow/process/", slow_process_tests }, { "slow/util/", slow_util_tests }, END_OF_GROUPS }; From 289ed0849d9b473cd0fbafada30c03638d73e541 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Mon, 26 Nov 2018 04:59:12 +0100 Subject: [PATCH 0295/2557] Add tests for process environment functionality of process_t. This patch adds tests for the process_environment_t interaction in process_t. See: https://bugs.torproject.org/28179 --- src/test/test_process.c | 52 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/src/test/test_process.c b/src/test/test_process.c index 4f86e786cb..2fcb3f6686 100644 --- a/src/test/test_process.c +++ b/src/test/test_process.c @@ -9,6 +9,7 @@ #include "orconfig.h" #include "core/or/or.h" #include "test/test.h" +#include "lib/process/env.h" #define PROCESS_PRIVATE #include "lib/process/process.h" @@ -172,6 +173,56 @@ test_default_values(void *arg) process_free_all(); } +static void +test_environment(void *arg) +{ + (void)arg; + process_init(); + + process_t *process = process_new(""); + process_environment_t *env = NULL; + + process_set_environment(process, "E", "F"); + process_set_environment(process, "C", "D"); + process_set_environment(process, "A", "B"); + + env = process_get_environment(process); + tt_mem_op(env->windows_environment_block, OP_EQ, + "A=B\0C=D\0E=F\0", 12); + tt_str_op(env->unixoid_environment_block[0], OP_EQ, + "A=B"); + tt_str_op(env->unixoid_environment_block[1], OP_EQ, + "C=D"); + tt_str_op(env->unixoid_environment_block[2], OP_EQ, + "E=F"); + tt_ptr_op(env->unixoid_environment_block[3], OP_EQ, + NULL); + process_environment_free(env); + + /* Reset our environment. */ + smartlist_t *new_env = smartlist_new(); + smartlist_add(new_env, (char *)"FOO=bar"); + smartlist_add(new_env, (char *)"HELLO=world"); + + process_reset_environment(process, new_env); + smartlist_free(new_env); + + env = process_get_environment(process); + tt_mem_op(env->windows_environment_block, OP_EQ, + "FOO=bar\0HELLO=world\0", 20); + tt_str_op(env->unixoid_environment_block[0], OP_EQ, + "FOO=bar"); + tt_str_op(env->unixoid_environment_block[1], OP_EQ, + "HELLO=world"); + tt_ptr_op(env->unixoid_environment_block[2], OP_EQ, + NULL); + + done: + process_environment_free(env); + process_free(process); + process_free_all(); +} + static void test_stringified_types(void *arg) { @@ -589,6 +640,7 @@ test_win32(void *arg) struct testcase_t process_tests[] = { { "default_values", test_default_values, TT_FORK, NULL, NULL }, + { "environment", test_environment, TT_FORK, NULL, NULL }, { "stringified_types", test_stringified_types, TT_FORK, NULL, NULL }, { "line_protocol_simple", test_line_protocol_simple, TT_FORK, NULL, NULL }, { "line_protocol_multi", test_line_protocol_multi, TT_FORK, NULL, NULL }, From f7d13425fc738d7d56d5582b9be8510ac236c076 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Mon, 26 Nov 2018 05:34:30 +0100 Subject: [PATCH 0296/2557] Delete old process_handle_t code. This patch removes the old process_handle_t code. Everything should by now be using the process_t interface. See: https://bugs.torproject.org/28179 --- .gitignore | 2 - src/feature/client/transports.c | 16 - src/lib/process/subprocess.c | 1078 ------------------------------- src/lib/process/subprocess.h | 118 +--- src/test/Makefile.nmake | 7 +- src/test/include.am | 2 - src/test/test-child.c | 61 -- src/test/test.h | 1 - src/test/test_logging.c | 21 +- src/test/test_slow.c | 1 - src/test/test_util.c | 262 -------- src/test/test_util_slow.c | 396 ------------ 12 files changed, 17 insertions(+), 1948 deletions(-) delete mode 100644 src/test/test-child.c delete mode 100644 src/test/test_util_slow.c diff --git a/.gitignore b/.gitignore index df8db1113a..60635263ad 100644 --- a/.gitignore +++ b/.gitignore @@ -239,7 +239,6 @@ uptime-*.json /src/test/test /src/test/test-slow /src/test/test-bt-cl -/src/test/test-child /src/test/test-process /src/test/test-memwipe /src/test/test-ntor-cl @@ -250,7 +249,6 @@ uptime-*.json /src/test/test.exe /src/test/test-slow.exe /src/test/test-bt-cl.exe -/src/test/test-child.exe /src/test/test-process.exe /src/test/test-ntor-cl.exe /src/test/test-hs-ntor-cl.exe diff --git a/src/feature/client/transports.c b/src/feature/client/transports.c index dc76e9c245..fd4bec7807 100644 --- a/src/feature/client/transports.c +++ b/src/feature/client/transports.c @@ -913,22 +913,6 @@ handle_proxy_line(const char *line, managed_proxy_t *mp) } else if (!strcmpstart(line, PROTO_LOG)) { parse_log_line(line); return; - } else if (!strcmpstart(line, SPAWN_ERROR_MESSAGE)) { - /* managed proxy launch failed: parse error message to learn why. */ - int retval, child_state, saved_errno; - retval = tor_sscanf(line, SPAWN_ERROR_MESSAGE "%x/%x", - &child_state, &saved_errno); - if (retval == 2) { - log_warn(LD_GENERAL, - "Could not launch managed proxy executable at '%s' ('%s').", - mp->argv[0], strerror(saved_errno)); - } else { /* failed to parse error message */ - log_warn(LD_GENERAL,"Could not launch managed proxy executable at '%s'.", - mp->argv[0]); - } - - mp->conf_state = PT_PROTO_FAILED_LAUNCH; - return; } log_notice(LD_GENERAL, "Unknown line received by managed proxy (%s).", line); diff --git a/src/lib/process/subprocess.c b/src/lib/process/subprocess.c index 70851c15e0..c163790155 100644 --- a/src/lib/process/subprocess.c +++ b/src/lib/process/subprocess.c @@ -142,236 +142,6 @@ tor_join_win_cmdline(const char *argv[]) return joined_argv; } -#ifndef _WIN32 -/** Format child_state and saved_errno as a hex string placed in - * hex_errno. Called between fork and _exit, so must be signal-handler - * safe. - * - * hex_errno must have at least HEX_ERRNO_SIZE+1 bytes available. - * - * The format of hex_errno is: "CHILD_STATE/ERRNO\n", left-padded - * with spaces. CHILD_STATE indicates where - * in the process of starting the child process did the failure occur (see - * CHILD_STATE_* macros for definition), and SAVED_ERRNO is the value of - * errno when the failure occurred. - * - * On success return the number of characters added to hex_errno, not counting - * the terminating NUL; return -1 on error. - */ -STATIC int -format_helper_exit_status(unsigned char child_state, int saved_errno, - char *hex_errno) -{ - unsigned int unsigned_errno; - int written, left; - char *cur; - size_t i; - int res = -1; - - /* Fill hex_errno with spaces, and a trailing newline (memset may - not be signal handler safe, so we can't use it) */ - for (i = 0; i < (HEX_ERRNO_SIZE - 1); i++) - hex_errno[i] = ' '; - hex_errno[HEX_ERRNO_SIZE - 1] = '\n'; - - /* Convert errno to be unsigned for hex conversion */ - if (saved_errno < 0) { - // Avoid overflow on the cast to unsigned int when result is INT_MIN - // by adding 1 to the signed int negative value, - // then, after it has been negated and cast to unsigned, - // adding the original 1 back (the double-addition is intentional). - // Otherwise, the cast to signed could cause a temporary int - // to equal INT_MAX + 1, which is undefined. - unsigned_errno = ((unsigned int) -(saved_errno + 1)) + 1; - } else { - unsigned_errno = (unsigned int) saved_errno; - } - - /* - * Count how many chars of space we have left, and keep a pointer into the - * current point in the buffer. - */ - left = HEX_ERRNO_SIZE+1; - cur = hex_errno; - - /* Emit child_state */ - written = format_hex_number_sigsafe(child_state, cur, left); - - if (written <= 0) - goto err; - - /* Adjust left and cur */ - left -= written; - cur += written; - if (left <= 0) - goto err; - - /* Now the '/' */ - *cur = '/'; - - /* Adjust left and cur */ - ++cur; - --left; - if (left <= 0) - goto err; - - /* Need minus? */ - if (saved_errno < 0) { - *cur = '-'; - ++cur; - --left; - if (left <= 0) - goto err; - } - - /* Emit unsigned_errno */ - written = format_hex_number_sigsafe(unsigned_errno, cur, left); - - if (written <= 0) - goto err; - - /* Adjust left and cur */ - left -= written; - cur += written; - - /* Check that we have enough space left for a newline and a NUL */ - if (left <= 1) - goto err; - - /* Emit the newline and NUL */ - *cur++ = '\n'; - *cur++ = '\0'; - - res = (int)(cur - hex_errno - 1); - - goto done; - - err: - /* - * In error exit, just write a '\0' in the first char so whatever called - * this at least won't fall off the end. - */ - *hex_errno = '\0'; - - done: - return res; -} -#endif /* !defined(_WIN32) */ - -/* Maximum number of file descriptors, if we cannot get it via sysconf() */ -#define DEFAULT_MAX_FD 256 - -/** Terminate the process of process_handle, if that process has not - * already exited. - * - * Return 0 if we succeeded in terminating the process (or if the process - * already exited), and -1 if we tried to kill the process but failed. - * - * Based on code originally borrowed from Python's os.kill. */ -int -tor_terminate_process(process_handle_t *process_handle) -{ -#ifdef _WIN32 - if (tor_get_exit_code(process_handle, 0, NULL) == PROCESS_EXIT_RUNNING) { - HANDLE handle = process_handle->pid.hProcess; - - if (!TerminateProcess(handle, 0)) - return -1; - else - return 0; - } -#else /* !(defined(_WIN32)) */ - if (process_handle->waitpid_cb) { - /* We haven't got a waitpid yet, so we can just kill off the process. */ - return kill(process_handle->pid, SIGTERM); - } -#endif /* defined(_WIN32) */ - - return 0; /* We didn't need to kill the process, so report success */ -} - -/** Return the Process ID of process_handle. */ -int -tor_process_get_pid(process_handle_t *process_handle) -{ -#ifdef _WIN32 - return (int) process_handle->pid.dwProcessId; -#else - return (int) process_handle->pid; -#endif -} - -#ifdef _WIN32 -HANDLE -tor_process_get_stdout_pipe(process_handle_t *process_handle) -{ - return process_handle->stdout_pipe; -} -#else /* !(defined(_WIN32)) */ -/* DOCDOC tor_process_get_stdout_pipe */ -int -tor_process_get_stdout_pipe(process_handle_t *process_handle) -{ - return process_handle->stdout_pipe; -} -#endif /* defined(_WIN32) */ - -/* DOCDOC process_handle_new */ -static process_handle_t * -process_handle_new(void) -{ - process_handle_t *out = tor_malloc_zero(sizeof(process_handle_t)); - -#ifdef _WIN32 - out->stdin_pipe = INVALID_HANDLE_VALUE; - out->stdout_pipe = INVALID_HANDLE_VALUE; - out->stderr_pipe = INVALID_HANDLE_VALUE; -#else - out->stdin_pipe = -1; - out->stdout_pipe = -1; - out->stderr_pipe = -1; -#endif /* defined(_WIN32) */ - - return out; -} - -#ifndef _WIN32 -/** Invoked when a process that we've launched via tor_spawn_background() has - * been found to have terminated. - */ -static void -process_handle_waitpid_cb(int status, void *arg) -{ - process_handle_t *process_handle = arg; - - process_handle->waitpid_exit_status = status; - clear_waitpid_callback(process_handle->waitpid_cb); - if (process_handle->status == PROCESS_STATUS_RUNNING) - process_handle->status = PROCESS_STATUS_NOTRUNNING; - process_handle->waitpid_cb = 0; -} -#endif /* !defined(_WIN32) */ - -/** - * @name child-process states - * - * Each of these values represents a possible state that a child process can - * be in. They're used to determine what to say when telling the parent how - * far along we were before failure. - * - * @{ - */ -#define CHILD_STATE_INIT 0 -#define CHILD_STATE_PIPE 1 -#define CHILD_STATE_MAXFD 2 -#define CHILD_STATE_FORK 3 -#define CHILD_STATE_DUPOUT 4 -#define CHILD_STATE_DUPERR 5 -#define CHILD_STATE_DUPIN 6 -#define CHILD_STATE_CLOSEFD 7 -#define CHILD_STATE_EXEC 8 -#define CHILD_STATE_FAILEXEC 9 -/** @} */ /** * Boolean. If true, then Tor may call execve or CreateProcess via * tor_spawn_background. @@ -386,851 +156,3 @@ tor_disable_spawning_background_processes(void) { may_spawn_background_process = 0; } -/** Start a program in the background. If filename contains a '/', then - * it will be treated as an absolute or relative path. Otherwise, on - * non-Windows systems, the system path will be searched for filename. - * On Windows, only the current directory will be searched. Here, to search the - * system path (as well as the application directory, current working - * directory, and system directories), set filename to NULL. - * - * The strings in argv will be passed as the command line arguments of - * the child program (following convention, argv[0] should normally be the - * filename of the executable, and this must be the case if filename is - * NULL). The last element of argv must be NULL. A handle to the child process - * will be returned in process_handle (which must be non-NULL). Read - * process_handle.status to find out if the process was successfully launched. - * For convenience, process_handle.status is returned by this function. - * - * Some parts of this code are based on the POSIX subprocess module from - * Python, and example code from - * http://msdn.microsoft.com/en-us/library/ms682499%28v=vs.85%29.aspx. - */ -int -tor_spawn_background(const char *const filename, const char **argv, - process_environment_t *env, - process_handle_t **process_handle_out) -{ - if (BUG(may_spawn_background_process == 0)) { - /* We should never reach this point if we're forbidden to spawn - * processes. Instead we should have caught the attempt earlier. */ - return PROCESS_STATUS_ERROR; - } - -#ifdef _WIN32 - HANDLE stdout_pipe_read = NULL; - HANDLE stdout_pipe_write = NULL; - HANDLE stderr_pipe_read = NULL; - HANDLE stderr_pipe_write = NULL; - HANDLE stdin_pipe_read = NULL; - HANDLE stdin_pipe_write = NULL; - process_handle_t *process_handle; - int status; - - STARTUPINFOA siStartInfo; - BOOL retval = FALSE; - - SECURITY_ATTRIBUTES saAttr; - char *joined_argv; - - saAttr.nLength = sizeof(SECURITY_ATTRIBUTES); - saAttr.bInheritHandle = TRUE; - /* TODO: should we set explicit security attributes? (#2046, comment 5) */ - saAttr.lpSecurityDescriptor = NULL; - - /* Assume failure to start process */ - status = PROCESS_STATUS_ERROR; - - /* Set up pipe for stdout */ - if (!CreatePipe(&stdout_pipe_read, &stdout_pipe_write, &saAttr, 0)) { - log_warn(LD_GENERAL, - "Failed to create pipe for stdout communication with child process: %s", - format_win32_error(GetLastError())); - return status; - } - if (!SetHandleInformation(stdout_pipe_read, HANDLE_FLAG_INHERIT, 0)) { - log_warn(LD_GENERAL, - "Failed to configure pipe for stdout communication with child " - "process: %s", format_win32_error(GetLastError())); - return status; - } - - /* Set up pipe for stderr */ - if (!CreatePipe(&stderr_pipe_read, &stderr_pipe_write, &saAttr, 0)) { - log_warn(LD_GENERAL, - "Failed to create pipe for stderr communication with child process: %s", - format_win32_error(GetLastError())); - return status; - } - if (!SetHandleInformation(stderr_pipe_read, HANDLE_FLAG_INHERIT, 0)) { - log_warn(LD_GENERAL, - "Failed to configure pipe for stderr communication with child " - "process: %s", format_win32_error(GetLastError())); - return status; - } - - /* Set up pipe for stdin */ - if (!CreatePipe(&stdin_pipe_read, &stdin_pipe_write, &saAttr, 0)) { - log_warn(LD_GENERAL, - "Failed to create pipe for stdin communication with child process: %s", - format_win32_error(GetLastError())); - return status; - } - if (!SetHandleInformation(stdin_pipe_write, HANDLE_FLAG_INHERIT, 0)) { - log_warn(LD_GENERAL, - "Failed to configure pipe for stdin communication with child " - "process: %s", format_win32_error(GetLastError())); - return status; - } - - /* Create the child process */ - - /* Windows expects argv to be a whitespace delimited string, so join argv up - */ - joined_argv = tor_join_win_cmdline(argv); - - process_handle = process_handle_new(); - process_handle->status = status; - - ZeroMemory(&(process_handle->pid), sizeof(PROCESS_INFORMATION)); - ZeroMemory(&siStartInfo, sizeof(STARTUPINFO)); - siStartInfo.cb = sizeof(STARTUPINFO); - siStartInfo.hStdError = stderr_pipe_write; - siStartInfo.hStdOutput = stdout_pipe_write; - siStartInfo.hStdInput = stdin_pipe_read; - siStartInfo.dwFlags |= STARTF_USESTDHANDLES; - - /* Create the child process */ - - retval = CreateProcessA(filename, // module name - joined_argv, // command line - /* TODO: should we set explicit security attributes? (#2046, comment 5) */ - NULL, // process security attributes - NULL, // primary thread security attributes - TRUE, // handles are inherited - /*(TODO: set CREATE_NEW CONSOLE/PROCESS_GROUP to make GetExitCodeProcess() - * work?) */ - CREATE_NO_WINDOW, // creation flags - (env==NULL) ? NULL : env->windows_environment_block, - NULL, // use parent's current directory - &siStartInfo, // STARTUPINFO pointer - &(process_handle->pid)); // receives PROCESS_INFORMATION - - tor_free(joined_argv); - - if (!retval) { - log_warn(LD_GENERAL, - "Failed to create child process %s: %s", filename?filename:argv[0], - format_win32_error(GetLastError())); - tor_free(process_handle); - } else { - /* TODO: Close hProcess and hThread in process_handle->pid? */ - process_handle->stdout_pipe = stdout_pipe_read; - process_handle->stderr_pipe = stderr_pipe_read; - process_handle->stdin_pipe = stdin_pipe_write; - status = process_handle->status = PROCESS_STATUS_RUNNING; - } - - /* TODO: Close pipes on exit */ - *process_handle_out = process_handle; - return status; -#else /* !(defined(_WIN32)) */ - pid_t pid; - int stdout_pipe[2]; - int stderr_pipe[2]; - int stdin_pipe[2]; - int fd, retval; - process_handle_t *process_handle; - int status; - - const char *error_message = SPAWN_ERROR_MESSAGE; - size_t error_message_length; - - /* Represents where in the process of spawning the program is; - this is used for printing out the error message */ - unsigned char child_state = CHILD_STATE_INIT; - - char hex_errno[HEX_ERRNO_SIZE + 2]; /* + 1 should be sufficient actually */ - - static int max_fd = -1; - - status = PROCESS_STATUS_ERROR; - - /* We do the strlen here because strlen() is not signal handler safe, - and we are not allowed to use unsafe functions between fork and exec */ - error_message_length = strlen(error_message); - - // child_state = CHILD_STATE_PIPE; - - /* Set up pipe for redirecting stdout, stderr, and stdin of child */ - retval = pipe(stdout_pipe); - if (-1 == retval) { - log_warn(LD_GENERAL, - "Failed to set up pipe for stdout communication with child process: %s", - strerror(errno)); - return status; - } - - retval = pipe(stderr_pipe); - if (-1 == retval) { - log_warn(LD_GENERAL, - "Failed to set up pipe for stderr communication with child process: %s", - strerror(errno)); - - close(stdout_pipe[0]); - close(stdout_pipe[1]); - - return status; - } - - retval = pipe(stdin_pipe); - if (-1 == retval) { - log_warn(LD_GENERAL, - "Failed to set up pipe for stdin communication with child process: %s", - strerror(errno)); - - close(stdout_pipe[0]); - close(stdout_pipe[1]); - close(stderr_pipe[0]); - close(stderr_pipe[1]); - - return status; - } - - // child_state = CHILD_STATE_MAXFD; - -#ifdef _SC_OPEN_MAX - if (-1 == max_fd) { - max_fd = (int) sysconf(_SC_OPEN_MAX); - if (max_fd == -1) { - max_fd = DEFAULT_MAX_FD; - log_warn(LD_GENERAL, - "Cannot find maximum file descriptor, assuming %d", max_fd); - } - } -#else /* !(defined(_SC_OPEN_MAX)) */ - max_fd = DEFAULT_MAX_FD; -#endif /* defined(_SC_OPEN_MAX) */ - - // child_state = CHILD_STATE_FORK; - - pid = fork(); - if (0 == pid) { - /* In child */ - -#if defined(HAVE_SYS_PRCTL_H) && defined(__linux__) - /* Attempt to have the kernel issue a SIGTERM if the parent - * goes away. Certain attributes of the binary being execve()ed - * will clear this during the execve() call, but it's better - * than nothing. - */ - prctl(PR_SET_PDEATHSIG, SIGTERM); -#endif /* defined(HAVE_SYS_PRCTL_H) && defined(__linux__) */ - - child_state = CHILD_STATE_DUPOUT; - - /* Link child stdout to the write end of the pipe */ - retval = dup2(stdout_pipe[1], STDOUT_FILENO); - if (-1 == retval) - goto error; - - child_state = CHILD_STATE_DUPERR; - - /* Link child stderr to the write end of the pipe */ - retval = dup2(stderr_pipe[1], STDERR_FILENO); - if (-1 == retval) - goto error; - - child_state = CHILD_STATE_DUPIN; - - /* Link child stdin to the read end of the pipe */ - retval = dup2(stdin_pipe[0], STDIN_FILENO); - if (-1 == retval) - goto error; - - // child_state = CHILD_STATE_CLOSEFD; - - close(stderr_pipe[0]); - close(stderr_pipe[1]); - close(stdout_pipe[0]); - close(stdout_pipe[1]); - close(stdin_pipe[0]); - close(stdin_pipe[1]); - - /* Close all other fds, including the read end of the pipe */ - /* XXX: We should now be doing enough FD_CLOEXEC setting to make - * this needless. */ - for (fd = STDERR_FILENO + 1; fd < max_fd; fd++) { - close(fd); - } - - // child_state = CHILD_STATE_EXEC; - - /* Call the requested program. We need the cast because - execvp doesn't define argv as const, even though it - does not modify the arguments */ - if (env) - execve(filename, (char *const *) argv, env->unixoid_environment_block); - else { - static char *new_env[] = { NULL }; - execve(filename, (char *const *) argv, new_env); - } - - /* If we got here, the exec or open(/dev/null) failed */ - - child_state = CHILD_STATE_FAILEXEC; - - error: - { - /* XXX: are we leaking fds from the pipe? */ - int n, err=0; - ssize_t nbytes; - - n = format_helper_exit_status(child_state, errno, hex_errno); - - if (n >= 0) { - /* Write the error message. GCC requires that we check the return - value, but there is nothing we can do if it fails */ - /* TODO: Don't use STDOUT, use a pipe set up just for this purpose */ - nbytes = write(STDOUT_FILENO, error_message, error_message_length); - err = (nbytes < 0); - nbytes = write(STDOUT_FILENO, hex_errno, n); - err += (nbytes < 0); - } - - _exit(err?254:255); // exit ok: in child. - } - - /* Never reached, but avoids compiler warning */ - return status; // LCOV_EXCL_LINE - } - - /* In parent */ - - if (-1 == pid) { - log_warn(LD_GENERAL, "Failed to fork child process: %s", strerror(errno)); - close(stdin_pipe[0]); - close(stdin_pipe[1]); - close(stdout_pipe[0]); - close(stdout_pipe[1]); - close(stderr_pipe[0]); - close(stderr_pipe[1]); - return status; - } - - process_handle = process_handle_new(); - process_handle->status = status; - process_handle->pid = pid; - - /* TODO: If the child process forked but failed to exec, waitpid it */ - - /* Return read end of the pipes to caller, and close write end */ - process_handle->stdout_pipe = stdout_pipe[0]; - retval = close(stdout_pipe[1]); - - if (-1 == retval) { - log_warn(LD_GENERAL, - "Failed to close write end of stdout pipe in parent process: %s", - strerror(errno)); - } - - process_handle->waitpid_cb = set_waitpid_callback(pid, - process_handle_waitpid_cb, - process_handle); - - process_handle->stderr_pipe = stderr_pipe[0]; - retval = close(stderr_pipe[1]); - - if (-1 == retval) { - log_warn(LD_GENERAL, - "Failed to close write end of stderr pipe in parent process: %s", - strerror(errno)); - } - - /* Return write end of the stdin pipe to caller, and close the read end */ - process_handle->stdin_pipe = stdin_pipe[1]; - retval = close(stdin_pipe[0]); - - if (-1 == retval) { - log_warn(LD_GENERAL, - "Failed to close read end of stdin pipe in parent process: %s", - strerror(errno)); - } - - status = process_handle->status = PROCESS_STATUS_RUNNING; - /* Set stdin/stdout/stderr pipes to be non-blocking */ - if (fcntl(process_handle->stdout_pipe, F_SETFL, O_NONBLOCK) < 0 || - fcntl(process_handle->stderr_pipe, F_SETFL, O_NONBLOCK) < 0 || - fcntl(process_handle->stdin_pipe, F_SETFL, O_NONBLOCK) < 0) { - log_warn(LD_GENERAL, "Failed to set stderror/stdout/stdin pipes " - "nonblocking in parent process: %s", strerror(errno)); - } - - *process_handle_out = process_handle; - return status; -#endif /* defined(_WIN32) */ -} - -/** Destroy all resources allocated by the process handle in - * process_handle. - * If also_terminate_process is true, also terminate the - * process of the process handle. */ -MOCK_IMPL(void, -tor_process_handle_destroy,(process_handle_t *process_handle, - int also_terminate_process)) -{ - if (!process_handle) - return; - - if (also_terminate_process) { - if (tor_terminate_process(process_handle) < 0) { - const char *errstr = -#ifdef _WIN32 - format_win32_error(GetLastError()); -#else - strerror(errno); -#endif - log_notice(LD_GENERAL, "Failed to terminate process with " - "PID '%d' ('%s').", tor_process_get_pid(process_handle), - errstr); - } else { - log_info(LD_GENERAL, "Terminated process with PID '%d'.", - tor_process_get_pid(process_handle)); - } - } - - process_handle->status = PROCESS_STATUS_NOTRUNNING; - -#ifdef _WIN32 - if (process_handle->stdout_pipe) - CloseHandle(process_handle->stdout_pipe); - - if (process_handle->stderr_pipe) - CloseHandle(process_handle->stderr_pipe); - - if (process_handle->stdin_pipe) - CloseHandle(process_handle->stdin_pipe); -#else /* !(defined(_WIN32)) */ - close(process_handle->stdout_pipe); - close(process_handle->stderr_pipe); - close(process_handle->stdin_pipe); - - clear_waitpid_callback(process_handle->waitpid_cb); -#endif /* defined(_WIN32) */ - - memset(process_handle, 0x0f, sizeof(process_handle_t)); - tor_free(process_handle); -} - -/** Get the exit code of a process specified by process_handle and store - * it in exit_code, if set to a non-NULL value. If block is set - * to true, the call will block until the process has exited. Otherwise if - * the process is still running, the function will return - * PROCESS_EXIT_RUNNING, and exit_code will be left unchanged. Returns - * PROCESS_EXIT_EXITED if the process did exit. If there is a failure, - * PROCESS_EXIT_ERROR will be returned and the contents of exit_code (if - * non-NULL) will be undefined. N.B. Under *nix operating systems, this will - * probably not work in Tor, because waitpid() is called in main.c to reap any - * terminated child processes.*/ -int -tor_get_exit_code(process_handle_t *process_handle, - int block, int *exit_code) -{ -#ifdef _WIN32 - DWORD retval; - BOOL success; - - if (block) { - /* Wait for the process to exit */ - retval = WaitForSingleObject(process_handle->pid.hProcess, INFINITE); - if (retval != WAIT_OBJECT_0) { - log_warn(LD_GENERAL, "WaitForSingleObject() failed (%d): %s", - (int)retval, format_win32_error(GetLastError())); - return PROCESS_EXIT_ERROR; - } - } else { - retval = WaitForSingleObject(process_handle->pid.hProcess, 0); - if (WAIT_TIMEOUT == retval) { - /* Process has not exited */ - return PROCESS_EXIT_RUNNING; - } else if (retval != WAIT_OBJECT_0) { - log_warn(LD_GENERAL, "WaitForSingleObject() failed (%d): %s", - (int)retval, format_win32_error(GetLastError())); - return PROCESS_EXIT_ERROR; - } - } - - if (exit_code != NULL) { - success = GetExitCodeProcess(process_handle->pid.hProcess, - (PDWORD)exit_code); - if (!success) { - log_warn(LD_GENERAL, "GetExitCodeProcess() failed: %s", - format_win32_error(GetLastError())); - return PROCESS_EXIT_ERROR; - } - } -#else /* !(defined(_WIN32)) */ - int stat_loc; - int retval; - - if (process_handle->waitpid_cb) { - /* We haven't processed a SIGCHLD yet. */ - retval = waitpid(process_handle->pid, &stat_loc, block?0:WNOHANG); - if (retval == process_handle->pid) { - clear_waitpid_callback(process_handle->waitpid_cb); - process_handle->waitpid_cb = NULL; - process_handle->waitpid_exit_status = stat_loc; - } - } else { - /* We already got a SIGCHLD for this process, and handled it. */ - retval = process_handle->pid; - stat_loc = process_handle->waitpid_exit_status; - } - - if (!block && 0 == retval) { - /* Process has not exited */ - return PROCESS_EXIT_RUNNING; - } else if (retval != process_handle->pid) { - log_warn(LD_GENERAL, "waitpid() failed for PID %d: %s", - (int)process_handle->pid, strerror(errno)); - return PROCESS_EXIT_ERROR; - } - - if (!WIFEXITED(stat_loc)) { - log_warn(LD_GENERAL, "Process %d did not exit normally", - (int)process_handle->pid); - return PROCESS_EXIT_ERROR; - } - - if (exit_code != NULL) - *exit_code = WEXITSTATUS(stat_loc); -#endif /* defined(_WIN32) */ - - return PROCESS_EXIT_EXITED; -} - -#ifdef _WIN32 -/** Read from a handle h into buf, up to count bytes. If - * hProcess is NULL, the function will return immediately if there is - * nothing more to read. Otherwise hProcess should be set to the handle - * to the process owning the h. In this case, the function will exit - * only once the process has exited, or count bytes are read. Returns - * the number of bytes read, or -1 on error. */ -ssize_t -tor_read_all_handle(HANDLE h, char *buf, size_t count, - const process_handle_t *process) -{ - size_t numread = 0; - BOOL retval; - DWORD byte_count; - BOOL process_exited = FALSE; - - if (count > SIZE_T_CEILING || count > SSIZE_MAX) - return -1; - - while (numread < count) { - /* Check if there is anything to read */ - retval = PeekNamedPipe(h, NULL, 0, NULL, &byte_count, NULL); - if (!retval) { - log_warn(LD_GENERAL, - "Failed to peek from handle: %s", - format_win32_error(GetLastError())); - return -1; - } else if (0 == byte_count) { - /* Nothing available: process exited or it is busy */ - - /* Exit if we don't know whether the process is running */ - if (NULL == process) - break; - - /* The process exited and there's nothing left to read from it */ - if (process_exited) - break; - - /* If process is not running, check for output one more time in case - it wrote something after the peek was performed. Otherwise keep on - waiting for output */ - tor_assert(process != NULL); - byte_count = WaitForSingleObject(process->pid.hProcess, 0); - if (WAIT_TIMEOUT != byte_count) - process_exited = TRUE; - - continue; - } - - /* There is data to read; read it */ - retval = ReadFile(h, buf+numread, count-numread, &byte_count, NULL); - tor_assert(byte_count + numread <= count); - if (!retval) { - log_warn(LD_GENERAL, "Failed to read from handle: %s", - format_win32_error(GetLastError())); - return -1; - } else if (0 == byte_count) { - /* End of file */ - break; - } - numread += byte_count; - } - return (ssize_t)numread; -} -#else /* !(defined(_WIN32)) */ -/** Read from a handle fd into buf, up to count bytes. If - * process is NULL, the function will return immediately if there is - * nothing more to read. Otherwise data will be read until end of file, or - * count bytes are read. Returns the number of bytes read, or -1 on - * error. Sets eof to true if eof is not NULL and the end of the - * file has been reached. */ -ssize_t -tor_read_all_handle(int fd, char *buf, size_t count, - const process_handle_t *process, - int *eof) -{ - size_t numread = 0; - ssize_t result; - - if (eof) - *eof = 0; - - if (count > SIZE_T_CEILING || count > SSIZE_MAX) - return -1; - - while (numread < count) { - result = read(fd, buf+numread, count-numread); - - if (result == 0) { - log_debug(LD_GENERAL, "read() reached end of file"); - if (eof) - *eof = 1; - break; - } else if (result < 0 && errno == EAGAIN) { - if (process) - continue; - else - break; - } else if (result < 0) { - log_warn(LD_GENERAL, "read() failed: %s", strerror(errno)); - return -1; - } - - numread += result; - } - - log_debug(LD_GENERAL, "read() read %d bytes from handle", (int)numread); - return (ssize_t)numread; -} -#endif /* defined(_WIN32) */ - -/** Read from stdout of a process until the process exits. */ -ssize_t -tor_read_all_from_process_stdout(const process_handle_t *process_handle, - char *buf, size_t count) -{ -#ifdef _WIN32 - return tor_read_all_handle(process_handle->stdout_pipe, buf, count, - process_handle); -#else - return tor_read_all_handle(process_handle->stdout_pipe, buf, count, - process_handle, NULL); -#endif /* defined(_WIN32) */ -} - -/** Read from stdout of a process until the process exits. */ -ssize_t -tor_read_all_from_process_stderr(const process_handle_t *process_handle, - char *buf, size_t count) -{ -#ifdef _WIN32 - return tor_read_all_handle(process_handle->stderr_pipe, buf, count, - process_handle); -#else - return tor_read_all_handle(process_handle->stderr_pipe, buf, count, - process_handle, NULL); -#endif /* defined(_WIN32) */ -} - -/** Return a string corresponding to stream_status. */ -const char * -stream_status_to_string(enum stream_status stream_status) -{ - switch (stream_status) { - case IO_STREAM_OKAY: - return "okay"; - case IO_STREAM_EAGAIN: - return "temporarily unavailable"; - case IO_STREAM_TERM: - return "terminated"; - case IO_STREAM_CLOSED: - return "closed"; - default: - tor_fragile_assert(); - return "unknown"; - } -} - -/** Split buf into lines, and add to smartlist. The buffer buf will be - * modified. The resulting smartlist will consist of pointers to buf, so there - * is no need to free the contents of sl. buf must be a NUL-terminated - * string. len should be set to the length of the buffer excluding the - * NUL. Non-printable characters (including NUL) will be replaced with "." */ -int -tor_split_lines(smartlist_t *sl, char *buf, int len) -{ - /* Index in buf of the start of the current line */ - int start = 0; - /* Index in buf of the current character being processed */ - int cur = 0; - /* Are we currently in a line */ - char in_line = 0; - - /* Loop over string */ - while (cur < len) { - /* Loop until end of line or end of string */ - for (; cur < len; cur++) { - if (in_line) { - if ('\r' == buf[cur] || '\n' == buf[cur]) { - /* End of line */ - buf[cur] = '\0'; - /* Point cur to the next line */ - cur++; - /* Line starts at start and ends with a nul */ - break; - } else { - if (!TOR_ISPRINT(buf[cur])) - buf[cur] = '.'; - } - } else { - if ('\r' == buf[cur] || '\n' == buf[cur]) { - /* Skip leading vertical space */ - ; - } else { - in_line = 1; - start = cur; - if (!TOR_ISPRINT(buf[cur])) - buf[cur] = '.'; - } - } - } - /* We are at the end of the line or end of string. If in_line is true there - * is a line which starts at buf+start and ends at a NUL. cur points to - * the character after the NUL. */ - if (in_line) - smartlist_add(sl, (void *)(buf+start)); - in_line = 0; - } - return smartlist_len(sl); -} - -#ifdef _WIN32 - -/** Return a smartlist containing lines outputted from - * handle. Return NULL on error, and set - * stream_status_out appropriately. */ -MOCK_IMPL(smartlist_t *, -tor_get_lines_from_handle, (HANDLE *handle, - enum stream_status *stream_status_out)) -{ - int pos; - char stdout_buf[600] = {0}; - smartlist_t *lines = NULL; - - tor_assert(stream_status_out); - - *stream_status_out = IO_STREAM_TERM; - - pos = tor_read_all_handle(handle, stdout_buf, sizeof(stdout_buf) - 1, NULL); - if (pos < 0) { - *stream_status_out = IO_STREAM_TERM; - return NULL; - } - if (pos == 0) { - *stream_status_out = IO_STREAM_EAGAIN; - return NULL; - } - - /* End with a null even if there isn't a \r\n at the end */ - /* TODO: What if this is a partial line? */ - stdout_buf[pos] = '\0'; - - /* Split up the buffer */ - lines = smartlist_new(); - tor_split_lines(lines, stdout_buf, pos); - - /* Currently 'lines' is populated with strings residing on the - stack. Replace them with their exact copies on the heap: */ - SMARTLIST_FOREACH(lines, char *, line, - SMARTLIST_REPLACE_CURRENT(lines, line, tor_strdup(line))); - - *stream_status_out = IO_STREAM_OKAY; - - return lines; -} - -#else /* !(defined(_WIN32)) */ - -/** Return a smartlist containing lines outputted from - * fd. Return NULL on error, and set - * stream_status_out appropriately. */ -MOCK_IMPL(smartlist_t *, -tor_get_lines_from_handle, (int fd, enum stream_status *stream_status_out)) -{ - enum stream_status stream_status; - char stdout_buf[400]; - smartlist_t *lines = NULL; - - while (1) { - memset(stdout_buf, 0, sizeof(stdout_buf)); - - stream_status = get_string_from_pipe(fd, - stdout_buf, sizeof(stdout_buf) - 1); - if (stream_status != IO_STREAM_OKAY) - goto done; - - if (!lines) lines = smartlist_new(); - smartlist_split_string(lines, stdout_buf, "\n", 0, 0); - } - - done: - *stream_status_out = stream_status; - return lines; -} - -#endif /* defined(_WIN32) */ - -/** Reads from fd and stores input in buf_out making - * sure it's below count bytes. - * If the string has a trailing newline, we strip it off. - * - * This function is specifically created to handle input from managed - * proxies, according to the pluggable transports spec. Make sure it - * fits your needs before using it. - * - * Returns: - * IO_STREAM_CLOSED: If the stream is closed. - * IO_STREAM_EAGAIN: If there is nothing to read and we should check back - * later. - * IO_STREAM_TERM: If something is wrong with the stream. - * IO_STREAM_OKAY: If everything went okay and we got a string - * in buf_out. */ -enum stream_status -get_string_from_pipe(int fd, char *buf_out, size_t count) -{ - ssize_t ret; - - tor_assert(count <= INT_MAX); - - ret = read(fd, buf_out, count); - - if (ret == 0) - return IO_STREAM_CLOSED; - else if (ret < 0 && errno == EAGAIN) - return IO_STREAM_EAGAIN; - else if (ret < 0) - return IO_STREAM_TERM; - - if (buf_out[ret - 1] == '\n') { - /* Remove the trailing newline */ - buf_out[ret - 1] = '\0'; - } else - buf_out[ret] = '\0'; - - return IO_STREAM_OKAY; -} diff --git a/src/lib/process/subprocess.h b/src/lib/process/subprocess.h index 5b4318ef2b..eb91626343 100644 --- a/src/lib/process/subprocess.h +++ b/src/lib/process/subprocess.h @@ -11,124 +11,10 @@ #ifndef TOR_SUBPROCESS_H #define TOR_SUBPROCESS_H -#include "lib/cc/torint.h" -#include "lib/testsupport/testsupport.h" -#include -#ifdef _WIN32 -#include -#endif - -struct smartlist_t; - void tor_disable_spawning_background_processes(void); -typedef struct process_handle_t process_handle_t; -struct process_environment_t; -int tor_spawn_background(const char *const filename, const char **argv, - struct process_environment_t *env, - process_handle_t **process_handle_out); - -#define SPAWN_ERROR_MESSAGE "ERR: Failed to spawn background process - code " - -/** Status of an I/O stream. */ -enum stream_status { - IO_STREAM_OKAY, - IO_STREAM_EAGAIN, - IO_STREAM_TERM, - IO_STREAM_CLOSED -}; - -const char *stream_status_to_string(enum stream_status stream_status); - -enum stream_status get_string_from_pipe(int fd, char *buf, size_t count); - -/* Values of process_handle_t.status. */ -#define PROCESS_STATUS_NOTRUNNING 0 -#define PROCESS_STATUS_RUNNING 1 -#define PROCESS_STATUS_ERROR -1 - -#ifdef SUBPROCESS_PRIVATE -struct waitpid_callback_t; - -/** Structure to represent the state of a process with which Tor is - * communicating. The contents of this structure are private to util.c */ -struct process_handle_t { - /** One of the PROCESS_STATUS_* values */ - int status; -#ifdef _WIN32 - HANDLE stdin_pipe; - HANDLE stdout_pipe; - HANDLE stderr_pipe; - PROCESS_INFORMATION pid; -#else /* !(defined(_WIN32)) */ - int stdin_pipe; - int stdout_pipe; - int stderr_pipe; - pid_t pid; - /** If the process has not given us a SIGCHLD yet, this has the - * waitpid_callback_t that gets invoked once it has. Otherwise this - * contains NULL. */ - struct waitpid_callback_t *waitpid_cb; - /** The exit status reported by waitpid. */ - int waitpid_exit_status; -#endif /* defined(_WIN32) */ -}; -#endif /* defined(SUBPROCESS_PRIVATE) */ - -/* Return values of tor_get_exit_code() */ -#define PROCESS_EXIT_RUNNING 1 -#define PROCESS_EXIT_EXITED 0 -#define PROCESS_EXIT_ERROR -1 -int tor_get_exit_code(process_handle_t *process_handle, - int block, int *exit_code); -int tor_split_lines(struct smartlist_t *sl, char *buf, int len); -#ifdef _WIN32 -ssize_t tor_read_all_handle(HANDLE h, char *buf, size_t count, - const process_handle_t *process); -#else -ssize_t tor_read_all_handle(int fd, char *buf, size_t count, - const process_handle_t *process, - int *eof); -#endif /* defined(_WIN32) */ -ssize_t tor_read_all_from_process_stdout( - const process_handle_t *process_handle, char *buf, size_t count); -ssize_t tor_read_all_from_process_stderr( - const process_handle_t *process_handle, char *buf, size_t count); -char *tor_join_win_cmdline(const char *argv[]); - -int tor_process_get_pid(process_handle_t *process_handle); -#ifdef _WIN32 -HANDLE tor_process_get_stdout_pipe(process_handle_t *process_handle); -#else -int tor_process_get_stdout_pipe(process_handle_t *process_handle); -#endif - -#ifdef _WIN32 -MOCK_DECL(struct smartlist_t *, tor_get_lines_from_handle,(HANDLE *handle, - enum stream_status *stream_status)); -#else -MOCK_DECL(struct smartlist_t *, tor_get_lines_from_handle,(int fd, - enum stream_status *stream_status)); -#endif /* defined(_WIN32) */ - -int tor_terminate_process(process_handle_t *process_handle); - -MOCK_DECL(void, tor_process_handle_destroy,(process_handle_t *process_handle, - int also_terminate_process)); - -#ifdef SUBPROCESS_PRIVATE -/* Prototypes for private functions only used by util.c (and unit tests) */ - #ifndef _WIN32 -STATIC int format_helper_exit_status(unsigned char child_state, - int saved_errno, char *hex_errno); - -/* Space for hex values of child state, a slash, saved_errno (with - leading minus) and newline (no null) */ -#define HEX_ERRNO_SIZE (sizeof(char) * 2 + 1 + \ - 1 + sizeof(int) * 2 + 1) -#endif /* !defined(_WIN32) */ - -#endif /* defined(SUBPROCESS_PRIVATE) */ +char *tor_join_win_cmdline(const char *argv[]); +#endif #endif diff --git a/src/test/Makefile.nmake b/src/test/Makefile.nmake index cfbe281b94..aa16a22b52 100644 --- a/src/test/Makefile.nmake +++ b/src/test/Makefile.nmake @@ -1,4 +1,4 @@ -all: test.exe test-child.exe bench.exe +all: test.exe bench.exe CFLAGS = /I ..\win32 /I ..\..\..\build-alpha\include /I ..\common /I ..\or \ /I ..\ext @@ -30,8 +30,5 @@ test.exe: $(TEST_OBJECTS) bench.exe: bench.obj $(CC) $(CFLAGS) bench.obj $(LIBS) ..\common\*.lib /Fe$@ -test-child.exe: test-child.obj - $(CC) $(CFLAGS) test-child.obj /Fe$@ - clean: - del *.obj *.lib test.exe bench.exe test-child.exe + del *.obj *.lib test.exe bench.exe diff --git a/src/test/include.am b/src/test/include.am index 9df2fd481d..e1f55e5d03 100644 --- a/src/test/include.am +++ b/src/test/include.am @@ -65,7 +65,6 @@ noinst_PROGRAMS+= \ src/test/test \ src/test/test-slow \ src/test/test-memwipe \ - src/test/test-child \ src/test/test-process \ src/test/test_workqueue \ src/test/test-switch-id \ @@ -204,7 +203,6 @@ src_test_test_slow_SOURCES += \ src/test/test_slow.c \ src/test/test_crypto_slow.c \ src/test/test_process_slow.c \ - src/test/test_util_slow.c \ src/test/testing_common.c \ src/test/testing_rsakeys.c \ src/ext/tinytest.c diff --git a/src/test/test-child.c b/src/test/test-child.c deleted file mode 100644 index 14df1a9b76..0000000000 --- a/src/test/test-child.c +++ /dev/null @@ -1,61 +0,0 @@ -/* Copyright (c) 2011-2018, The Tor Project, Inc. */ -/* See LICENSE for licensing information */ - -#include "orconfig.h" -#include -#ifdef _WIN32 -#define WINDOWS_LEAN_AND_MEAN -#include -#else -#include -#endif /* defined(_WIN32) */ -#include - -#ifdef _WIN32 -#define SLEEP(sec) Sleep((sec)*1000) -#else -#define SLEEP(sec) sleep(sec) -#endif - -/** Trivial test program which prints out its command line arguments so we can - * check if tor_spawn_background() works */ -int -main(int argc, char **argv) -{ - int i; - int delay = 1; - int fast = 0; - - if (argc > 1) { - if (!strcmp(argv[1], "--hang")) { - delay = 60; - } else if (!strcmp(argv[1], "--fast")) { - fast = 1; - delay = 0; - } - } - - fprintf(stdout, "OUT\n"); - fprintf(stderr, "ERR\n"); - for (i = 1; i < argc; i++) - fprintf(stdout, "%s\n", argv[i]); - if (!fast) - fprintf(stdout, "SLEEPING\n"); - /* We need to flush stdout so that test_util_spawn_background_partial_read() - succeed. Otherwise ReadFile() will get the entire output in one */ - // XXX: Can we make stdio flush on newline? - fflush(stdout); - if (!fast) - SLEEP(1); - fprintf(stdout, "DONE\n"); - fflush(stdout); - if (fast) - return 0; - - while (--delay) { - SLEEP(1); - } - - return 0; -} - diff --git a/src/test/test.h b/src/test/test.h index 9f72623968..ce50fa89d9 100644 --- a/src/test/test.h +++ b/src/test/test.h @@ -271,7 +271,6 @@ extern struct testcase_t x509_tests[]; extern struct testcase_t slow_crypto_tests[]; extern struct testcase_t slow_process_tests[]; -extern struct testcase_t slow_util_tests[]; extern struct testgroup_t testgroups[]; diff --git a/src/test/test_logging.c b/src/test/test_logging.c index cd685a4af7..9d3ee5d55f 100644 --- a/src/test/test_logging.c +++ b/src/test/test_logging.c @@ -117,22 +117,27 @@ test_sigsafe_err(void *arg) content = read_file_to_str(fn, 0, NULL); tt_ptr_op(content, OP_NE, NULL); - tor_split_lines(lines, content, (int)strlen(content)); + smartlist_split_string(lines, content, "\n", 0, 0); tt_int_op(smartlist_len(lines), OP_GE, 5); - if (strstr(smartlist_get(lines, 0), "opening new log file")) + if (strstr(smartlist_get(lines, 0), "opening new log file")) { + void *item = smartlist_get(lines, 0); smartlist_del_keeporder(lines, 0); + tor_free(item); + } + tt_assert(strstr(smartlist_get(lines, 0), "Say, this isn't too cool")); - /* Next line is blank. */ - tt_assert(!strcmpstart(smartlist_get(lines, 1), "==============")); - tt_assert(!strcmpstart(smartlist_get(lines, 2), "Minimal.")); - /* Next line is blank. */ - tt_assert(!strcmpstart(smartlist_get(lines, 3), "==============")); - tt_str_op(smartlist_get(lines, 4), OP_EQ, + tt_str_op(smartlist_get(lines, 1), OP_EQ, ""); + tt_assert(!strcmpstart(smartlist_get(lines, 2), "==============")); + tt_assert(!strcmpstart(smartlist_get(lines, 3), "Minimal.")); + tt_str_op(smartlist_get(lines, 4), OP_EQ, ""); + tt_assert(!strcmpstart(smartlist_get(lines, 5), "==============")); + tt_str_op(smartlist_get(lines, 6), OP_EQ, "Testing any attempt to manually log from a signal."); done: tor_free(content); + SMARTLIST_FOREACH(lines, char *, x, tor_free(x)); smartlist_free(lines); } diff --git a/src/test/test_slow.c b/src/test/test_slow.c index 0c66eff930..97c2912af6 100644 --- a/src/test/test_slow.c +++ b/src/test/test_slow.c @@ -21,7 +21,6 @@ struct testgroup_t testgroups[] = { { "slow/crypto/", slow_crypto_tests }, { "slow/process/", slow_process_tests }, - { "slow/util/", slow_util_tests }, END_OF_GROUPS }; diff --git a/src/test/test_util.c b/src/test/test_util.c index bcface64fd..4fa67b6419 100644 --- a/src/test/test_util.c +++ b/src/test/test_util.c @@ -4301,204 +4301,6 @@ test_util_load_win_lib(void *ptr) } #endif /* defined(_WIN32) */ -#ifndef _WIN32 -static void -clear_hex_errno(char *hex_errno) -{ - memset(hex_errno, '\0', HEX_ERRNO_SIZE + 1); -} - -static void -test_util_exit_status(void *ptr) -{ - /* Leave an extra byte for a \0 so we can do string comparison */ - char hex_errno[HEX_ERRNO_SIZE + 1]; - int n; - - (void)ptr; - - clear_hex_errno(hex_errno); - tt_str_op("",OP_EQ, hex_errno); - - clear_hex_errno(hex_errno); - n = format_helper_exit_status(0, 0, hex_errno); - tt_str_op("0/0\n",OP_EQ, hex_errno); - tt_int_op(n,OP_EQ, strlen(hex_errno)); - -#if SIZEOF_INT == 4 - - clear_hex_errno(hex_errno); - n = format_helper_exit_status(0, 0x7FFFFFFF, hex_errno); - tt_str_op("0/7FFFFFFF\n",OP_EQ, hex_errno); - tt_int_op(n,OP_EQ, strlen(hex_errno)); - - clear_hex_errno(hex_errno); - n = format_helper_exit_status(0xFF, -0x80000000, hex_errno); - tt_str_op("FF/-80000000\n",OP_EQ, hex_errno); - tt_int_op(n,OP_EQ, strlen(hex_errno)); - tt_int_op(n,OP_EQ, HEX_ERRNO_SIZE); - -#elif SIZEOF_INT == 8 - - clear_hex_errno(hex_errno); - n = format_helper_exit_status(0, 0x7FFFFFFFFFFFFFFF, hex_errno); - tt_str_op("0/7FFFFFFFFFFFFFFF\n",OP_EQ, hex_errno); - tt_int_op(n,OP_EQ, strlen(hex_errno)); - - clear_hex_errno(hex_errno); - n = format_helper_exit_status(0xFF, -0x8000000000000000, hex_errno); - tt_str_op("FF/-8000000000000000\n",OP_EQ, hex_errno); - tt_int_op(n,OP_EQ, strlen(hex_errno)); - tt_int_op(n,OP_EQ, HEX_ERRNO_SIZE); - -#endif /* SIZEOF_INT == 4 || ... */ - - clear_hex_errno(hex_errno); - n = format_helper_exit_status(0x7F, 0, hex_errno); - tt_str_op("7F/0\n",OP_EQ, hex_errno); - tt_int_op(n,OP_EQ, strlen(hex_errno)); - - clear_hex_errno(hex_errno); - n = format_helper_exit_status(0x08, -0x242, hex_errno); - tt_str_op("8/-242\n",OP_EQ, hex_errno); - tt_int_op(n,OP_EQ, strlen(hex_errno)); - - clear_hex_errno(hex_errno); - tt_str_op("",OP_EQ, hex_errno); - - done: - ; -} -#endif /* !defined(_WIN32) */ - -#ifndef _WIN32 -static void -test_util_string_from_pipe(void *ptr) -{ - int test_pipe[2] = {-1, -1}; - int retval = 0; - enum stream_status status = IO_STREAM_TERM; - ssize_t retlen; - char buf[4] = { 0 }; - - (void)ptr; - - errno = 0; - - /* Set up a pipe to test on */ - retval = pipe(test_pipe); - tt_int_op(retval, OP_EQ, 0); - - /* Send in a string. */ - retlen = write(test_pipe[1], "ABC", 3); - tt_int_op(retlen, OP_EQ, 3); - - status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1); - tt_int_op(errno, OP_EQ, 0); - tt_int_op(status, OP_EQ, IO_STREAM_OKAY); - tt_str_op(buf, OP_EQ, "ABC"); - errno = 0; - - /* Send in a string that contains a nul. */ - retlen = write(test_pipe[1], "AB\0", 3); - tt_int_op(retlen, OP_EQ, 3); - - status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1); - tt_int_op(errno, OP_EQ, 0); - tt_int_op(status, OP_EQ, IO_STREAM_OKAY); - tt_str_op(buf, OP_EQ, "AB"); - errno = 0; - - /* Send in a string that contains a nul only. */ - retlen = write(test_pipe[1], "\0", 1); - tt_int_op(retlen, OP_EQ, 1); - - status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1); - tt_int_op(errno, OP_EQ, 0); - tt_int_op(status, OP_EQ, IO_STREAM_OKAY); - tt_str_op(buf, OP_EQ, ""); - errno = 0; - - /* Send in a string that contains a trailing newline. */ - retlen = write(test_pipe[1], "AB\n", 3); - tt_int_op(retlen, OP_EQ, 3); - - status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1); - tt_int_op(errno, OP_EQ, 0); - tt_int_op(status, OP_EQ, IO_STREAM_OKAY); - tt_str_op(buf, OP_EQ, "AB"); - errno = 0; - - /* Send in a string that contains a newline only. */ - retlen = write(test_pipe[1], "\n", 1); - tt_int_op(retlen, OP_EQ, 1); - - status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1); - tt_int_op(errno, OP_EQ, 0); - tt_int_op(status, OP_EQ, IO_STREAM_OKAY); - tt_str_op(buf, OP_EQ, ""); - errno = 0; - - /* Send in a string and check that we nul terminate return values. */ - retlen = write(test_pipe[1], "AAA", 3); - tt_int_op(retlen, OP_EQ, 3); - - status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1); - tt_int_op(errno, OP_EQ, 0); - tt_int_op(status, OP_EQ, IO_STREAM_OKAY); - tt_str_op(buf, OP_EQ, "AAA"); - tt_mem_op(buf, OP_EQ, "AAA\0", sizeof(buf)); - errno = 0; - - retlen = write(test_pipe[1], "B", 1); - tt_int_op(retlen, OP_EQ, 1); - - memset(buf, '\xff', sizeof(buf)); - status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1); - tt_int_op(errno, OP_EQ, 0); - tt_int_op(status, OP_EQ, IO_STREAM_OKAY); - tt_str_op(buf, OP_EQ, "B"); - tt_mem_op(buf, OP_EQ, "B\0\xff\xff", sizeof(buf)); - errno = 0; - - /* Send in multiple lines. */ - retlen = write(test_pipe[1], "A\nB", 3); - tt_int_op(retlen, OP_EQ, 3); - - status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1); - tt_int_op(errno, OP_EQ, 0); - tt_int_op(status, OP_EQ, IO_STREAM_OKAY); - tt_str_op(buf, OP_EQ, "A\nB"); - errno = 0; - - /* Send in a line and close */ - retlen = write(test_pipe[1], "AB", 2); - tt_int_op(retlen, OP_EQ, 2); - retval = close(test_pipe[1]); - tt_int_op(retval, OP_EQ, 0); - test_pipe[1] = -1; - - status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1); - tt_int_op(errno, OP_EQ, 0); - tt_int_op(status, OP_EQ, IO_STREAM_OKAY); - tt_str_op(buf, OP_EQ, "AB"); - errno = 0; - - /* Check for EOF */ - status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1); - tt_int_op(errno, OP_EQ, 0); - tt_int_op(status, OP_EQ, IO_STREAM_CLOSED); - errno = 0; - - done: - if (test_pipe[0] != -1) - close(test_pipe[0]); - if (test_pipe[1] != -1) - close(test_pipe[1]); -} - -#endif /* !defined(_WIN32) */ - /** * Test for format_hex_number_sigsafe() */ @@ -4651,67 +4453,6 @@ struct split_lines_test_t { const char *split_line[MAX_SPLIT_LINE_COUNT]; // Split lines }; -/** - * Test that we properly split a buffer into lines - */ -static void -test_util_split_lines(void *ptr) -{ - /* Test cases. orig_line of last test case must be NULL. - * The last element of split_line[i] must be NULL. */ - struct split_lines_test_t tests[] = { - {"", 0, {NULL}}, - {"foo", 3, {"foo", NULL}}, - {"\n\rfoo\n\rbar\r\n", 12, {"foo", "bar", NULL}}, - {"fo o\r\nb\tar", 10, {"fo o", "b.ar", NULL}}, - {"\x0f""f\0o\0\n\x01""b\0r\0\r", 12, {".f.o.", ".b.r.", NULL}}, - {"line 1\r\nline 2", 14, {"line 1", "line 2", NULL}}, - {"line 1\r\n\r\nline 2", 16, {"line 1", "line 2", NULL}}, - {"line 1\r\n\r\r\r\nline 2", 18, {"line 1", "line 2", NULL}}, - {"line 1\r\n\n\n\n\rline 2", 18, {"line 1", "line 2", NULL}}, - {"line 1\r\n\r\t\r\nline 3", 18, {"line 1", ".", "line 3", NULL}}, - {"\n\t\r\t\nline 3", 11, {".", ".", "line 3", NULL}}, - {NULL, 0, { NULL }} - }; - - int i, j; - char *orig_line=NULL; - smartlist_t *sl=NULL; - - (void)ptr; - - for (i=0; tests[i].orig_line; i++) { - sl = smartlist_new(); - /* Allocate space for string and trailing NULL */ - orig_line = tor_memdup(tests[i].orig_line, tests[i].orig_length + 1); - tor_split_lines(sl, orig_line, tests[i].orig_length); - - j = 0; - log_info(LD_GENERAL, "Splitting test %d of length %d", - i, tests[i].orig_length); - SMARTLIST_FOREACH_BEGIN(sl, const char *, line) { - /* Check we have not got too many lines */ - tt_int_op(MAX_SPLIT_LINE_COUNT, OP_GT, j); - /* Check that there actually should be a line here */ - tt_ptr_op(tests[i].split_line[j], OP_NE, NULL); - log_info(LD_GENERAL, "Line %d of test %d, should be <%s>", - j, i, tests[i].split_line[j]); - /* Check that the line is as expected */ - tt_str_op(line,OP_EQ, tests[i].split_line[j]); - j++; - } SMARTLIST_FOREACH_END(line); - /* Check that we didn't miss some lines */ - tt_ptr_op(NULL,OP_EQ, tests[i].split_line[j]); - tor_free(orig_line); - smartlist_free(sl); - sl = NULL; - } - - done: - tor_free(orig_line); - smartlist_free(sl); -} - static void test_util_di_ops(void *arg) { @@ -6483,12 +6224,9 @@ struct testcase_t util_tests[] = { UTIL_TEST(nowrap_math, 0), UTIL_TEST(num_cpus, 0), UTIL_TEST_WIN_ONLY(load_win_lib, 0), - UTIL_TEST_NO_WIN(exit_status, 0), - UTIL_TEST_NO_WIN(string_from_pipe, 0), UTIL_TEST(format_hex_number, 0), UTIL_TEST(format_dec_number, 0), UTIL_TEST(join_win_cmdline, 0), - UTIL_TEST(split_lines, 0), UTIL_TEST(n_bits_set, 0), UTIL_TEST(eat_whitespace, 0), UTIL_TEST(sl_new_from_text_lines, 0), diff --git a/src/test/test_util_slow.c b/src/test/test_util_slow.c deleted file mode 100644 index c7b3e3e2a4..0000000000 --- a/src/test/test_util_slow.c +++ /dev/null @@ -1,396 +0,0 @@ -/* Copyright (c) 2001-2004, Roger Dingledine. - * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ -/* See LICENSE for licensing information */ - -#include "orconfig.h" -#define UTIL_PRIVATE -#define SUBPROCESS_PRIVATE -#include "lib/crypt_ops/crypto_cipher.h" -#include "lib/log/log.h" -#include "lib/process/subprocess.h" -#include "lib/process/waitpid.h" -#include "lib/string/printf.h" -#include "lib/time/compat_time.h" -#include "test/test.h" - -#include -#include - -#ifndef BUILDDIR -#define BUILDDIR "." -#endif - -#ifdef _WIN32 -#define notify_pending_waitpid_callbacks() STMT_NIL -#define TEST_CHILD "test-child.exe" -#define EOL "\r\n" -#else -#define TEST_CHILD (BUILDDIR "/src/test/test-child") -#define EOL "\n" -#endif /* defined(_WIN32) */ - -#ifdef _WIN32 -/* I've assumed Windows doesn't have the gap between fork and exec - * that causes the race condition on unix-like platforms */ -#define MATCH_PROCESS_STATUS(s1,s2) ((s1) == (s2)) - -#else /* !(defined(_WIN32)) */ -/* work around a race condition of the timing of SIGCHLD handler updates - * to the process_handle's fields, and checks of those fields - * - * TODO: Once we can signal failure to exec, change PROCESS_STATUS_RUNNING to - * PROCESS_STATUS_ERROR (and similarly with *_OR_NOTRUNNING) */ -#define PROCESS_STATUS_RUNNING_OR_NOTRUNNING (PROCESS_STATUS_RUNNING+1) -#define IS_RUNNING_OR_NOTRUNNING(s) \ - ((s) == PROCESS_STATUS_RUNNING || (s) == PROCESS_STATUS_NOTRUNNING) -/* well, this is ugly */ -#define MATCH_PROCESS_STATUS(s1,s2) \ - ( (s1) == (s2) \ - ||((s1) == PROCESS_STATUS_RUNNING_OR_NOTRUNNING \ - && IS_RUNNING_OR_NOTRUNNING(s2)) \ - ||((s2) == PROCESS_STATUS_RUNNING_OR_NOTRUNNING \ - && IS_RUNNING_OR_NOTRUNNING(s1))) - -#endif /* defined(_WIN32) */ - -/** Helper function for testing tor_spawn_background */ -static void -run_util_spawn_background(const char *argv[], const char *expected_out, - const char *expected_err, int expected_exit, - int expected_status) -{ - int retval, exit_code; - ssize_t pos; - process_handle_t *process_handle=NULL; - char stdout_buf[100], stderr_buf[100]; - int status; - - /* Start the program */ -#ifdef _WIN32 - status = tor_spawn_background(NULL, argv, NULL, &process_handle); -#else - status = tor_spawn_background(argv[0], argv, NULL, &process_handle); -#endif - - notify_pending_waitpid_callbacks(); - - /* the race condition doesn't affect status, - * because status isn't updated by the SIGCHLD handler, - * but we still need to handle PROCESS_STATUS_RUNNING_OR_NOTRUNNING */ - tt_assert(MATCH_PROCESS_STATUS(expected_status, status)); - if (status == PROCESS_STATUS_ERROR) { - tt_ptr_op(process_handle, OP_EQ, NULL); - return; - } - - tt_ptr_op(process_handle, OP_NE, NULL); - - /* When a spawned process forks, fails, then exits very quickly, - * (this typically occurs when exec fails) - * there is a race condition between the SIGCHLD handler - * updating the process_handle's fields, and this test - * checking the process status in those fields. - * The SIGCHLD update can occur before or after the code below executes. - * This causes intermittent failures in spawn_background_fail(), - * typically when the machine is under load. - * We use PROCESS_STATUS_RUNNING_OR_NOTRUNNING to avoid this issue. */ - - /* the race condition affects the change in - * process_handle->status from RUNNING to NOTRUNNING */ - tt_assert(MATCH_PROCESS_STATUS(expected_status, process_handle->status)); - -#ifndef _WIN32 - notify_pending_waitpid_callbacks(); - /* the race condition affects the change in - * process_handle->waitpid_cb to NULL, - * so we skip the check if expected_status is ambiguous, - * that is, PROCESS_STATUS_RUNNING_OR_NOTRUNNING */ - tt_assert(process_handle->waitpid_cb != NULL - || expected_status == PROCESS_STATUS_RUNNING_OR_NOTRUNNING); -#endif /* !defined(_WIN32) */ - -#ifdef _WIN32 - tt_assert(process_handle->stdout_pipe != INVALID_HANDLE_VALUE); - tt_assert(process_handle->stderr_pipe != INVALID_HANDLE_VALUE); - tt_assert(process_handle->stdin_pipe != INVALID_HANDLE_VALUE); -#else - tt_assert(process_handle->stdout_pipe >= 0); - tt_assert(process_handle->stderr_pipe >= 0); - tt_assert(process_handle->stdin_pipe >= 0); -#endif /* defined(_WIN32) */ - - /* Check stdout */ - pos = tor_read_all_from_process_stdout(process_handle, stdout_buf, - sizeof(stdout_buf) - 1); - tt_assert(pos >= 0); - stdout_buf[pos] = '\0'; - tt_int_op(strlen(expected_out),OP_EQ, pos); - tt_str_op(expected_out,OP_EQ, stdout_buf); - - notify_pending_waitpid_callbacks(); - - /* Check it terminated correctly */ - retval = tor_get_exit_code(process_handle, 1, &exit_code); - tt_int_op(PROCESS_EXIT_EXITED,OP_EQ, retval); - tt_int_op(expected_exit,OP_EQ, exit_code); - // TODO: Make test-child exit with something other than 0 - -#ifndef _WIN32 - notify_pending_waitpid_callbacks(); - tt_ptr_op(process_handle->waitpid_cb, OP_EQ, NULL); -#endif - - /* Check stderr */ - pos = tor_read_all_from_process_stderr(process_handle, stderr_buf, - sizeof(stderr_buf) - 1); - tt_assert(pos >= 0); - stderr_buf[pos] = '\0'; - tt_str_op(expected_err,OP_EQ, stderr_buf); - tt_int_op(strlen(expected_err),OP_EQ, pos); - - notify_pending_waitpid_callbacks(); - - done: - if (process_handle) - tor_process_handle_destroy(process_handle, 1); -} - -/** Check that we can launch a process and read the output */ -static void -test_util_spawn_background_ok(void *ptr) -{ - const char *argv[] = {TEST_CHILD, "--test", NULL}; - const char *expected_out = "OUT"EOL "--test"EOL "SLEEPING"EOL "DONE" EOL; - const char *expected_err = "ERR"EOL; - - (void)ptr; - - run_util_spawn_background(argv, expected_out, expected_err, 0, - PROCESS_STATUS_RUNNING); -} - -/** Check that failing to find the executable works as expected */ -static void -test_util_spawn_background_fail(void *ptr) -{ - const char *argv[] = {BUILDDIR "/src/test/no-such-file", "--test", NULL}; - const char *expected_err = ""; - char expected_out[1024]; - char code[32]; -#ifdef _WIN32 - const int expected_status = PROCESS_STATUS_ERROR; -#else - /* TODO: Once we can signal failure to exec, set this to be - * PROCESS_STATUS_RUNNING_OR_ERROR */ - const int expected_status = PROCESS_STATUS_RUNNING_OR_NOTRUNNING; -#endif /* defined(_WIN32) */ - - memset(expected_out, 0xf0, sizeof(expected_out)); - memset(code, 0xf0, sizeof(code)); - - (void)ptr; - - tor_snprintf(code, sizeof(code), "%x/%x", - 9 /* CHILD_STATE_FAILEXEC */ , ENOENT); - tor_snprintf(expected_out, sizeof(expected_out), - "ERR: Failed to spawn background process - code %s\n", code); - - run_util_spawn_background(argv, expected_out, expected_err, 255, - expected_status); -} - -/** Test that reading from a handle returns a partial read rather than - * blocking */ -static void -test_util_spawn_background_partial_read_impl(int exit_early) -{ - const int expected_exit = 0; - const int expected_status = PROCESS_STATUS_RUNNING; - - int retval, exit_code; - ssize_t pos = -1; - process_handle_t *process_handle=NULL; - int status; - char stdout_buf[100], stderr_buf[100]; - - const char *argv[] = {TEST_CHILD, "--test", NULL}; - const char *expected_out[] = { "OUT" EOL "--test" EOL "SLEEPING" EOL, - "DONE" EOL, - NULL }; - const char *expected_err = "ERR" EOL; - -#ifndef _WIN32 - int eof = 0; -#endif - int expected_out_ctr; - - if (exit_early) { - argv[1] = "--hang"; - expected_out[0] = "OUT"EOL "--hang"EOL "SLEEPING" EOL; - } - - /* Start the program */ -#ifdef _WIN32 - status = tor_spawn_background(NULL, argv, NULL, &process_handle); -#else - status = tor_spawn_background(argv[0], argv, NULL, &process_handle); -#endif - tt_int_op(expected_status,OP_EQ, status); - tt_assert(process_handle); - tt_int_op(expected_status,OP_EQ, process_handle->status); - - /* Check stdout */ - for (expected_out_ctr = 0; expected_out[expected_out_ctr] != NULL;) { -#ifdef _WIN32 - pos = tor_read_all_handle(process_handle->stdout_pipe, stdout_buf, - sizeof(stdout_buf) - 1, NULL); -#else - /* Check that we didn't read the end of file last time */ - tt_assert(!eof); - pos = tor_read_all_handle(process_handle->stdout_pipe, stdout_buf, - sizeof(stdout_buf) - 1, NULL, &eof); -#endif /* defined(_WIN32) */ - log_info(LD_GENERAL, "tor_read_all_handle() returned %d", (int)pos); - - /* We would have blocked, keep on trying */ - if (0 == pos) - continue; - - tt_assert(pos > 0); - stdout_buf[pos] = '\0'; - tt_str_op(expected_out[expected_out_ctr],OP_EQ, stdout_buf); - tt_int_op(strlen(expected_out[expected_out_ctr]),OP_EQ, pos); - expected_out_ctr++; - } - - if (exit_early) { - tor_process_handle_destroy(process_handle, 1); - process_handle = NULL; - goto done; - } - - /* The process should have exited without writing more */ -#ifdef _WIN32 - pos = tor_read_all_handle(process_handle->stdout_pipe, stdout_buf, - sizeof(stdout_buf) - 1, - process_handle); - tt_int_op(0,OP_EQ, pos); -#else /* !(defined(_WIN32)) */ - if (!eof) { - /* We should have got all the data, but maybe not the EOF flag */ - pos = tor_read_all_handle(process_handle->stdout_pipe, stdout_buf, - sizeof(stdout_buf) - 1, - process_handle, &eof); - tt_int_op(0,OP_EQ, pos); - tt_assert(eof); - } - /* Otherwise, we got the EOF on the last read */ -#endif /* defined(_WIN32) */ - - /* Check it terminated correctly */ - retval = tor_get_exit_code(process_handle, 1, &exit_code); - tt_int_op(PROCESS_EXIT_EXITED,OP_EQ, retval); - tt_int_op(expected_exit,OP_EQ, exit_code); - - // TODO: Make test-child exit with something other than 0 - - /* Check stderr */ - pos = tor_read_all_from_process_stderr(process_handle, stderr_buf, - sizeof(stderr_buf) - 1); - tt_assert(pos >= 0); - stderr_buf[pos] = '\0'; - tt_str_op(expected_err,OP_EQ, stderr_buf); - tt_int_op(strlen(expected_err),OP_EQ, pos); - - done: - tor_process_handle_destroy(process_handle, 1); -} - -static void -test_util_spawn_background_partial_read(void *arg) -{ - (void)arg; - test_util_spawn_background_partial_read_impl(0); -} - -static void -test_util_spawn_background_exit_early(void *arg) -{ - (void)arg; - test_util_spawn_background_partial_read_impl(1); -} - -static void -test_util_spawn_background_waitpid_notify(void *arg) -{ - int retval, exit_code; - process_handle_t *process_handle=NULL; - int status; - int ms_timer; - - const char *argv[] = {TEST_CHILD, "--fast", NULL}; - - (void) arg; - -#ifdef _WIN32 - status = tor_spawn_background(NULL, argv, NULL, &process_handle); -#else - status = tor_spawn_background(argv[0], argv, NULL, &process_handle); -#endif - - tt_int_op(status, OP_EQ, PROCESS_STATUS_RUNNING); - tt_ptr_op(process_handle, OP_NE, NULL); - - /* We're not going to look at the stdout/stderr output this time. Instead, - * we're testing whether notify_pending_waitpid_calbacks() can report the - * process exit (on unix) and/or whether tor_get_exit_code() can notice it - * (on windows) */ - -#ifndef _WIN32 - ms_timer = 30*1000; - tt_ptr_op(process_handle->waitpid_cb, OP_NE, NULL); - while (process_handle->waitpid_cb && ms_timer > 0) { - tor_sleep_msec(100); - ms_timer -= 100; - notify_pending_waitpid_callbacks(); - } - tt_int_op(ms_timer, OP_GT, 0); - tt_ptr_op(process_handle->waitpid_cb, OP_EQ, NULL); -#endif /* !defined(_WIN32) */ - - ms_timer = 30*1000; - while (((retval = tor_get_exit_code(process_handle, 0, &exit_code)) - == PROCESS_EXIT_RUNNING) && ms_timer > 0) { - tor_sleep_msec(100); - ms_timer -= 100; - } - tt_int_op(ms_timer, OP_GT, 0); - - tt_int_op(retval, OP_EQ, PROCESS_EXIT_EXITED); - - done: - tor_process_handle_destroy(process_handle, 1); -} - -#undef TEST_CHILD -#undef EOL - -#undef MATCH_PROCESS_STATUS - -#ifndef _WIN32 -#undef PROCESS_STATUS_RUNNING_OR_NOTRUNNING -#undef IS_RUNNING_OR_NOTRUNNING -#endif - -#define UTIL_TEST(name, flags) \ - { #name, test_util_ ## name, flags, NULL, NULL } - -struct testcase_t slow_util_tests[] = { - UTIL_TEST(spawn_background_ok, 0), - UTIL_TEST(spawn_background_fail, 0), - UTIL_TEST(spawn_background_partial_read, 0), - UTIL_TEST(spawn_background_exit_early, 0), - UTIL_TEST(spawn_background_waitpid_notify, 0), - END_OF_TESTCASES -}; From ccc1963890bc7448383cd4c45370139697973d64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Mon, 26 Nov 2018 06:14:47 +0100 Subject: [PATCH 0297/2557] Move remaining code from subprocess.{h,c} to more appropriate places. This patch moves the remaining code from subprocess.{h,c} to more appropriate places in the process.c and process_win32.c module. We also delete the now empty subprocess module files. See: https://bugs.torproject.org/28179 --- src/app/config/config.c | 2 +- src/feature/client/transports.c | 1 - src/lib/process/include.am | 2 - src/lib/process/process.c | 19 ++++ src/lib/process/process.h | 2 + src/lib/process/process_win32.c | 105 ++++++++++++++++++++- src/lib/process/process_win32.h | 3 + src/lib/process/subprocess.c | 158 -------------------------------- src/lib/process/subprocess.h | 20 ---- src/test/test_logging.c | 1 - src/test/test_process.c | 40 ++++++++ src/test/test_pt.c | 2 - src/test/test_util.c | 56 +---------- 13 files changed, 171 insertions(+), 240 deletions(-) delete mode 100644 src/lib/process/subprocess.c delete mode 100644 src/lib/process/subprocess.h diff --git a/src/app/config/config.c b/src/app/config/config.c index 90eae50fdd..e33c6b26e8 100644 --- a/src/app/config/config.c +++ b/src/app/config/config.c @@ -142,7 +142,7 @@ #include "lib/process/pidfile.h" #include "lib/process/restrict.h" #include "lib/process/setuid.h" -#include "lib/process/subprocess.h" +#include "lib/process/process.h" #include "lib/net/gethostname.h" #include "lib/thread/numcpus.h" diff --git a/src/feature/client/transports.c b/src/feature/client/transports.c index fd4bec7807..0e326f90e9 100644 --- a/src/feature/client/transports.c +++ b/src/feature/client/transports.c @@ -103,7 +103,6 @@ #include "feature/control/control.h" #include "lib/process/process.h" -#include "lib/process/subprocess.h" #include "lib/process/env.h" static smartlist_t * diff --git a/src/lib/process/include.am b/src/lib/process/include.am index aa73356146..a2d54b6238 100644 --- a/src/lib/process/include.am +++ b/src/lib/process/include.am @@ -14,7 +14,6 @@ src_lib_libtor_process_a_SOURCES = \ src/lib/process/process_win32.c \ src/lib/process/restrict.c \ src/lib/process/setuid.c \ - src/lib/process/subprocess.c \ src/lib/process/waitpid.c \ src/lib/process/winprocess_sys.c @@ -32,6 +31,5 @@ noinst_HEADERS += \ src/lib/process/process_win32.h \ src/lib/process/restrict.h \ src/lib/process/setuid.h \ - src/lib/process/subprocess.h \ src/lib/process/waitpid.h \ src/lib/process/winprocess_sys.h diff --git a/src/lib/process/process.c b/src/lib/process/process.c index 915217e132..7275d0a21b 100644 --- a/src/lib/process/process.c +++ b/src/lib/process/process.c @@ -26,6 +26,12 @@ /** A list of all process_t instances currently allocated. */ static smartlist_t *processes; +/** + * Boolean. If true, then Tor may call execve or CreateProcess via + * tor_spawn_background. + **/ +static int may_spawn_background_process = 1; + /** Structure to represent a child process. */ struct process_t { /** Process status. */ @@ -114,6 +120,16 @@ process_protocol_to_string(process_protocol_t protocol) /* LCOV_EXCL_STOP */ } +/** + * Turn off may_spawn_background_process, so that all future calls to + * tor_spawn_background are guaranteed to fail. + **/ +void +tor_disable_spawning_background_processes(void) +{ + may_spawn_background_process = 0; +} + /** Initialize the Process subsystem. This function initializes the Process * subsystem's global state. For cleaning up, process_free_all() should * be called. */ @@ -234,6 +250,9 @@ process_exec(process_t *process) { tor_assert(process); + if (BUG(may_spawn_background_process == 0)) + return PROCESS_STATUS_ERROR; + process_status_t status = PROCESS_STATUS_NOT_RUNNING; log_info(LD_PROCESS, "Starting new process: %s", process->command); diff --git a/src/lib/process/process.h b/src/lib/process/process.h index 6092c2da7d..cb5bccbf7b 100644 --- a/src/lib/process/process.h +++ b/src/lib/process/process.h @@ -45,6 +45,8 @@ typedef enum { const char *process_protocol_to_string(process_protocol_t protocol); +void tor_disable_spawning_background_processes(void); + struct process_t; typedef struct process_t process_t; diff --git a/src/lib/process/process_win32.c b/src/lib/process/process_win32.c index 7b18903c70..73e19d518f 100644 --- a/src/lib/process/process_win32.c +++ b/src/lib/process/process_win32.c @@ -18,13 +18,16 @@ #include "lib/log/win32err.h" #include "lib/process/process.h" #include "lib/process/process_win32.h" -#include "lib/process/subprocess.h" #include "lib/process/env.h" #ifdef HAVE_SYS_TIME_H #include #endif +#ifdef HAVE_STRING_H +#include +#endif + #ifdef _WIN32 /** The size of our intermediate buffers. */ @@ -889,4 +892,104 @@ process_win32_handle_read_completion(process_win32_handle_t *handle, return false; } +/** Format a single argument for being put on a Windows command line. + * Returns a newly allocated string */ +STATIC char * +format_win_cmdline_argument(const char *arg) +{ + char *formatted_arg; + char need_quotes; + const char *c; + int i; + int bs_counter = 0; + /* Backslash we can point to when one is inserted into the string */ + const char backslash = '\\'; + + /* Smartlist of *char */ + smartlist_t *arg_chars; + arg_chars = smartlist_new(); + + /* Quote string if it contains whitespace or is empty */ + need_quotes = (strchr(arg, ' ') || strchr(arg, '\t') || '\0' == arg[0]); + + /* Build up smartlist of *chars */ + for (c=arg; *c != '\0'; c++) { + if ('"' == *c) { + /* Double up backslashes preceding a quote */ + for (i=0; i<(bs_counter*2); i++) + smartlist_add(arg_chars, (void*)&backslash); + bs_counter = 0; + /* Escape the quote */ + smartlist_add(arg_chars, (void*)&backslash); + smartlist_add(arg_chars, (void*)c); + } else if ('\\' == *c) { + /* Count backslashes until we know whether to double up */ + bs_counter++; + } else { + /* Don't double up slashes preceding a non-quote */ + for (i=0; i -#endif -#ifdef HAVE_SYS_PRCTL_H -#include -#endif -#ifdef HAVE_UNISTD_H -#include -#endif -#ifdef HAVE_SIGNAL_H -#include -#endif -#ifdef HAVE_FCNTL_H -#include -#endif -#ifdef HAVE_SYS_WAIT_H -#include -#endif -#include -#include - -/** Format a single argument for being put on a Windows command line. - * Returns a newly allocated string */ -static char * -format_win_cmdline_argument(const char *arg) -{ - char *formatted_arg; - char need_quotes; - const char *c; - int i; - int bs_counter = 0; - /* Backslash we can point to when one is inserted into the string */ - const char backslash = '\\'; - - /* Smartlist of *char */ - smartlist_t *arg_chars; - arg_chars = smartlist_new(); - - /* Quote string if it contains whitespace or is empty */ - need_quotes = (strchr(arg, ' ') || strchr(arg, '\t') || '\0' == arg[0]); - - /* Build up smartlist of *chars */ - for (c=arg; *c != '\0'; c++) { - if ('"' == *c) { - /* Double up backslashes preceding a quote */ - for (i=0; i<(bs_counter*2); i++) - smartlist_add(arg_chars, (void*)&backslash); - bs_counter = 0; - /* Escape the quote */ - smartlist_add(arg_chars, (void*)&backslash); - smartlist_add(arg_chars, (void*)c); - } else if ('\\' == *c) { - /* Count backslashes until we know whether to double up */ - bs_counter++; - } else { - /* Don't double up slashes preceding a non-quote */ - for (i=0; i diff --git a/src/test/test_process.c b/src/test/test_process.c index 2fcb3f6686..3640b86688 100644 --- a/src/test/test_process.c +++ b/src/test/test_process.c @@ -628,11 +628,51 @@ test_win32(void *arg) process_init(); process_t *process = process_new(""); + char *joined_argv = NULL; /* On Win32 all processes should have a Win32 process handle. */ tt_ptr_op(NULL, OP_NE, process_get_win32_process(process)); + /* Based on some test cases from "Parsing C++ Command-Line Arguments" in + * MSDN but we don't exercise all quoting rules because tor_join_win_cmdline + * will try to only generate simple cases for the child process to parse; + * i.e. we never embed quoted strings in arguments. */ + + const char *argvs[][4] = { + {"a", "bb", "CCC", NULL}, // Normal + {NULL, NULL, NULL, NULL}, // Empty argument list + {"", NULL, NULL, NULL}, // Empty argument + {"\"a", "b\"b", "CCC\"", NULL}, // Quotes + {"a\tbc", "dd dd", "E", NULL}, // Whitespace + {"a\\\\\\b", "de fg", "H", NULL}, // Backslashes + {"a\\\"b", "\\c", "D\\", NULL}, // Backslashes before quote + {"a\\\\b c", "d", "E", NULL}, // Backslashes not before quote + { NULL } // Terminator + }; + + const char *cmdlines[] = { + "a bb CCC", + "", + "\"\"", + "\\\"a b\\\"b CCC\\\"", + "\"a\tbc\" \"dd dd\" E", + "a\\\\\\b \"de fg\" H", + "a\\\\\\\"b \\c D\\", + "\"a\\\\b c\" d E", + NULL // Terminator + }; + + int i; + + for (i=0; cmdlines[i]!=NULL; i++) { + log_info(LD_GENERAL, "Joining argvs[%d], expecting <%s>", i, cmdlines[i]); + joined_argv = tor_join_win_cmdline(argvs[i]); + tt_str_op(cmdlines[i],OP_EQ, joined_argv); + tor_free(joined_argv); + } + done: + tor_free(joined_argv); process_free(process); process_free_all(); #endif diff --git a/src/test/test_pt.c b/src/test/test_pt.c index c980276b14..b5572ef80c 100644 --- a/src/test/test_pt.c +++ b/src/test/test_pt.c @@ -8,7 +8,6 @@ #define UTIL_PRIVATE #define STATEFILE_PRIVATE #define CONTROL_PRIVATE -#define SUBPROCESS_PRIVATE #define PROCESS_PRIVATE #include "core/or/or.h" #include "app/config/config.h" @@ -18,7 +17,6 @@ #include "core/or/circuitbuild.h" #include "app/config/statefile.h" #include "test/test.h" -#include "lib/process/subprocess.h" #include "lib/encoding/confline.h" #include "lib/net/resolve.h" #include "lib/process/process.h" diff --git a/src/test/test_util.c b/src/test/test_util.c index 4fa67b6419..a94153f2dc 100644 --- a/src/test/test_util.c +++ b/src/test/test_util.c @@ -10,7 +10,7 @@ #define UTIL_PRIVATE #define UTIL_MALLOC_PRIVATE #define SOCKET_PRIVATE -#define SUBPROCESS_PRIVATE +#define PROCESS_WIN32_PRIVATE #include "lib/testsupport/testsupport.h" #include "core/or/or.h" #include "lib/container/buffers.h" @@ -22,6 +22,7 @@ #include "test/test.h" #include "lib/memarea/memarea.h" #include "lib/process/waitpid.h" +#include "lib/process/process_win32.h" #include "test/log_test_helpers.h" #include "lib/compress/compress.h" #include "lib/compress/compress_zstd.h" @@ -30,7 +31,6 @@ #include "lib/fs/winlib.h" #include "lib/process/env.h" #include "lib/process/pidfile.h" -#include "lib/process/subprocess.h" #include "lib/intmath/weakrng.h" #include "lib/thread/numcpus.h" #include "lib/math/fp.h" @@ -4395,57 +4395,6 @@ test_util_format_dec_number(void *ptr) return; } -/** - * Test that we can properly format a Windows command line - */ -static void -test_util_join_win_cmdline(void *ptr) -{ - /* Based on some test cases from "Parsing C++ Command-Line Arguments" in - * MSDN but we don't exercise all quoting rules because tor_join_win_cmdline - * will try to only generate simple cases for the child process to parse; - * i.e. we never embed quoted strings in arguments. */ - - const char *argvs[][4] = { - {"a", "bb", "CCC", NULL}, // Normal - {NULL, NULL, NULL, NULL}, // Empty argument list - {"", NULL, NULL, NULL}, // Empty argument - {"\"a", "b\"b", "CCC\"", NULL}, // Quotes - {"a\tbc", "dd dd", "E", NULL}, // Whitespace - {"a\\\\\\b", "de fg", "H", NULL}, // Backslashes - {"a\\\"b", "\\c", "D\\", NULL}, // Backslashes before quote - {"a\\\\b c", "d", "E", NULL}, // Backslashes not before quote - { NULL } // Terminator - }; - - const char *cmdlines[] = { - "a bb CCC", - "", - "\"\"", - "\\\"a b\\\"b CCC\\\"", - "\"a\tbc\" \"dd dd\" E", - "a\\\\\\b \"de fg\" H", - "a\\\\\\\"b \\c D\\", - "\"a\\\\b c\" d E", - NULL // Terminator - }; - - int i; - char *joined_argv = NULL; - - (void)ptr; - - for (i=0; cmdlines[i]!=NULL; i++) { - log_info(LD_GENERAL, "Joining argvs[%d], expecting <%s>", i, cmdlines[i]); - joined_argv = tor_join_win_cmdline(argvs[i]); - tt_str_op(cmdlines[i],OP_EQ, joined_argv); - tor_free(joined_argv); - } - - done: - tor_free(joined_argv); -} - #define MAX_SPLIT_LINE_COUNT 4 struct split_lines_test_t { const char *orig_line; // Line to be split (may contain \0's) @@ -6226,7 +6175,6 @@ struct testcase_t util_tests[] = { UTIL_TEST_WIN_ONLY(load_win_lib, 0), UTIL_TEST(format_hex_number, 0), UTIL_TEST(format_dec_number, 0), - UTIL_TEST(join_win_cmdline, 0), UTIL_TEST(n_bits_set, 0), UTIL_TEST(eat_whitespace, 0), UTIL_TEST(sl_new_from_text_lines, 0), From 6e508e9eb48c88f94cd9431aca6ac5743beb41d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Mon, 26 Nov 2018 07:16:28 +0100 Subject: [PATCH 0298/2557] Fix tests on kqueue() based platforms. This patch disables fork()'ing of the slow process tests. This fixes the tests on the MacOS and other kqueue() based platforms. Without this patch the main loop exits eearly with EBADF as error. See: https://bugs.torproject.org/28179 --- src/test/test_process_slow.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/test_process_slow.c b/src/test/test_process_slow.c index b4c5a2d5e2..2ec9ff28a0 100644 --- a/src/test/test_process_slow.c +++ b/src/test/test_process_slow.c @@ -324,7 +324,7 @@ test_callbacks_terminate(void *arg) } struct testcase_t slow_process_tests[] = { - { "callbacks", test_callbacks, TT_FORK, NULL, NULL }, - { "callbacks_terminate", test_callbacks_terminate, TT_FORK, NULL, NULL }, + { "callbacks", test_callbacks, 0, NULL, NULL }, + { "callbacks_terminate", test_callbacks_terminate, 0, NULL, NULL }, END_OF_TESTCASES }; From f983a60a6c9ce70e1c674458e468b0d7dcd80c01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Tue, 27 Nov 2018 17:36:10 +0100 Subject: [PATCH 0299/2557] Copy (zlib1|libssp-0).dll to \src\test\ to run test-process.exe. This patch ensures that AppVeyor copies over libssp-0.dll and zlib1.dll to src/test/ to make sure we can run text-process.exe from our slow tests. See: https://bugs.torproject.org/28179 --- .appveyor.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.appveyor.yml b/.appveyor.yml index 837cded43c..63d49536fa 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -73,6 +73,8 @@ test_script: $buildpath = @("C:\msys64\${env:compiler_path}\bin") + $oldpath $env:Path = $buildpath -join ';' Set-Location "${env:build}" + Copy-Item "C:/msys64/${env:compiler_path}/bin/libssp-0.dll" -Destination "${env:build}/src/test" + Copy-Item "C:/msys64/${env:compiler_path}/bin/zlib1.dll" -Destination "${env:build}/src/test" Execute-Bash "VERBOSE=1 make -j2 check" } From bc6983afed24c0b83a49d2cef531dcc036245a04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Wed, 28 Nov 2018 18:10:02 +0100 Subject: [PATCH 0300/2557] Use run_main_loop_until_done() for process_t tests. This patch changes the slow process_t tests to use run_main_loop_until_done() instead of do_main_loop() since do_main_loop() initializes a lot of subsystem callbacks that we don't need to run in our tests. See: https://bugs.torproject.org/28179 --- src/core/mainloop/mainloop.c | 11 +++++------ src/core/mainloop/mainloop.h | 1 + src/test/test_process_slow.c | 3 ++- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/core/mainloop/mainloop.c b/src/core/mainloop/mainloop.c index aaaa5009cb..9e58448f33 100644 --- a/src/core/mainloop/mainloop.c +++ b/src/core/mainloop/mainloop.c @@ -200,7 +200,6 @@ static int can_complete_circuits = 0; #define LAZY_DESCRIPTOR_RETRY_INTERVAL (60) static int conn_close_if_marked(int i); -static int run_main_loop_until_done(void); static void connection_start_reading_from_linked_conn(connection_t *conn); static int connection_should_read_from_linked_conn(connection_t *conn); static void conn_read_callback(evutil_socket_t fd, short event, void *_conn); @@ -2847,10 +2846,6 @@ do_main_loop(void) } } #endif /* defined(HAVE_SYSTEMD_209) */ - - main_loop_should_exit = 0; - main_loop_exit_value = 0; - #ifdef ENABLE_RESTART_DEBUGGING { static int first_time = 1; @@ -2976,10 +2971,14 @@ run_main_loop_once(void) * * Shadow won't invoke this function, so don't fill it up with things. */ -static int +STATIC int run_main_loop_until_done(void) { int loop_result = 1; + + main_loop_should_exit = 0; + main_loop_exit_value = 0; + do { loop_result = run_main_loop_once(); } while (loop_result == 1); diff --git a/src/core/mainloop/mainloop.h b/src/core/mainloop/mainloop.h index 14e80ebb21..730234857d 100644 --- a/src/core/mainloop/mainloop.h +++ b/src/core/mainloop/mainloop.h @@ -100,6 +100,7 @@ extern struct token_bucket_rw_t global_bucket; extern struct token_bucket_rw_t global_relayed_bucket; #ifdef MAINLOOP_PRIVATE +STATIC int run_main_loop_until_done(void); STATIC void close_closeable_connections(void); STATIC void initialize_periodic_events(void); STATIC void teardown_periodic_events(void); diff --git a/src/test/test_process_slow.c b/src/test/test_process_slow.c index 2ec9ff28a0..ad84127bba 100644 --- a/src/test/test_process_slow.c +++ b/src/test/test_process_slow.c @@ -6,6 +6,7 @@ * \brief Slow test cases for the Process API. */ +#define MAINLOOP_PRIVATE #include "orconfig.h" #include "core/or/or.h" #include "core/mainloop/mainloop.h" @@ -168,7 +169,7 @@ run_main_loop(void) NULL); /* Run our main loop. */ - ret = do_main_loop(); + ret = run_main_loop_until_done(); /* Clean up our main loop timeout timer. */ tt_int_op(ret, OP_EQ, 0); From 22cb3c6ce9164920ff81013d5f8dce3c26911af4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Wed, 28 Nov 2018 18:12:30 +0100 Subject: [PATCH 0301/2557] Call close() on stdin/stdout/stderr in process_terminate(). Call close() on all process handles after we have called kill(pid, SIGTERM). See: https://bugs.torproject.org/28179 --- src/lib/process/process_unix.c | 59 +++++++++++++++++++++++++++++++-- src/lib/process/process_unix.h | 1 + src/lib/process/process_win32.c | 5 +++ 3 files changed, 62 insertions(+), 3 deletions(-) diff --git a/src/lib/process/process_unix.c b/src/lib/process/process_unix.c index 4a9aaa2edd..04d6381aa3 100644 --- a/src/lib/process/process_unix.c +++ b/src/lib/process/process_unix.c @@ -119,7 +119,11 @@ process_unix_free_(process_unix_t *unix_process) if (! unix_process->stderr_handle.reached_eof) process_unix_stop_reading(&unix_process->stderr_handle); - process_unix_stop_writing(&unix_process->stdin_handle); + if (unix_process->stdin_handle.is_writing) + process_unix_stop_writing(&unix_process->stdin_handle); + + /* Close all our file descriptors. */ + process_unix_close_file_descriptors(unix_process); tor_event_free(unix_process->stdout_handle.event); tor_event_free(unix_process->stderr_handle.event); @@ -368,6 +372,8 @@ process_unix_terminate(process_t *process) if (BUG(unix_process->waitpid == NULL)) return false; + bool success = true; + /* Send a SIGTERM to our child process. */ int ret; @@ -376,10 +382,14 @@ process_unix_terminate(process_t *process) if (ret == -1) { log_warn(LD_PROCESS, "Unable to terminate process: %s", strerror(errno)); - return false; + success = false; } - return ret == 0; + /* Close all our FD's. */ + if (! process_unix_close_file_descriptors(unix_process)) + success = false; + + return success; } /** Returns the unique process identifier for the given process. */ @@ -648,4 +658,47 @@ process_unix_read_handle(process_t *process, return ret; } +/** Close the standard in, out, and error handles of the given + * unix_process. */ +STATIC bool +process_unix_close_file_descriptors(process_unix_t *unix_process) +{ + tor_assert(unix_process); + + int ret; + bool success = true; + + if (unix_process->stdin_handle.fd != -1) { + ret = close(unix_process->stdin_handle.fd); + if (ret == -1) { + log_warn(LD_PROCESS, "Unable to close standard in"); + success = false; + } + + unix_process->stdin_handle.fd = -1; + } + + if (unix_process->stdout_handle.fd != -1) { + ret = close(unix_process->stdout_handle.fd); + if (ret == -1) { + log_warn(LD_PROCESS, "Unable to close standard out"); + success = false; + } + + unix_process->stdout_handle.fd = -1; + } + + if (unix_process->stderr_handle.fd != -1) { + ret = close(unix_process->stderr_handle.fd); + if (ret == -1) { + log_warn(LD_PROCESS, "Unable to close standard error"); + success = false; + } + + unix_process->stderr_handle.fd = -1; + } + + return success; +} + #endif /* defined(_WIN32). */ diff --git a/src/lib/process/process_unix.h b/src/lib/process/process_unix.h index e17c59ea81..86c10d7449 100644 --- a/src/lib/process/process_unix.h +++ b/src/lib/process/process_unix.h @@ -60,6 +60,7 @@ STATIC void process_unix_setup_handle(process_t *process, STATIC int process_unix_read_handle(process_t *, process_unix_handle_t *, buf_t *); +STATIC bool process_unix_close_file_descriptors(process_unix_t *); #endif /* defined(PROCESS_UNIX_PRIVATE). */ #endif /* defined(_WIN32). */ diff --git a/src/lib/process/process_win32.c b/src/lib/process/process_win32.c index 73e19d518f..e75328f3e8 100644 --- a/src/lib/process/process_win32.c +++ b/src/lib/process/process_win32.c @@ -293,6 +293,11 @@ process_win32_terminate(process_t *process) return false; } + /* Cleanup our handles. */ + process_win32_cleanup_handle(&win32_process->stdin_handle); + process_win32_cleanup_handle(&win32_process->stdout_handle); + process_win32_cleanup_handle(&win32_process->stderr_handle); + return true; } From 5585cbd08f54f732c32feea276c1a47ec8446c5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Wed, 28 Nov 2018 21:55:04 +0100 Subject: [PATCH 0302/2557] Change the Process exit_callback to return bool. This patch changes our process_t's exit_callback to return a boolean value. If the returned value is true, the process subsystem will call process_free() on the given process_t. See: https://bugs.torproject.org/28179 --- src/feature/client/transports.c | 11 +++++--- src/feature/client/transports.h | 2 +- src/lib/process/process.c | 14 +++++++--- src/lib/process/process.h | 4 +-- src/lib/process/process_win32.c | 48 +++++++++++++++++++++++++-------- src/lib/process/process_win32.h | 2 +- src/test/test_process.c | 5 ++-- src/test/test_process_slow.c | 5 ++-- 8 files changed, 65 insertions(+), 26 deletions(-) diff --git a/src/feature/client/transports.c b/src/feature/client/transports.c index 0e326f90e9..e3cc679411 100644 --- a/src/feature/client/transports.c +++ b/src/feature/client/transports.c @@ -1759,8 +1759,9 @@ managed_proxy_stderr_callback(process_t *process, char *line, size_t size) /** Callback function that is called when our PT process terminates. The * process exit code can be found in exit_code and our process can be - * found in process. */ -STATIC void + * found in process. Returns true iff we want the process subsystem to + * free our process_t handle for us. */ +STATIC bool managed_proxy_exit_callback(process_t *process, process_exit_code_t exit_code) { tor_assert(process); @@ -1772,10 +1773,14 @@ managed_proxy_exit_callback(process_t *process, process_exit_code_t exit_code) /* We detach ourself from the MP (if we are attached) and free ourself. */ managed_proxy_t *mp = process_get_data(process); + /* If we are still attached to the process, it is probably because our PT + * process crashed before we got to call process_set_data(p, NULL); */ if (BUG(mp != NULL)) { + /* FIXME(ahf): Our process stopped without us having told it to stop + * (crashed). Should we restart it here? */ mp->process = NULL; process_set_data(process, NULL); } - process_free(process); + return true; } diff --git a/src/feature/client/transports.h b/src/feature/client/transports.h index fbb720aac6..ba8cbf7105 100644 --- a/src/feature/client/transports.h +++ b/src/feature/client/transports.h @@ -145,7 +145,7 @@ STATIC void free_execve_args(char **arg); STATIC void managed_proxy_stdout_callback(process_t *, char *, size_t); STATIC void managed_proxy_stderr_callback(process_t *, char *, size_t); -STATIC void managed_proxy_exit_callback(process_t *, process_exit_code_t); +STATIC bool managed_proxy_exit_callback(process_t *, process_exit_code_t); #endif /* defined(PT_PRIVATE) */ diff --git a/src/lib/process/process.c b/src/lib/process/process.c index 7275d0a21b..75bffe35b9 100644 --- a/src/lib/process/process.c +++ b/src/lib/process/process.c @@ -612,7 +612,8 @@ process_notify_event_stdin(process_t *process) /** This function is called by the Process backend when a given process have * terminated. The exit status code is passed in exit_code. We mark the * process as no longer running and calls the exit_callback with - * information about the process termination. */ + * information about the process termination. The given process is + * free'd iff the exit_callback returns true. */ void process_notify_event_exit(process_t *process, process_exit_code_t exit_code) { @@ -626,9 +627,14 @@ process_notify_event_exit(process_t *process, process_exit_code_t exit_code) process->exit_code = exit_code; /* Call our exit callback, if it exists. */ - if (process->exit_callback) { - process->exit_callback(process, exit_code); - } + bool free_process_handle = false; + + /* The exit callback will tell us if we should process_free() our handle. */ + if (process->exit_callback) + free_process_handle = process->exit_callback(process, exit_code); + + if (free_process_handle) + process_free(process); } /** This function is called whenever the Process backend have notified us that diff --git a/src/lib/process/process.h b/src/lib/process/process.h index cb5bccbf7b..4b0fae4250 100644 --- a/src/lib/process/process.h +++ b/src/lib/process/process.h @@ -56,8 +56,8 @@ typedef uint64_t process_pid_t; typedef void (*process_read_callback_t)(process_t *, char *, size_t); -typedef void (*process_exit_callback_t)(process_t *, - process_exit_code_t); +typedef bool +(*process_exit_callback_t)(process_t *, process_exit_code_t); void process_init(void); void process_free_all(void); diff --git a/src/lib/process/process_win32.c b/src/lib/process/process_win32.c index e75328f3e8..911bad3933 100644 --- a/src/lib/process/process_win32.c +++ b/src/lib/process/process_win32.c @@ -474,28 +474,47 @@ process_win32_timer_callback(periodic_timer_t *timer, void *data) tor_assert(timer == periodic_timer); tor_assert(data == NULL); - log_debug(LD_PROCESS, "Windows Process I/O timer ticked"); - /* Move the process into an alertable state. */ process_win32_trigger_completion_callbacks(); /* Check if our processes are still alive. */ - const smartlist_t *processes = process_get_all_processes(); - SMARTLIST_FOREACH(processes, process_t *, p, - process_win32_timer_test_process(p)); + /* Since the call to process_win32_timer_test_process() might call + * process_notify_event_exit() which again might call process_free() which + * updates the list of processes returned by process_get_all_processes() it + * is important here that we make sure to not touch the list of processes if + * the call to process_win32_timer_test_process() returns true. */ + bool done; + + do { + const smartlist_t *processes = process_get_all_processes(); + done = true; + + SMARTLIST_FOREACH_BEGIN(processes, process_t *, process) { + /* If process_win32_timer_test_process() returns true, it means that + * smartlist_remove() might have been called on the list returned by + * process_get_all_processes(). We start the loop over again until we + * have a succesful run over the entire list where the list was not + * modified. */ + if (process_win32_timer_test_process(process)) { + done = false; + break; + } + } SMARTLIST_FOREACH_END(process); + } while (! done); } /** Test whether a given process is still alive. Notify the Process subsystem - * if our process have died. */ -STATIC void + * if our process have died. Returns true iff the given process have + * terminated. */ +STATIC bool process_win32_timer_test_process(process_t *process) { tor_assert(process); /* No need to look at processes that don't claim they are running. */ if (process_get_status(process) != PROCESS_STATUS_RUNNING) - return; + return false; process_win32_t *win32_process = process_get_win32_process(process); BOOL ret = FALSE; @@ -508,12 +527,19 @@ process_win32_timer_test_process(process_t *process) if (! ret) { log_warn(LD_PROCESS, "GetExitCodeProcess() failed: %s", format_win32_error(GetLastError())); - return; + return false; } - /* Notify our process_t that our process have terminated. */ - if (exit_code != STILL_ACTIVE) + /* Notify our process_t that our process have terminated. Since our + * exit_callback might decide to process_free() our process handle it is very + * important that we do not touch the process_t after the call to + * process_notify_event_exit(). */ + if (exit_code != STILL_ACTIVE) { process_notify_event_exit(process, exit_code); + return true; + } + + return false; } /** Create a new overlapped named pipe. This function creates a new connected, diff --git a/src/lib/process/process_win32.h b/src/lib/process/process_win32.h index 3c809dfd23..00de8c949b 100644 --- a/src/lib/process/process_win32.h +++ b/src/lib/process/process_win32.h @@ -49,7 +49,7 @@ STATIC void process_win32_timer_start(void); STATIC void process_win32_timer_stop(void); STATIC bool process_win32_timer_running(void); STATIC void process_win32_timer_callback(periodic_timer_t *, void *); -STATIC void process_win32_timer_test_process(process_t *); +STATIC bool process_win32_timer_test_process(process_t *); /* I/O pipe handling. */ struct process_win32_handle_t; diff --git a/src/test/test_process.c b/src/test/test_process.c index 3640b86688..17481097be 100644 --- a/src/test/test_process.c +++ b/src/test/test_process.c @@ -125,7 +125,7 @@ process_stderr_callback(process_t *process, char *data, size_t size) return; } -static void +static bool process_exit_callback(process_t *process, process_exit_code_t exit_code) { tt_ptr_op(process, OP_NE, NULL); @@ -134,7 +134,8 @@ process_exit_callback(process_t *process, process_exit_code_t exit_code) process_data->exit_code = exit_code; done: - return; + /* Do not free up our process_t. */ + return false; } static void diff --git a/src/test/test_process_slow.c b/src/test/test_process_slow.c index ad84127bba..b23492e972 100644 --- a/src/test/test_process_slow.c +++ b/src/test/test_process_slow.c @@ -94,7 +94,7 @@ process_stderr_callback(process_t *process, char *data, size_t size) return; } -static void +static bool process_exit_callback(process_t *process, process_exit_code_t exit_code) { tt_ptr_op(process, OP_NE, NULL); @@ -106,7 +106,8 @@ process_exit_callback(process_t *process, process_exit_code_t exit_code) tor_shutdown_event_loop_and_exit(0); done: - return; + /* Do not free up our process_t. */ + return false; } #ifdef _WIN32 From ec2ae3ed8b542950dc774223607f14f524375a50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Thu, 29 Nov 2018 16:42:56 +0100 Subject: [PATCH 0303/2557] Change EVENT_TRANSPORT_LOG to EVENT_PT_LOG. This patch changes our EVENT_TRANSPORT_LOG event to be EVENT_PT_LOG. The new message includes the path to the PT executable instead of the transport name, since one PT binary can include multiple transport they sometimes might need to log messages that are not specific to a given transport. See: https://bugs.torproject.org/28179 --- src/feature/client/transports.c | 33 ++++++++++----------------------- src/feature/client/transports.h | 2 +- src/feature/control/control.c | 10 +++++----- src/feature/control/control.h | 5 ++--- src/test/test_pt.c | 6 +++--- 5 files changed, 21 insertions(+), 35 deletions(-) diff --git a/src/feature/client/transports.c b/src/feature/client/transports.c index e3cc679411..df7991846c 100644 --- a/src/feature/client/transports.c +++ b/src/feature/client/transports.c @@ -910,7 +910,7 @@ handle_proxy_line(const char *line, managed_proxy_t *mp) parse_proxy_error(line); goto err; } else if (!strcmpstart(line, PROTO_LOG)) { - parse_log_line(line); + parse_log_line(line, mp); return; } @@ -1135,40 +1135,27 @@ parse_proxy_error(const char *line) /** Parses a LOG line and emit log events accordingly. */ STATIC void -parse_log_line(const char *line) +parse_log_line(const char *line, managed_proxy_t *mp) { - smartlist_t *items = smartlist_new(); + tor_assert(line); + tor_assert(mp); if (strlen(line) < (strlen(PROTO_LOG) + 1)) { log_warn(LD_PT, "Managed proxy sent us a %s line " - "with missing arguments.", PROTO_LOG); + "with missing argument.", PROTO_LOG); goto done; } - const char *arguments = line + strlen(PROTO_LOG) + 1; + const char *message = line + strlen(PROTO_LOG) + 1; - /* The format is 'LOG '. We accept empty messages. */ - smartlist_split_string(items, arguments, NULL, - SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 2); - - if (smartlist_len(items) < 2) { - log_warn(LD_PT, "Managed proxy sent us a %s line " - "with too few arguments.", PROTO_LOG); - goto done; - } - - const char *transport_name = smartlist_get(items, 0); - const char *message = smartlist_get(items, 1); - - log_info(LD_PT, "Managed proxy transport \"%s\" says: %s", - transport_name, message); + log_info(LD_PT, "Managed proxy \"%s\" says: %s", + mp->argv[0], message); /* Emit control port event. */ - control_event_transport_log(transport_name, message); + control_event_pt_log(mp->argv[0], message); done: - SMARTLIST_FOREACH(items, char *, s, tor_free(s)); - smartlist_free(items); + return; } /** Return a newly allocated string that tor should place in diff --git a/src/feature/client/transports.h b/src/feature/client/transports.h index ba8cbf7105..88735a7211 100644 --- a/src/feature/client/transports.h +++ b/src/feature/client/transports.h @@ -128,7 +128,7 @@ STATIC int parse_version(const char *line, managed_proxy_t *mp); STATIC void parse_env_error(const char *line); STATIC void parse_proxy_error(const char *line); STATIC void handle_proxy_line(const char *line, managed_proxy_t *mp); -STATIC void parse_log_line(const char *line); +STATIC void parse_log_line(const char *line, managed_proxy_t *mp); STATIC char *get_transport_options_for_server_proxy(const managed_proxy_t *mp); STATIC void managed_proxy_destroy(managed_proxy_t *mp, diff --git a/src/feature/control/control.c b/src/feature/control/control.c index b6505a85d6..ebda51781a 100644 --- a/src/feature/control/control.c +++ b/src/feature/control/control.c @@ -7395,14 +7395,14 @@ control_event_transport_launched(const char *mode, const char *transport_name, mode, transport_name, fmt_addr(addr), port); } -/** A pluggable transport called transport_name has emitted a log +/** A pluggable transport called pt_name has emitted a log * message found in message. */ void -control_event_transport_log(const char *transport_name, const char *message) +control_event_pt_log(const char *pt_name, const char *message) { - send_control_event(EVENT_TRANSPORT_LOG, - "650 TRANSPORT_LOG %s %s\r\n", - transport_name, + send_control_event(EVENT_PT_LOG, + "650 PT_LOG %s %s\r\n", + pt_name, message); } diff --git a/src/feature/control/control.h b/src/feature/control/control.h index eb2b5676ea..c554c32539 100644 --- a/src/feature/control/control.h +++ b/src/feature/control/control.h @@ -205,8 +205,7 @@ void control_event_clients_seen(const char *controller_str); void control_event_transport_launched(const char *mode, const char *transport_name, tor_addr_t *addr, uint16_t port); -void control_event_transport_log(const char *transport_name, - const char *message); +void control_event_pt_log(const char *pt_name, const char *message); const char *rend_auth_type_to_string(rend_auth_type_t auth_type); MOCK_DECL(const char *, node_describe_longname_by_id,(const char *id_digest)); void control_event_hs_descriptor_requested(const char *onion_address, @@ -295,7 +294,7 @@ void control_free_all(void); #define EVENT_HS_DESC 0x0021 #define EVENT_HS_DESC_CONTENT 0x0022 #define EVENT_NETWORK_LIVENESS 0x0023 -#define EVENT_TRANSPORT_LOG 0x0024 +#define EVENT_PT_LOG 0x0024 #define EVENT_MAX_ 0x0024 /* sizeof(control_connection_t.event_mask) in bits, currently a uint64_t */ diff --git a/src/test/test_pt.c b/src/test/test_pt.c index b5572ef80c..8fcdd5c1e8 100644 --- a/src/test/test_pt.c +++ b/src/test/test_pt.c @@ -310,7 +310,7 @@ process_read_stdout_replacement(process_t *process, buf_t *buffer) } else if (times_called <= 6) { buf_add_string(buffer, "SMETHODS DONE\n"); } else if (times_called <= 7) { - buf_add_string(buffer, "LOG mock3 Oh noes, something bad happened. " + buf_add_string(buffer, "LOG Oh noes, something bad happened. " "What do we do!?\n"); } @@ -417,10 +417,10 @@ test_pt_configure_proxy(void *arg) process_notify_event_stdout(mp->process); tt_int_op(controlevent_n, OP_EQ, 6); - tt_int_op(controlevent_event, OP_EQ, EVENT_TRANSPORT_LOG); + tt_int_op(controlevent_event, OP_EQ, EVENT_PT_LOG); tt_int_op(smartlist_len(controlevent_msgs), OP_EQ, 6); tt_str_op(smartlist_get(controlevent_msgs, 5), OP_EQ, - "650 TRANSPORT_LOG mock3 Oh noes, something bad happened. " + "650 PT_LOG Oh noes, something bad happened. " "What do we do!?\r\n"); { /* check that the transport info were saved properly in the tor state */ From cacdd290872420d51f880d75466f6d6e22430466 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Fri, 7 Dec 2018 02:09:22 +0100 Subject: [PATCH 0304/2557] Use `const char *` instead of `char *` for line parameter for process callbacks. This patch changes the type definition of the process callbacks to use `const char *` instead of `char *`. See: https://bugs.torproject.org/28179 --- src/feature/client/transports.c | 8 ++++++-- src/feature/client/transports.h | 4 ++-- src/lib/process/process.h | 2 +- src/test/test_process.c | 4 ++-- src/test/test_process_slow.c | 4 ++-- 5 files changed, 13 insertions(+), 9 deletions(-) diff --git a/src/feature/client/transports.c b/src/feature/client/transports.c index df7991846c..0f456ba1e6 100644 --- a/src/feature/client/transports.c +++ b/src/feature/client/transports.c @@ -1711,7 +1711,9 @@ tor_escape_str_for_pt_args(const char *string, const char *chars_to_escape) * stdout. Our process can be found in process, the data can be found in * line and the length of our line is given in size. */ STATIC void -managed_proxy_stdout_callback(process_t *process, char *line, size_t size) +managed_proxy_stdout_callback(process_t *process, + const char *line, + size_t size) { tor_assert(process); tor_assert(line); @@ -1732,7 +1734,9 @@ managed_proxy_stdout_callback(process_t *process, char *line, size_t size) * stderr. Our process can be found in process, the data can be found in * line and the length of our line is given in size. */ STATIC void -managed_proxy_stderr_callback(process_t *process, char *line, size_t size) +managed_proxy_stderr_callback(process_t *process, + const char *line, + size_t size) { tor_assert(process); tor_assert(line); diff --git a/src/feature/client/transports.h b/src/feature/client/transports.h index 88735a7211..a3994a0099 100644 --- a/src/feature/client/transports.h +++ b/src/feature/client/transports.h @@ -143,8 +143,8 @@ STATIC char* get_pt_proxy_uri(void); STATIC void free_execve_args(char **arg); -STATIC void managed_proxy_stdout_callback(process_t *, char *, size_t); -STATIC void managed_proxy_stderr_callback(process_t *, char *, size_t); +STATIC void managed_proxy_stdout_callback(process_t *, const char *, size_t); +STATIC void managed_proxy_stderr_callback(process_t *, const char *, size_t); STATIC bool managed_proxy_exit_callback(process_t *, process_exit_code_t); #endif /* defined(PT_PRIVATE) */ diff --git a/src/lib/process/process.h b/src/lib/process/process.h index 4b0fae4250..179db19aeb 100644 --- a/src/lib/process/process.h +++ b/src/lib/process/process.h @@ -54,7 +54,7 @@ typedef uint64_t process_exit_code_t; typedef uint64_t process_pid_t; typedef void (*process_read_callback_t)(process_t *, - char *, + const char *, size_t); typedef bool (*process_exit_callback_t)(process_t *, process_exit_code_t); diff --git a/src/test/test_process.c b/src/test/test_process.c index 17481097be..9b62862f04 100644 --- a/src/test/test_process.c +++ b/src/test/test_process.c @@ -98,7 +98,7 @@ process_mocked_write_stdin(process_t *process, buf_t *buffer) } static void -process_stdout_callback(process_t *process, char *data, size_t size) +process_stdout_callback(process_t *process, const char *data, size_t size) { tt_ptr_op(process, OP_NE, NULL); tt_ptr_op(data, OP_NE, NULL); @@ -112,7 +112,7 @@ process_stdout_callback(process_t *process, char *data, size_t size) } static void -process_stderr_callback(process_t *process, char *data, size_t size) +process_stderr_callback(process_t *process, const char *data, size_t size) { tt_ptr_op(process, OP_NE, NULL); tt_ptr_op(data, OP_NE, NULL); diff --git a/src/test/test_process_slow.c b/src/test/test_process_slow.c index b23492e972..cc7e8c5a9c 100644 --- a/src/test/test_process_slow.c +++ b/src/test/test_process_slow.c @@ -67,7 +67,7 @@ process_data_free(process_data_t *process_data) } static void -process_stdout_callback(process_t *process, char *data, size_t size) +process_stdout_callback(process_t *process, const char *data, size_t size) { tt_ptr_op(process, OP_NE, NULL); tt_ptr_op(data, OP_NE, NULL); @@ -81,7 +81,7 @@ process_stdout_callback(process_t *process, char *data, size_t size) } static void -process_stderr_callback(process_t *process, char *data, size_t size) +process_stderr_callback(process_t *process, const char *data, size_t size) { tt_ptr_op(process, OP_NE, NULL); tt_ptr_op(data, OP_NE, NULL); From 0d796cce17bdda605f5e768971935d7f876e33fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Wed, 12 Dec 2018 18:48:45 +0100 Subject: [PATCH 0305/2557] Use errno directly if we are not reading/writing from/to a socket. See: https://bugs.torproject.org/28179 --- src/lib/net/buffers_net.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/lib/net/buffers_net.c b/src/lib/net/buffers_net.c index da7043d5cb..9430a6b8c4 100644 --- a/src/lib/net/buffers_net.c +++ b/src/lib/net/buffers_net.c @@ -51,7 +51,8 @@ read_to_chunk(buf_t *buf, chunk_t *chunk, tor_socket_t fd, size_t at_most, read_result = read(fd, CHUNK_WRITE_PTR(chunk), at_most); if (read_result < 0) { - int e = tor_socket_errno(fd); + int e = is_socket ? tor_socket_errno(fd) : errno; + if (!ERRNO_IS_EAGAIN(e)) { /* it's a real error */ #ifdef _WIN32 if (e == WSAENOBUFS) @@ -152,7 +153,8 @@ flush_chunk(tor_socket_t fd, buf_t *buf, chunk_t *chunk, size_t sz, write_result = write(fd, chunk->data, sz); if (write_result < 0) { - int e = tor_socket_errno(fd); + int e = is_socket ? tor_socket_errno(fd) : errno; + if (!ERRNO_IS_EAGAIN(e)) { /* it's a real error */ #ifdef _WIN32 if (e == WSAENOBUFS) From a33a77d9cd3c06b5a871e99631b7f1c40bed23c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Thu, 13 Dec 2018 00:48:33 +0100 Subject: [PATCH 0306/2557] Document the format of process_t::arguments. See: https://bugs.torproject.org/28179 --- src/lib/process/process.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/lib/process/process.c b/src/lib/process/process.c index 75bffe35b9..fb76a0a725 100644 --- a/src/lib/process/process.c +++ b/src/lib/process/process.c @@ -55,7 +55,11 @@ struct process_t { /** Name of the command we want to execute (for example: /bin/ls). */ char *command; - /** The arguments used for the new process. */ + /** The arguments used for the new process. The format here is one argument + * per element of the smartlist_t. On Windows these arguments are combined + * together using the tor_join_win_cmdline function. On Unix the + * process name (argv[0]) and the trailing NULL is added automatically before + * the process is executed. */ smartlist_t *arguments; /** The environment used for the new process. */ From fab22509d70ff3289f7300788bd9243659268f69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Thu, 13 Dec 2018 00:52:39 +0100 Subject: [PATCH 0307/2557] Make Windows process event timer API available for dormant interface. This patch changes the API of the Windows backend of the Process subsystem to allow the dormant interface to disable the Process event timer. See: https://bugs.torproject.org/28179 --- src/lib/process/process_win32.c | 6 +++--- src/lib/process/process_win32.h | 9 +++++---- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/lib/process/process_win32.c b/src/lib/process/process_win32.c index 911bad3933..e9367dafc9 100644 --- a/src/lib/process/process_win32.c +++ b/src/lib/process/process_win32.c @@ -429,7 +429,7 @@ process_win32_trigger_completion_callbacks(void) /** Start the periodic timer which is reponsible for checking whether processes * are still alive and to make sure that the Tor process is periodically being * moved into an alertable state. */ -STATIC void +void process_win32_timer_start(void) { /* Make sure we never start our timer if it's already running. */ @@ -447,7 +447,7 @@ process_win32_timer_start(void) } /** Stops the periodic timer. */ -STATIC void +void process_win32_timer_stop(void) { if (BUG(periodic_timer == NULL)) @@ -458,7 +458,7 @@ process_win32_timer_stop(void) } /** Returns true iff the periodic timer is running. */ -STATIC bool +bool process_win32_timer_running(void) { return periodic_timer != NULL; diff --git a/src/lib/process/process_win32.h b/src/lib/process/process_win32.h index 00de8c949b..8ab4880fbd 100644 --- a/src/lib/process/process_win32.h +++ b/src/lib/process/process_win32.h @@ -43,11 +43,12 @@ int process_win32_read_stderr(struct process_t *process, buf_t *buffer); void process_win32_trigger_completion_callbacks(void); -#ifdef PROCESS_WIN32_PRIVATE /* Timer handling. */ -STATIC void process_win32_timer_start(void); -STATIC void process_win32_timer_stop(void); -STATIC bool process_win32_timer_running(void); +void process_win32_timer_start(void); +void process_win32_timer_stop(void); +bool process_win32_timer_running(void); + +#ifdef PROCESS_WIN32_PRIVATE STATIC void process_win32_timer_callback(periodic_timer_t *, void *); STATIC bool process_win32_timer_test_process(process_t *); From 651cdd05b7071bff4da7c3335f697d90c82c9f3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Fri, 14 Dec 2018 02:17:00 +0100 Subject: [PATCH 0308/2557] Add an additional space when we check for the PROTO_LOG handler. See: https://bugs.torproject.org/28179 --- src/feature/client/transports.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/feature/client/transports.c b/src/feature/client/transports.c index 0f456ba1e6..de53fe3469 100644 --- a/src/feature/client/transports.c +++ b/src/feature/client/transports.c @@ -909,7 +909,11 @@ handle_proxy_line(const char *line, managed_proxy_t *mp) parse_proxy_error(line); goto err; - } else if (!strcmpstart(line, PROTO_LOG)) { + + /* We check for the additional " " after the PROTO_LOG string to make sure + * we can later extend this big if/else-if table with something that begins + * with "LOG" without having to get the order right. */ + } else if (!strcmpstart(line, PROTO_LOG " ")) { parse_log_line(line, mp); return; } From c8b8b15f0eb2651dea694a057e70e6b8c34dbe05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Fri, 14 Dec 2018 03:31:56 +0100 Subject: [PATCH 0309/2557] Ensure that line_size >= 1 before trying to trim input string. See: https://bugs.torproject.org/28179 --- src/lib/process/process.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/process/process.c b/src/lib/process/process.c index fb76a0a725..b3370e919f 100644 --- a/src/lib/process/process.c +++ b/src/lib/process/process.c @@ -766,13 +766,13 @@ process_read_lines(process_t *process, tor_assert(ret != -1); /* Remove \n from the end of the line. */ - if (data[line_size - 1] == '\n') { + if (line_size >= 1 && data[line_size - 1] == '\n') { data[line_size - 1] = '\0'; --line_size; } /* Remove \r from the end of the line. */ - if (data[line_size - 1] == '\r') { + if (line_size >= 1 && data[line_size - 1] == '\r') { data[line_size - 1] = '\0'; --line_size; } From 4ad59bfbc274ff0edfe56ff78df0eeba0841415f Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 17 Dec 2018 17:01:50 -0500 Subject: [PATCH 0310/2557] Update location of buffers.h --- src/lib/process/process.c | 2 +- src/lib/process/process_unix.c | 2 +- src/lib/process/process_win32.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/lib/process/process.c b/src/lib/process/process.c index b3370e919f..ae345ceeae 100644 --- a/src/lib/process/process.c +++ b/src/lib/process/process.c @@ -9,7 +9,7 @@ **/ #define PROCESS_PRIVATE -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "lib/net/buffers_net.h" #include "lib/container/smartlist.h" #include "lib/log/log.h" diff --git a/src/lib/process/process_unix.c b/src/lib/process/process_unix.c index 04d6381aa3..4fc1b7a519 100644 --- a/src/lib/process/process_unix.c +++ b/src/lib/process/process_unix.c @@ -10,7 +10,7 @@ #define PROCESS_UNIX_PRIVATE #include "lib/intmath/cmp.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "lib/net/buffers_net.h" #include "lib/container/smartlist.h" #include "lib/evloop/compat_libevent.h" diff --git a/src/lib/process/process_win32.c b/src/lib/process/process_win32.c index e9367dafc9..c92b0975a7 100644 --- a/src/lib/process/process_win32.c +++ b/src/lib/process/process_win32.c @@ -10,7 +10,7 @@ #define PROCESS_WIN32_PRIVATE #include "lib/intmath/cmp.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "lib/net/buffers_net.h" #include "lib/container/smartlist.h" #include "lib/log/log.h" From 508837b62bcb31afbba711ec3dd7b7c18630d604 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 17 Dec 2018 17:54:01 -0500 Subject: [PATCH 0311/2557] Document the output of --version Closes 28889 --- changes/document_version | 2 ++ doc/tor.1.txt | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 changes/document_version diff --git a/changes/document_version b/changes/document_version new file mode 100644 index 0000000000..a45992b6b5 --- /dev/null +++ b/changes/document_version @@ -0,0 +1,2 @@ + o Documentation: + - Document the exact output of "tor --version". Closes ticket 28889. diff --git a/doc/tor.1.txt b/doc/tor.1.txt index 8c419b1a63..4ecc4ee382 100644 --- a/doc/tor.1.txt +++ b/doc/tor.1.txt @@ -89,7 +89,8 @@ COMMAND-LINE OPTIONS future version. (This is a warning, not a promise.) [[opt-version]] **--version**:: - Display Tor version and exit. + Display Tor version and exit. The output is a line of the format, + "Tor version [version number]." [[opt-quiet]] **--quiet**|**--hush**:: Override the default console log. By default, Tor starts out logging From 8a01f0eaabe8d5975b12328a745d2a6e921ef868 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 17 Dec 2018 17:58:49 -0500 Subject: [PATCH 0312/2557] lib/process may include lib/buf. --- src/lib/process/.may_include | 1 + 1 file changed, 1 insertion(+) diff --git a/src/lib/process/.may_include b/src/lib/process/.may_include index 2e90f4d8df..ce1b6ecf59 100644 --- a/src/lib/process/.may_include +++ b/src/lib/process/.may_include @@ -1,5 +1,6 @@ orconfig.h +lib/buf/*.h lib/cc/*.h lib/container/*.h lib/ctime/*.h From c92c0cbc9f7a1ad889cfdda6943ca484d110438e Mon Sep 17 00:00:00 2001 From: rl1987 Date: Tue, 18 Dec 2018 11:58:40 +0200 Subject: [PATCH 0313/2557] Actually allow unrecognized address types in NETINFO cell Ignore the address value instead of failing with error condition in case unrecognized address type is found. --- src/test/include.am | 1 + src/test/test.c | 1 + src/test/test.h | 1 + src/test/test_netinfo.c | 48 +++++++++++++++++ src/trunnel/netinfo.c | 100 +++++++++++++++++++++--------------- src/trunnel/netinfo.trunnel | 4 +- 6 files changed, 113 insertions(+), 42 deletions(-) create mode 100644 src/test/test_netinfo.c diff --git a/src/test/include.am b/src/test/include.am index dd2986c67c..d90d78d58e 100644 --- a/src/test/include.am +++ b/src/test/include.am @@ -147,6 +147,7 @@ src_test_test_SOURCES += \ src/test/test_logging.c \ src/test/test_mainloop.c \ src/test/test_microdesc.c \ + src/test/test_netinfo.c \ src/test/test_nodelist.c \ src/test/test_oom.c \ src/test/test_oos.c \ diff --git a/src/test/test.c b/src/test/test.c index 17b736d305..7f4f6017ef 100644 --- a/src/test/test.c +++ b/src/test/test.c @@ -890,6 +890,7 @@ struct testgroup_t testgroups[] = { { "legacy_hs/", hs_tests }, { "link-handshake/", link_handshake_tests }, { "mainloop/", mainloop_tests }, + { "netinfo/", netinfo_tests }, { "nodelist/", nodelist_tests }, { "oom/", oom_tests }, { "oos/", oos_tests }, diff --git a/src/test/test.h b/src/test/test.h index 092356f0fb..970a9b555b 100644 --- a/src/test/test.h +++ b/src/test/test.h @@ -232,6 +232,7 @@ extern struct testcase_t link_handshake_tests[]; extern struct testcase_t logging_tests[]; extern struct testcase_t mainloop_tests[]; extern struct testcase_t microdesc_tests[]; +extern struct testcase_t netinfo_tests[]; extern struct testcase_t nodelist_tests[]; extern struct testcase_t oom_tests[]; extern struct testcase_t oos_tests[]; diff --git a/src/test/test_netinfo.c b/src/test/test_netinfo.c new file mode 100644 index 0000000000..1fee2a1eba --- /dev/null +++ b/src/test/test_netinfo.c @@ -0,0 +1,48 @@ +/* Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "orconfig.h" +#include "core/or/or.h" // XXX: is this needed? +#include "trunnel/netinfo.h" +#include "test/test.h" + +static void +test_netinfo_unsupported_addr(void *arg) +{ + const uint8_t wire_data[] = + { // TIME + 0x00, 0x00, 0x00, 0x01, + // OTHERADDR + 0x04, // ATYPE + 0x04, // ALEN + 0x08, 0x08, 0x08, 0x08, // AVAL + 0x01, // NMYADDR + 0x03, // ATYPE (unsupported) + 0x05, // ALEN + 'a', 'd', 'r', 'r', '!' // AVAL (unsupported) + }; + + (void)arg; + + netinfo_cell_t *parsed_cell = NULL; + + ssize_t parsed = netinfo_cell_parse(&parsed_cell, wire_data, + sizeof(wire_data)); + + tt_assert(parsed == sizeof(wire_data)); + + netinfo_addr_t *addr = netinfo_cell_get_my_addrs(parsed_cell, 0); + tt_assert(addr); + + tt_int_op(3, OP_EQ, netinfo_addr_get_addr_type(addr)); + tt_int_op(5, OP_EQ, netinfo_addr_get_len(addr)); + + done: + netinfo_cell_free(parsed_cell); +} + +struct testcase_t netinfo_tests[] = { + { "unsupported_addr", test_netinfo_unsupported_addr, 0, NULL, NULL }, + END_OF_TESTCASES +}; + diff --git a/src/trunnel/netinfo.c b/src/trunnel/netinfo.c index de389eb13f..5d815b9b12 100644 --- a/src/trunnel/netinfo.c +++ b/src/trunnel/netinfo.c @@ -140,7 +140,6 @@ netinfo_addr_check(const netinfo_addr_t *obj) break; default: - return "Bad tag for union"; break; } return NULL; @@ -175,7 +174,6 @@ netinfo_addr_encoded_len(const netinfo_addr_t *obj) break; default: - trunnel_assert(0); break; } return result; @@ -198,6 +196,8 @@ netinfo_addr_encode(uint8_t *output, const size_t avail, const netinfo_addr_t *o const ssize_t encoded_len = netinfo_addr_encoded_len(obj); #endif + uint8_t *backptr_len = NULL; + if (NULL != (msg = netinfo_addr_check(obj))) goto check_failed; @@ -213,39 +213,49 @@ netinfo_addr_encode(uint8_t *output, const size_t avail, const netinfo_addr_t *o written += 1; ptr += 1; /* Encode u8 len */ + backptr_len = ptr; trunnel_assert(written <= avail); if (avail - written < 1) goto truncated; trunnel_set_uint8(ptr, (obj->len)); written += 1; ptr += 1; + { + size_t written_before_union = written; - /* Encode union addr[addr_type] */ - trunnel_assert(written <= avail); - switch (obj->addr_type) { + /* Encode union addr[addr_type] */ + trunnel_assert(written <= avail); + switch (obj->addr_type) { - case NETINFO_ADDR_TYPE_IPV4: + case NETINFO_ADDR_TYPE_IPV4: - /* Encode u32 addr_ipv4 */ - trunnel_assert(written <= avail); - if (avail - written < 4) - goto truncated; - trunnel_set_uint32(ptr, trunnel_htonl(obj->addr_ipv4)); - written += 4; ptr += 4; - break; + /* Encode u32 addr_ipv4 */ + trunnel_assert(written <= avail); + if (avail - written < 4) + goto truncated; + trunnel_set_uint32(ptr, trunnel_htonl(obj->addr_ipv4)); + written += 4; ptr += 4; + break; - case NETINFO_ADDR_TYPE_IPV6: + case NETINFO_ADDR_TYPE_IPV6: - /* Encode u8 addr_ipv6[16] */ - trunnel_assert(written <= avail); - if (avail - written < 16) - goto truncated; - memcpy(ptr, obj->addr_ipv6, 16); - written += 16; ptr += 16; - break; + /* Encode u8 addr_ipv6[16] */ + trunnel_assert(written <= avail); + if (avail - written < 16) + goto truncated; + memcpy(ptr, obj->addr_ipv6, 16); + written += 16; ptr += 16; + break; - default: - trunnel_assert(0); - break; + default: + break; + } + /* Write the length field back to len */ + trunnel_assert(written >= written_before_union); +#if UINT8_MAX < SIZE_MAX + if (written - written_before_union > UINT8_MAX) + goto check_failed; +#endif + trunnel_set_uint8(backptr_len, (written - written_before_union)); } @@ -291,29 +301,39 @@ netinfo_addr_parse_into(netinfo_addr_t *obj, const uint8_t *input, const size_t CHECK_REMAINING(1, truncated); obj->len = (trunnel_get_uint8(ptr)); remaining -= 1; ptr += 1; + { + size_t remaining_after; + CHECK_REMAINING(obj->len, truncated); + remaining_after = remaining - obj->len; + remaining = obj->len; - /* Parse union addr[addr_type] */ - switch (obj->addr_type) { + /* Parse union addr[addr_type] */ + switch (obj->addr_type) { - case NETINFO_ADDR_TYPE_IPV4: + case NETINFO_ADDR_TYPE_IPV4: - /* Parse u32 addr_ipv4 */ - CHECK_REMAINING(4, truncated); - obj->addr_ipv4 = trunnel_ntohl(trunnel_get_uint32(ptr)); - remaining -= 4; ptr += 4; - break; + /* Parse u32 addr_ipv4 */ + CHECK_REMAINING(4, fail); + obj->addr_ipv4 = trunnel_ntohl(trunnel_get_uint32(ptr)); + remaining -= 4; ptr += 4; + break; - case NETINFO_ADDR_TYPE_IPV6: + case NETINFO_ADDR_TYPE_IPV6: - /* Parse u8 addr_ipv6[16] */ - CHECK_REMAINING(16, truncated); - memcpy(obj->addr_ipv6, ptr, 16); - remaining -= 16; ptr += 16; - break; + /* Parse u8 addr_ipv6[16] */ + CHECK_REMAINING(16, fail); + memcpy(obj->addr_ipv6, ptr, 16); + remaining -= 16; ptr += 16; + break; - default: + default: + /* Skip to end of union */ + ptr += remaining; remaining = 0; + break; + } + if (remaining != 0) goto fail; - break; + remaining = remaining_after; } trunnel_assert(ptr + remaining == input + len_in); return len_in - remaining; diff --git a/src/trunnel/netinfo.trunnel b/src/trunnel/netinfo.trunnel index 83c3a9e40f..2c4b7a7591 100644 --- a/src/trunnel/netinfo.trunnel +++ b/src/trunnel/netinfo.trunnel @@ -7,10 +7,10 @@ const NETINFO_ADDR_TYPE_IPV6 = 6; struct netinfo_addr { u8 addr_type; u8 len; - union addr[addr_type] { + union addr[addr_type] with length len { NETINFO_ADDR_TYPE_IPV4: u32 ipv4; NETINFO_ADDR_TYPE_IPV6: u8 ipv6[16]; - default: fail; + default: ignore; }; } From c659603ac5a428d93ea625aa28b6b51ef9f2f9b3 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Tue, 18 Dec 2018 12:11:33 +0200 Subject: [PATCH 0314/2557] Unit test to check that we can parse NETINFO cell with unsupported address type --- src/test/test_netinfo.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/test_netinfo.c b/src/test/test_netinfo.c index 1fee2a1eba..8fc5330a46 100644 --- a/src/test/test_netinfo.c +++ b/src/test/test_netinfo.c @@ -2,7 +2,7 @@ /* See LICENSE for licensing information */ #include "orconfig.h" -#include "core/or/or.h" // XXX: is this needed? +#include "core/or/or.h" #include "trunnel/netinfo.h" #include "test/test.h" From 90187b1bfc377a177a62043b1959640848c6bcf7 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 18 Dec 2018 08:15:38 -0500 Subject: [PATCH 0315/2557] Remove changes that are already merged in 0.3.5.x releases --- changes/bug27197 | 3 --- changes/bug27740 | 4 ---- changes/bug27741 | 5 ----- changes/bug27750 | 6 ------ changes/bug27800 | 4 ---- changes/bug27804 | 3 --- changes/bug27841 | 7 ------- changes/bug27948 | 6 ------ changes/bug27963_timeradd | 4 ---- changes/bug27968 | 3 --- changes/bug28096 | 13 ------------- changes/bug28115 | 3 --- changes/bug28127 | 7 ------- changes/bug28183 | 4 ---- changes/bug28202 | 4 ---- changes/bug28245 | 6 ------ changes/bug28298 | 4 ---- changes/bug28303 | 3 --- changes/bug28348_034 | 5 ----- changes/bug28399 | 4 ---- changes/bug28413 | 4 ---- changes/bug28419 | 3 --- changes/bug28435 | 3 --- changes/bug28441 | 4 ---- changes/bug28454 | 4 ---- changes/bug28485 | 3 --- changes/bug28524 | 4 ---- changes/bug28554 | 3 --- changes/bug28562 | 5 ----- changes/bug28619 | 6 ------ changes/geoip-2018-11-06 | 4 ---- changes/geoip-2018-12-05 | 4 ---- changes/rust_asan | 8 -------- changes/ticket19566 | 6 ------ changes/ticket24803 | 5 ----- changes/ticket27471 | 5 ----- changes/ticket27751 | 2 -- changes/ticket27838 | 4 ---- changes/ticket27913 | 3 --- changes/ticket27995 | 4 ---- changes/ticket28026 | 3 --- changes/ticket28113 | 5 ----- changes/ticket28128 | 4 ---- changes/ticket28229_diag | 3 --- changes/ticket28275 | 4 ---- changes/ticket28318 | 3 --- changes/ticket28459 | 4 ---- changes/ticket28574 | 4 ---- changes/ticket28731 | 4 ---- 49 files changed, 216 deletions(-) delete mode 100644 changes/bug27197 delete mode 100644 changes/bug27740 delete mode 100644 changes/bug27741 delete mode 100644 changes/bug27750 delete mode 100644 changes/bug27800 delete mode 100644 changes/bug27804 delete mode 100644 changes/bug27841 delete mode 100644 changes/bug27948 delete mode 100644 changes/bug27963_timeradd delete mode 100644 changes/bug27968 delete mode 100644 changes/bug28096 delete mode 100644 changes/bug28115 delete mode 100644 changes/bug28127 delete mode 100644 changes/bug28183 delete mode 100644 changes/bug28202 delete mode 100644 changes/bug28245 delete mode 100644 changes/bug28298 delete mode 100644 changes/bug28303 delete mode 100644 changes/bug28348_034 delete mode 100644 changes/bug28399 delete mode 100644 changes/bug28413 delete mode 100644 changes/bug28419 delete mode 100644 changes/bug28435 delete mode 100644 changes/bug28441 delete mode 100644 changes/bug28454 delete mode 100644 changes/bug28485 delete mode 100644 changes/bug28524 delete mode 100644 changes/bug28554 delete mode 100644 changes/bug28562 delete mode 100644 changes/bug28619 delete mode 100644 changes/geoip-2018-11-06 delete mode 100644 changes/geoip-2018-12-05 delete mode 100644 changes/rust_asan delete mode 100644 changes/ticket19566 delete mode 100644 changes/ticket24803 delete mode 100644 changes/ticket27471 delete mode 100644 changes/ticket27751 delete mode 100644 changes/ticket27838 delete mode 100644 changes/ticket27913 delete mode 100644 changes/ticket27995 delete mode 100644 changes/ticket28026 delete mode 100644 changes/ticket28113 delete mode 100644 changes/ticket28128 delete mode 100644 changes/ticket28229_diag delete mode 100644 changes/ticket28275 delete mode 100644 changes/ticket28318 delete mode 100644 changes/ticket28459 delete mode 100644 changes/ticket28574 delete mode 100644 changes/ticket28731 diff --git a/changes/bug27197 b/changes/bug27197 deleted file mode 100644 index e389f85065..0000000000 --- a/changes/bug27197 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor bugfixes (protover, rust): - - Reject extra commas in version string. Fixes bug 27197; bugfix on - 0.3.3.3-alpha. diff --git a/changes/bug27740 b/changes/bug27740 deleted file mode 100644 index 76a17b7dda..0000000000 --- a/changes/bug27740 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (rust): - - Return a string that can be safely freed by C code, not one created by - the rust allocator, in protover_all_supported(). Fixes bug 27740; bugfix - on 0.3.3.1-alpha. diff --git a/changes/bug27741 b/changes/bug27741 deleted file mode 100644 index 531e264b63..0000000000 --- a/changes/bug27741 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor bugfixes (rust, directory authority): - - Fix an API mismatch in the rust implementation of - protover_compute_vote(). This bug could have caused crashes on any - directory authorities running Tor with Rust (which we do not yet - recommend). Fixes bug 27741; bugfix on 0.3.3.6. diff --git a/changes/bug27750 b/changes/bug27750 deleted file mode 100644 index c234788b1c..0000000000 --- a/changes/bug27750 +++ /dev/null @@ -1,6 +0,0 @@ - o Minor bugfixes (connection, relay): - - Avoid a wrong BUG() stacktrace in case a closing connection is being held - open because the write side is rate limited but not the read side. Now, - the connection read side is simply shutdown instead of kept open until tor - is able to flush the connection and then fully close it. Fixes bug 27750; - bugfix on 0.3.4.1-alpha. diff --git a/changes/bug27800 b/changes/bug27800 deleted file mode 100644 index 63d5dbc681..0000000000 --- a/changes/bug27800 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (directory authority): - - Log additional info when we get a relay that shares an ed25519 - ID with a different relay, instead making a BUG() warning. - Fixes bug 27800; bugfix on 0.3.2.1-alpha. diff --git a/changes/bug27804 b/changes/bug27804 deleted file mode 100644 index fa7fec0bc5..0000000000 --- a/changes/bug27804 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor bugfixes (rust): - - Fix a potential null dereference in protover_all_supported(). - Add a test for it. Fixes bug 27804; bugfix on 0.3.3.1-alpha. diff --git a/changes/bug27841 b/changes/bug27841 deleted file mode 100644 index 9cd1da7275..0000000000 --- a/changes/bug27841 +++ /dev/null @@ -1,7 +0,0 @@ - o Minor bugfixes (onion services): - - On an intro point for a version 3 onion service, we do not close - an introduction circuit on an NACK. This lets the client decide - whether to reuse the circuit or discard it. Previously, we closed - intro circuits on NACKs. Fixes bug 27841; bugfix on 0.3.2.1-alpha. - Patch by Neel Chaunan - diff --git a/changes/bug27948 b/changes/bug27948 deleted file mode 100644 index fea16f3d0f..0000000000 --- a/changes/bug27948 +++ /dev/null @@ -1,6 +0,0 @@ - o Minor bugfixes (tests): - - Treat backtrace test failures as expected on BSD-derived systems - (NetBSD, OpenBSD, and macOS/Darwin) until we solve bug 17808. - (FreeBSD failures have been treated as expected since 18204 in 0.2.8.) - Fixes bug 27948; bugfix on 0.2.5.2-alpha. - diff --git a/changes/bug27963_timeradd b/changes/bug27963_timeradd deleted file mode 100644 index 34b361cf8d..0000000000 --- a/changes/bug27963_timeradd +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (compilation, opensolaris): - - Add a missing include to compat_pthreads.c, to fix compilation - on OpenSolaris and its descendants. Fixes bug 27963; bugfix - on 0.3.5.1-alpha. diff --git a/changes/bug27968 b/changes/bug27968 deleted file mode 100644 index 78c8eee33a..0000000000 --- a/changes/bug27968 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor bugfixes (testing): - - Avoid hangs and race conditions in test_rebind.py. - Fixes bug 27968; bugfix on 0.3.5.1-alpha. diff --git a/changes/bug28096 b/changes/bug28096 deleted file mode 100644 index 6847df9798..0000000000 --- a/changes/bug28096 +++ /dev/null @@ -1,13 +0,0 @@ - o Minor bugfixes (Windows): - - Correctly identify Windows 8.1, Windows 10, and Windows Server 2008 - and later from their NT versions. - Fixes bug 28096; bugfix on 0.2.2.34; reported by Keifer Bly. - - On recent Windows versions, the GetVersionEx() function may report - an earlier Windows version than the running OS. To avoid user - confusion, add "[or later]" to Tor's version string on affected - versions of Windows. - Fixes bug 28096; bugfix on 0.2.2.34; reported by Keifer Bly. - - Remove Windows versions that were never supported by the - GetVersionEx() function. Stop duplicating the latest Windows - version in get_uname(). - Fixes bug 28096; bugfix on 0.2.2.34; reported by Keifer Bly. diff --git a/changes/bug28115 b/changes/bug28115 deleted file mode 100644 index e3e29968eb..0000000000 --- a/changes/bug28115 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor bugfixes (portability): - - Make the OPE code (which is used for v3 onion services) run correctly - on big-endian platforms. Fixes bug 28115; bugfix on 0.3.5.1-alpha. diff --git a/changes/bug28127 b/changes/bug28127 deleted file mode 100644 index 541128c88e..0000000000 --- a/changes/bug28127 +++ /dev/null @@ -1,7 +0,0 @@ - o Minor bugfixes (onion services): - - Unless we have explicitly set HiddenServiceVersion, detect the onion - service version and then look for invalid options. Previously, we - did the reverse, but that broke existing configs which were pointed - to a v2 hidden service and had options like HiddenServiceAuthorizeClient - set Fixes bug 28127; bugfix on 0.3.5.1-alpha. Patch by Neel Chauhan. - diff --git a/changes/bug28183 b/changes/bug28183 deleted file mode 100644 index 8d35dcdc01..0000000000 --- a/changes/bug28183 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (Linux seccomp2 sandbox): - - Permit the "shutdown()" system call, which is apparently - used by OpenSSL under some circumstances. Fixes bug 28183; - bugfix on 0.2.5.1-alpha. diff --git a/changes/bug28202 b/changes/bug28202 deleted file mode 100644 index 182daac4f1..0000000000 --- a/changes/bug28202 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (C correctness): - - Avoid undefined behavior in an end-of-string check when parsing the - BEGIN line in a directory object. Fixes bug 28202; bugfix on - 0.2.0.3-alpha. diff --git a/changes/bug28245 b/changes/bug28245 deleted file mode 100644 index d7e6deb810..0000000000 --- a/changes/bug28245 +++ /dev/null @@ -1,6 +0,0 @@ - o Major bugfixes (OpenSSL, portability): - - Fix our usage of named groups when running as a TLS 1.3 client in - OpenSSL 1.1.1. Previously, we only initialized EC groups when running - as a server, which caused clients to fail to negotiate TLS 1.3 with - relays. Fixes bug 28245; bugfix on 0.2.9.15 when TLS 1.3 support was - added. diff --git a/changes/bug28298 b/changes/bug28298 deleted file mode 100644 index 8db340f3df..0000000000 --- a/changes/bug28298 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (configuration): - - Resume refusing to start with relative file paths and RunAsDaemon - set (regression from the fix for bug 22731). Fixes bug 28298; - bugfix on 0.3.3.1-alpha. diff --git a/changes/bug28303 b/changes/bug28303 deleted file mode 100644 index 80f1302e5e..0000000000 --- a/changes/bug28303 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor bugfixes (compilation): - - Fix a pair of missing headers on OpenBSD. Fixes bug 28303; - bugfix on 0.3.5.1-alpha. Patch from Kris Katterjohn. diff --git a/changes/bug28348_034 b/changes/bug28348_034 deleted file mode 100644 index 3913c03a4c..0000000000 --- a/changes/bug28348_034 +++ /dev/null @@ -1,5 +0,0 @@ - o Major bugfixes (embedding, main loop): - - When DisableNetwork becomes set, actually disable periodic events that - are already enabled. (Previously, we would refrain from enabling new - ones, but we would leave the old ones turned on.) - Fixes bug 28348; bugfix on 0.3.4.1-alpha. diff --git a/changes/bug28399 b/changes/bug28399 deleted file mode 100644 index 9096db70b0..0000000000 --- a/changes/bug28399 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (continuous integration, Windows): - - Stop using an external OpenSSL install, and stop installing MSYS2 - packages, when building using mingw on Appveyor Windows CI. - Fixes bug 28399; bugfix on 0.3.4.1-alpha. diff --git a/changes/bug28413 b/changes/bug28413 deleted file mode 100644 index 4c88bea7e7..0000000000 --- a/changes/bug28413 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (compilation): - - Initialize a variable in aes_new_cipher(), since some compilers - cannot tell that we always initialize it before use. Fixes bug 28413; - bugfix on 0.2.9.3-alpha. diff --git a/changes/bug28419 b/changes/bug28419 deleted file mode 100644 index 52ceb0a2a7..0000000000 --- a/changes/bug28419 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor bugfixes (memory leaks): - - Fix a harmless memory leak in libtorrunner.a. Fixes bug 28419; - bugfix on 0.3.3.1-alpha. Patch from Martin Kepplinger. \ No newline at end of file diff --git a/changes/bug28435 b/changes/bug28435 deleted file mode 100644 index 2a886cb8b7..0000000000 --- a/changes/bug28435 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor bugfixes (documentation): - - Make Doxygen work again after the 0.3.5 source tree moves. - Fixes bug 28435; bugfix on 0.3.5.1-alpha. diff --git a/changes/bug28441 b/changes/bug28441 deleted file mode 100644 index d259b9f742..0000000000 --- a/changes/bug28441 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (logging): - - Stop talking about the Named flag in log messages. Clients have - ignored the Named flag since 0.3.2. Fixes bug 28441; - bugfix on 0.3.2.1-alpha. diff --git a/changes/bug28454 b/changes/bug28454 deleted file mode 100644 index ca46ae2777..0000000000 --- a/changes/bug28454 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (continuous integration, Windows): - - Manually configure the zstd compiler options, when building using - mingw on Appveyor Windows CI. The MSYS2 mingw zstd package does not - come with a pkg-config file. Fixes bug 28454; bugfix on 0.3.4.1-alpha. diff --git a/changes/bug28485 b/changes/bug28485 deleted file mode 100644 index a8309ae21f..0000000000 --- a/changes/bug28485 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor bugfixes (compilation): - - Add missing dependency on libgdi32.dll for tor-print-ed-signing-cert.exe - on Windows. Fixes bug 28485; bugfix on 0.3.5.1-alpha. diff --git a/changes/bug28524 b/changes/bug28524 deleted file mode 100644 index 1cad700422..0000000000 --- a/changes/bug28524 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (restart-in-process, boostrap): - - Add missing resets of bootstrap tracking state when shutting - down (regression caused by ticket 27169). Fixes bug 28524; - bugfix on 0.3.5.1-alpha. diff --git a/changes/bug28554 b/changes/bug28554 deleted file mode 100644 index 9a0b281406..0000000000 --- a/changes/bug28554 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor bugfixes (unit tests, guard selection): - - Stop leaking memory in an entry guard unit test. Fixes bug 28554; - bugfix on 0.3.0.1-alpha. diff --git a/changes/bug28562 b/changes/bug28562 deleted file mode 100644 index e14362164d..0000000000 --- a/changes/bug28562 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor bugfixes (testing): - - Use a separate DataDirectory for the test_rebind script. - Previously, this script would run using the default DataDirectory, - and sometimes fail. Fixes bug 28562; bugfix on 0.3.5.1-alpha. - Patch from Taylor R Campbell. diff --git a/changes/bug28619 b/changes/bug28619 deleted file mode 100644 index 86be8cb2fb..0000000000 --- a/changes/bug28619 +++ /dev/null @@ -1,6 +0,0 @@ - o Minor bugfixes (hidden service v3): - - When deleting an ephemeral onion service (DEL_ONION), do not close any - rendezvous circuits in order to let the existing client connections - finish by themselves or closed by the application. The HS v2 is doing - that already so now we have the same behavior for all versions. Fixes - bug 28619; bugfix on 0.3.3.1-alpha. diff --git a/changes/geoip-2018-11-06 b/changes/geoip-2018-11-06 deleted file mode 100644 index 5c18ea4244..0000000000 --- a/changes/geoip-2018-11-06 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor features (geoip): - - Update geoip and geoip6 to the November 6 2018 Maxmind GeoLite2 - Country database. Closes ticket 28395. - diff --git a/changes/geoip-2018-12-05 b/changes/geoip-2018-12-05 deleted file mode 100644 index 20ccf2d8a5..0000000000 --- a/changes/geoip-2018-12-05 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor features (geoip): - - Update geoip and geoip6 to the December 5 2018 Maxmind GeoLite2 - Country database. Closes ticket 28744. - diff --git a/changes/rust_asan b/changes/rust_asan deleted file mode 100644 index 1ca7ae6888..0000000000 --- a/changes/rust_asan +++ /dev/null @@ -1,8 +0,0 @@ - o Major bugfixes (compilation, rust): - - Rust tests can now build and run successfully with the - --enable-fragile-hardening option enabled. - Doing this currently requires the rust beta channel; it will - be possible with stable rust as of rust version 1.31 is out. - Patch from Alex Crichton. - Fixes bugs 27272, 27273, and 27274. - Bugfix on 0.3.1.1-alpha. diff --git a/changes/ticket19566 b/changes/ticket19566 deleted file mode 100644 index bf7071e660..0000000000 --- a/changes/ticket19566 +++ /dev/null @@ -1,6 +0,0 @@ - o Code simplification and refactoring (shared random, dirauth): - - Change many tor_assert() to use BUG() instead. The idea is to not crash - a dirauth but rather scream loudly with a stacktrace and let it continue - run. The shared random subsystem is very resilient and if anything wrong - happens with it, at worst a non coherent value will be put in the vote - and discarded by the other authorities. Closes ticket 19566. diff --git a/changes/ticket24803 b/changes/ticket24803 deleted file mode 100644 index e76a9eeab9..0000000000 --- a/changes/ticket24803 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor features (fallback directory list): - - Replace the 150 fallbacks originally introduced in Tor 0.3.3.1-alpha in - January 2018 (of which ~115 were still functional), with a list of - 157 fallbacks (92 new, 65 existing, 85 removed) generated in - December 2018. Closes ticket 24803. diff --git a/changes/ticket27471 b/changes/ticket27471 deleted file mode 100644 index ffe77d268e..0000000000 --- a/changes/ticket27471 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor bugfixes (hidden service v3, client): - - When replacing a descriptor in the client cache with a newer descriptor, - make sure to close all client introduction circuits of the old - descriptor so we don't end up with unusable leftover circuits. Fixes bug - 27471; bugfix on 0.3.2.1-alpha. diff --git a/changes/ticket27751 b/changes/ticket27751 deleted file mode 100644 index 593c473b61..0000000000 --- a/changes/ticket27751 +++ /dev/null @@ -1,2 +0,0 @@ - o Minor features (continuous integration): - - Add a Travis CI build for --enable-nss on Linux gcc. Closes ticket 27751. diff --git a/changes/ticket27838 b/changes/ticket27838 deleted file mode 100644 index 1699730d7a..0000000000 --- a/changes/ticket27838 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (hidden service v3): - - Build the service descriptor signing key certificate before uploading so - we always have a fresh one leaving no chances for it to expire service - side. Fixes bug 27838; bugfix on 0.3.2.1-alpha. diff --git a/changes/ticket27913 b/changes/ticket27913 deleted file mode 100644 index 81ce725932..0000000000 --- a/changes/ticket27913 +++ /dev/null @@ -1,3 +0,0 @@ - o Testing: - - Add new CI job to Travis configuration that runs stem-based - integration tests. Closes ticket 27913. diff --git a/changes/ticket27995 b/changes/ticket27995 deleted file mode 100644 index 8c75425749..0000000000 --- a/changes/ticket27995 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (hidden service v3, client authorization): - - Fix an assert() when adding a client authorization for the first time - and then sending a HUP signal to the service. Before that, tor would - stop abruptly. Fixes bug 27995; bugfix on 0.3.5.1-alpha. diff --git a/changes/ticket28026 b/changes/ticket28026 deleted file mode 100644 index a6911c2cab..0000000000 --- a/changes/ticket28026 +++ /dev/null @@ -1,3 +0,0 @@ - o Documentation (hidden service manpage): - - Improve HSv3 client authorization by making some options more explicit - and detailed. Closes ticket 28026. Patch by "mtigas". diff --git a/changes/ticket28113 b/changes/ticket28113 deleted file mode 100644 index 30dd825a9b..0000000000 --- a/changes/ticket28113 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor bugfixes (relay shutdown, systemd): - - Notify systemd of ShutdownWaitLength so it can be set to longer than - systemd's TimeoutStopSec. In tor's systemd service file, set - TimeoutSec to 60 seconds, to allow tor some time to shut down. - Fixes bug 28113; bugfix on 0.2.6.2-alpha. diff --git a/changes/ticket28128 b/changes/ticket28128 deleted file mode 100644 index 6d08c74242..0000000000 --- a/changes/ticket28128 +++ /dev/null @@ -1,4 +0,0 @@ - o Documentation (hidden service manpage, sandbox): - - Document in the man page that changing ClientOnionAuthDir value or - adding a new file in the directory will not work at runtime upon sending - a HUP if Sandbox 1. Closes ticket 28128. diff --git a/changes/ticket28229_diag b/changes/ticket28229_diag deleted file mode 100644 index cd02b81faa..0000000000 --- a/changes/ticket28229_diag +++ /dev/null @@ -1,3 +0,0 @@ - o Testing: - - Increase logging and tag all log entries with timestamps - in test_rebind.py. Provides diagnostics for issue 28229. diff --git a/changes/ticket28275 b/changes/ticket28275 deleted file mode 100644 index eadca86b7b..0000000000 --- a/changes/ticket28275 +++ /dev/null @@ -1,4 +0,0 @@ - o Documentation (hidden service v3, man page): - - Note in the man page that the only real way to fully revoke an onion - service v3 client authorization is by restarting the tor process. Closes - ticket 28275. diff --git a/changes/ticket28318 b/changes/ticket28318 deleted file mode 100644 index 24dc1e9580..0000000000 --- a/changes/ticket28318 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor features (Windows, continuous integration): - - Build tor on Windows Server 2012 R2 and Windows Server 2016 using - Appveyor's CI. Closes ticket 28318. diff --git a/changes/ticket28459 b/changes/ticket28459 deleted file mode 100644 index 6b5839b52b..0000000000 --- a/changes/ticket28459 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor features (continuous integration, Windows): - - Always show the configure and test logs, and upload them as build - artifacts, when building for Windows using Appveyor CI. - Implements 28459. diff --git a/changes/ticket28574 b/changes/ticket28574 deleted file mode 100644 index 562810f511..0000000000 --- a/changes/ticket28574 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (continuous integration, Windows): - - Explicitly specify the path to the OpenSSL library and do not download - OpenSSL from Pacman, but instead use the library that is already provided - by AppVeyor. Fixes bug 28574; bugfix on master. diff --git a/changes/ticket28731 b/changes/ticket28731 deleted file mode 100644 index f8b116cc12..0000000000 --- a/changes/ticket28731 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor features (bootstrap): - - Add the bootstrap tag name to the log messages, so people - troubleshooting connection problems can look up a symbol instead - of a number. Closes ticket 28731. From cd14f98115c2110b9f38930b290716aa91ffd950 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 18 Dec 2018 13:33:26 -0500 Subject: [PATCH 0316/2557] Forward-port changelog from 0.3.5.6-rc --- ChangeLog | 84 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/ChangeLog b/ChangeLog index 4c776f7163..a1667aa30a 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,87 @@ +Changes in version 0.3.5.6-rc - 2018-12-18 + Tor 0.3.5.6-rc fixes numerous small bugs in earlier versions of Tor. + It is the first release candidate in the 0.3.5.x series; if no further + huge bugs are found, our next release may be the stable 0.3.5.x. + + o Minor features (continuous integration, Windows): + - Always show the configure and test logs, and upload them as build + artifacts, when building for Windows using Appveyor CI. + Implements 28459. + + o Minor features (fallback directory list): + - Replace the 150 fallbacks originally introduced in Tor + 0.3.3.1-alpha in January 2018 (of which ~115 were still + functional), with a list of 157 fallbacks (92 new, 65 existing, 85 + removed) generated in December 2018. Closes ticket 24803. + + o Minor features (geoip): + - Update geoip and geoip6 to the December 5 2018 Maxmind GeoLite2 + Country database. Closes ticket 28744. + + o Minor bugfixes (compilation): + - Add missing dependency on libgdi32.dll for tor-print-ed-signing- + cert.exe on Windows. Fixes bug 28485; bugfix on 0.3.5.1-alpha. + + o Minor bugfixes (continuous integration, Windows): + - Explicitly specify the path to the OpenSSL library and do not + download OpenSSL from Pacman, but instead use the library that is + already provided by AppVeyor. Fixes bug 28574; bugfix on master. + + o Minor bugfixes (onion service v3): + - When deleting an ephemeral onion service (DEL_ONION), do not close + any rendezvous circuits in order to let the existing client + connections finish by themselves or closed by the application. The + HS v2 is doing that already so now we have the same behavior for + all versions. Fixes bug 28619; bugfix on 0.3.3.1-alpha. + + o Minor bugfixes (restart-in-process, boostrap): + - Add missing resets of bootstrap tracking state when shutting down + (regression caused by ticket 27169). Fixes bug 28524; bugfix + on 0.3.5.1-alpha. + + o Minor bugfixes (testing): + - Use a separate DataDirectory for the test_rebind script. + Previously, this script would run using the default DataDirectory, + and sometimes fail. Fixes bug 28562; bugfix on 0.3.5.1-alpha. + Patch from Taylor R Campbell. + - Stop leaking memory in an entry guard unit test. Fixes bug 28554; + bugfix on 0.3.0.1-alpha. + + o Minor bugfixes (Windows): + - Correctly identify Windows 8.1, Windows 10, and Windows Server + 2008 and later from their NT versions. Fixes bug 28096; bugfix on + 0.2.2.34; reported by Keifer Bly. + - On recent Windows versions, the GetVersionEx() function may report + an earlier Windows version than the running OS. To avoid user + confusion, add "[or later]" to Tor's version string on affected + versions of Windows. Fixes bug 28096; bugfix on 0.2.2.34; reported + by Keifer Bly. + - Remove Windows versions that were never supported by the + GetVersionEx() function. Stop duplicating the latest Windows + version in get_uname(). Fixes bug 28096; bugfix on 0.2.2.34; + reported by Keifer Bly. + + o Testing: + - Increase logging and tag all log entries with timestamps in + test_rebind.py. Provides diagnostics for issue 28229. + + o Code simplification and refactoring (shared random, dirauth): + - Change many tor_assert() to use BUG() instead. The idea is to not + crash a dirauth but rather scream loudly with a stacktrace and let + it continue run. The shared random subsystem is very resilient and + if anything wrong happens with it, at worst a non coherent value + will be put in the vote and discarded by the other authorities. + Closes ticket 19566. + + o Documentation (onion services): + - Document in the man page that changing ClientOnionAuthDir value or + adding a new file in the directory will not work at runtime upon + sending a HUP if Sandbox 1. Closes ticket 28128. + - Note in the man page that the only real way to fully revoke an + onion service v3 client authorization is by restarting the tor + process. Closes ticket 28275. + + Changes in version 0.3.5.5-alpha - 2018-11-16 Tor 0.3.5.5-alpha includes numerous bugfixes on earlier releases, including several that we hope to backport to older release series in From ca7a2ecc512145589964577f2f2ff6133ab146f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Tue, 18 Dec 2018 15:18:22 +0100 Subject: [PATCH 0317/2557] Avoid breaking the event loop prematurely. This patch makes sure that we terminate the event loop from the event loop timer instead of directly in the process' exit handler. This allows us to run the event loop an additional time to ensure that the SleepEx() call on Windows is called and the data from stdout/stderr is delivered to us. Additionally we ensure that we don't try to read or write data from a Unix process that have been terminated in the main loop, since its file descriptors are closed at that time. See: https://bugs.torproject.org/28179 --- src/lib/process/process_unix.c | 21 +++++++++-------- src/test/test_process_slow.c | 42 +++++++++++++++++++++------------- 2 files changed, 37 insertions(+), 26 deletions(-) diff --git a/src/lib/process/process_unix.c b/src/lib/process/process_unix.c index 4fc1b7a519..57ca69a768 100644 --- a/src/lib/process/process_unix.c +++ b/src/lib/process/process_unix.c @@ -112,16 +112,6 @@ process_unix_free_(process_unix_t *unix_process) /* FIXME(ahf): Refactor waitpid code? */ unix_process->waitpid = NULL; - /* Cleanup our events. */ - if (! unix_process->stdout_handle.reached_eof) - process_unix_stop_reading(&unix_process->stdout_handle); - - if (! unix_process->stderr_handle.reached_eof) - process_unix_stop_reading(&unix_process->stderr_handle); - - if (unix_process->stdin_handle.is_writing) - process_unix_stop_writing(&unix_process->stdin_handle); - /* Close all our file descriptors. */ process_unix_close_file_descriptors(unix_process); @@ -668,6 +658,17 @@ process_unix_close_file_descriptors(process_unix_t *unix_process) int ret; bool success = true; + /* Stop reading and writing before we close() our + * file descriptors. */ + if (! unix_process->stdout_handle.reached_eof) + process_unix_stop_reading(&unix_process->stdout_handle); + + if (! unix_process->stderr_handle.reached_eof) + process_unix_stop_reading(&unix_process->stderr_handle); + + if (unix_process->stdin_handle.is_writing) + process_unix_stop_writing(&unix_process->stdin_handle); + if (unix_process->stdin_handle.fd != -1) { ret = close(unix_process->stdin_handle.fd); if (ret == -1) { diff --git a/src/test/test_process_slow.c b/src/test/test_process_slow.c index cc7e8c5a9c..a1f99bff0f 100644 --- a/src/test/test_process_slow.c +++ b/src/test/test_process_slow.c @@ -36,6 +36,7 @@ struct process_data_t { smartlist_t *stderr_data; smartlist_t *stdin_data; process_exit_code_t exit_code; + bool did_exit; }; typedef struct process_data_t process_data_t; @@ -97,13 +98,17 @@ process_stderr_callback(process_t *process, const char *data, size_t size) static bool process_exit_callback(process_t *process, process_exit_code_t exit_code) { + process_status_t status; + tt_ptr_op(process, OP_NE, NULL); process_data_t *process_data = process_get_data(process); process_data->exit_code = exit_code; + process_data->did_exit = true; - /* Our process died. Let's check the values it returned. */ - tor_shutdown_event_loop_and_exit(0); + /* Check if our process is still running? */ + status = process_get_status(process); + tt_int_op(status, OP_EQ, PROCESS_STATUS_NOT_RUNNING); done: /* Do not free up our process_t. */ @@ -137,9 +142,16 @@ main_loop_timeout_cb(periodic_timer_t *timer, void *data) { /* Sanity check. */ tt_ptr_op(timer, OP_EQ, main_loop_timeout_timer); - tt_ptr_op(data, OP_EQ, NULL); + tt_ptr_op(data, OP_NE, NULL); - /* Have we been called 10 times we exit. */ + /* Our process data. */ + process_data_t *process_data = data; + + /* Our process did exit. */ + if (process_data->did_exit) + tor_shutdown_event_loop_and_exit(0); + + /* Have we been called 10 times we exit the main loop. */ timer_tick_count++; tt_int_op(timer_tick_count, OP_LT, 10); @@ -156,7 +168,7 @@ main_loop_timeout_cb(periodic_timer_t *timer, void *data) } static void -run_main_loop(void) +run_main_loop(process_data_t *process_data) { int ret; @@ -167,7 +179,7 @@ run_main_loop(void) main_loop_timeout_timer = periodic_timer_new(tor_libevent_get_base(), &interval, main_loop_timeout_cb, - NULL); + process_data); /* Run our main loop. */ ret = run_main_loop_until_done(); @@ -225,11 +237,7 @@ test_callbacks(void *arg) process_printf(process, " lines?\r\n"); /* Start our main loop. */ - run_main_loop(); - - /* Check if our process is still running? */ - status = process_get_status(process); - tt_int_op(status, OP_EQ, PROCESS_STATUS_NOT_RUNNING); + run_main_loop(process_data); /* We returned. Let's see what our event loop said. */ tt_int_op(smartlist_len(process_data->stdout_data), OP_EQ, 12); @@ -310,14 +318,16 @@ test_callbacks_terminate(void *arg) tt_int_op(status, OP_EQ, PROCESS_STATUS_RUNNING); /* Zap our process. */ - process_terminate(process); + bool success; + + success = process_terminate(process); + tt_assert(success); /* Start our main loop. */ - run_main_loop(); + run_main_loop(process_data); - /* Check if our process is still running? */ - status = process_get_status(process); - tt_int_op(status, OP_EQ, PROCESS_STATUS_NOT_RUNNING); + /* Check if we did exit. */ + tt_assert(process_data->did_exit); done: process_data_free(process_data); From 44a80bb36115ebf865ba688a0f0d0f8023d992d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Mon, 17 Dec 2018 23:41:53 +0100 Subject: [PATCH 0318/2557] Add missing changes files for #26360, #28179, #28180. See: https://bugs.torproject.org/26179 See: https://bugs.torproject.org/28180 See: https://bugs.torproject.org/28360 --- changes/ticket26360 | 4 ++++ changes/ticket28179 | 5 +++++ changes/ticket28180 | 3 +++ 3 files changed, 12 insertions(+) create mode 100644 changes/ticket26360 create mode 100644 changes/ticket28179 create mode 100644 changes/ticket28180 diff --git a/changes/ticket26360 b/changes/ticket26360 new file mode 100644 index 0000000000..80afbd1c17 --- /dev/null +++ b/changes/ticket26360 @@ -0,0 +1,4 @@ + o Minor bugfixes (pluggable transports): + - Make sure that data is continously read from standard out and error of the + PT child-process to avoid deadlocking when the pipes' buffer is full. + Fixes bug 26360; bugfix on 0.2.3.6-alpha. diff --git a/changes/ticket28179 b/changes/ticket28179 new file mode 100644 index 0000000000..f548c4a79a --- /dev/null +++ b/changes/ticket28179 @@ -0,0 +1,5 @@ + o Minor features (process): + - Add new Process API for handling child processes. This + new API allows Tor to have bi-directional + communication with child processes on both Unix and Windows. + Closes ticket 28179. diff --git a/changes/ticket28180 b/changes/ticket28180 new file mode 100644 index 0000000000..2ec547bd47 --- /dev/null +++ b/changes/ticket28180 @@ -0,0 +1,3 @@ + o Minor features (pluggable transports): + - Add support for logging to Tor's logging subsystem from a pluggable + transport process. Partial implementation for ticket 28180 From 0556942284d7dcdf0a5e7a31e94b925378a338a8 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 14 Dec 2018 13:17:17 -0500 Subject: [PATCH 0319/2557] Use a single path for all PEM-like objects in get_next_token() Previously, we would decode the PEM wrapper for keys twice: once in get_next_token, and once later in PEM decode. Now we just do all of the wrapper and base64 stuff in get_next_token, and store the base64-decoded part in the token object for keys and non-keys alike. This change should speed up parsing slightly by letting us skip a bunch of stuff in crypto_pk_read_*from_string(), including the tag detection parts of pem_decode(), and an extra key allocation and deallocation pair. Retaining the base64-decoded part in the token object will allow us to speed up our microdesc parsing, since it is the asn1 portion that we actually want to retain. --- src/feature/dirparse/parsecommon.c | 24 +++++++++++++----------- src/test/test_parsecommon.c | 8 ++++---- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/src/feature/dirparse/parsecommon.c b/src/feature/dirparse/parsecommon.c index 91b775533b..2e7cea8169 100644 --- a/src/feature/dirparse/parsecommon.c +++ b/src/feature/dirparse/parsecommon.c @@ -266,7 +266,7 @@ get_next_token(memarea_t *area, * attack, a bug, or some other nonsense. */ #define MAX_LINE_LENGTH (128*1024) - const char *next, *eol, *obstart; + const char *next, *eol; size_t obname_len; int i; directory_token_t *tok; @@ -352,7 +352,6 @@ get_next_token(memarea_t *area, if (!eol || eol-*s<11 || strcmpstart(*s, "-----BEGIN ")) /* No object. */ goto check_object; - obstart = *s; /* Set obstart to start of object spec */ if (eol - *s <= 16 || memchr(*s+11,'\0',eol-*s-16) || /* no short lines, */ strcmp_len(eol-5, "-----", 5) || /* nuls or invalid endings */ (eol-*s) > MAX_UNPARSED_OBJECT_SIZE) { /* name too long */ @@ -383,15 +382,7 @@ get_next_token(memarea_t *area, if (next - *s > MAX_UNPARSED_OBJECT_SIZE) RET_ERR("Couldn't parse object: missing footer or object much too big."); - if (!strcmp(tok->object_type, "RSA PUBLIC KEY")) { /* If it's a public key */ - tok->key = crypto_pk_new(); - if (crypto_pk_read_public_key_from_string(tok->key, obstart, eol-obstart)) - RET_ERR("Couldn't parse public key."); - } else if (!strcmp(tok->object_type, "RSA PRIVATE KEY")) { /* private key */ - tok->key = crypto_pk_new(); - if (crypto_pk_read_private_key_from_string(tok->key, obstart, eol-obstart)) - RET_ERR("Couldn't parse private key."); - } else { /* If it's something else, try to base64-decode it */ + { int r; size_t maxsize = base64_decode_maxsize(next-*s); tok->object_body = ALLOC(maxsize); @@ -400,6 +391,17 @@ get_next_token(memarea_t *area, RET_ERR("Malformed object: bad base64-encoded data"); tok->object_size = r; } + + if (!strcmp(tok->object_type, "RSA PUBLIC KEY")) { /* If it's a public key */ + tok->key = crypto_pk_asn1_decode(tok->object_body, tok->object_size); + if (! tok->key) + RET_ERR("Couldn't parse public key."); + } else if (!strcmp(tok->object_type, "RSA PRIVATE KEY")) { /* private key */ + tok->key = crypto_pk_asn1_decode_private(tok->object_body, + tok->object_size); + if (! tok->key) + RET_ERR("Couldn't parse private key."); + } *s = eol; check_object: diff --git a/src/test/test_parsecommon.c b/src/test/test_parsecommon.c index 6da125dd0a..8e74fcdb4d 100644 --- a/src/test/test_parsecommon.c +++ b/src/test/test_parsecommon.c @@ -300,8 +300,8 @@ test_parsecommon_get_next_token_parse_keys(void *arg) tt_int_op(token->tp, OP_EQ, R_IPO_ONION_KEY); tt_int_op(token->n_args, OP_EQ, 0); tt_str_op(token->object_type, OP_EQ, "RSA PUBLIC KEY"); - tt_int_op(token->object_size, OP_EQ, 0); - tt_assert(!token->object_body); + tt_int_op(token->object_size, OP_EQ, 140); + tt_assert(token->object_body); tt_assert(token->key); tt_assert(!token->error); @@ -335,8 +335,8 @@ test_parsecommon_get_next_token_parse_keys(void *arg) tt_int_op(token2->tp, OP_EQ, C_CLIENT_KEY); tt_int_op(token2->n_args, OP_EQ, 0); tt_str_op(token2->object_type, OP_EQ, "RSA PRIVATE KEY"); - tt_int_op(token2->object_size, OP_EQ, 0); - tt_assert(!token2->object_body); + tt_int_op(token2->object_size, OP_EQ, 608); + tt_assert(token2->object_body); tt_assert(token2->key); tt_assert(!token->error); From 7113a339dc7c680b9e48565bfaf97a10185a5244 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 14 Dec 2018 13:28:07 -0500 Subject: [PATCH 0320/2557] Avoid a needless decode/re-encode step in assigning onion keys Previously we had decoded the asn.1 to get a public key, and then discarded the asn.1 so that we had to re-encode the key to store it in the onion_pkey field of a microdesc_t or routerinfo_t. Now we can just do a tor_memdup() instead, which should be loads faster. --- src/feature/dirparse/microdesc_parse.c | 4 ++-- src/feature/dirparse/routerparse.c | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/feature/dirparse/microdesc_parse.c b/src/feature/dirparse/microdesc_parse.c index 8ad9626377..165945e392 100644 --- a/src/feature/dirparse/microdesc_parse.c +++ b/src/feature/dirparse/microdesc_parse.c @@ -177,8 +177,8 @@ microdescs_parse_from_string(const char *s, const char *eos, "Relay's onion key had invalid exponent."); goto next; } - router_set_rsa_onion_pkey(tok->key, &md->onion_pkey, - &md->onion_pkey_len); + md->onion_pkey = tor_memdup(tok->object_body, tok->object_size); + md->onion_pkey_len = tok->object_size; crypto_pk_free(tok->key); if ((tok = find_opt_by_keyword(tokens, K_ONION_KEY_NTOR))) { diff --git a/src/feature/dirparse/routerparse.c b/src/feature/dirparse/routerparse.c index 2249ab7cbc..358f6e44e8 100644 --- a/src/feature/dirparse/routerparse.c +++ b/src/feature/dirparse/routerparse.c @@ -588,8 +588,8 @@ router_parse_entry_from_string(const char *s, const char *end, "Relay's onion key had invalid exponent."); goto err; } - router_set_rsa_onion_pkey(tok->key, &router->onion_pkey, - &router->onion_pkey_len); + router->onion_pkey = tor_memdup(tok->object_body, tok->object_size); + router->onion_pkey_len = tok->object_size; crypto_pk_free(tok->key); if ((tok = find_opt_by_keyword(tokens, K_ONION_KEY_NTOR))) { From 976c62e62a38c9f30c32ca742a43d59633a0e6ab Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 14 Dec 2018 13:34:32 -0500 Subject: [PATCH 0321/2557] Changes file for ticket28839 --- changes/ticket28839 | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 changes/ticket28839 diff --git a/changes/ticket28839 b/changes/ticket28839 new file mode 100644 index 0000000000..e9f81dc405 --- /dev/null +++ b/changes/ticket28839 @@ -0,0 +1,3 @@ + o Minor features (performance): + - Speed up microdesriptor parsing by about 30%, to help + improve startup time. Closes ticket 28839. From 426c52b377057dc5f4428c664ee56ca77d648c9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Thu, 20 Dec 2018 02:10:42 +0100 Subject: [PATCH 0322/2557] Use K/V parser to handle LOG messages for pluggable transports. This patch changes the LOG pluggable transport message to use the recent K/V parser that landed in Tor. This allows PT's to specify the log severity level as well as the message. A mapping between the PT log severity levels and Tor's log serverity level is provided. See: https://bugs.torproject.org/28846 --- src/feature/client/transports.c | 80 ++++++++++++++++++++++++++++++--- src/feature/client/transports.h | 2 + src/feature/control/control.c | 11 +++-- src/feature/control/control.h | 2 +- src/test/test_pt.c | 29 +++++++++--- 5 files changed, 106 insertions(+), 18 deletions(-) diff --git a/src/feature/client/transports.c b/src/feature/client/transports.c index de53fe3469..6a3479a470 100644 --- a/src/feature/client/transports.c +++ b/src/feature/client/transports.c @@ -101,6 +101,8 @@ #include "core/or/connection_or.h" #include "feature/relay/ext_orport.h" #include "feature/control/control.h" +#include "lib/encoding/confline.h" +#include "lib/encoding/kvline.h" #include "lib/process/process.h" #include "lib/process/env.h" @@ -1144,22 +1146,63 @@ parse_log_line(const char *line, managed_proxy_t *mp) tor_assert(line); tor_assert(mp); + config_line_t *values = NULL; + char *log_message = NULL; + if (strlen(line) < (strlen(PROTO_LOG) + 1)) { log_warn(LD_PT, "Managed proxy sent us a %s line " "with missing argument.", PROTO_LOG); goto done; } - const char *message = line + strlen(PROTO_LOG) + 1; + const char *data = line + strlen(PROTO_LOG) + 1; + values = kvline_parse(data, KV_QUOTED); - log_info(LD_PT, "Managed proxy \"%s\" says: %s", - mp->argv[0], message); + if (! values) { + log_warn(LD_PT, "Managed proxy \"%s\" wrote an invalid LOG message: %s", + mp->argv[0], data); + goto done; + } + + const config_line_t *severity = config_line_find(values, "SEVERITY"); + const config_line_t *message = config_line_find(values, "MESSAGE"); + + /* Check if we got a message. */ + if (! message) { + log_warn(LD_PT, "Managed proxy \"%s\" wrote a LOG line without " + "MESSAGE: %s", mp->argv[0], data); + goto done; + } + + /* Check if severity is there and whether it's valid. */ + if (! severity) { + log_warn(LD_PT, "Managed proxy \"%s\" wrote a LOG line without " + "SEVERITY: %s", mp->argv[0], data); + goto done; + } + + int log_severity = managed_proxy_severity_parse(severity->value); + + if (log_severity == -1) { + log_warn(LD_PT, "Managed proxy \"%s\" wrote a LOG line with an " + "invalid severity level: %s", + mp->argv[0], severity->value); + goto done; + } + + tor_log(log_severity, LD_PT, "Managed proxy \"%s\": %s", + mp->argv[0], message->value); + + /* Prepend the PT name. */ + config_line_prepend(&values, "PT", mp->argv[0]); + log_message = kvline_encode(values, KV_QUOTED); /* Emit control port event. */ - control_event_pt_log(mp->argv[0], message); + control_event_pt_log(log_message); done: - return; + config_free_lines(values); + tor_free(log_message); } /** Return a newly allocated string that tor should place in @@ -1779,3 +1822,30 @@ managed_proxy_exit_callback(process_t *process, process_exit_code_t exit_code) return true; } + +/** Returns a valid integer log severity level from severity that + * is compatible with Tor's logging functions. Returns -1 on + * error. */ +STATIC int +managed_proxy_severity_parse(const char *severity) +{ + tor_assert(severity); + + /* Slightly different than log.c's parse_log_level :-( */ + if (! strcmp(severity, "debug")) + return LOG_DEBUG; + + if (! strcmp(severity, "info")) + return LOG_INFO; + + if (! strcmp(severity, "notice")) + return LOG_NOTICE; + + if (! strcmp(severity, "warning")) + return LOG_WARN; + + if (! strcmp(severity, "error")) + return LOG_ERR; + + return -1; +} diff --git a/src/feature/client/transports.h b/src/feature/client/transports.h index a3994a0099..b8d1bb0081 100644 --- a/src/feature/client/transports.h +++ b/src/feature/client/transports.h @@ -147,6 +147,8 @@ STATIC void managed_proxy_stdout_callback(process_t *, const char *, size_t); STATIC void managed_proxy_stderr_callback(process_t *, const char *, size_t); STATIC bool managed_proxy_exit_callback(process_t *, process_exit_code_t); +STATIC int managed_proxy_severity_parse(const char *); + #endif /* defined(PT_PRIVATE) */ #endif /* !defined(TOR_TRANSPORTS_H) */ diff --git a/src/feature/control/control.c b/src/feature/control/control.c index 7fae3b7a1b..849f11707e 100644 --- a/src/feature/control/control.c +++ b/src/feature/control/control.c @@ -7033,15 +7033,14 @@ control_event_transport_launched(const char *mode, const char *transport_name, mode, transport_name, fmt_addr(addr), port); } -/** A pluggable transport called pt_name has emitted a log - * message found in message. */ +/** A pluggable transport called pt_name has emitted a log message + * found in message at severity log level. */ void -control_event_pt_log(const char *pt_name, const char *message) +control_event_pt_log(const char *log) { send_control_event(EVENT_PT_LOG, - "650 PT_LOG %s %s\r\n", - pt_name, - message); + "650 PT_LOG %s\r\n", + log); } /** Convert rendezvous auth type to string for HS_DESC control events diff --git a/src/feature/control/control.h b/src/feature/control/control.h index a1609b0f06..b3a2707672 100644 --- a/src/feature/control/control.h +++ b/src/feature/control/control.h @@ -207,7 +207,7 @@ void control_event_clients_seen(const char *controller_str); void control_event_transport_launched(const char *mode, const char *transport_name, tor_addr_t *addr, uint16_t port); -void control_event_pt_log(const char *pt_name, const char *message); +void control_event_pt_log(const char *log); const char *rend_auth_type_to_string(rend_auth_type_t auth_type); MOCK_DECL(const char *, node_describe_longname_by_id,(const char *id_digest)); void control_event_hs_descriptor_requested(const char *onion_address, diff --git a/src/test/test_pt.c b/src/test/test_pt.c index 8fcdd5c1e8..271d74f26a 100644 --- a/src/test/test_pt.c +++ b/src/test/test_pt.c @@ -310,8 +310,12 @@ process_read_stdout_replacement(process_t *process, buf_t *buffer) } else if (times_called <= 6) { buf_add_string(buffer, "SMETHODS DONE\n"); } else if (times_called <= 7) { - buf_add_string(buffer, "LOG Oh noes, something bad happened. " - "What do we do!?\n"); + buf_add_string(buffer, "LOG SEVERITY=error MESSAGE=\"Oh noes, something " + "bad happened. What do we do!?\"\n"); + buf_add_string(buffer, "LOG SEVERITY=warning MESSAGE=\"warning msg\"\n"); + buf_add_string(buffer, "LOG SEVERITY=notice MESSAGE=\"notice msg\"\n"); + buf_add_string(buffer, "LOG SEVERITY=info MESSAGE=\"info msg\"\n"); + buf_add_string(buffer, "LOG SEVERITY=debug MESSAGE=\"debug msg\"\n"); } return (int)buf_datalen(buffer); @@ -416,12 +420,25 @@ test_pt_configure_proxy(void *arg) /* Get the log message out. */ process_notify_event_stdout(mp->process); - tt_int_op(controlevent_n, OP_EQ, 6); + tt_int_op(controlevent_n, OP_EQ, 10); tt_int_op(controlevent_event, OP_EQ, EVENT_PT_LOG); - tt_int_op(smartlist_len(controlevent_msgs), OP_EQ, 6); + tt_int_op(smartlist_len(controlevent_msgs), OP_EQ, 10); tt_str_op(smartlist_get(controlevent_msgs, 5), OP_EQ, - "650 PT_LOG Oh noes, something bad happened. " - "What do we do!?\r\n"); + "650 PT_LOG PT= SEVERITY=error " + "MESSAGE=\"Oh noes, " + "something bad happened. What do we do!?\"\r\n"); + tt_str_op(smartlist_get(controlevent_msgs, 6), OP_EQ, + "650 PT_LOG PT= SEVERITY=warning " + "MESSAGE=\"warning msg\"\r\n"); + tt_str_op(smartlist_get(controlevent_msgs, 7), OP_EQ, + "650 PT_LOG PT= SEVERITY=notice " + "MESSAGE=\"notice msg\"\r\n"); + tt_str_op(smartlist_get(controlevent_msgs, 8), OP_EQ, + "650 PT_LOG PT= SEVERITY=info " + "MESSAGE=\"info msg\"\r\n"); + tt_str_op(smartlist_get(controlevent_msgs, 9), OP_EQ, + "650 PT_LOG PT= SEVERITY=debug " + "MESSAGE=\"debug msg\"\r\n"); { /* check that the transport info were saved properly in the tor state */ config_line_t *transport_in_state = NULL; From 4efe4cc2f918ce45075b010d2cdc09ec7791ac6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Thu, 20 Dec 2018 03:55:02 +0100 Subject: [PATCH 0323/2557] Add support for STATUS messages from Pluggable Transports. This patch adds support for the new STATUS message that PT's can emit from their standard out. The STATUS message uses the `config_line_t` K/V format that was recently added in Tor. See: https://bugs.torproject.org/28846 --- src/feature/client/transports.c | 60 +++++++++++++++++++++++++++++++-- src/feature/client/transports.h | 1 + src/feature/control/control.c | 10 ++++++ src/feature/control/control.h | 4 ++- src/test/test_pt.c | 21 ++++++++++++ 5 files changed, 92 insertions(+), 4 deletions(-) diff --git a/src/feature/client/transports.c b/src/feature/client/transports.c index 6a3479a470..45dbd0c888 100644 --- a/src/feature/client/transports.c +++ b/src/feature/client/transports.c @@ -130,6 +130,7 @@ static void parse_method_error(const char *line, int is_server_method); #define PROTO_PROXY_DONE "PROXY DONE" #define PROTO_PROXY_ERROR "PROXY-ERROR" #define PROTO_LOG "LOG" +#define PROTO_STATUS "STATUS" /** The first and only supported - at the moment - configuration protocol version. */ @@ -912,12 +913,16 @@ handle_proxy_line(const char *line, managed_proxy_t *mp) parse_proxy_error(line); goto err; - /* We check for the additional " " after the PROTO_LOG string to make sure - * we can later extend this big if/else-if table with something that begins - * with "LOG" without having to get the order right. */ + /* We check for the additional " " after the PROTO_LOG * PROTO_STATUS + * string to make sure we can later extend this big if/else-if table with + * something that begins with "LOG" without having to get the order right. + * */ } else if (!strcmpstart(line, PROTO_LOG " ")) { parse_log_line(line, mp); return; + } else if (!strcmpstart(line, PROTO_STATUS " ")) { + parse_status_line(line, mp); + return; } log_notice(LD_GENERAL, "Unknown line received by managed proxy (%s).", line); @@ -1205,6 +1210,55 @@ parse_log_line(const char *line, managed_proxy_t *mp) tor_free(log_message); } +/** Parses a STATUS line and emit control events accordingly. */ +STATIC void +parse_status_line(const char *line, managed_proxy_t *mp) +{ + tor_assert(line); + tor_assert(mp); + + config_line_t *values = NULL; + char *status_message = NULL; + + if (strlen(line) < (strlen(PROTO_STATUS) + 1)) { + log_warn(LD_PT, "Managed proxy sent us a %s line " + "with missing argument.", PROTO_STATUS); + goto done; + } + + const char *data = line + strlen(PROTO_STATUS) + 1; + + values = kvline_parse(data, KV_QUOTED); + + if (! values) { + log_warn(LD_PT, "Managed proxy \"%s\" wrote an invalid " + "STATUS message: %s", mp->argv[0], data); + goto done; + } + + /* We check if we received the TYPE parameter, which is the only *required* + * value. */ + const config_line_t *type = config_line_find(values, "TYPE"); + + if (! type) { + log_warn(LD_PT, "Managed proxy \"%s\" wrote a STATUS line without " + "TYPE: %s", mp->argv[0], data); + goto done; + } + + /* Prepend the PT name. */ + config_line_prepend(&values, "PT", mp->argv[0]); + status_message = kvline_encode(values, KV_QUOTED); + + /* We have checked that TYPE is there, we can now emit the STATUS event via + * the control port. */ + control_event_pt_status(status_message); + + done: + config_free_lines(values); + tor_free(status_message); +} + /** Return a newly allocated string that tor should place in * TOR_PT_SERVER_TRANSPORT_OPTIONS while configuring the server * manged proxy in mp. Return NULL if no such options are found. */ diff --git a/src/feature/client/transports.h b/src/feature/client/transports.h index b8d1bb0081..1a910ae82c 100644 --- a/src/feature/client/transports.h +++ b/src/feature/client/transports.h @@ -129,6 +129,7 @@ STATIC void parse_env_error(const char *line); STATIC void parse_proxy_error(const char *line); STATIC void handle_proxy_line(const char *line, managed_proxy_t *mp); STATIC void parse_log_line(const char *line, managed_proxy_t *mp); +STATIC void parse_status_line(const char *line, managed_proxy_t *mp); STATIC char *get_transport_options_for_server_proxy(const managed_proxy_t *mp); STATIC void managed_proxy_destroy(managed_proxy_t *mp, diff --git a/src/feature/control/control.c b/src/feature/control/control.c index 849f11707e..4ef550c919 100644 --- a/src/feature/control/control.c +++ b/src/feature/control/control.c @@ -7043,6 +7043,16 @@ control_event_pt_log(const char *log) log); } +/** A pluggable transport has emitted a STATUS message found in + * status. */ +void +control_event_pt_status(const char *status) +{ + send_control_event(EVENT_PT_STATUS, + "650 PT_STATUS %s\r\n", + status); +} + /** Convert rendezvous auth type to string for HS_DESC control events */ const char * diff --git a/src/feature/control/control.h b/src/feature/control/control.h index b3a2707672..d78ce4d87c 100644 --- a/src/feature/control/control.h +++ b/src/feature/control/control.h @@ -208,6 +208,7 @@ void control_event_transport_launched(const char *mode, const char *transport_name, tor_addr_t *addr, uint16_t port); void control_event_pt_log(const char *log); +void control_event_pt_status(const char *status); const char *rend_auth_type_to_string(rend_auth_type_t auth_type); MOCK_DECL(const char *, node_describe_longname_by_id,(const char *id_digest)); void control_event_hs_descriptor_requested(const char *onion_address, @@ -297,7 +298,8 @@ void control_free_all(void); #define EVENT_HS_DESC_CONTENT 0x0022 #define EVENT_NETWORK_LIVENESS 0x0023 #define EVENT_PT_LOG 0x0024 -#define EVENT_MAX_ 0x0024 +#define EVENT_PT_STATUS 0x0025 +#define EVENT_MAX_ 0x0025 /* sizeof(control_connection_t.event_mask) in bits, currently a uint64_t */ #define EVENT_CAPACITY_ 0x0040 diff --git a/src/test/test_pt.c b/src/test/test_pt.c index 271d74f26a..8de687c866 100644 --- a/src/test/test_pt.c +++ b/src/test/test_pt.c @@ -316,6 +316,10 @@ process_read_stdout_replacement(process_t *process, buf_t *buffer) buf_add_string(buffer, "LOG SEVERITY=notice MESSAGE=\"notice msg\"\n"); buf_add_string(buffer, "LOG SEVERITY=info MESSAGE=\"info msg\"\n"); buf_add_string(buffer, "LOG SEVERITY=debug MESSAGE=\"debug msg\"\n"); + } else if (times_called <= 8) { + buf_add_string(buffer, "STATUS TYPE=a K_1=a K_2=b K_3=\"foo bar\"\n"); + buf_add_string(buffer, "STATUS TYPE=b K_1=a K_2=b K_3=\"foo bar\"\n"); + buf_add_string(buffer, "STATUS TYPE=c K_1=a K_2=b K_3=\"foo bar\"\n"); } return (int)buf_datalen(buffer); @@ -440,6 +444,23 @@ test_pt_configure_proxy(void *arg) "650 PT_LOG PT= SEVERITY=debug " "MESSAGE=\"debug msg\"\r\n"); + /* Get the STATUS messages out. */ + process_notify_event_stdout(mp->process); + + tt_int_op(controlevent_n, OP_EQ, 13); + tt_int_op(controlevent_event, OP_EQ, EVENT_PT_STATUS); + tt_int_op(smartlist_len(controlevent_msgs), OP_EQ, 13); + + tt_str_op(smartlist_get(controlevent_msgs, 10), OP_EQ, + "650 PT_STATUS " + "PT= TYPE=a K_1=a K_2=b K_3=\"foo bar\"\r\n"); + tt_str_op(smartlist_get(controlevent_msgs, 11), OP_EQ, + "650 PT_STATUS " + "PT= TYPE=b K_1=a K_2=b K_3=\"foo bar\"\r\n"); + tt_str_op(smartlist_get(controlevent_msgs, 12), OP_EQ, + "650 PT_STATUS " + "PT= TYPE=c K_1=a K_2=b K_3=\"foo bar\"\r\n"); + { /* check that the transport info were saved properly in the tor state */ config_line_t *transport_in_state = NULL; smartlist_t *transport_info_sl = smartlist_new(); From e2c36b9ca064580208ac0976aaad28f08f3a744b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Thu, 20 Dec 2018 04:08:04 +0100 Subject: [PATCH 0324/2557] Add and update changes files for #28846. This ticket finishes the implementatoin of #28180 and adds the new STATUS message from #28846. See: https://bugs.torproject.org/28846 --- changes/ticket28180 | 2 +- changes/ticket28846 | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 changes/ticket28846 diff --git a/changes/ticket28180 b/changes/ticket28180 index 2ec547bd47..59de1c6251 100644 --- a/changes/ticket28180 +++ b/changes/ticket28180 @@ -1,3 +1,3 @@ o Minor features (pluggable transports): - Add support for logging to Tor's logging subsystem from a pluggable - transport process. Partial implementation for ticket 28180 + transport process. Closes ticket 28180 diff --git a/changes/ticket28846 b/changes/ticket28846 new file mode 100644 index 0000000000..efb5b9938e --- /dev/null +++ b/changes/ticket28846 @@ -0,0 +1,3 @@ + o Minor features (pluggable transports): + - Add support for emitting STATUS updates to Tor's control port from a + pluggable transport process. Closes ticket 28846. From fe2f4f3ec58ef03e8d3fe802283b848d54b5eb76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Thu, 20 Dec 2018 12:45:52 +0100 Subject: [PATCH 0325/2557] Remember to check for whether we actually did exit in tests. See: https://bugs.torproject.org/28179 --- src/test/test_process_slow.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test/test_process_slow.c b/src/test/test_process_slow.c index a1f99bff0f..3f8c07f312 100644 --- a/src/test/test_process_slow.c +++ b/src/test/test_process_slow.c @@ -242,6 +242,7 @@ test_callbacks(void *arg) /* We returned. Let's see what our event loop said. */ tt_int_op(smartlist_len(process_data->stdout_data), OP_EQ, 12); tt_int_op(smartlist_len(process_data->stderr_data), OP_EQ, 3); + tt_assert(process_data->did_exit); tt_int_op(process_data->exit_code, OP_EQ, 0); /* Check stdout output. */ From 1d8dcb416c989ad86a1e3ae2aa92f4c2c1339183 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Thu, 20 Dec 2018 12:47:04 +0100 Subject: [PATCH 0326/2557] Remember to close the child process' ends of the pipes. This prevents us from leaking the HANDLE for stdout, stderr, and stdin. See: https://bugs.torproject.org/28179 --- src/lib/process/process_win32.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/lib/process/process_win32.c b/src/lib/process/process_win32.c index c92b0975a7..a09664b501 100644 --- a/src/lib/process/process_win32.c +++ b/src/lib/process/process_win32.c @@ -248,6 +248,11 @@ process_win32_exec(process_t *process) win32_process->stderr_handle.pipe = stderr_pipe_read; win32_process->stdin_handle.pipe = stdin_pipe_write; + /* Close our ends of the pipes that is now owned by the child process. */ + CloseHandle(stdout_pipe_write); + CloseHandle(stderr_pipe_write); + CloseHandle(stdin_pipe_read); + /* Used by the callback functions from ReadFileEx() and WriteFileEx() such * that we can figure out which process_t that was responsible for the event. * From 44586a89ef636b0d3f736e44a1d2fc6497080bfc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Thu, 20 Dec 2018 12:53:28 +0100 Subject: [PATCH 0327/2557] Delay checking process for termination until both stdout and stderr are closed. This patch makes us delay checking for whether we have an exit code value (via GetExitCodeProcess()) until both stdout and stderr have been closed by the operating system either by the process itself or by process cleanup after termination. See: https://bugs.torproject.org/28179 --- src/lib/process/process_win32.c | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/lib/process/process_win32.c b/src/lib/process/process_win32.c index a09664b501..52acf49370 100644 --- a/src/lib/process/process_win32.c +++ b/src/lib/process/process_win32.c @@ -525,6 +525,28 @@ process_win32_timer_test_process(process_t *process) BOOL ret = FALSE; DWORD exit_code = 0; + /* Sometimes the Windows kernel wont give us the EOF/Broken Pipe error + * message until some time after the process have actually terminated. We + * make sure that our ReadFileEx() calls for the process have *all* returned + * and both standard out and error have been marked as EOF before we try to + * see if the process terminated. + * + * This ensures that we *never* call the exit callback of the `process_t`, + * which potentially ends up calling `process_free()` on our `process_t`, + * before all data have been received from the process. + * + * We do NOT have a check here for whether standard in reached EOF since + * standard in's WriteFileEx() function is only called on-demand when we have + * something to write and is thus usually not awaiting to finish any + * operations. If we WriteFileEx() to a file that has terminated we'll simply + * get an error from ReadFileEx() or its completion routine and move on with + * life. */ + if (! win32_process->stdout_handle.reached_eof) + return false; + + if (! win32_process->stderr_handle.reached_eof) + return false; + /* We start by testing whether our process is still running. */ ret = GetExitCodeProcess(win32_process->process_information.hProcess, &exit_code); From c6e041e3d8dcc6f887014f9dd8887faebf5f4a49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Thu, 20 Dec 2018 12:57:20 +0100 Subject: [PATCH 0328/2557] Handle errors even after success from ReadFileEx() and WriteFileEx(). This patch adds some additional error checking after calls to ReadFileEx() and WriteFileEx(). I have not managed to get this code to reach the branch where `error_code` is NOT `ERROR_SUCCESS`, but MSDN says one should check for this condition so we do so just to be safe. See: https://bugs.torproject.org/28179 --- src/lib/process/process_win32.c | 50 +++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/src/lib/process/process_win32.c b/src/lib/process/process_win32.c index 52acf49370..71dd4001c8 100644 --- a/src/lib/process/process_win32.c +++ b/src/lib/process/process_win32.c @@ -329,6 +329,7 @@ process_win32_write(struct process_t *process, buf_t *buffer) process_win32_t *win32_process = process_get_win32_process(process); BOOL ret = FALSE; + DWORD error_code = 0; const size_t buffer_size = buf_datalen(buffer); /* Windows is still writing our buffer. */ @@ -350,6 +351,12 @@ process_win32_write(struct process_t *process, buf_t *buffer) /* Read data from the process_t buffer into our intermediate buffer. */ buf_get_bytes(buffer, win32_process->stdin_handle.buffer, write_size); + /* Because of the slightly weird API for WriteFileEx() we must set this to 0 + * before we call WriteFileEx() because WriteFileEx() does not reset the last + * error itself when it's succesful. See comment below after the call to + * GetLastError(). */ + SetLastError(0); + /* Schedule our write. */ ret = WriteFileEx(win32_process->stdin_handle.pipe, win32_process->stdin_handle.buffer, @@ -363,6 +370,24 @@ process_win32_write(struct process_t *process, buf_t *buffer) return 0; } + /* Here be dragons: According to MSDN's documentation for WriteFileEx() we + * should check GetLastError() after a call to WriteFileEx() even though the + * `ret` return value was successful. If everything is good, GetLastError() + * returns `ERROR_SUCCESS` and nothing happens. + * + * XXX(ahf): I have not managed to trigger this code while stress-testing + * this code. */ + error_code = GetLastError(); + + if (error_code != ERROR_SUCCESS) { + /* LCOV_EXCL_START */ + log_warn(LD_PROCESS, "WriteFileEx() failed after returning success: %s", + format_win32_error(error_code)); + win32_process->stdin_handle.reached_eof = true; + return 0; + /* LCOV_EXCL_STOP */ + } + /* This cast should be safe since our buffer can maximum be BUFFER_SIZE * large. */ return (int)write_size; @@ -864,6 +889,7 @@ process_win32_read_from_handle(process_win32_handle_t *handle, BOOL ret = FALSE; int bytes_available = 0; + DWORD error_code = 0; /* We already have a request to read data that isn't complete yet. */ if (BUG(handle->busy)) @@ -887,6 +913,12 @@ process_win32_read_from_handle(process_win32_handle_t *handle, memset(handle->buffer, 0, sizeof(handle->buffer)); } + /* Because of the slightly weird API for ReadFileEx() we must set this to 0 + * before we call ReadFileEx() because ReadFileEx() does not reset the last + * error itself when it's succesful. See comment below after the call to + * GetLastError(). */ + SetLastError(0); + /* Ask the Windows kernel to read data from our pipe into our buffer and call * the callback function when it is done. */ ret = ReadFileEx(handle->pipe, @@ -901,6 +933,24 @@ process_win32_read_from_handle(process_win32_handle_t *handle, return bytes_available; } + /* Here be dragons: According to MSDN's documentation for ReadFileEx() we + * should check GetLastError() after a call to ReadFileEx() even though the + * `ret` return value was successful. If everything is good, GetLastError() + * returns `ERROR_SUCCESS` and nothing happens. + * + * XXX(ahf): I have not managed to trigger this code while stress-testing + * this code. */ + error_code = GetLastError(); + + if (error_code != ERROR_SUCCESS) { + /* LCOV_EXCL_START */ + log_warn(LD_PROCESS, "ReadFileEx() failed after returning success: %s", + format_win32_error(error_code)); + handle->reached_eof = true; + return bytes_available; + /* LCOV_EXCL_STOP */ + } + /* We mark our handle as having a pending I/O request. */ handle->busy = true; From 36e24782f87c25d13b4c918c66cd4affaab8df79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Thu, 20 Dec 2018 13:02:22 +0100 Subject: [PATCH 0329/2557] Remember to set `reached_eof` when our handles are reporting errors. This patch adds some missing calls to set `reached_eof` of our handles when various error conditions happens or when we close our handle (which happens at `process_terminate()`. See: https://bugs.torproject.org/28179 --- src/lib/process/process_win32.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/lib/process/process_win32.c b/src/lib/process/process_win32.c index 71dd4001c8..358765f790 100644 --- a/src/lib/process/process_win32.c +++ b/src/lib/process/process_win32.c @@ -367,6 +367,7 @@ process_win32_write(struct process_t *process, buf_t *buffer) if (! ret) { log_warn(LD_PROCESS, "WriteFileEx() failed: %s", format_win32_error(GetLastError())); + win32_process->stdin_handle.reached_eof = true; return 0; } @@ -749,6 +750,7 @@ process_win32_cleanup_handle(process_win32_handle_t *handle) if (handle->pipe != INVALID_HANDLE_VALUE) { CloseHandle(handle->pipe); handle->pipe = INVALID_HANDLE_VALUE; + handle->reached_eof = true; } } @@ -930,6 +932,7 @@ process_win32_read_from_handle(process_win32_handle_t *handle, if (! ret) { log_warn(LD_PROCESS, "ReadFileEx() failed: %s", format_win32_error(GetLastError())); + handle->reached_eof = true; return bytes_available; } From f58e597d42c204e25101143f9618d55aef666edc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Thu, 20 Dec 2018 13:04:49 +0100 Subject: [PATCH 0330/2557] Handle ERROR_BROKEN_PIPE in completion routines. Handle `ERROR_BROKEN_PIPE` from ReadFileEx() and WriteFileEx() in process_win32_stdin_write_done() and process_win32_handle_read_completion() instead of in the early handler. This most importantmly makes sure that `reached_eof` is set to true when these errors appears. See: https://bugs.torproject.org/28179 --- src/lib/process/process_win32.c | 37 ++++++--------------------------- 1 file changed, 6 insertions(+), 31 deletions(-) diff --git a/src/lib/process/process_win32.c b/src/lib/process/process_win32.c index 358765f790..3b4373f425 100644 --- a/src/lib/process/process_win32.c +++ b/src/lib/process/process_win32.c @@ -765,15 +765,6 @@ process_win32_stdout_read_done(DWORD error_code, tor_assert(overlapped); tor_assert(overlapped->hEvent); - /* This happens when we have asked ReadFileEx() to read some data, but we - * then decided to call CloseHandle() on the HANDLE. This can happen if - * someone runs process_free() in the exit_callback of process_t, which means - * we cannot call process_get_win32_process() here. */ - if (error_code == ERROR_BROKEN_PIPE) { - log_debug(LD_PROCESS, "Process reported broken pipe on standard out"); - return; - } - /* Extract our process_t from the hEvent member of OVERLAPPED. */ process_t *process = (process_t *)overlapped->hEvent; process_win32_t *win32_process = process_get_win32_process(process); @@ -797,15 +788,6 @@ process_win32_stderr_read_done(DWORD error_code, tor_assert(overlapped); tor_assert(overlapped->hEvent); - /* This happens when we have asked ReadFileEx() to read some data, but we - * then decided to call CloseHandle() on the HANDLE. This can happen if - * someone runs process_free() in the exit_callback of process_t, which means - * we cannot call process_get_win32_process() here. */ - if (error_code == ERROR_BROKEN_PIPE) { - log_debug(LD_PROCESS, "Process reported broken pipe on standard error"); - return; - } - /* Extract our process_t from the hEvent member of OVERLAPPED. */ process_t *process = (process_t *)overlapped->hEvent; process_win32_t *win32_process = process_get_win32_process(process); @@ -831,15 +813,6 @@ process_win32_stdin_write_done(DWORD error_code, (void)byte_count; - /* This happens when we have asked WriteFileEx() to write some data, but we - * then decided to call CloseHandle() on the HANDLE. This can happen if - * someone runs process_free() in the exit_callback of process_t, which means - * we cannot call process_get_win32_process() here. */ - if (error_code == ERROR_BROKEN_PIPE) { - log_debug(LD_PROCESS, "Process reported broken pipe on standard input"); - return; - } - process_t *process = (process_t *)overlapped->hEvent; process_win32_t *win32_process = process_get_win32_process(process); @@ -860,7 +833,8 @@ process_win32_stdin_write_done(DWORD error_code, /* Schedule the next write. */ process_notify_event_stdin(process); - } else if (error_code == ERROR_HANDLE_EOF) { + } else if (error_code == ERROR_HANDLE_EOF || + error_code == ERROR_BROKEN_PIPE) { /* Our WriteFileEx() call was succesful, but we reached the end of our * file. We mark our handle as having reached EOF and returns. */ tor_assert(byte_count == 0); @@ -983,9 +957,10 @@ process_win32_handle_read_completion(process_win32_handle_t *handle, /* Tell our caller to schedule the next read. */ return true; - } else if (error_code == ERROR_HANDLE_EOF) { - /* Our ReadFileEx() call was succesful, but we reached the end of our file. - * We mark our handle as having reached EOF and returns. */ + } else if (error_code == ERROR_HANDLE_EOF || + error_code == ERROR_BROKEN_PIPE) { + /* Our ReadFileEx() finished, but we reached the end of our file. We mark + * our handle as having reached EOF and returns. */ tor_assert(byte_count == 0); handle->reached_eof = true; From 412fbe9f177f742e0e56df612f44261f6771973a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Thu, 20 Dec 2018 13:11:24 +0100 Subject: [PATCH 0331/2557] Make example CancelIoEx() code use CancelIo(). This patch changes the CancelIoEx() example code to use CancelIo(), which is available for older versions of Windows too. I still think the kernel handles this nicely by sending broken pipes if either side closes the pipe while I/O operations are pending. See: https://bugs.torproject.org/28179 --- src/lib/process/process_win32.c | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/src/lib/process/process_win32.c b/src/lib/process/process_win32.c index 3b4373f425..641af2bb0f 100644 --- a/src/lib/process/process_win32.c +++ b/src/lib/process/process_win32.c @@ -720,30 +720,23 @@ process_win32_cleanup_handle(process_win32_handle_t *handle) tor_assert(handle); #if 0 - /* FIXME(ahf): My compiler does not set _WIN32_WINNT to a high enough value - * for this code to be available. Should we force it? CancelIoEx() is - * available from Windows 7 and above. If we decide to require this, we need - * to update the checks in all the three I/O completion callbacks to handle - * the ERROR_OPERATION_ABORTED as well as ERROR_BROKEN_PIPE. */ - -#if _WIN32_WINNT >= 0x0600 - /* This code is only supported from Windows 7 and onwards. */ BOOL ret; DWORD error_code; - /* Cancel any pending I/O requests. */ - ret = CancelIoEx(handle->pipe, &handle->overlapped); + /* Cancel any pending I/O requests: This means that instead of getting + * ERROR_BROKEN_PIPE we get ERROR_OPERATION_ABORTED, but it doesn't seem + * like this is needed. */ + ret = CancelIo(handle->pipe); if (! ret) { error_code = GetLastError(); /* There was no pending I/O requests for our handle. */ if (error_code != ERROR_NOT_FOUND) { - log_warn(LD_PROCESS, "CancelIoEx() failed: %s", + log_warn(LD_PROCESS, "CancelIo() failed: %s", format_win32_error(error_code)); } } -#endif #endif /* Close our handle. */ From 776208896756651519fd6540103c73a369b6d485 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Thu, 20 Dec 2018 13:12:53 +0100 Subject: [PATCH 0332/2557] No need to log ordinary EOF conditions as LOG_WARN. Let's not use log_warn() when a pipe is closed under what should be considered normal conditions. See: https://bugs.torproject.org/28179 --- src/lib/process/process_win32.c | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/src/lib/process/process_win32.c b/src/lib/process/process_win32.c index 641af2bb0f..d8a895875c 100644 --- a/src/lib/process/process_win32.c +++ b/src/lib/process/process_win32.c @@ -365,8 +365,17 @@ process_win32_write(struct process_t *process, buf_t *buffer) process_win32_stdin_write_done); if (! ret) { - log_warn(LD_PROCESS, "WriteFileEx() failed: %s", - format_win32_error(GetLastError())); + error_code = GetLastError(); + + /* No need to log at warning level for these two. */ + if (error_code == ERROR_HANDLE_EOF || error_code == ERROR_BROKEN_PIPE) { + log_debug(LD_PROCESS, "WriteFileEx() returned EOF from pipe: %s", + format_win32_error(error_code)); + } else { + log_warn(LD_PROCESS, "WriteFileEx() failed: %s", + format_win32_error(error_code)); + } + win32_process->stdin_handle.reached_eof = true; return 0; } @@ -897,8 +906,17 @@ process_win32_read_from_handle(process_win32_handle_t *handle, callback); if (! ret) { - log_warn(LD_PROCESS, "ReadFileEx() failed: %s", - format_win32_error(GetLastError())); + error_code = GetLastError(); + + /* No need to log at warning level for these two. */ + if (error_code == ERROR_HANDLE_EOF || error_code == ERROR_BROKEN_PIPE) { + log_debug(LD_PROCESS, "ReadFileEx() returned EOF from pipe: %s", + format_win32_error(error_code)); + } else { + log_warn(LD_PROCESS, "ReadFileEx() failed: %s", + format_win32_error(error_code)); + } + handle->reached_eof = true; return bytes_available; } From f7e175db57c3c0a0acc2ca424163213544f34633 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Thu, 20 Dec 2018 14:36:04 +0100 Subject: [PATCH 0333/2557] Forward declare smartlist_t in process.h This allows other libraries to include process.h without including the smartlist_t headers first. See: https://bugs.torproject.org/28847 --- src/lib/process/process.h | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/lib/process/process.h b/src/lib/process/process.h index 179db19aeb..956d34ab29 100644 --- a/src/lib/process/process.h +++ b/src/lib/process/process.h @@ -47,6 +47,8 @@ const char *process_protocol_to_string(process_protocol_t protocol); void tor_disable_spawning_background_processes(void); +struct smartlist_t; + struct process_t; typedef struct process_t process_t; @@ -61,7 +63,7 @@ typedef bool void process_init(void); void process_free_all(void); -const smartlist_t *process_get_all_processes(void); +const struct smartlist_t *process_get_all_processes(void); process_t *process_new(const char *command); void process_free_(process_t *process); @@ -82,10 +84,11 @@ void process_set_exit_callback(process_t *, const char *process_get_command(const process_t *process); void process_append_argument(process_t *process, const char *argument); -const smartlist_t *process_get_arguments(const process_t *process); +const struct smartlist_t *process_get_arguments(const process_t *process); char **process_get_argv(const process_t *process); -void process_reset_environment(process_t *process, const smartlist_t *env); +void process_reset_environment(process_t *process, + const struct smartlist_t *env); void process_set_environment(process_t *process, const char *key, const char *value); From a517daa56f5848d25ba79617a1a7b82ed2b0a7c0 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 20 Dec 2018 08:36:25 -0500 Subject: [PATCH 0334/2557] base32_decode(): Return number of bytes written on success. This makes it consistent with base64_decode(). Closes ticket 28913. --- changes/ticket28913 | 4 ++++ src/feature/control/control.c | 3 ++- src/feature/rend/rendcache.c | 3 +-- src/lib/encoding/binascii.c | 4 ++-- src/test/test_crypto.c | 8 ++++---- src/test/test_util_format.c | 4 ++-- 6 files changed, 15 insertions(+), 11 deletions(-) create mode 100644 changes/ticket28913 diff --git a/changes/ticket28913 b/changes/ticket28913 new file mode 100644 index 0000000000..e09847464d --- /dev/null +++ b/changes/ticket28913 @@ -0,0 +1,4 @@ + o Code simplification and refactoring: + - Make the base32_decode() API return the number of bytes written, + for consistency with base64_decode(). + Closes ticket 28913. diff --git a/src/feature/control/control.c b/src/feature/control/control.c index 7fae3b7a1b..1e3db6337e 100644 --- a/src/feature/control/control.c +++ b/src/feature/control/control.c @@ -4428,7 +4428,8 @@ handle_control_hsfetch(control_connection_t *conn, uint32_t len, } else if (strcmpstart(arg1, v2_str) == 0 && rend_valid_descriptor_id(arg1 + v2_str_len) && base32_decode(digest, sizeof(digest), arg1 + v2_str_len, - REND_DESC_ID_V2_LEN_BASE32) == 0) { + REND_DESC_ID_V2_LEN_BASE32) == + REND_DESC_ID_V2_LEN_BASE32) { /* We have a well formed version 2 descriptor ID. Keep the decoded value * of the id. */ desc_id = digest; diff --git a/src/feature/rend/rendcache.c b/src/feature/rend/rendcache.c index b851e71959..ecd85e4a5a 100644 --- a/src/feature/rend/rendcache.c +++ b/src/feature/rend/rendcache.c @@ -854,7 +854,7 @@ rend_cache_store_v2_desc_as_client(const char *desc, *entry = NULL; } if (base32_decode(want_desc_id, sizeof(want_desc_id), - desc_id_base32, strlen(desc_id_base32)) != 0) { + desc_id_base32, strlen(desc_id_base32)) < 0) { log_warn(LD_BUG, "Couldn't decode base32 %s for descriptor id.", escaped_safe_str_client(desc_id_base32)); goto err; @@ -1005,4 +1005,3 @@ rend_cache_store_v2_desc_as_client(const char *desc, tor_free(intro_content); return retval; } - diff --git a/src/lib/encoding/binascii.c b/src/lib/encoding/binascii.c index 067db075ad..a7662658f0 100644 --- a/src/lib/encoding/binascii.c +++ b/src/lib/encoding/binascii.c @@ -84,7 +84,7 @@ base32_encode(char *dest, size_t destlen, const char *src, size_t srclen) } /** Implements base32 decoding as in RFC 4648. - * Returns 0 if successful, -1 otherwise. + * Return the number of bytes decoded if successful; -1 otherwise. */ int base32_decode(char *dest, size_t destlen, const char *src, size_t srclen) @@ -147,7 +147,7 @@ base32_decode(char *dest, size_t destlen, const char *src, size_t srclen) memset(tmp, 0, srclen); /* on the heap, this should be safe */ tor_free(tmp); tmp = NULL; - return 0; + return i; } #define BASE64_OPENSSL_LINELEN 64 diff --git a/src/test/test_crypto.c b/src/test/test_crypto.c index 81d2fa6f33..e924cea423 100644 --- a/src/test/test_crypto.c +++ b/src/test/test_crypto.c @@ -1865,13 +1865,13 @@ test_crypto_base32_decode(void *arg) /* Encode and decode a random string. */ base32_encode(encoded, 96 + 1, plain, 60); res = base32_decode(decoded, 60, encoded, 96); - tt_int_op(res,OP_EQ, 0); + tt_int_op(res, OP_EQ, 60); tt_mem_op(plain,OP_EQ, decoded, 60); /* Encode, uppercase, and decode a random string. */ base32_encode(encoded, 96 + 1, plain, 60); tor_strupper(encoded); res = base32_decode(decoded, 60, encoded, 96); - tt_int_op(res,OP_EQ, 0); + tt_int_op(res, OP_EQ, 60); tt_mem_op(plain,OP_EQ, decoded, 60); /* Change encoded string and decode. */ if (encoded[0] == 'A' || encoded[0] == 'a') @@ -1879,12 +1879,12 @@ test_crypto_base32_decode(void *arg) else encoded[0] = 'A'; res = base32_decode(decoded, 60, encoded, 96); - tt_int_op(res,OP_EQ, 0); + tt_int_op(res, OP_EQ, 60); tt_mem_op(plain,OP_NE, decoded, 60); /* Bad encodings. */ encoded[0] = '!'; res = base32_decode(decoded, 60, encoded, 96); - tt_int_op(0, OP_GT, res); + tt_int_op(res, OP_LT, 0); done: ; diff --git a/src/test/test_util_format.c b/src/test/test_util_format.c index fd57125b86..f91171dc28 100644 --- a/src/test/test_util_format.c +++ b/src/test/test_util_format.c @@ -346,7 +346,7 @@ test_util_format_base32_decode(void *arg) const char *src = "mjwgc2dcnrswqmjs"; ret = base32_decode(dst, strlen(expected), src, strlen(src)); - tt_int_op(ret, OP_EQ, 0); + tt_int_op(ret, OP_EQ, 10); tt_str_op(expected, OP_EQ, dst); } @@ -357,7 +357,7 @@ test_util_format_base32_decode(void *arg) const char *src = "mjwgc2dcnrswq"; ret = base32_decode(dst, strlen(expected), src, strlen(src)); - tt_int_op(ret, OP_EQ, 0); + tt_int_op(ret, OP_EQ, 8); tt_mem_op(expected, OP_EQ, dst, strlen(expected)); } From 01819faaba0b4817ce15d29be0944e23268e76c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Thu, 20 Dec 2018 14:36:59 +0100 Subject: [PATCH 0335/2557] Remove Process initializer/shutdown function from main.c. See: https://bugs.torproject.org/28847 --- src/app/main/main.c | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/app/main/main.c b/src/app/main/main.c index aa322046fe..d71e43ec30 100644 --- a/src/app/main/main.c +++ b/src/app/main/main.c @@ -74,7 +74,6 @@ #include "lib/net/resolve.h" #include "lib/process/waitpid.h" -#include "lib/process/process.h" #include "lib/meminfo/meminfo.h" #include "lib/osinfo/uname.h" @@ -560,9 +559,6 @@ tor_init(int argc, char *argv[]) addressmap_init(); /* Init the client dns cache. Do it always, since it's * cheap. */ - /* Initialize Process subsystem. */ - process_init(); - /* Initialize the HS subsystem. */ hs_init(); @@ -790,7 +786,6 @@ tor_free_all(int postfork) circuitmux_ewma_free_all(); accounting_free_all(); protover_summary_cache_free_all(); - process_free_all(); if (!postfork) { config_free_all(); From 22c5ad682cdd2e6a9a2124585302b2335361ab80 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 20 Dec 2018 08:37:04 -0500 Subject: [PATCH 0336/2557] Add base32 to the round-trip fuzzer --- src/test/fuzz/fuzz_strops.c | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/test/fuzz/fuzz_strops.c b/src/test/fuzz/fuzz_strops.c index 5da590acfa..cfc7fc5063 100644 --- a/src/test/fuzz/fuzz_strops.c +++ b/src/test/fuzz/fuzz_strops.c @@ -86,15 +86,13 @@ b16_enc(const chunk_t *inp) return ch; } -#if 0 static chunk_t * b32_dec(const chunk_t *inp) { chunk_t *ch = chunk_new(inp->len);//XXXX int r = base32_decode((char *)ch->buf, ch->len, (char *)inp->buf, inp->len); if (r >= 0) { - ch->len = r; // XXXX we need some way to get the actual length of - // XXXX the output here. + ch->len = r; } else { chunk_free(ch); } @@ -108,7 +106,6 @@ b32_enc(const chunk_t *inp) ch->len = strlen((char *) ch->buf); return ch; } -#endif static chunk_t * b64_dec(const chunk_t *inp) @@ -222,10 +219,7 @@ fuzz_main(const uint8_t *stdin_buf, size_t data_size) ENCODE_ROUNDTRIP(b16_enc, b16_dec, chunk_free_); break; case 1: - /* - XXXX see notes above about our base-32 functions. ENCODE_ROUNDTRIP(b32_enc, b32_dec, chunk_free_); - */ break; case 2: ENCODE_ROUNDTRIP(b64_enc, b64_dec, chunk_free_); From 4d9b55907c4a5223ea7ef603a87ba6089bf5b3e9 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 20 Dec 2018 11:50:40 -0500 Subject: [PATCH 0337/2557] Even more clarify on --version. --- doc/tor.1.txt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/doc/tor.1.txt b/doc/tor.1.txt index 4ecc4ee382..5c01381176 100644 --- a/doc/tor.1.txt +++ b/doc/tor.1.txt @@ -89,8 +89,9 @@ COMMAND-LINE OPTIONS future version. (This is a warning, not a promise.) [[opt-version]] **--version**:: - Display Tor version and exit. The output is a line of the format, - "Tor version [version number]." + Display Tor version and exit. The output is a single line of the format + "Tor version [version number]." (The version number format + is as specified in version-spec.txt.) [[opt-quiet]] **--quiet**|**--hush**:: Override the default console log. By default, Tor starts out logging From ab0d7d2dd470a447535eaf7c35bd4877c43bce26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Thu, 20 Dec 2018 19:05:08 +0100 Subject: [PATCH 0338/2557] Escape the PT K/V data before sending it to the logger. See: https://bugs.torproject.org/28846 --- src/feature/client/transports.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/feature/client/transports.c b/src/feature/client/transports.c index 45dbd0c888..8a8bcd9f7f 100644 --- a/src/feature/client/transports.c +++ b/src/feature/client/transports.c @@ -1175,14 +1175,14 @@ parse_log_line(const char *line, managed_proxy_t *mp) /* Check if we got a message. */ if (! message) { log_warn(LD_PT, "Managed proxy \"%s\" wrote a LOG line without " - "MESSAGE: %s", mp->argv[0], data); + "MESSAGE: %s", mp->argv[0], escaped(data)); goto done; } /* Check if severity is there and whether it's valid. */ if (! severity) { log_warn(LD_PT, "Managed proxy \"%s\" wrote a LOG line without " - "SEVERITY: %s", mp->argv[0], data); + "SEVERITY: %s", mp->argv[0], escaped(data)); goto done; } @@ -1232,7 +1232,7 @@ parse_status_line(const char *line, managed_proxy_t *mp) if (! values) { log_warn(LD_PT, "Managed proxy \"%s\" wrote an invalid " - "STATUS message: %s", mp->argv[0], data); + "STATUS message: %s", mp->argv[0], escaped(data)); goto done; } @@ -1242,7 +1242,7 @@ parse_status_line(const char *line, managed_proxy_t *mp) if (! type) { log_warn(LD_PT, "Managed proxy \"%s\" wrote a STATUS line without " - "TYPE: %s", mp->argv[0], data); + "TYPE: %s", mp->argv[0], escaped(data)); goto done; } From 308dde0c386158971a66c6e035b3f1ad67019eb5 Mon Sep 17 00:00:00 2001 From: Taylor Yu Date: Thu, 13 Dec 2018 17:18:49 -0600 Subject: [PATCH 0339/2557] Remove unused old_state var in connection_or.c connection_or_change_state() saved an old_state to pass to channel_tls_handle_state_change_on_orconn(), which promptly cast it to void. Remove this unused variable and parameter. --- src/core/or/channeltls.c | 3 --- src/core/or/channeltls.h | 1 - src/core/or/connection_or.c | 6 +----- 3 files changed, 1 insertion(+), 9 deletions(-) diff --git a/src/core/or/channeltls.c b/src/core/or/channeltls.c index 8f407d5e15..cf33feec01 100644 --- a/src/core/or/channeltls.c +++ b/src/core/or/channeltls.c @@ -950,7 +950,6 @@ channel_tls_listener_describe_transport_method(channel_listener_t *chan_l) void channel_tls_handle_state_change_on_orconn(channel_tls_t *chan, or_connection_t *conn, - uint8_t old_state, uint8_t state) { channel_t *base_chan; @@ -959,8 +958,6 @@ channel_tls_handle_state_change_on_orconn(channel_tls_t *chan, tor_assert(conn); tor_assert(conn->chan == chan); tor_assert(chan->conn == conn); - /* Shut the compiler up without triggering -Wtautological-compare */ - (void)old_state; base_chan = TLS_CHAN_TO_BASE(chan); diff --git a/src/core/or/channeltls.h b/src/core/or/channeltls.h index 12715450b9..2ec7fe5453 100644 --- a/src/core/or/channeltls.h +++ b/src/core/or/channeltls.h @@ -49,7 +49,6 @@ channel_tls_t * channel_tls_from_base(channel_t *chan); void channel_tls_handle_cell(cell_t *cell, or_connection_t *conn); void channel_tls_handle_state_change_on_orconn(channel_tls_t *chan, or_connection_t *conn, - uint8_t old_state, uint8_t state); void channel_tls_handle_var_cell(var_cell_t *var_cell, or_connection_t *conn); diff --git a/src/core/or/connection_or.c b/src/core/or/connection_or.c index 962ec51c36..04ab4fe4a9 100644 --- a/src/core/or/connection_or.c +++ b/src/core/or/connection_or.c @@ -408,16 +408,12 @@ connection_or_report_broken_states(int severity, int domain) static void connection_or_change_state(or_connection_t *conn, uint8_t state) { - uint8_t old_state; - tor_assert(conn); - old_state = conn->base_.state; conn->base_.state = state; if (conn->chan) - channel_tls_handle_state_change_on_orconn(conn->chan, conn, - old_state, state); + channel_tls_handle_state_change_on_orconn(conn->chan, conn, state); } /** Return the number of circuits using an or_connection_t; this used to From 271b50f54abac7af44e3e54589ff965d3cdac816 Mon Sep 17 00:00:00 2001 From: Taylor Yu Date: Sun, 16 Dec 2018 16:05:58 -0600 Subject: [PATCH 0340/2557] Add ORCONN event pubsub system Add a publish-subscribe subsystem to publish messages about changes to OR connections. connection_or_change_state() in connection_or.c and control_event_or_conn_event() in control.c publish messages to this subsystem via helper functions. Move state constants from connection_or.h to orconn_state.h so that subscribers don't have to include all of connection_or.h to take actions based on changes in OR connection state. Move event constants from control.h for similar reasons. Part of ticket 27167. --- src/app/main/subsystem_list.c | 2 + src/core/include.am | 3 + src/core/mainloop/connection.c | 2 +- src/core/or/connection_or.c | 73 +++++++++++++++++--- src/core/or/connection_or.h | 30 ++------- src/core/or/orconn_event.c | 81 ++++++++++++++++++++++ src/core/or/orconn_event.h | 120 +++++++++++++++++++++++++++++++++ src/core/or/orconn_event_sys.h | 12 ++++ src/feature/control/control.h | 12 +--- src/feature/relay/ext_orport.c | 2 +- 10 files changed, 288 insertions(+), 49 deletions(-) create mode 100644 src/core/or/orconn_event.c create mode 100644 src/core/or/orconn_event.h create mode 100644 src/core/or/orconn_event_sys.h diff --git a/src/app/main/subsystem_list.c b/src/app/main/subsystem_list.c index 8640329e92..1858738096 100644 --- a/src/app/main/subsystem_list.c +++ b/src/app/main/subsystem_list.c @@ -8,6 +8,7 @@ #include "lib/cc/compat_compiler.h" #include "lib/cc/torint.h" +#include "core/or/orconn_event_sys.h" #include "lib/compress/compress_sys.h" #include "lib/crypt_ops/crypto_sys.h" #include "lib/err/torerr_sys.h" @@ -35,6 +36,7 @@ const subsys_fns_t *tor_subsystems[] = { &sys_compress, /* -70 */ &sys_crypto, /* -60 */ &sys_tortls, /* -50 */ + &sys_orconn_event, /* -40 */ }; const unsigned n_tor_subsystems = ARRAY_LENGTH(tor_subsystems); diff --git a/src/core/include.am b/src/core/include.am index 3cd6e83ed5..a1fae73609 100644 --- a/src/core/include.am +++ b/src/core/include.am @@ -39,6 +39,7 @@ LIBTOR_APP_A_SOURCES = \ src/core/or/connection_or.c \ src/core/or/dos.c \ src/core/or/onion.c \ + src/core/or/orconn_event.c \ src/core/or/policies.c \ src/core/or/protover.c \ src/core/or/protover_rust.c \ @@ -238,6 +239,8 @@ noinst_HEADERS += \ src/core/or/listener_connection_st.h \ src/core/or/onion.h \ src/core/or/or.h \ + src/core/or/orconn_event.h \ + src/core/or/orconn_event_sys.h \ src/core/or/or_circuit_st.h \ src/core/or/or_connection_st.h \ src/core/or/or_handshake_certs_st.h \ diff --git a/src/core/mainloop/connection.c b/src/core/mainloop/connection.c index 601c872c98..6cc6122702 100644 --- a/src/core/mainloop/connection.c +++ b/src/core/mainloop/connection.c @@ -1875,7 +1875,7 @@ connection_init_accepted_conn(connection_t *conn, /* Initiate Extended ORPort authentication. */ return connection_ext_or_start_auth(TO_OR_CONN(conn)); case CONN_TYPE_OR: - control_event_or_conn_status(TO_OR_CONN(conn), OR_CONN_EVENT_NEW, 0); + connection_or_event_status(TO_OR_CONN(conn), OR_CONN_EVENT_NEW, 0); rv = connection_tls_start_handshake(TO_OR_CONN(conn), 1); if (rv < 0) { connection_or_close_for_error(TO_OR_CONN(conn), 0); diff --git a/src/core/or/connection_or.c b/src/core/or/connection_or.c index 04ab4fe4a9..a12ea55c46 100644 --- a/src/core/or/connection_or.c +++ b/src/core/or/connection_or.c @@ -29,6 +29,7 @@ */ #define TOR_CHANNEL_INTERNAL_ #define CONNECTION_OR_PRIVATE +#define ORCONN_EVENT_PRIVATE #include "core/or/channel.h" #include "core/or/channeltls.h" #include "core/or/circuitbuild.h" @@ -79,6 +80,8 @@ #include "lib/tls/tortls.h" #include "lib/tls/x509.h" +#include "core/or/orconn_event.h" + static int connection_tls_finish_handshake(or_connection_t *conn); static int connection_or_launch_v3_or_handshake(or_connection_t *conn); static int connection_or_process_cells_from_inbuf(or_connection_t *conn); @@ -401,6 +404,49 @@ connection_or_report_broken_states(int severity, int domain) smartlist_free(items); } +/** + * Helper function to publish an OR connection status event + * + * Publishes a messages to subscribers of ORCONN messages, and sends + * the control event. + **/ +void +connection_or_event_status(or_connection_t *conn, or_conn_status_event_t tp, + int reason) +{ + orconn_event_msg_t msg; + + msg.type = ORCONN_MSGTYPE_STATUS; + msg.u.status.gid = conn->base_.global_identifier; + msg.u.status.status = tp; + msg.u.status.reason = reason; + orconn_event_publish(&msg); + control_event_or_conn_status(conn, tp, reason); +} + +/** + * Helper function to publish a state change message + * + * connection_or_change_state() calls this to notify subscribers about + * a change of an OR connection state. + **/ +static void +connection_or_state_publish(const or_connection_t *conn, uint8_t state) +{ + orconn_event_msg_t msg; + + msg.type = ORCONN_MSGTYPE_STATE; + msg.u.state.gid = conn->base_.global_identifier; + msg.u.state.proxy_type = conn->proxy_type; + msg.u.state.state = state; + if (conn->chan) { + msg.u.state.chan = TLS_CHAN_TO_BASE(conn->chan)->global_identifier; + } else { + msg.u.state.chan = 0; + } + orconn_event_publish(&msg); +} + /** Call this to change or_connection_t states, so the owning channel_tls_t can * be notified. */ @@ -412,6 +458,7 @@ connection_or_change_state(or_connection_t *conn, uint8_t state) conn->base_.state = state; + connection_or_state_publish(conn, state); if (conn->chan) channel_tls_handle_state_change_on_orconn(conn->chan, conn, state); } @@ -755,8 +802,8 @@ connection_or_about_to_close(or_connection_t *or_conn) entry_guard_chan_failed(TLS_CHAN_TO_BASE(or_conn->chan)); if (conn->state >= OR_CONN_STATE_TLS_HANDSHAKING) { int reason = tls_error_to_orconn_end_reason(or_conn->tls_error); - control_event_or_conn_status(or_conn, OR_CONN_EVENT_FAILED, - reason); + connection_or_event_status(or_conn, OR_CONN_EVENT_FAILED, + reason); if (!authdir_mode_tests_reachability(options)) control_event_bootstrap_prob_or( orconn_end_reason_to_control_string(reason), @@ -766,10 +813,10 @@ connection_or_about_to_close(or_connection_t *or_conn) } else if (conn->hold_open_until_flushed) { /* We only set hold_open_until_flushed when we're intentionally * closing a connection. */ - control_event_or_conn_status(or_conn, OR_CONN_EVENT_CLOSED, + connection_or_event_status(or_conn, OR_CONN_EVENT_CLOSED, tls_error_to_orconn_end_reason(or_conn->tls_error)); } else if (!tor_digest_is_zero(or_conn->identity_digest)) { - control_event_or_conn_status(or_conn, OR_CONN_EVENT_CLOSED, + connection_or_event_status(or_conn, OR_CONN_EVENT_CLOSED, tls_error_to_orconn_end_reason(or_conn->tls_error)); } } @@ -1361,7 +1408,7 @@ void connection_or_connect_failed(or_connection_t *conn, int reason, const char *msg) { - control_event_or_conn_status(conn, OR_CONN_EVENT_FAILED, reason); + connection_or_event_status(conn, OR_CONN_EVENT_FAILED, reason); if (!authdir_mode_tests_reachability(get_options())) control_event_bootstrap_prob_or(msg, reason, conn); note_or_connect_failed(conn); @@ -1468,9 +1515,6 @@ connection_or_connect, (const tor_addr_t *_addr, uint16_t port, return NULL; } - connection_or_change_state(conn, OR_CONN_STATE_CONNECTING); - control_event_or_conn_status(conn, OR_CONN_EVENT_LAUNCHED, 0); - conn->is_outgoing = 1; /* If we are using a proxy server, find it and use it. */ @@ -1482,7 +1526,14 @@ connection_or_connect, (const tor_addr_t *_addr, uint16_t port, port = proxy_port; conn->base_.proxy_state = PROXY_INFANT; } + connection_or_change_state(conn, OR_CONN_STATE_CONNECTING); + connection_or_event_status(conn, OR_CONN_EVENT_LAUNCHED, 0); } else { + /* This duplication of state change calls is necessary in case we + * run into an error condition below */ + connection_or_change_state(conn, OR_CONN_STATE_CONNECTING); + connection_or_event_status(conn, OR_CONN_EVENT_LAUNCHED, 0); + /* get_proxy_addrport() might fail if we have a Bridge line that references a transport, but no ClientTransportPlugin lines defining its transport proxy. If this is the case, let's try to @@ -1978,8 +2029,8 @@ connection_or_client_learned_peer_id(or_connection_t *conn, /* Tell the new guard API about the channel failure */ entry_guard_chan_failed(TLS_CHAN_TO_BASE(conn->chan)); - control_event_or_conn_status(conn, OR_CONN_EVENT_FAILED, - END_OR_CONN_REASON_OR_IDENTITY); + connection_or_event_status(conn, OR_CONN_EVENT_FAILED, + END_OR_CONN_REASON_OR_IDENTITY); if (!authdir_mode_tests_reachability(options)) control_event_bootstrap_prob_or( "Unexpected identity in router certificate", @@ -2219,7 +2270,7 @@ int connection_or_set_state_open(or_connection_t *conn) { connection_or_change_state(conn, OR_CONN_STATE_OPEN); - control_event_or_conn_status(conn, OR_CONN_EVENT_CONNECTED, 0); + connection_or_event_status(conn, OR_CONN_EVENT_CONNECTED, 0); /* Link protocol 3 appeared in Tor 0.2.3.6-alpha, so any connection * that uses an earlier link protocol should not be treated as a relay. */ diff --git a/src/core/or/connection_or.h b/src/core/or/connection_or.h index d4bcdd93e5..5f4856d51f 100644 --- a/src/core/or/connection_or.h +++ b/src/core/or/connection_or.h @@ -17,32 +17,7 @@ struct ed25519_keypair_t; or_connection_t *TO_OR_CONN(connection_t *); -#define OR_CONN_STATE_MIN_ 1 -/** State for a connection to an OR: waiting for connect() to finish. */ -#define OR_CONN_STATE_CONNECTING 1 -/** State for a connection to an OR: waiting for proxy handshake to complete */ -#define OR_CONN_STATE_PROXY_HANDSHAKING 2 -/** State for an OR connection client: SSL is handshaking, not done - * yet. */ -#define OR_CONN_STATE_TLS_HANDSHAKING 3 -/** State for a connection to an OR: We're doing a second SSL handshake for - * renegotiation purposes. (V2 handshake only.) */ -#define OR_CONN_STATE_TLS_CLIENT_RENEGOTIATING 4 -/** State for a connection at an OR: We're waiting for the client to - * renegotiate (to indicate a v2 handshake) or send a versions cell (to - * indicate a v3 handshake) */ -#define OR_CONN_STATE_TLS_SERVER_RENEGOTIATING 5 -/** State for an OR connection: We're done with our SSL handshake, we've done - * renegotiation, but we haven't yet negotiated link protocol versions and - * sent a netinfo cell. */ -#define OR_CONN_STATE_OR_HANDSHAKING_V2 6 -/** State for an OR connection: We're done with our SSL handshake, but we - * haven't yet negotiated link protocol versions, done a V3 handshake, and - * sent a netinfo cell. */ -#define OR_CONN_STATE_OR_HANDSHAKING_V3 7 -/** State for an OR connection: Ready to send/receive cells. */ -#define OR_CONN_STATE_OPEN 8 -#define OR_CONN_STATE_MAX_ 8 +#include "core/or/orconn_event.h" void connection_or_clear_identity(or_connection_t *conn); void connection_or_clear_identity_map(void); @@ -81,6 +56,9 @@ MOCK_DECL(void,connection_or_close_for_error, void connection_or_report_broken_states(int severity, int domain); +void connection_or_event_status(or_connection_t *conn, + or_conn_status_event_t tp, int reason); + MOCK_DECL(int,connection_tls_start_handshake,(or_connection_t *conn, int receiving)); int connection_tls_continue_handshake(or_connection_t *conn); diff --git a/src/core/or/orconn_event.c b/src/core/or/orconn_event.c new file mode 100644 index 0000000000..11f5ed9662 --- /dev/null +++ b/src/core/or/orconn_event.c @@ -0,0 +1,81 @@ +/* Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file orconn_event.c + * \brief Publish state change messages for OR connections + * + * Implements a basic publish-subscribe framework for messages about + * the state of OR connections. The publisher calls the subscriber + * callback functions synchronously. + * + * Although the synchronous calls might not simplify the call graph, + * this approach improves data isolation because the publisher doesn't + * need knowledge about the internals of subscribing subsystems. It + * also avoids race conditions that might occur in asynchronous + * frameworks. + **/ + +#include "core/or/or.h" +#include "lib/subsys/subsys.h" + +#define ORCONN_EVENT_PRIVATE +#include "core/or/orconn_event.h" +#include "core/or/orconn_event_sys.h" + +/** List of subscribers */ +static smartlist_t *orconn_event_rcvrs; + +/** Initialize subscriber list */ +static int +orconn_event_init(void) +{ + orconn_event_rcvrs = smartlist_new(); + return 0; +} + +/** Free subscriber list */ +static void +orconn_event_fini(void) +{ + smartlist_free(orconn_event_rcvrs); +} + +/** + * Subscribe to messages about OR connection events + * + * Register a callback function to receive messages about ORCONNs. + * The publisher calls this function synchronously. + **/ +void +orconn_event_subscribe(orconn_event_rcvr_t fn) +{ + tor_assert(fn); + /* Don't duplicate subscriptions. */ + if (smartlist_contains(orconn_event_rcvrs, fn)) + return; + + smartlist_add(orconn_event_rcvrs, fn); +} + +/** + * Publish a message about OR connection events + * + * This calls the subscriber receiver function synchronously. + **/ +void +orconn_event_publish(const orconn_event_msg_t *msg) +{ + SMARTLIST_FOREACH_BEGIN(orconn_event_rcvrs, orconn_event_rcvr_t, fn) { + tor_assert(fn); + (*fn)(msg); + } SMARTLIST_FOREACH_END(fn); +} + +const subsys_fns_t sys_orconn_event = { + .name = "orconn_event", + .supported = true, + .level = -40, + .initialize = orconn_event_init, + .shutdown = orconn_event_fini, +}; diff --git a/src/core/or/orconn_event.h b/src/core/or/orconn_event.h new file mode 100644 index 0000000000..4c999e53be --- /dev/null +++ b/src/core/or/orconn_event.h @@ -0,0 +1,120 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file orconn_event.h + * \brief Header file for orconn_event.c + * + * The OR_CONN_STATE_* symbols are here to make it easier for + * subscribers to make decisions based on the messages that they + * receive. + **/ + +#ifndef TOR_ORCONN_EVENT_H +#define TOR_ORCONN_EVENT_H + +/** + * @name States of OR connections + * + * These must be in a partial ordering such that usually no OR + * connection will transition from a higher-numbered state to a + * lower-numbered one. Code such as bto_update_best() depends on this + * ordering to determine the best state it's seen so far. + * @{ */ +#define OR_CONN_STATE_MIN_ 1 +/** State for a connection to an OR: waiting for connect() to finish. */ +#define OR_CONN_STATE_CONNECTING 1 +/** State for a connection to an OR: waiting for proxy handshake to complete */ +#define OR_CONN_STATE_PROXY_HANDSHAKING 2 +/** State for an OR connection client: SSL is handshaking, not done + * yet. */ +#define OR_CONN_STATE_TLS_HANDSHAKING 3 +/** State for a connection to an OR: We're doing a second SSL handshake for + * renegotiation purposes. (V2 handshake only.) */ +#define OR_CONN_STATE_TLS_CLIENT_RENEGOTIATING 4 +/** State for a connection at an OR: We're waiting for the client to + * renegotiate (to indicate a v2 handshake) or send a versions cell (to + * indicate a v3 handshake) */ +#define OR_CONN_STATE_TLS_SERVER_RENEGOTIATING 5 +/** State for an OR connection: We're done with our SSL handshake, we've done + * renegotiation, but we haven't yet negotiated link protocol versions and + * sent a netinfo cell. */ +#define OR_CONN_STATE_OR_HANDSHAKING_V2 6 +/** State for an OR connection: We're done with our SSL handshake, but we + * haven't yet negotiated link protocol versions, done a V3 handshake, and + * sent a netinfo cell. */ +#define OR_CONN_STATE_OR_HANDSHAKING_V3 7 +/** State for an OR connection: Ready to send/receive cells. */ +#define OR_CONN_STATE_OPEN 8 +#define OR_CONN_STATE_MAX_ 8 +/** @} */ + +/** Used to indicate the type of an OR connection event passed to the + * controller. The various types are defined in control-spec.txt */ +typedef enum or_conn_status_event_t { + OR_CONN_EVENT_LAUNCHED = 0, + OR_CONN_EVENT_CONNECTED = 1, + OR_CONN_EVENT_FAILED = 2, + OR_CONN_EVENT_CLOSED = 3, + OR_CONN_EVENT_NEW = 4, +} or_conn_status_event_t; + +/** Discriminant values for orconn event message */ +typedef enum orconn_msgtype_t { + ORCONN_MSGTYPE_STATE, + ORCONN_MSGTYPE_STATUS, +} orconn_msgtype_t; + +/** + * Message for orconn state update + * + * This contains information about internal state changes of + * or_connection_t objects. The chan and proxy_type fields are + * additional information that a subscriber may need to make + * decisions. + **/ +typedef struct orconn_state_msg_t { + uint64_t gid; /**< connection's global ID */ + uint64_t chan; /**< associated channel ID */ + int proxy_type; /**< connection's proxy type */ + uint8_t state; /**< new connection state */ +} orconn_state_msg_t; + +/** + * Message for orconn status event + * + * This contains information that ends up in ORCONN control protocol + * events. + **/ +typedef struct orconn_status_msg_t { + uint64_t gid; /**< connection's global ID */ + int status; /**< or_conn_status_event_t */ + int reason; /**< reason */ +} orconn_status_msg_t; + +/** Discriminated union for the actual message */ +typedef struct orconn_event_msg_t { + int type; + union { + orconn_state_msg_t state; + orconn_status_msg_t status; + } u; +} orconn_event_msg_t; + +/** + * Receiver function pointer for OR subscribers + * + * This function gets called synchronously by the publisher. + **/ +typedef void (*orconn_event_rcvr_t)(const orconn_event_msg_t *); + +void orconn_event_subscribe(orconn_event_rcvr_t); + +#ifdef ORCONN_EVENT_PRIVATE +void orconn_event_publish(const orconn_event_msg_t *); +#endif + +#endif /* defined(TOR_ORCONN_EVENT_H) */ diff --git a/src/core/or/orconn_event_sys.h b/src/core/or/orconn_event_sys.h new file mode 100644 index 0000000000..7639023386 --- /dev/null +++ b/src/core/or/orconn_event_sys.h @@ -0,0 +1,12 @@ +/* Copyright (c) 2007-2018, The Tor Project, Inc. */ + +/** + * \file orconn_event_sys.h + * \brief Declare subsystem object for the OR connection event module. + **/ +#ifndef TOR_ORCONN_EVENT_SYS_H +#define TOR_ORCONN_EVENT_SYS_H + +extern const struct subsys_fns_t sys_orconn_event; + +#endif /* defined(TOR_ORCONN_SYS_H) */ diff --git a/src/feature/control/control.h b/src/feature/control/control.h index d78ce4d87c..d3245fcaf1 100644 --- a/src/feature/control/control.h +++ b/src/feature/control/control.h @@ -29,6 +29,8 @@ typedef enum circuit_status_minor_event_t { CIRC_MINOR_EVENT_CANNIBALIZED, } circuit_status_minor_event_t; +#include "core/or/orconn_event.h" + /** Used to indicate the type of a stream event passed to the controller. * The various types are defined in control-spec.txt */ typedef enum stream_status_event_t { @@ -43,16 +45,6 @@ typedef enum stream_status_event_t { STREAM_EVENT_REMAP = 8 } stream_status_event_t; -/** Used to indicate the type of an OR connection event passed to the - * controller. The various types are defined in control-spec.txt */ -typedef enum or_conn_status_event_t { - OR_CONN_EVENT_LAUNCHED = 0, - OR_CONN_EVENT_CONNECTED = 1, - OR_CONN_EVENT_FAILED = 2, - OR_CONN_EVENT_CLOSED = 3, - OR_CONN_EVENT_NEW = 4, -} or_conn_status_event_t; - /** Used to indicate the type of a buildtime event */ typedef enum buildtimeout_set_event_t { BUILDTIMEOUT_SET_EVENT_COMPUTED = 0, diff --git a/src/feature/relay/ext_orport.c b/src/feature/relay/ext_orport.c index 3607bdede4..0a649f2743 100644 --- a/src/feature/relay/ext_orport.c +++ b/src/feature/relay/ext_orport.c @@ -90,7 +90,7 @@ connection_ext_or_transition(or_connection_t *conn) conn->base_.type = CONN_TYPE_OR; TO_CONN(conn)->state = 0; // set the state to a neutral value - control_event_or_conn_status(conn, OR_CONN_EVENT_NEW, 0); + connection_or_event_status(conn, OR_CONN_EVENT_NEW, 0); connection_tls_start_handshake(conn, 1); } From a0b4fa1f167f9b013ee1a8326e325ec3f97e4700 Mon Sep 17 00:00:00 2001 From: Taylor Yu Date: Sun, 16 Dec 2018 17:01:25 -0600 Subject: [PATCH 0341/2557] Add origin circuit event pubsub system Add a publish-subscribe subsystem to publish messages about changes to origin circuits. Functions in circuitbuild.c and circuitlist.c publish messages to this subsystem. Move circuit event constants out of control.h so that subscribers don't have to include all of control.h to take actions based on messages they receive. Part of ticket 27167. --- src/app/main/subsystem_list.c | 2 + src/core/include.am | 3 ++ src/core/or/circuitbuild.c | 30 +++++++++++- src/core/or/circuitlist.c | 57 +++++++++++++++++++++- src/core/or/circuitlist.h | 3 ++ src/core/or/circuitstats.c | 6 +-- src/core/or/circuituse.c | 2 +- src/core/or/ocirc_event.c | 84 +++++++++++++++++++++++++++++++++ src/core/or/ocirc_event.h | 89 +++++++++++++++++++++++++++++++++++ src/core/or/ocirc_event_sys.h | 13 +++++ src/feature/control/control.c | 5 +- src/feature/control/control.h | 10 +--- 12 files changed, 287 insertions(+), 17 deletions(-) create mode 100644 src/core/or/ocirc_event.c create mode 100644 src/core/or/ocirc_event.h create mode 100644 src/core/or/ocirc_event_sys.h diff --git a/src/app/main/subsystem_list.c b/src/app/main/subsystem_list.c index 1858738096..ef9b8142d9 100644 --- a/src/app/main/subsystem_list.c +++ b/src/app/main/subsystem_list.c @@ -8,6 +8,7 @@ #include "lib/cc/compat_compiler.h" #include "lib/cc/torint.h" +#include "core/or/ocirc_event_sys.h" #include "core/or/orconn_event_sys.h" #include "lib/compress/compress_sys.h" #include "lib/crypt_ops/crypto_sys.h" @@ -37,6 +38,7 @@ const subsys_fns_t *tor_subsystems[] = { &sys_crypto, /* -60 */ &sys_tortls, /* -50 */ &sys_orconn_event, /* -40 */ + &sys_ocirc_event, /* -39 */ }; const unsigned n_tor_subsystems = ARRAY_LENGTH(tor_subsystems); diff --git a/src/core/include.am b/src/core/include.am index a1fae73609..e171e4a6f1 100644 --- a/src/core/include.am +++ b/src/core/include.am @@ -39,6 +39,7 @@ LIBTOR_APP_A_SOURCES = \ src/core/or/connection_or.c \ src/core/or/dos.c \ src/core/or/onion.c \ + src/core/or/ocirc_event.c \ src/core/or/orconn_event.c \ src/core/or/policies.c \ src/core/or/protover.c \ @@ -245,6 +246,8 @@ noinst_HEADERS += \ src/core/or/or_connection_st.h \ src/core/or/or_handshake_certs_st.h \ src/core/or/or_handshake_state_st.h \ + src/core/or/ocirc_event.h \ + src/core/or/ocirc_event_sys.h \ src/core/or/origin_circuit_st.h \ src/core/or/policies.h \ src/core/or/port_cfg_st.h \ diff --git a/src/core/or/circuitbuild.c b/src/core/or/circuitbuild.c index d3744dc1c1..3de8da9cde 100644 --- a/src/core/or/circuitbuild.c +++ b/src/core/or/circuitbuild.c @@ -26,6 +26,7 @@ **/ #define CIRCUITBUILD_PRIVATE +#define OCIRC_EVENT_PRIVATE #include "core/or/or.h" #include "app/config/config.h" @@ -46,6 +47,7 @@ #include "core/or/connection_edge.h" #include "core/or/connection_or.h" #include "core/or/onion.h" +#include "core/or/ocirc_event.h" #include "core/or/policies.h" #include "core/or/relay.h" #include "feature/client/bridges.h" @@ -492,7 +494,7 @@ circuit_establish_circuit(uint8_t purpose, extend_info_t *exit_ei, int flags) return NULL; } - control_event_circuit_status(circ, CIRC_EVENT_LAUNCHED, 0); + circuit_event_status(circ, CIRC_EVENT_LAUNCHED, 0); if ((err_reason = circuit_handle_first_hop(circ)) < 0) { circuit_mark_for_close(TO_CIRCUIT(circ), -err_reason); @@ -508,6 +510,28 @@ origin_circuit_get_guard_state(origin_circuit_t *circ) return circ->guard_state; } +/** + * Helper function to publish a channel association message + * + * circuit_handle_first_hop() calls this to notify subscribers about a + * channel launch event, which associates a circuit with a channel. + * This doesn't always correspond to an assignment of the circuit's + * n_chan field, because that seems to be only for fully-open + * channels. + **/ +static void +circuit_chan_publish(const origin_circuit_t *circ, const channel_t *chan) +{ + ocirc_event_msg_t msg; + + msg.type = OCIRC_MSGTYPE_CHAN; + msg.u.chan.gid = circ->global_identifier; + msg.u.chan.chan = chan->global_identifier; + msg.u.chan.onehop = circ->build_state->onehop_tunnel; + + ocirc_event_publish(&msg); +} + /** Start establishing the first hop of our circuit. Figure out what * OR we should connect to, and if necessary start the connection to * it. If we're already connected, then send the 'create' cell. @@ -570,6 +594,7 @@ circuit_handle_first_hop(origin_circuit_t *circ) log_info(LD_CIRC,"connect to firsthop failed. Closing."); return -END_CIRC_REASON_CONNECTFAILED; } + circuit_chan_publish(circ, n_chan); } log_debug(LD_CIRC,"connecting in progress (or finished). Good."); @@ -581,6 +606,7 @@ circuit_handle_first_hop(origin_circuit_t *circ) } else { /* it's already open. use it. */ tor_assert(!circ->base_.n_hop); circ->base_.n_chan = n_chan; + circuit_chan_publish(circ, n_chan); log_debug(LD_CIRC,"Conn open. Delivering first onion skin."); if ((err_reason = circuit_send_next_onion_skin(circ)) < 0) { log_info(LD_CIRC,"circuit_send_next_onion_skin failed."); @@ -1416,7 +1442,7 @@ circuit_finish_handshake(origin_circuit_t *circ, hop->state = CPATH_STATE_OPEN; log_info(LD_CIRC,"Finished building circuit hop:"); circuit_log_path(LOG_INFO,LD_CIRC,circ); - control_event_circuit_status(circ, CIRC_EVENT_EXTENDED, 0); + circuit_event_status(circ, CIRC_EVENT_EXTENDED, 0); return 0; } diff --git a/src/core/or/circuitlist.c b/src/core/or/circuitlist.c index 0aa21000a1..c4b5f7ee3e 100644 --- a/src/core/or/circuitlist.c +++ b/src/core/or/circuitlist.c @@ -51,6 +51,7 @@ * logic, which was originally circuit-focused. **/ #define CIRCUITLIST_PRIVATE +#define OCIRC_EVENT_PRIVATE #include "lib/cc/torint.h" /* TOR_PRIuSZ */ #include "core/or/or.h" @@ -96,6 +97,9 @@ #include "lib/compress/compress_zstd.h" #include "lib/buf/buffers.h" +#define OCIRC_EVENT_PRIVATE +#include "core/or/ocirc_event.h" + #include "ht.h" #include "core/or/cpath_build_state_st.h" @@ -481,6 +485,56 @@ circuit_set_n_circid_chan(circuit_t *circ, circid_t id, } } +/** + * Helper function to publish a message about events on an origin circuit + * + * Publishes a message to subscribers of origin circuit events, and + * sends the control event. + **/ +int +circuit_event_status(origin_circuit_t *circ, circuit_status_event_t tp, + int reason_code) +{ + ocirc_event_msg_t msg; + + tor_assert(circ); + + msg.type = OCIRC_MSGTYPE_CEVENT; + msg.u.cevent.gid = circ->global_identifier; + msg.u.cevent.evtype = tp; + msg.u.cevent.reason = reason_code; + msg.u.cevent.onehop = circ->build_state->onehop_tunnel; + + ocirc_event_publish(&msg); + return control_event_circuit_status(circ, tp, reason_code); +} + +/** + * Helper function to publish a state change message + * + * circuit_set_state() calls this to notify subscribers about a change + * of the state of an origin circuit. + **/ +static void +circuit_state_publish(const circuit_t *circ) +{ + ocirc_event_msg_t msg; + const origin_circuit_t *ocirc; + + if (!CIRCUIT_IS_ORIGIN(circ)) + return; + ocirc = CONST_TO_ORIGIN_CIRCUIT(circ); + /* Only inbound OR circuits can be in this state, not origin circuits. */ + tor_assert(circ->state != CIRCUIT_STATE_ONIONSKIN_PENDING); + + msg.type = OCIRC_MSGTYPE_STATE; + msg.u.state.gid = ocirc->global_identifier; + msg.u.state.state = circ->state; + msg.u.state.onehop = ocirc->build_state->onehop_tunnel; + + ocirc_event_publish(&msg); +} + /** Change the state of circ to state, adding it to or removing * it from lists as appropriate. */ void @@ -510,6 +564,7 @@ circuit_set_state(circuit_t *circ, uint8_t state) if (state == CIRCUIT_STATE_GUARD_WAIT || state == CIRCUIT_STATE_OPEN) tor_assert(!circ->n_chan_create_cell); circ->state = state; + circuit_state_publish(circ); } /** Append to out all circuits in state CHAN_WAIT waiting for @@ -2270,7 +2325,7 @@ circuit_about_to_free(circuit_t *circ) smartlist_remove(circuits_pending_other_guards, circ); } if (CIRCUIT_IS_ORIGIN(circ)) { - control_event_circuit_status(TO_ORIGIN_CIRCUIT(circ), + circuit_event_status(TO_ORIGIN_CIRCUIT(circ), (circ->state == CIRCUIT_STATE_OPEN || circ->state == CIRCUIT_STATE_GUARD_WAIT) ? CIRC_EVENT_CLOSED:CIRC_EVENT_FAILED, diff --git a/src/core/or/circuitlist.h b/src/core/or/circuitlist.h index cb89d1820d..37d37a089d 100644 --- a/src/core/or/circuitlist.h +++ b/src/core/or/circuitlist.h @@ -14,6 +14,7 @@ #include "lib/testsupport/testsupport.h" #include "feature/hs/hs_ident.h" +#include "core/or/ocirc_event.h" /** Circuit state: I'm the origin, still haven't done all my handshakes. */ #define CIRCUIT_STATE_BUILDING 0 @@ -184,6 +185,8 @@ void channel_mark_circid_unusable(channel_t *chan, circid_t id); void channel_mark_circid_usable(channel_t *chan, circid_t id); time_t circuit_id_when_marked_unusable_on_channel(circid_t circ_id, channel_t *chan); +int circuit_event_status(origin_circuit_t *circ, circuit_status_event_t tp, + int reason_code); void circuit_set_state(circuit_t *circ, uint8_t state); void circuit_close_all_marked(void); int32_t circuit_initial_package_window(void); diff --git a/src/core/or/circuitstats.c b/src/core/or/circuitstats.c index 0429f2c86e..61d5e18a45 100644 --- a/src/core/or/circuitstats.c +++ b/src/core/or/circuitstats.c @@ -639,9 +639,9 @@ circuit_build_times_rewind_history(circuit_build_times_t *cbt, int n) void circuit_build_times_mark_circ_as_measurement_only(origin_circuit_t *circ) { - control_event_circuit_status(circ, - CIRC_EVENT_FAILED, - END_CIRC_REASON_TIMEOUT); + circuit_event_status(circ, + CIRC_EVENT_FAILED, + END_CIRC_REASON_TIMEOUT); circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT); /* Record this event to check for too many timeouts diff --git a/src/core/or/circuituse.c b/src/core/or/circuituse.c index 088358a4d6..e230ad1005 100644 --- a/src/core/or/circuituse.c +++ b/src/core/or/circuituse.c @@ -1664,7 +1664,7 @@ circuit_testing_failed(origin_circuit_t *circ, int at_last_hop) void circuit_has_opened(origin_circuit_t *circ) { - control_event_circuit_status(circ, CIRC_EVENT_BUILT, 0); + circuit_event_status(circ, CIRC_EVENT_BUILT, 0); /* Remember that this circuit has finished building. Now if we start * it building again later (e.g. by extending it), we will know not diff --git a/src/core/or/ocirc_event.c b/src/core/or/ocirc_event.c new file mode 100644 index 0000000000..f9f8af279f --- /dev/null +++ b/src/core/or/ocirc_event.c @@ -0,0 +1,84 @@ +/* Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file ocirc_event.c + * \brief Publish state change messages for origin circuits + * + * Implements a basic publish-subscribe framework for messages about + * the state of origin circuits. The publisher calls the subscriber + * callback functions synchronously. + * + * Although the synchronous calls might not simplify the call graph, + * this approach improves data isolation because the publisher doesn't + * need knowledge about the internals of subscribing subsystems. It + * also avoids race conditions that might occur in asynchronous + * frameworks. + **/ + +#include "core/or/or.h" + +#define OCIRC_EVENT_PRIVATE + +#include "core/or/cpath_build_state_st.h" +#include "core/or/ocirc_event.h" +#include "core/or/ocirc_event_sys.h" +#include "core/or/origin_circuit_st.h" +#include "lib/subsys/subsys.h" + +/** List of subscribers */ +static smartlist_t *ocirc_event_rcvrs; + +/** Initialize subscriber list */ +static int +ocirc_event_init(void) +{ + ocirc_event_rcvrs = smartlist_new(); + return 0; +} + +/** Free subscriber list */ +static void +ocirc_event_fini(void) +{ + smartlist_free(ocirc_event_rcvrs); +} + +/** + * Subscribe to messages about origin circuit events + * + * Register a callback function to receive messages about origin + * circuits. The publisher calls this function synchronously. + **/ +void +ocirc_event_subscribe(ocirc_event_rcvr_t fn) +{ + tor_assert(fn); + /* Don't duplicate subscriptions. */ + if (smartlist_contains(ocirc_event_rcvrs, fn)) + return; + + smartlist_add(ocirc_event_rcvrs, fn); +} + +/** + * Publish a message about OR connection events + * + * This calls the subscriber receiver function synchronously. + **/ +void +ocirc_event_publish(const ocirc_event_msg_t *msg) +{ + SMARTLIST_FOREACH_BEGIN(ocirc_event_rcvrs, ocirc_event_rcvr_t, fn) { + tor_assert(fn); + (*fn)(msg); + } SMARTLIST_FOREACH_END(fn); +} + +const subsys_fns_t sys_ocirc_event = { + .name = "ocirc_event", + .supported = true, + .level = -39, + .initialize = ocirc_event_init, + .shutdown = ocirc_event_fini, +}; diff --git a/src/core/or/ocirc_event.h b/src/core/or/ocirc_event.h new file mode 100644 index 0000000000..19a237d7df --- /dev/null +++ b/src/core/or/ocirc_event.h @@ -0,0 +1,89 @@ +/* Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file ocirc_event.h + * \brief Header file for ocirc_event.c + **/ + +#ifndef TOR_OCIRC_EVENT_H +#define TOR_OCIRC_EVENT_H + +#include + +#include "lib/cc/torint.h" + +/** Used to indicate the type of a circuit event passed to the controller. + * The various types are defined in control-spec.txt */ +typedef enum circuit_status_event_t { + CIRC_EVENT_LAUNCHED = 0, + CIRC_EVENT_BUILT = 1, + CIRC_EVENT_EXTENDED = 2, + CIRC_EVENT_FAILED = 3, + CIRC_EVENT_CLOSED = 4, +} circuit_status_event_t; + +/** Message for origin circuit state update */ +typedef struct ocirc_state_msg_t { + uint32_t gid; /**< global ID (only origin circuits have them) */ + int state; /**< new circuit state */ + bool onehop; /**< one-hop circuit? */ +} ocirc_state_msg_t; + +/** + * Message when a channel gets associated to a circuit. + * + * This doesn't always correspond to something in circuitbuild.c + * setting the n_chan field in the circuit. For some reason, if + * circuit_handle_first_hop() launches a new circuit, it doesn't set + * the n_chan field. + */ +typedef struct ocirc_chan_msg_t { + uint32_t gid; /**< global ID */ + uint64_t chan; /**< channel ID */ + bool onehop; /**< one-hop circuit? */ +} ocirc_chan_msg_t; + +/** + * Message for origin circuit status event + * + * This contains information that ends up in CIRC control protocol events. + */ +typedef struct ocirc_cevent_msg_t { + uint32_t gid; /**< global ID */ + int evtype; /**< event type */ + int reason; /**< reason */ + bool onehop; /**< one-hop circuit? */ +} ocirc_cevent_msg_t; + +/** Discriminant values for origin circuit event message */ +typedef enum ocirc_msgtype_t { + OCIRC_MSGTYPE_STATE, + OCIRC_MSGTYPE_CHAN, + OCIRC_MSGTYPE_CEVENT, +} ocirc_msgtype_t; + +/** Discriminated union for the actual message */ +typedef struct ocirc_event_msg_t { + int type; + union { + ocirc_state_msg_t state; + ocirc_chan_msg_t chan; + ocirc_cevent_msg_t cevent; + } u; +} ocirc_event_msg_t; + +/** + * Receiver function pointer for origin circuit subscribers + * + * This function gets called synchronously by the publisher. + **/ +typedef void (*ocirc_event_rcvr_t)(const ocirc_event_msg_t *); + +void ocirc_event_subscribe(ocirc_event_rcvr_t fn); + +#ifdef OCIRC_EVENT_PRIVATE +void ocirc_event_publish(const ocirc_event_msg_t *msg); +#endif + +#endif /* defined(TOR_OCIRC_EVENT_STATE_H) */ diff --git a/src/core/or/ocirc_event_sys.h b/src/core/or/ocirc_event_sys.h new file mode 100644 index 0000000000..0bc135ffaf --- /dev/null +++ b/src/core/or/ocirc_event_sys.h @@ -0,0 +1,13 @@ +/* Copyright (c) 2007-2018, The Tor Project, Inc. */ + +/** + * \file ocirc_event_sys.h + * \brief Declare subsystem object for the origin circuit event module. + **/ + +#ifndef TOR_OCIRC_EVENT_SYS_H +#define TOR_OCIRC_EVENT_SYS_H + +extern const struct subsys_fns_t sys_ocirc_event; + +#endif /* defined(TOR_OCIRC_EVENT_H) */ diff --git a/src/feature/control/control.c b/src/feature/control/control.c index 4ef550c919..da62c94981 100644 --- a/src/feature/control/control.c +++ b/src/feature/control/control.c @@ -34,6 +34,7 @@ **/ #define CONTROL_PRIVATE +#define OCIRC_EVENT_PRIVATE #include "core/or/or.h" #include "app/config/config.h" @@ -50,6 +51,7 @@ #include "core/or/command.h" #include "core/or/connection_edge.h" #include "core/or/connection_or.h" +#include "core/or/ocirc_event.h" #include "core/or/policies.h" #include "core/or/reasons.h" #include "core/or/versions.h" @@ -3749,7 +3751,7 @@ handle_control_extendcircuit(control_connection_t *conn, uint32_t len, connection_printf_to_buf(conn, "250 EXTENDED %lu\r\n", (unsigned long)circ->global_identifier); if (zero_circ) /* send a 'launched' event, for completeness */ - control_event_circuit_status(circ, CIRC_EVENT_LAUNCHED, 0); + circuit_event_status(circ, CIRC_EVENT_LAUNCHED, 0); done: SMARTLIST_FOREACH(router_nicknames, char *, n, tor_free(n)); smartlist_free(router_nicknames); @@ -5625,6 +5627,7 @@ control_event_circuit_status(origin_circuit_t *circ, circuit_status_event_t tp, { const char *status; char reasons[64] = ""; + if (!EVENT_IS_INTERESTING(EVENT_CIRCUIT_STATUS)) return 0; tor_assert(circ); diff --git a/src/feature/control/control.h b/src/feature/control/control.h index d3245fcaf1..68c9a6bed1 100644 --- a/src/feature/control/control.h +++ b/src/feature/control/control.h @@ -12,15 +12,7 @@ #ifndef TOR_CONTROL_H #define TOR_CONTROL_H -/** Used to indicate the type of a circuit event passed to the controller. - * The various types are defined in control-spec.txt */ -typedef enum circuit_status_event_t { - CIRC_EVENT_LAUNCHED = 0, - CIRC_EVENT_BUILT = 1, - CIRC_EVENT_EXTENDED = 2, - CIRC_EVENT_FAILED = 3, - CIRC_EVENT_CLOSED = 4, -} circuit_status_event_t; +#include "core/or/ocirc_event.h" /** Used to indicate the type of a CIRC_MINOR event passed to the controller. * The various types are defined in control-spec.txt . */ From b0f974633ac17e07c3ab93ecea9337506bb8d583 Mon Sep 17 00:00:00 2001 From: Taylor Yu Date: Wed, 19 Dec 2018 21:35:26 -0600 Subject: [PATCH 0342/2557] Add LD_BTRACK log domain for bootstrap tracker Part of ticket 27167. --- doc/tor.1.txt | 2 +- src/lib/log/log.c | 2 +- src/lib/log/log.h | 4 +++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/doc/tor.1.txt b/doc/tor.1.txt index 974a484541..4ff789a931 100644 --- a/doc/tor.1.txt +++ b/doc/tor.1.txt @@ -678,7 +678,7 @@ GENERAL OPTIONS The currently recognized domains are: general, crypto, net, config, fs, protocol, mm, http, app, control, circ, rend, bug, dir, dirserv, or, edge, acct, hist, handshake, heartbeat, channel, sched, guard, consdiff, dos, - process, and pt. + process, pt, and btrack. Domain names are case-insensitive. + + For example, "`Log [handshake]debug [~net,~mm]info notice stdout`" sends diff --git a/src/lib/log/log.c b/src/lib/log/log.c index 2c25ddf1aa..d032f57add 100644 --- a/src/lib/log/log.c +++ b/src/lib/log/log.c @@ -1268,7 +1268,7 @@ static const char *domain_list[] = { "GENERAL", "CRYPTO", "NET", "CONFIG", "FS", "PROTOCOL", "MM", "HTTP", "APP", "CONTROL", "CIRC", "REND", "BUG", "DIR", "DIRSERV", "OR", "EDGE", "ACCT", "HIST", "HANDSHAKE", "HEARTBEAT", "CHANNEL", - "SCHED", "GUARD", "CONSDIFF", "DOS", "PROCESS", "PT", NULL + "SCHED", "GUARD", "CONSDIFF", "DOS", "PROCESS", "PT", "BTRACK", NULL }; /** Return a bitmask for the log domain for which domain is the name, diff --git a/src/lib/log/log.h b/src/lib/log/log.h index c8820ee037..423e58e11b 100644 --- a/src/lib/log/log.h +++ b/src/lib/log/log.h @@ -111,8 +111,10 @@ #define LD_PROCESS (1u<<26) /** Pluggable Transports. */ #define LD_PT (1u<<27) +/** Bootstrap tracker. */ +#define LD_BTRACK (1u<<28) /** Number of logging domains in the code. */ -#define N_LOGGING_DOMAINS 28 +#define N_LOGGING_DOMAINS 29 /** This log message is not safe to send to a callback-based logger * immediately. Used as a flag, not a log domain. */ From b0ae6a332a0882a670b3a2bacf5c70b69936e4bc Mon Sep 17 00:00:00 2001 From: Taylor Yu Date: Mon, 17 Dec 2018 09:45:09 -0600 Subject: [PATCH 0343/2557] Add bootstrap tracker subsystem Add a tracker for bootstrap progress, tracking events related to origin circuit and ORCONN states. This uses the ocirc_event and orconn_event publish-subscribe subsystems. Part of ticket 27167. --- src/app/main/subsystem_list.c | 2 + src/core/include.am | 8 + src/feature/control/btrack.c | 53 ++++++ src/feature/control/btrack_circuit.c | 164 +++++++++++++++++ src/feature/control/btrack_circuit.h | 15 ++ src/feature/control/btrack_orconn.c | 196 ++++++++++++++++++++ src/feature/control/btrack_orconn.h | 38 ++++ src/feature/control/btrack_orconn_maps.c | 223 +++++++++++++++++++++++ src/feature/control/btrack_orconn_maps.h | 17 ++ src/feature/control/btrack_sys.h | 14 ++ 10 files changed, 730 insertions(+) create mode 100644 src/feature/control/btrack.c create mode 100644 src/feature/control/btrack_circuit.c create mode 100644 src/feature/control/btrack_circuit.h create mode 100644 src/feature/control/btrack_orconn.c create mode 100644 src/feature/control/btrack_orconn.h create mode 100644 src/feature/control/btrack_orconn_maps.c create mode 100644 src/feature/control/btrack_orconn_maps.h create mode 100644 src/feature/control/btrack_sys.h diff --git a/src/app/main/subsystem_list.c b/src/app/main/subsystem_list.c index ef9b8142d9..2f2586ac15 100644 --- a/src/app/main/subsystem_list.c +++ b/src/app/main/subsystem_list.c @@ -10,6 +10,7 @@ #include "core/or/ocirc_event_sys.h" #include "core/or/orconn_event_sys.h" +#include "feature/control/btrack_sys.h" #include "lib/compress/compress_sys.h" #include "lib/crypt_ops/crypto_sys.h" #include "lib/err/torerr_sys.h" @@ -39,6 +40,7 @@ const subsys_fns_t *tor_subsystems[] = { &sys_tortls, /* -50 */ &sys_orconn_event, /* -40 */ &sys_ocirc_event, /* -39 */ + &sys_btrack, /* -30 */ }; const unsigned n_tor_subsystems = ARRAY_LENGTH(tor_subsystems); diff --git a/src/core/include.am b/src/core/include.am index e171e4a6f1..93a8bca5d5 100644 --- a/src/core/include.am +++ b/src/core/include.am @@ -63,6 +63,10 @@ LIBTOR_APP_A_SOURCES = \ src/feature/client/dnsserv.c \ src/feature/client/entrynodes.c \ src/feature/client/transports.c \ + src/feature/control/btrack.c \ + src/feature/control/btrack_circuit.c \ + src/feature/control/btrack_orconn.c \ + src/feature/control/btrack_orconn_maps.c \ src/feature/control/control.c \ src/feature/control/control_bootstrap.c \ src/feature/control/fmt_serverstatus.c \ @@ -274,6 +278,10 @@ noinst_HEADERS += \ src/feature/client/dnsserv.h \ src/feature/client/entrynodes.h \ src/feature/client/transports.h \ + src/feature/control/btrack_circuit.h \ + src/feature/control/btrack_orconn.h \ + src/feature/control/btrack_orconn_maps.h \ + src/feature/control/btrack_sys.h \ src/feature/control/control.h \ src/feature/control/control_connection_st.h \ src/feature/control/fmt_serverstatus.h \ diff --git a/src/feature/control/btrack.c b/src/feature/control/btrack.c new file mode 100644 index 0000000000..14220faad1 --- /dev/null +++ b/src/feature/control/btrack.c @@ -0,0 +1,53 @@ +/* Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file btrack.c + * \brief Bootstrap trackers + * + * Initializes and shuts down the specific bootstrap trackers. These + * trackers help the reporting of bootstrap progress by maintaining + * state information about various subsystems within tor. When the + * correct state changes happen, these trackers emit controller + * events. + * + * These trackers avoid referring directly to the internals of state + * objects of other subsystems. + * + * btrack_circuit.c contains the tracker for origin circuits. + * + * btrack_orconn.c contains the tracker for OR connections. + * + * Eventually there will be a tracker for directory downloads as well. + **/ + +#include "feature/control/btrack_circuit.h" +#include "feature/control/btrack_orconn.h" +#include "feature/control/btrack_sys.h" +#include "lib/subsys/subsys.h" + +static int +btrack_init(void) +{ + if (btrack_orconn_init()) + return -1; + if (btrack_circ_init()) + return -1; + + return 0; +} + +static void +btrack_fini(void) +{ + btrack_orconn_fini(); + btrack_circ_fini(); +} + +const subsys_fns_t sys_btrack = { + .name = "btrack", + .supported = true, + .level = -30, + .initialize = btrack_init, + .shutdown = btrack_fini, +}; diff --git a/src/feature/control/btrack_circuit.c b/src/feature/control/btrack_circuit.c new file mode 100644 index 0000000000..bf09e0b99c --- /dev/null +++ b/src/feature/control/btrack_circuit.c @@ -0,0 +1,164 @@ +/* Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file btrack_circuit.c + * \brief Bootstrap tracker for origin circuits + * + * Track state changes of origin circuits, as published by the circuit + * subsystem. + **/ + +#include "core/or/or.h" + +#include "core/or/ocirc_event.h" + +#include "feature/control/btrack_circuit.h" +#include "feature/control/control.h" +#include "lib/log/log.h" + +/** Pair of a best origin circuit GID with its state or status */ +typedef struct btc_best_t { + uint32_t gid; + int val; +} btc_best_t; + +/** GID and state of the best origin circuit we've seen so far */ +static btc_best_t best_any_state = { 0, -1 }; +/** GID and state of the best application circuit we've seen so far */ +static btc_best_t best_ap_state = { 0, -1 }; +/** GID and status of the best origin circuit we've seen so far */ +static btc_best_t best_any_evtype = { 0, -1 }; +/** GID and status of the best application circuit we've seen so far */ +static btc_best_t best_ap_evtype = { 0, -1 }; + +/** Reset cached "best" values */ +static void +btc_reset_bests(void) +{ + best_any_state.gid = best_ap_state.gid = 0; + best_any_state.val = best_ap_state.val = -1; + best_any_evtype.gid = best_ap_state.gid = 0; + best_any_evtype.val = best_ap_evtype.val = -1; +} + +/** True if @a state is a "better" origin circuit state than @a best->val */ +static bool +btc_state_better(int state, const btc_best_t *best) +{ + return state > best->val; +} + +/** + * Definine an ordering on circuit status events + * + * The CIRC_EVENT_ constants aren't sorted in a useful order, so this + * array helps to decode them. This approach depends on the statuses + * being nonnegative and dense. + **/ +static int circ_event_order[] = { + [CIRC_EVENT_FAILED] = -1, + [CIRC_EVENT_CLOSED] = -1, + [CIRC_EVENT_LAUNCHED] = 1, + [CIRC_EVENT_EXTENDED] = 2, + [CIRC_EVENT_BUILT] = 3, +}; +#define N_CIRC_EVENT_ORDER \ + (sizeof(circ_event_order) / sizeof(circ_event_order[0])) + +/** True if @a state is a "better" origin circuit event status than @a + best->val */ +static bool +btc_evtype_better(int state, const btc_best_t *best) +{ + if (state < 0) + return false; + if (best->val < 0) + return true; + + tor_assert(state >= 0 && (unsigned)state < N_CIRC_EVENT_ORDER); + tor_assert(best->val >= 0 && (unsigned)best->val < N_CIRC_EVENT_ORDER); + return circ_event_order[state] > circ_event_order[best->val]; +} + +static bool +btc_update_state(const ocirc_state_msg_t *msg, btc_best_t *best, + const char *type) +{ + if (btc_state_better(msg->state, best)) { + log_info(LD_BTRACK, "CIRC BEST_%s state %d->%d gid=%"PRIu32, type, + best->val, msg->state, msg->gid); + best->gid = msg->gid; + best->val = msg->state; + return true; + } + return false; +} + +static bool +btc_update_evtype(const ocirc_cevent_msg_t *msg, btc_best_t *best, + const char *type) +{ + if (btc_evtype_better(msg->evtype, best)) { + log_info(LD_BTRACK, "CIRC BEST_%s evtype %d->%d gid=%"PRIu32, type, + best->val, msg->evtype, msg->gid); + best->gid = msg->gid; + best->val = msg->evtype; + return true; + } + return false; +} + +static void +btc_state_rcvr(const ocirc_state_msg_t *msg) +{ + log_debug(LD_BTRACK, "CIRC gid=%"PRIu32" state=%d onehop=%d", + msg->gid, msg->state, msg->onehop); + + btc_update_state(msg, &best_any_state, "ANY"); + if (msg->onehop) + return; + btc_update_state(msg, &best_ap_state, "AP"); +} + +static void +btc_cevent_rcvr(const ocirc_cevent_msg_t *msg) +{ + log_debug(LD_BTRACK, "CIRC gid=%"PRIu32" evtype=%d reason=%d onehop=%d", + msg->gid, msg->evtype, msg->reason, msg->onehop); + + btc_update_evtype(msg, &best_any_evtype, "ANY"); + if (msg->onehop) + return; + btc_update_evtype(msg, &best_ap_evtype, "AP"); +} + +static void +btc_event_rcvr(const ocirc_event_msg_t *msg) +{ + switch (msg->type) { + case OCIRC_MSGTYPE_STATE: + return btc_state_rcvr(&msg->u.state); + case OCIRC_MSGTYPE_CHAN: + log_debug(LD_BTRACK, "CIRC gid=%"PRIu32" chan=%"PRIu64" onehop=%d", + msg->u.chan.gid, msg->u.chan.chan, msg->u.chan.onehop); + break; + case OCIRC_MSGTYPE_CEVENT: + return btc_cevent_rcvr(&msg->u.cevent); + default: + break; + } +} + +int +btrack_circ_init(void) +{ + ocirc_event_subscribe(btc_event_rcvr); + return 0; +} + +void +btrack_circ_fini(void) +{ + btc_reset_bests(); +} diff --git a/src/feature/control/btrack_circuit.h b/src/feature/control/btrack_circuit.h new file mode 100644 index 0000000000..ab8b8b652c --- /dev/null +++ b/src/feature/control/btrack_circuit.h @@ -0,0 +1,15 @@ +/* Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file btrack_circuit.h + * \brief Header file for btrack_circuit.c + **/ + +#ifndef TOR_BTRACK_CIRCUIT_H +#define TOR_BTRACK_CIRCUIT_H + +int btrack_circ_init(void); +void btrack_circ_fini(void); + +#endif /* defined(TOR_BTRACK_CIRCUIT_H) */ diff --git a/src/feature/control/btrack_orconn.c b/src/feature/control/btrack_orconn.c new file mode 100644 index 0000000000..69344569bf --- /dev/null +++ b/src/feature/control/btrack_orconn.c @@ -0,0 +1,196 @@ +/* Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file btrack_orconn.c + * \brief Bootstrap tracker for OR connections + * + * Track state changes of OR connections, as published by the + * connection subsystem. Also track circuit launch events, because + * they're one of the few ways to discover the association between a + * channel (and OR connection) and a circuit. + * + * We track all OR connections that we receive events for, whether or + * not they're carrying origin circuits. (An OR connection might + * carry origin circuits only after we first find out about that + * connection.) + * + * All origin ORCONN events update the "any" state variables, while + * only application ORCONN events update the "ap" state variables (and + * also update the "any") variables. + * + * We do this because we want to report the first increments of + * connection progress as the earliest bootstrap phases. This results + * in a better user experience because failures here translate into + * zero or very small amounts of displayed progress, instead of + * progress stuck near completion. The first connection to a relay + * might be a one-hop circuit for directory lookups, or it might be a + * connection for an application circuit because we already have + * enough directory info to build an application circuit. + **/ + +#include + +#include "core/or/or.h" + +#define BTRACK_ORCONN_PRIVATE + +#include "core/or/ocirc_event.h" +#include "core/or/orconn_event.h" +#include "feature/control/btrack_orconn.h" +#include "feature/control/btrack_orconn_maps.h" +#include "lib/log/log.h" + +/** Pair of a best ORCONN GID and with its state */ +typedef struct bto_best_t { + uint64_t gid; + int state; +} bto_best_t; + +/** GID and state of the best ORCONN we've seen so far */ +static bto_best_t best_any = { 0, -1 }; +/** GID and state of the best application circuit ORCONN we've seen so far */ +static bto_best_t best_ap = { 0, -1 }; + +/** + * Update a cached state of a best ORCONN progress we've seen so far. + * + * Return true if the new state is better than the old. + **/ +static bool +bto_update_best(const bt_orconn_t *bto, bto_best_t *best, const char *type) +{ + if (bto->state < best->state) + return false; + best->gid = bto->gid; + if (bto->state > best->state) { + log_info(LD_BTRACK, "ORCONN BEST_%s state %d->%d gid=%"PRIu64, type, + best->state, bto->state, bto->gid); + best->state = bto->state; + return true; + } + return false; +} + +/** + * Update cached states of best ORCONN progress we've seen + * + * Only update the application ORCONN state if we know it's carrying + * an application circuit. + **/ +static void +bto_update_bests(const bt_orconn_t *bto) +{ + tor_assert(bto->is_orig); + + bto_update_best(bto, &best_any, "ANY"); + if (!bto->is_onehop) + bto_update_best(bto, &best_ap, "AP"); +} + +/** Reset cached "best" values */ +static void +bto_reset_bests(void) +{ + best_any.gid = best_ap.gid = 0; + best_any.state = best_ap.state = -1; +} + +/** + * Update cached states of ORCONNs from the incoming message. This + * message comes from code in connection_or.c. + **/ +static void +bto_state_rcvr(const orconn_state_msg_t *msg) +{ + bt_orconn_t *bto; + + bto = bto_find_or_new(msg->gid, msg->chan); + log_debug(LD_BTRACK, "ORCONN gid=%"PRIu64" chan=%"PRIu64 + " proxy_type=%d state=%d", + msg->gid, msg->chan, msg->proxy_type, msg->state); + bto->proxy_type = msg->proxy_type; + bto->state = msg->state; + if (bto->is_orig) + bto_update_bests(bto); +} + +/** + * Delete a cached ORCONN state if we get an incoming message saying + * the ORCONN is failed or closed. This message comes from code in + * control.c. + **/ +static void +bto_status_rcvr(const orconn_status_msg_t *msg) +{ + switch (msg->status) { + case OR_CONN_EVENT_FAILED: + case OR_CONN_EVENT_CLOSED: + log_info(LD_BTRACK, "ORCONN DELETE gid=%"PRIu64" status=%d reason=%d", + msg->gid, msg->status, msg->reason); + return bto_delete(msg->gid); + default: + break; + } +} + +/** Dispatch to individual ORCONN message handlers */ +static void +bto_event_rcvr(const orconn_event_msg_t *msg) +{ + switch (msg->type) { + case ORCONN_MSGTYPE_STATE: + return bto_state_rcvr(&msg->u.state); + case ORCONN_MSGTYPE_STATUS: + return bto_status_rcvr(&msg->u.status); + default: + tor_assert(false); + } +} + +/** + * Create or update a cached ORCONN state for a newly launched + * connection, including whether it's launched by an origin circuit + * and whether it's a one-hop circuit. + **/ +static void +bto_chan_rcvr(const ocirc_event_msg_t *msg) +{ + bt_orconn_t *bto; + + /* Ignore other kinds of origin circuit events; we don't need them */ + if (msg->type != OCIRC_MSGTYPE_CHAN) + return; + + bto = bto_find_or_new(0, msg->u.chan.chan); + if (!bto->is_orig || (bto->is_onehop && !msg->u.chan.onehop)) { + log_debug(LD_BTRACK, "ORCONN LAUNCH chan=%"PRIu64" onehop=%d", + msg->u.chan.chan, msg->u.chan.onehop); + } + bto->is_orig = true; + if (!msg->u.chan.onehop) + bto->is_onehop = false; + bto_update_bests(bto); +} + +/** + * Initialize the hash maps and subscribe to ORCONN and origin + * circuit events. + **/ +int +btrack_orconn_init(void) +{ + bto_init_maps(); + orconn_event_subscribe(bto_event_rcvr); + ocirc_event_subscribe(bto_chan_rcvr); + + return 0; +} + +/** Clear the hash maps and reset the "best" states */ +void +btrack_orconn_fini(void) +{ + bto_clear_maps(); + bto_reset_bests(); +} diff --git a/src/feature/control/btrack_orconn.h b/src/feature/control/btrack_orconn.h new file mode 100644 index 0000000000..4e514d4b04 --- /dev/null +++ b/src/feature/control/btrack_orconn.h @@ -0,0 +1,38 @@ +/* Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file btrack_orconn.h + * \brief Header file for btrack_orconn.c + **/ + +#ifndef TOR_BTRACK_ORCONN_H +#define TOR_BTRACK_ORCONN_H + +#ifdef BTRACK_ORCONN_PRIVATE + +#include "ht.h" + +/** + * Structure for tracking OR connection states + * + * This gets linked into two hash maps: one with connection IDs, and + * another with channel IDs. + **/ +typedef struct bt_orconn_t { + HT_ENTRY(bt_orconn_t) node; /**< Hash map entry indexed by gid */ + HT_ENTRY(bt_orconn_t) chan_node; /**< Hash map entry indexed by channel ID */ + uint64_t gid; /**< Global ID of this ORCONN */ + uint64_t chan; /**< Channel ID, if known */ + int proxy_type; /**< Proxy type */ + uint8_t state; /**< State of this ORCONN */ + bool is_orig; /**< Does this carry an origin circuit? */ + bool is_onehop; /**< Is this for a one-hop circuit? */ +} bt_orconn_t; + +#endif /* defined(BTRACK_ORCONN_PRIVATE) */ + +int btrack_orconn_init(void); +void btrack_orconn_fini(void); + +#endif /* defined(TOR_BTRACK_ORCONN_H) */ diff --git a/src/feature/control/btrack_orconn_maps.c b/src/feature/control/btrack_orconn_maps.c new file mode 100644 index 0000000000..b6bb23804c --- /dev/null +++ b/src/feature/control/btrack_orconn_maps.c @@ -0,0 +1,223 @@ +/* Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file btrack_orconn_maps.c + * \brief Hash map implementation for btrack_orconn.c + * + * These functions manipulate the hash maps that contain bt_orconn + * objects. + **/ + +#include + +#include "core/or/or.h" + +#include "ht.h" +#include "siphash.h" + +#define BTRACK_ORCONN_PRIVATE + +#include "feature/control/btrack_orconn.h" +#include "feature/control/btrack_orconn_maps.h" +#include "lib/log/log.h" + +static inline unsigned int +bto_gid_hash_(bt_orconn_t *elm) +{ + return (unsigned)siphash24g(&elm->gid, sizeof(elm->gid)); +} + +static inline int +bto_gid_eq_(bt_orconn_t *a, bt_orconn_t *b) +{ + return a->gid == b->gid; +} + +static inline unsigned int +bto_chan_hash_(bt_orconn_t *elm) +{ + return (unsigned)siphash24g(&elm->chan, sizeof(elm->chan)); +} + +static inline int +bto_chan_eq_(bt_orconn_t *a, bt_orconn_t *b) +{ + return a->chan == b->chan; +} + +HT_HEAD(bto_gid_ht, bt_orconn_t); +HT_PROTOTYPE(bto_gid_ht, bt_orconn_t, node, bto_gid_hash_, bto_gid_eq_) +HT_GENERATE2(bto_gid_ht, bt_orconn_t, node, + bto_gid_hash_, bto_gid_eq_, 0.6, + tor_reallocarray_, tor_free_) +static struct bto_gid_ht *bto_gid_map; + +HT_HEAD(bto_chan_ht, bt_orconn_t); +HT_PROTOTYPE(bto_chan_ht, bt_orconn_t, chan_node, bto_chan_hash_, bto_chan_eq_) +HT_GENERATE2(bto_chan_ht, bt_orconn_t, chan_node, + bto_chan_hash_, bto_chan_eq_, 0.6, + tor_reallocarray_, tor_free_) +static struct bto_chan_ht *bto_chan_map; + +/** Clear the GID hash map, freeing any bt_orconn_t objects that become + * unreferenced */ +static void +bto_gid_clear_map(void) +{ + bt_orconn_t **elt, **next, *c; + + for (elt = HT_START(bto_gid_ht, bto_gid_map); + elt; + elt = next) { + c = *elt; + next = HT_NEXT_RMV(bto_gid_ht, bto_gid_map, elt); + + c->gid = 0; + /* Don't delete if chan ID isn't zero: it's still in the chan hash map */ + if (!c->chan) + tor_free(c); + } + HT_CLEAR(bto_gid_ht, bto_gid_map); + tor_free(bto_gid_map); +} + +/** Clear the chan ID hash map, freeing any bt_orconn_t objects that + * become unreferenced */ +static void +bto_chan_clear_map(void) +{ + bt_orconn_t **elt, **next, *c; + + for (elt = HT_START(bto_chan_ht, bto_chan_map); + elt; + elt = next) { + c = *elt; + next = HT_NEXT_RMV(bto_chan_ht, bto_chan_map, elt); + + c->chan = 0; + /* Don't delete if GID isn't zero, it's still in the GID hash map */ + if (!c->gid) + tor_free(c); + } + HT_CLEAR(bto_chan_ht, bto_chan_map); + tor_free(bto_chan_map); +} + +/** Delete a bt_orconn from the hash maps by GID */ +void +bto_delete(uint64_t gid) +{ + bt_orconn_t key, *bto; + + key.gid = gid; + key.chan = 0; + bto = HT_FIND(bto_gid_ht, bto_gid_map, &key); + if (!bto) { + /* The orconn might be unregistered because it's an EXT_OR_CONN? */ + log_debug(LD_BTRACK, "tried to delete unregistered ORCONN gid=%"PRIu64, + gid); + return; + } + HT_REMOVE(bto_gid_ht, bto_gid_map, &key); + if (bto->chan) { + key.chan = bto->chan; + HT_REMOVE(bto_chan_ht, bto_chan_map, &key); + } + tor_free(bto); +} + +/** + * Helper for bto_find_or_new(). + * + * Update GID and chan ID of an existing bt_orconn object if needed, + * given a search key previously used within bto_find_or_new(). + **/ +static bt_orconn_t * +bto_update(bt_orconn_t *bto, const bt_orconn_t *key) +{ + /* ORCONN GIDs shouldn't change once assigned */ + tor_assert(!bto->gid || !key->gid || bto->gid == key->gid); + if (!bto->gid && key->gid) { + /* Got a gid when we didn't already have one; insert into gid map */ + log_debug(LD_BTRACK, "ORCONN chan=%"PRIu64" newgid=%"PRIu64, key->chan, + key->gid); + bto->gid = key->gid; + HT_INSERT(bto_gid_ht, bto_gid_map, bto); + } + /* association of ORCONN with channel shouldn't change */ + tor_assert(!bto->chan || !key->chan || bto->chan == key->chan); + if (!bto->chan && key->chan) { + /* Got a chan when we didn't already have one; insert into chan map */ + log_debug(LD_BTRACK, "ORCONN gid=%"PRIu64" newchan=%"PRIu64, + bto->gid, key->chan); + bto->chan = key->chan; + HT_INSERT(bto_chan_ht, bto_chan_map, bto); + } + return bto; +} + +/** Helper for bto_find_or_new() */ +static bt_orconn_t * +bto_new(const bt_orconn_t *key) +{ + struct bt_orconn_t *bto = tor_malloc(sizeof(*bto)); + + bto->gid = key->gid; + bto->chan = key->chan; + bto->state = 0; + bto->proxy_type = 0; + bto->is_orig = false; + bto->is_onehop = true; + + if (bto->gid) + HT_INSERT(bto_gid_ht, bto_gid_map, bto); + if (bto->chan) + HT_INSERT(bto_chan_ht, bto_chan_map, bto); + + return bto; +} + +/** + * Insert a new bt_orconn with the given GID and chan ID, or update + * the GID and chan ID if one already exists. + * + * Return the found or allocated bt_orconn. + **/ +bt_orconn_t * +bto_find_or_new(uint64_t gid, uint64_t chan) +{ + bt_orconn_t key, *bto = NULL; + + tor_assert(gid || chan); + key.gid = gid; + key.chan = chan; + if (key.gid) + bto = HT_FIND(bto_gid_ht, bto_gid_map, &key); + if (!bto && key.chan) { + /* Not found by GID; look up by chan ID */ + bto = HT_FIND(bto_chan_ht, bto_chan_map, &key); + } + if (bto) + return bto_update(bto, &key); + else + return bto_new(&key); +} + +/** Initialize the hash maps */ +void +bto_init_maps(void) +{ + bto_gid_map = tor_malloc(sizeof(*bto_gid_map)); + HT_INIT(bto_gid_ht, bto_gid_map); + bto_chan_map = tor_malloc(sizeof(*bto_chan_map)); + HT_INIT(bto_chan_ht, bto_chan_map); +} + +/** Clear the hash maps, freeing all associated storage */ +void +bto_clear_maps(void) +{ + bto_gid_clear_map(); + bto_chan_clear_map(); +} diff --git a/src/feature/control/btrack_orconn_maps.h b/src/feature/control/btrack_orconn_maps.h new file mode 100644 index 0000000000..b1c2c7aa08 --- /dev/null +++ b/src/feature/control/btrack_orconn_maps.h @@ -0,0 +1,17 @@ +/* Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file btrack_orconn_maps.h + * \brief Header file for btrack_orconn_maps.c + **/ + +#ifndef TOR_BTRACK_ORCONN_MAPS_H + +void bto_delete(uint64_t); +bt_orconn_t *bto_find_or_new(uint64_t, uint64_t); + +void bto_init_maps(void); +void bto_clear_maps(void); + +#endif /* defined(TOR_BTRACK_ORCONN_MAPS_H) */ diff --git a/src/feature/control/btrack_sys.h b/src/feature/control/btrack_sys.h new file mode 100644 index 0000000000..f80cf342e7 --- /dev/null +++ b/src/feature/control/btrack_sys.h @@ -0,0 +1,14 @@ +/* Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file btrack_sys.h + * \brief Declare subsystem object for the bootstrap tracker susbystem. + **/ + +#ifndef TOR_BTRACK_SYS_H +#define TOR_BTRACK_SYS_H + +extern const struct subsys_fns_t sys_btrack; + +#endif /* defined(TOR_BTRACK_SYS_H) */ From cf4b3dbd445799bfad21e84777eff91d3a409f21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Thu, 20 Dec 2018 15:07:05 +0100 Subject: [PATCH 0344/2557] Use the subsystem list to initialize and shutdown process module. This patch makes the process module use the subsystem list for initializing and shutting down. See: https://bugs.torproject.org/28847 --- changes/ticket28847 | 3 +++ src/app/main/subsystem_list.c | 2 ++ src/lib/process/include.am | 2 ++ src/lib/process/process_sys.c | 33 +++++++++++++++++++++++++++++++++ src/lib/process/process_sys.h | 14 ++++++++++++++ 5 files changed, 54 insertions(+) create mode 100644 changes/ticket28847 create mode 100644 src/lib/process/process_sys.c create mode 100644 src/lib/process/process_sys.h diff --git a/changes/ticket28847 b/changes/ticket28847 new file mode 100644 index 0000000000..63100c5813 --- /dev/null +++ b/changes/ticket28847 @@ -0,0 +1,3 @@ + o Minor features (process): + - Use the subsystem module to initialize and shut down the process module. + Closes ticket 28847. diff --git a/src/app/main/subsystem_list.c b/src/app/main/subsystem_list.c index 8640329e92..122f7af215 100644 --- a/src/app/main/subsystem_list.c +++ b/src/app/main/subsystem_list.c @@ -18,6 +18,7 @@ #include "lib/time/time_sys.h" #include "lib/tls/tortls_sys.h" #include "lib/wallclock/wallclock_sys.h" +#include "lib/process/process_sys.h" #include @@ -32,6 +33,7 @@ const subsys_fns_t *tor_subsystems[] = { &sys_logging, /* -90 */ &sys_time, /* -90 */ &sys_network, /* -90 */ + &sys_process, /* -80 */ &sys_compress, /* -70 */ &sys_crypto, /* -60 */ &sys_tortls, /* -50 */ diff --git a/src/lib/process/include.am b/src/lib/process/include.am index a2d54b6238..83b67bf029 100644 --- a/src/lib/process/include.am +++ b/src/lib/process/include.am @@ -10,6 +10,7 @@ src_lib_libtor_process_a_SOURCES = \ src/lib/process/env.c \ src/lib/process/pidfile.c \ src/lib/process/process.c \ + src/lib/process/process_sys.c \ src/lib/process/process_unix.c \ src/lib/process/process_win32.c \ src/lib/process/restrict.c \ @@ -27,6 +28,7 @@ noinst_HEADERS += \ src/lib/process/env.h \ src/lib/process/pidfile.h \ src/lib/process/process.h \ + src/lib/process/process_sys.h \ src/lib/process/process_unix.h \ src/lib/process/process_win32.h \ src/lib/process/restrict.h \ diff --git a/src/lib/process/process_sys.c b/src/lib/process/process_sys.c new file mode 100644 index 0000000000..a880ff146e --- /dev/null +++ b/src/lib/process/process_sys.c @@ -0,0 +1,33 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file process_sys.c + * \brief Subsystem object for process setup. + **/ + +#include "orconfig.h" +#include "lib/subsys/subsys.h" +#include "lib/process/process_sys.h" +#include "lib/process/process.h" + +static int +subsys_process_initialize(void) +{ + process_init(); + return 0; +} + +static void +subsys_process_shutdown(void) +{ + process_free_all(); +} + +const subsys_fns_t sys_process = { + .name = "process", + .level = -80, + .supported = true, + .initialize = subsys_process_initialize, + .shutdown = subsys_process_shutdown +}; diff --git a/src/lib/process/process_sys.h b/src/lib/process/process_sys.h new file mode 100644 index 0000000000..b299334b6b --- /dev/null +++ b/src/lib/process/process_sys.h @@ -0,0 +1,14 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file process_sys.h + * \brief Declare subsystem object for the process module. + **/ + +#ifndef TOR_PROCESS_SYS_H +#define TOR_PROCESS_SYS_H + +extern const struct subsys_fns_t sys_process; + +#endif /* !defined(TOR_PROCESS_SYS_H) */ From 2322b56389626969b85da605240cbaafbb98bb33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Thu, 20 Dec 2018 15:16:14 +0100 Subject: [PATCH 0345/2557] Fix typo in time_sys.h. --- src/lib/time/time_sys.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/time/time_sys.h b/src/lib/time/time_sys.h index 0f1aebc268..5f5982a33b 100644 --- a/src/lib/time/time_sys.h +++ b/src/lib/time/time_sys.h @@ -2,7 +2,7 @@ /* See LICENSE for licensing information */ /** - * \file log_time.h + * \file time_sys.h * \brief Declare subsystem object for the time module. **/ From bc836d559dce5756f8ad2150d2351220ba7610f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Thu, 20 Dec 2018 17:03:50 +0100 Subject: [PATCH 0346/2557] Don't initialize the process module manually in tests. It's not longer needed for us to initialize the process module in tests. See: https://bugs.torproject.org/28847 --- src/test/test_process.c | 27 --------------------------- src/test/test_process_slow.c | 8 -------- src/test/test_pt.c | 9 --------- 3 files changed, 44 deletions(-) diff --git a/src/test/test_process.c b/src/test/test_process.c index 9b62862f04..27c9935e0d 100644 --- a/src/test/test_process.c +++ b/src/test/test_process.c @@ -142,8 +142,6 @@ static void test_default_values(void *arg) { (void)arg; - process_init(); - process_t *process = process_new("/path/to/nothing"); /* We are not running by default. */ @@ -171,14 +169,12 @@ test_default_values(void *arg) done: process_free(process); - process_free_all(); } static void test_environment(void *arg) { (void)arg; - process_init(); process_t *process = process_new(""); process_environment_t *env = NULL; @@ -221,7 +217,6 @@ test_environment(void *arg) done: process_environment_free(env); process_free(process); - process_free_all(); } static void @@ -249,7 +244,6 @@ static void test_line_protocol_simple(void *arg) { (void)arg; - process_init(); process_data_t *process_data = process_data_new(); @@ -289,7 +283,6 @@ test_line_protocol_simple(void *arg) done: process_data_free(process_data); process_free(process); - process_free_all(); UNMOCK(process_read_stdout); UNMOCK(process_read_stderr); @@ -299,7 +292,6 @@ static void test_line_protocol_multi(void *arg) { (void)arg; - process_init(); process_data_t *process_data = process_data_new(); @@ -349,7 +341,6 @@ test_line_protocol_multi(void *arg) done: process_data_free(process_data); process_free(process); - process_free_all(); UNMOCK(process_read_stdout); UNMOCK(process_read_stderr); @@ -359,7 +350,6 @@ static void test_line_protocol_partial(void *arg) { (void)arg; - process_init(); process_data_t *process_data = process_data_new(); @@ -431,7 +421,6 @@ test_line_protocol_partial(void *arg) done: process_data_free(process_data); process_free(process); - process_free_all(); UNMOCK(process_read_stdout); UNMOCK(process_read_stderr); @@ -441,7 +430,6 @@ static void test_raw_protocol_simple(void *arg) { (void)arg; - process_init(); process_data_t *process_data = process_data_new(); @@ -499,7 +487,6 @@ test_raw_protocol_simple(void *arg) done: process_data_free(process_data); process_free(process); - process_free_all(); UNMOCK(process_read_stdout); UNMOCK(process_read_stderr); @@ -510,8 +497,6 @@ test_write_simple(void *arg) { (void)arg; - process_init(); - process_data_t *process_data = process_data_new(); process_t *process = process_new(""); @@ -530,7 +515,6 @@ test_write_simple(void *arg) done: process_data_free(process_data); process_free(process); - process_free_all(); UNMOCK(process_write_stdin); } @@ -540,8 +524,6 @@ test_exit_simple(void *arg) { (void)arg; - process_init(); - process_data_t *process_data = process_data_new(); process_t *process = process_new(""); @@ -566,14 +548,12 @@ test_exit_simple(void *arg) process_set_data(process, process_data); process_data_free(process_data); process_free(process); - process_free_all(); } static void test_argv_simple(void *arg) { (void)arg; - process_init(); process_t *process = process_new("/bin/cat"); char **argv = NULL; @@ -600,7 +580,6 @@ test_argv_simple(void *arg) done: tor_free(argv); process_free(process); - process_free_all(); } static void @@ -608,8 +587,6 @@ test_unix(void *arg) { (void)arg; #ifndef _WIN32 - process_init(); - process_t *process = process_new(""); /* On Unix all processes should have a Unix process handle. */ @@ -617,7 +594,6 @@ test_unix(void *arg) done: process_free(process); - process_free_all(); #endif } @@ -626,8 +602,6 @@ test_win32(void *arg) { (void)arg; #ifdef _WIN32 - process_init(); - process_t *process = process_new(""); char *joined_argv = NULL; @@ -675,7 +649,6 @@ test_win32(void *arg) done: tor_free(joined_argv); process_free(process); - process_free_all(); #endif } diff --git a/src/test/test_process_slow.c b/src/test/test_process_slow.c index a1f99bff0f..845662bde4 100644 --- a/src/test/test_process_slow.c +++ b/src/test/test_process_slow.c @@ -203,9 +203,6 @@ test_callbacks(void *arg) filename = TEST_PROCESS; #endif - /* Initialize Process subsystem. */ - process_init(); - /* Process callback data. */ process_data_t *process_data = process_data_new(); @@ -285,7 +282,6 @@ test_callbacks(void *arg) done: process_data_free(process_data); process_free(process); - process_free_all(); } static void @@ -300,9 +296,6 @@ test_callbacks_terminate(void *arg) filename = TEST_PROCESS; #endif - /* Initialize Process subsystem. */ - process_init(); - /* Process callback data. */ process_data_t *process_data = process_data_new(); @@ -332,7 +325,6 @@ test_callbacks_terminate(void *arg) done: process_data_free(process_data); process_free(process); - process_free_all(); } struct testcase_t slow_process_tests[] = { diff --git a/src/test/test_pt.c b/src/test/test_pt.c index 8fcdd5c1e8..c8b9d18823 100644 --- a/src/test/test_pt.c +++ b/src/test/test_pt.c @@ -151,8 +151,6 @@ test_pt_get_transport_options(void *arg) config_line_t *cl = NULL; (void)arg; - process_init(); - execve_args = tor_malloc(sizeof(char*)*2); execve_args[0] = tor_strdup("cheeseshop"); execve_args[1] = NULL; @@ -192,7 +190,6 @@ test_pt_get_transport_options(void *arg) config_free_lines(cl); managed_proxy_destroy(mp, 0); smartlist_free(transport_list); - process_free_all(); } static void @@ -256,8 +253,6 @@ test_pt_get_extrainfo_string(void *arg) char *s = NULL; (void) arg; - process_init(); - argv1 = tor_malloc_zero(sizeof(char*)*3); argv1[0] = tor_strdup("ewige"); argv1[1] = tor_strdup("Blumenkraft"); @@ -291,7 +286,6 @@ test_pt_get_extrainfo_string(void *arg) smartlist_free(t1); smartlist_free(t2); tor_free(s); - process_free_all(); } static int @@ -347,8 +341,6 @@ test_pt_configure_proxy(void *arg) managed_proxy_t *mp = NULL; (void) arg; - process_init(); - dummy_state = tor_malloc_zero(sizeof(or_state_t)); MOCK(process_read_stdout, process_read_stdout_replacement); @@ -463,7 +455,6 @@ test_pt_configure_proxy(void *arg) tor_free(mp->argv[0]); tor_free(mp->argv); tor_free(mp); - process_free_all(); } /* Test the get_pt_proxy_uri() function. */ From fd58e5e498f1c86732de8d9edf6777e7822abfe7 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 21 Dec 2018 14:12:20 -0500 Subject: [PATCH 0347/2557] Fix priority on process subsystem level: it uses "net" --- src/app/main/subsystem_list.c | 2 +- src/lib/process/process_sys.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/app/main/subsystem_list.c b/src/app/main/subsystem_list.c index 122f7af215..bd6be42145 100644 --- a/src/app/main/subsystem_list.c +++ b/src/app/main/subsystem_list.c @@ -33,10 +33,10 @@ const subsys_fns_t *tor_subsystems[] = { &sys_logging, /* -90 */ &sys_time, /* -90 */ &sys_network, /* -90 */ - &sys_process, /* -80 */ &sys_compress, /* -70 */ &sys_crypto, /* -60 */ &sys_tortls, /* -50 */ + &sys_process, /* -35 */ }; const unsigned n_tor_subsystems = ARRAY_LENGTH(tor_subsystems); diff --git a/src/lib/process/process_sys.c b/src/lib/process/process_sys.c index a880ff146e..d0a94f35f7 100644 --- a/src/lib/process/process_sys.c +++ b/src/lib/process/process_sys.c @@ -26,7 +26,7 @@ subsys_process_shutdown(void) const subsys_fns_t sys_process = { .name = "process", - .level = -80, + .level = -35, .supported = true, .initialize = subsys_process_initialize, .shutdown = subsys_process_shutdown From 9d29abb34e005f4e836976a4c00115a1e8977071 Mon Sep 17 00:00:00 2001 From: Taylor Yu Date: Fri, 21 Dec 2018 11:38:58 -0600 Subject: [PATCH 0348/2557] Add a comment about bto_update_best. --- src/feature/control/btrack_orconn.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/feature/control/btrack_orconn.c b/src/feature/control/btrack_orconn.c index 69344569bf..28f8509fe2 100644 --- a/src/feature/control/btrack_orconn.c +++ b/src/feature/control/btrack_orconn.c @@ -62,6 +62,9 @@ bto_update_best(const bt_orconn_t *bto, bto_best_t *best, const char *type) { if (bto->state < best->state) return false; + /* Update even if we won't change best->state, because it's more + * recent information that a particular connection transitioned to + * that state. */ best->gid = bto->gid; if (bto->state > best->state) { log_info(LD_BTRACK, "ORCONN BEST_%s state %d->%d gid=%"PRIu64, type, From 936c93e562deaba62f0d32f7e7fda770c5604318 Mon Sep 17 00:00:00 2001 From: Taylor Yu Date: Sat, 15 Dec 2018 22:14:46 -0600 Subject: [PATCH 0349/2557] Hook up control_event_bootstrap() to btrack_orconn Replace a few invocations of control_event_bootstrap() with calls from the bootstrap tracker subsystem. This mostly leaves behavior unchanged. The actual behavior changes come in the next commit. Part of ticket 27167. --- src/core/include.am | 2 + src/core/or/circuitbuild.c | 2 - src/core/or/connection_or.c | 2 - src/feature/control/btrack_orconn.c | 13 ++- src/feature/control/btrack_orconn_cevent.c | 102 +++++++++++++++++++++ src/feature/control/btrack_orconn_cevent.h | 17 ++++ src/feature/control/control.h | 1 - src/feature/control/control_bootstrap.c | 11 --- src/feature/nodelist/nodelist.c | 1 - src/test/test_controller_events.c | 4 +- 10 files changed, 133 insertions(+), 22 deletions(-) create mode 100644 src/feature/control/btrack_orconn_cevent.c create mode 100644 src/feature/control/btrack_orconn_cevent.h diff --git a/src/core/include.am b/src/core/include.am index 93a8bca5d5..5e69cb9ada 100644 --- a/src/core/include.am +++ b/src/core/include.am @@ -66,6 +66,7 @@ LIBTOR_APP_A_SOURCES = \ src/feature/control/btrack.c \ src/feature/control/btrack_circuit.c \ src/feature/control/btrack_orconn.c \ + src/feature/control/btrack_orconn_cevent.c \ src/feature/control/btrack_orconn_maps.c \ src/feature/control/control.c \ src/feature/control/control_bootstrap.c \ @@ -280,6 +281,7 @@ noinst_HEADERS += \ src/feature/client/transports.h \ src/feature/control/btrack_circuit.h \ src/feature/control/btrack_orconn.h \ + src/feature/control/btrack_orconn_cevent.h \ src/feature/control/btrack_orconn_maps.h \ src/feature/control/btrack_sys.h \ src/feature/control/control.h \ diff --git a/src/core/or/circuitbuild.c b/src/core/or/circuitbuild.c index 3de8da9cde..b89ec09a99 100644 --- a/src/core/or/circuitbuild.c +++ b/src/core/or/circuitbuild.c @@ -583,8 +583,6 @@ circuit_handle_first_hop(origin_circuit_t *circ) circ->base_.n_hop = extend_info_dup(firsthop->extend_info); if (should_launch) { - if (circ->build_state->onehop_tunnel) - control_event_bootstrap(BOOTSTRAP_STATUS_CONN_DIR, 0); n_chan = channel_connect_for_circuit( &firsthop->extend_info->addr, firsthop->extend_info->port, diff --git a/src/core/or/connection_or.c b/src/core/or/connection_or.c index a12ea55c46..5d0874869d 100644 --- a/src/core/or/connection_or.c +++ b/src/core/or/connection_or.c @@ -751,8 +751,6 @@ connection_or_finished_connecting(or_connection_t *or_conn) log_debug(LD_HANDSHAKE,"OR connect() to router at %s:%u finished.", conn->address,conn->port); - control_event_bootstrap(BOOTSTRAP_STATUS_HANDSHAKE, 0); - control_event_boot_first_orconn(); if (proxy_type != PROXY_NONE) { /* start proxy handshake */ diff --git a/src/feature/control/btrack_orconn.c b/src/feature/control/btrack_orconn.c index 28f8509fe2..0fbf521000 100644 --- a/src/feature/control/btrack_orconn.c +++ b/src/feature/control/btrack_orconn.c @@ -27,6 +27,10 @@ * might be a one-hop circuit for directory lookups, or it might be a * connection for an application circuit because we already have * enough directory info to build an application circuit. + * + * We call functions in btrack_orconn_cevent.c to generate the actual + * controller events, because some of the state decoding we need to do + * is complicated. **/ #include @@ -38,6 +42,7 @@ #include "core/or/ocirc_event.h" #include "core/or/orconn_event.h" #include "feature/control/btrack_orconn.h" +#include "feature/control/btrack_orconn_cevent.h" #include "feature/control/btrack_orconn_maps.h" #include "lib/log/log.h" @@ -86,9 +91,10 @@ bto_update_bests(const bt_orconn_t *bto) { tor_assert(bto->is_orig); - bto_update_best(bto, &best_any, "ANY"); - if (!bto->is_onehop) - bto_update_best(bto, &best_ap, "AP"); + if (bto_update_best(bto, &best_any, "ANY")) + bto_cevent_anyconn(bto); + if (!bto->is_onehop && bto_update_best(bto, &best_ap, "AP")) + bto_cevent_apconn(bto); } /** Reset cached "best" values */ @@ -196,4 +202,5 @@ btrack_orconn_fini(void) { bto_clear_maps(); bto_reset_bests(); + bto_cevent_reset(); } diff --git a/src/feature/control/btrack_orconn_cevent.c b/src/feature/control/btrack_orconn_cevent.c new file mode 100644 index 0000000000..fbbd34aca5 --- /dev/null +++ b/src/feature/control/btrack_orconn_cevent.c @@ -0,0 +1,102 @@ +/* Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file btrack_orconn_cevent.c + * \brief Emit bootstrap status events for OR connections + **/ + +#include + +#include "core/or/or.h" + +#define BTRACK_ORCONN_PRIVATE + +#include "core/or/orconn_event.h" +#include "feature/control/btrack_orconn.h" +#include "feature/control/btrack_orconn_cevent.h" +#include "feature/control/control.h" + +/** + * Have we completed our first OR connection? + * + * Block display of application circuit progress until we do, to avoid + * some misleading behavior of jumping to high progress. + **/ +static bool bto_first_orconn = false; + +/** + * Emit control events when we have updated our idea of the best state + * that any OR connection has reached. + **/ +void +bto_cevent_anyconn(const bt_orconn_t *bto) +{ + switch (bto->state) { + case OR_CONN_STATE_CONNECTING: + case OR_CONN_STATE_PROXY_HANDSHAKING: + /* XXX This isn't quite right, because this isn't necessarily a + directory server we're talking to, but we'll improve the + bootstrap tags and messages later */ + control_event_bootstrap(BOOTSTRAP_STATUS_CONN_DIR, 0); + break; + case OR_CONN_STATE_TLS_HANDSHAKING: + /* Here we should report a connection completed (TCP or proxied), + if we had the states */ + break; + case OR_CONN_STATE_TLS_CLIENT_RENEGOTIATING: + case OR_CONN_STATE_OR_HANDSHAKING_V2: + case OR_CONN_STATE_OR_HANDSHAKING_V3: + /* XXX Again, this isn't quite right, because it's not necessarily + a directory server we're talking to. */ + control_event_bootstrap(BOOTSTRAP_STATUS_HANDSHAKE_DIR, 0); + break; + case OR_CONN_STATE_OPEN: + /* Unblock directory progress display */ + control_event_boot_first_orconn(); + /* Unblock apconn progress display */ + bto_first_orconn = true; + break; + default: + break; + } +} + +/** + * Emit control events when we have updated our idea of the best state + * that any application circuit OR connection has reached. + **/ +void +bto_cevent_apconn(const bt_orconn_t *bto) +{ + if (!bto_first_orconn) + return; + + switch (bto->state) { + case OR_CONN_STATE_CONNECTING: + case OR_CONN_STATE_PROXY_HANDSHAKING: + control_event_bootstrap(BOOTSTRAP_STATUS_CONN_OR, 0); + break; + case OR_CONN_STATE_TLS_HANDSHAKING: + /* Here we should report a connection completed (TCP or proxied) */ + break; + case OR_CONN_STATE_TLS_CLIENT_RENEGOTIATING: + case OR_CONN_STATE_OR_HANDSHAKING_V2: + case OR_CONN_STATE_OR_HANDSHAKING_V3: + control_event_bootstrap(BOOTSTRAP_STATUS_HANDSHAKE_OR, 0); + break; + case OR_CONN_STATE_OPEN: + /* XXX Not quite right, because it implictly reports the next + state, but we'll improve it later. */ + control_event_bootstrap(BOOTSTRAP_STATUS_CIRCUIT_CREATE, 0); + default: + break; + } +} + +/** Forget that we completed our first OR connection */ +void +bto_cevent_reset(void) +{ + bto_first_orconn = false; +} diff --git a/src/feature/control/btrack_orconn_cevent.h b/src/feature/control/btrack_orconn_cevent.h new file mode 100644 index 0000000000..165ff69cdb --- /dev/null +++ b/src/feature/control/btrack_orconn_cevent.h @@ -0,0 +1,17 @@ +/* Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file btrack_orconn_cevent.h + * \brief Header file for btrack_orconn_cevent.c + **/ + +#ifndef TOR_BTRACK_ORCONN_CEVENT_H + +#include "feature/control/btrack_orconn.h" + +void bto_cevent_anyconn(const bt_orconn_t *); +void bto_cevent_apconn(const bt_orconn_t *); +void bto_cevent_reset(void); + +#endif /* defined(TOR_BTRACK_ORCONN_CEVENT_H) */ diff --git a/src/feature/control/control.h b/src/feature/control/control.h index 68c9a6bed1..8180c4b603 100644 --- a/src/feature/control/control.h +++ b/src/feature/control/control.h @@ -52,7 +52,6 @@ typedef enum { BOOTSTRAP_STATUS_UNDEF=-1, BOOTSTRAP_STATUS_STARTING=0, BOOTSTRAP_STATUS_CONN_DIR=5, - BOOTSTRAP_STATUS_HANDSHAKE=-2, BOOTSTRAP_STATUS_HANDSHAKE_DIR=10, BOOTSTRAP_STATUS_ONEHOP_CREATE=15, BOOTSTRAP_STATUS_REQUESTING_STATUS=20, diff --git a/src/feature/control/control_bootstrap.c b/src/feature/control/control_bootstrap.c index 0756e208e0..0478f0783d 100644 --- a/src/feature/control/control_bootstrap.c +++ b/src/feature/control/control_bootstrap.c @@ -34,7 +34,6 @@ static const struct { { BOOTSTRAP_STATUS_UNDEF, "undef", "Undefined" }, { BOOTSTRAP_STATUS_STARTING, "starting", "Starting" }, { BOOTSTRAP_STATUS_CONN_DIR, "conn_dir", "Connecting to directory server" }, - { BOOTSTRAP_STATUS_HANDSHAKE, "status_handshake", "Finishing handshake" }, { BOOTSTRAP_STATUS_HANDSHAKE_DIR, "handshake_dir", "Finishing handshake with directory server" }, { BOOTSTRAP_STATUS_ONEHOP_CREATE, "onehop_create", @@ -151,16 +150,6 @@ control_event_bootstrap(bootstrap_status_t status, int progress) if (bootstrap_percent == BOOTSTRAP_STATUS_DONE) return; /* already bootstrapped; nothing to be done here. */ - /* special case for handshaking status, since our TLS handshaking code - * can't distinguish what the connection is going to be for. */ - if (status == BOOTSTRAP_STATUS_HANDSHAKE) { - if (bootstrap_percent < BOOTSTRAP_STATUS_CONN_OR) { - status = BOOTSTRAP_STATUS_HANDSHAKE_DIR; - } else { - status = BOOTSTRAP_STATUS_HANDSHAKE_OR; - } - } - if (status <= bootstrap_percent) { /* If there's no new progress, return early. */ if (!progress || progress <= bootstrap_percent) diff --git a/src/feature/nodelist/nodelist.c b/src/feature/nodelist/nodelist.c index f93ecd5bfe..e1d5e4d3fa 100644 --- a/src/feature/nodelist/nodelist.c +++ b/src/feature/nodelist/nodelist.c @@ -2633,7 +2633,6 @@ update_router_have_minimum_dir_info(void) /* If paths have just become available in this update. */ if (res && !have_min_dir_info) { control_event_client_status(LOG_NOTICE, "ENOUGH_DIR_INFO"); - control_event_boot_dir(BOOTSTRAP_STATUS_CONN_OR, 0); log_info(LD_DIR, "We now have enough directory information to build circuits."); } diff --git a/src/test/test_controller_events.c b/src/test/test_controller_events.c index 4c404876b0..63013390e4 100644 --- a/src/test/test_controller_events.c +++ b/src/test/test_controller_events.c @@ -353,7 +353,7 @@ test_cntev_dirboot_defer_desc(void *arg) assert_bootmsg("0 TAG=starting"); control_event_bootstrap(BOOTSTRAP_STATUS_CONN_DIR, 0); assert_bootmsg("5 TAG=conn_dir"); - control_event_bootstrap(BOOTSTRAP_STATUS_HANDSHAKE, 0); + control_event_bootstrap(BOOTSTRAP_STATUS_HANDSHAKE_DIR, 0); assert_bootmsg("10 TAG=handshake_dir"); /* The deferred event should appear */ control_event_boot_first_orconn(); @@ -378,7 +378,7 @@ test_cntev_dirboot_defer_orconn(void *arg) assert_bootmsg("0 TAG=starting"); control_event_bootstrap(BOOTSTRAP_STATUS_CONN_DIR, 0); assert_bootmsg("5 TAG=conn_dir"); - control_event_bootstrap(BOOTSTRAP_STATUS_HANDSHAKE, 0); + control_event_bootstrap(BOOTSTRAP_STATUS_HANDSHAKE_DIR, 0); assert_bootmsg("10 TAG=handshake_dir"); /* The deferred event should appear */ control_event_boot_first_orconn(); From 85542ee5a0b37bf5b572b9beeda3cb8038ecd88e Mon Sep 17 00:00:00 2001 From: Taylor Yu Date: Mon, 17 Dec 2018 09:31:16 -0600 Subject: [PATCH 0350/2557] The big bootstrap phase redefinition Redefine the set of bootstrap phases to allow display of finer-grained progress in the early connection stages of connecting to a relay. This includes adding intermediate phases for proxy and PT connections. Also add a separate new phase to indicate obtaining enough directory info to build circuits so we can report that independently of actually initiating an ORCONN to build the first application circuit. Previously, we would claim to be connecting to a relay when we had merely finished obtaining directory info. Part of ticket 27167. --- src/feature/control/btrack_orconn_cevent.c | 87 ++++++++++++++++++---- src/feature/control/control.h | 41 ++++++++-- src/feature/control/control_bootstrap.c | 48 ++++++++++-- src/feature/nodelist/nodelist.c | 3 +- src/test/test_controller_events.c | 20 ++--- 5 files changed, 159 insertions(+), 40 deletions(-) diff --git a/src/feature/control/btrack_orconn_cevent.c b/src/feature/control/btrack_orconn_cevent.c index fbbd34aca5..c7970dca4d 100644 --- a/src/feature/control/btrack_orconn_cevent.c +++ b/src/feature/control/btrack_orconn_cevent.c @@ -4,6 +4,11 @@ /** * \file btrack_orconn_cevent.c * \brief Emit bootstrap status events for OR connections + * + * We do some decoding of the raw OR_CONN_STATE_* values. For + * example, OR_CONN_STATE_CONNECTING means the first TCP connect() + * completing, regardless of whether it's directly to a relay instead + * of a proxy or a PT. **/ #include @@ -25,33 +30,68 @@ **/ static bool bto_first_orconn = false; +/** Is the ORCONN using a pluggable transport? */ +static bool +using_pt(const bt_orconn_t *bto) +{ + return bto->proxy_type == PROXY_PLUGGABLE; +} + +/** Is the ORCONN using a non-PT proxy? */ +static bool +using_proxy(const bt_orconn_t *bto) +{ + switch (bto->proxy_type) { + case PROXY_CONNECT: + case PROXY_SOCKS4: + case PROXY_SOCKS5: + return true; + default: + return false; + } +} + /** * Emit control events when we have updated our idea of the best state * that any OR connection has reached. + * + * Do some decoding of the ORCONN states depending on whether a PT or + * a proxy is in use. **/ void bto_cevent_anyconn(const bt_orconn_t *bto) { switch (bto->state) { case OR_CONN_STATE_CONNECTING: + /* Exactly what kind of thing we're connecting to isn't + * information we directly get from the states in connection_or.c, + * so decode it here. */ + if (using_pt(bto)) + control_event_bootstrap(BOOTSTRAP_STATUS_CONN_PT, 0); + else if (using_proxy(bto)) + control_event_bootstrap(BOOTSTRAP_STATUS_CONN_PROXY, 0); + else + control_event_bootstrap(BOOTSTRAP_STATUS_CONN, 0); + break; case OR_CONN_STATE_PROXY_HANDSHAKING: - /* XXX This isn't quite right, because this isn't necessarily a - directory server we're talking to, but we'll improve the - bootstrap tags and messages later */ - control_event_bootstrap(BOOTSTRAP_STATUS_CONN_DIR, 0); + /* Similarly, starting a proxy handshake means the TCP connect() + * succeeded to the proxy. Let's be specific about what kind of + * proxy. */ + if (using_pt(bto)) + control_event_bootstrap(BOOTSTRAP_STATUS_CONN_DONE_PT, 0); + else if (using_proxy(bto)) + control_event_bootstrap(BOOTSTRAP_STATUS_CONN_DONE_PROXY, 0); break; case OR_CONN_STATE_TLS_HANDSHAKING: - /* Here we should report a connection completed (TCP or proxied), - if we had the states */ + control_event_bootstrap(BOOTSTRAP_STATUS_CONN_DONE, 0); break; case OR_CONN_STATE_TLS_CLIENT_RENEGOTIATING: case OR_CONN_STATE_OR_HANDSHAKING_V2: case OR_CONN_STATE_OR_HANDSHAKING_V3: - /* XXX Again, this isn't quite right, because it's not necessarily - a directory server we're talking to. */ - control_event_bootstrap(BOOTSTRAP_STATUS_HANDSHAKE_DIR, 0); + control_event_bootstrap(BOOTSTRAP_STATUS_HANDSHAKE, 0); break; case OR_CONN_STATE_OPEN: + control_event_bootstrap(BOOTSTRAP_STATUS_HANDSHAKE_DONE, 0); /* Unblock directory progress display */ control_event_boot_first_orconn(); /* Unblock apconn progress display */ @@ -65,6 +105,9 @@ bto_cevent_anyconn(const bt_orconn_t *bto) /** * Emit control events when we have updated our idea of the best state * that any application circuit OR connection has reached. + * + * Do some decoding of the ORCONN states depending on whether a PT or + * a proxy is in use. **/ void bto_cevent_apconn(const bt_orconn_t *bto) @@ -74,21 +117,35 @@ bto_cevent_apconn(const bt_orconn_t *bto) switch (bto->state) { case OR_CONN_STATE_CONNECTING: + /* Exactly what kind of thing we're connecting to isn't + * information we directly get from the states in connection_or.c, + * so decode it here. */ + if (using_pt(bto)) + control_event_bootstrap(BOOTSTRAP_STATUS_AP_CONN_PT, 0); + else if (using_proxy(bto)) + control_event_bootstrap(BOOTSTRAP_STATUS_AP_CONN_PROXY, 0); + else + control_event_bootstrap(BOOTSTRAP_STATUS_AP_CONN, 0); + break; case OR_CONN_STATE_PROXY_HANDSHAKING: - control_event_bootstrap(BOOTSTRAP_STATUS_CONN_OR, 0); + /* Similarly, starting a proxy handshake means the TCP connect() + * succeeded to the proxy. Let's be specific about what kind of + * proxy. */ + if (using_pt(bto)) + control_event_bootstrap(BOOTSTRAP_STATUS_AP_CONN_DONE_PT, 0); + else if (using_proxy(bto)) + control_event_bootstrap(BOOTSTRAP_STATUS_AP_CONN_DONE_PROXY, 0); break; case OR_CONN_STATE_TLS_HANDSHAKING: - /* Here we should report a connection completed (TCP or proxied) */ + control_event_bootstrap(BOOTSTRAP_STATUS_AP_CONN_DONE, 0); break; case OR_CONN_STATE_TLS_CLIENT_RENEGOTIATING: case OR_CONN_STATE_OR_HANDSHAKING_V2: case OR_CONN_STATE_OR_HANDSHAKING_V3: - control_event_bootstrap(BOOTSTRAP_STATUS_HANDSHAKE_OR, 0); + control_event_bootstrap(BOOTSTRAP_STATUS_AP_HANDSHAKE, 0); break; case OR_CONN_STATE_OPEN: - /* XXX Not quite right, because it implictly reports the next - state, but we'll improve it later. */ - control_event_bootstrap(BOOTSTRAP_STATUS_CIRCUIT_CREATE, 0); + control_event_bootstrap(BOOTSTRAP_STATUS_AP_HANDSHAKE_DONE, 0); default: break; } diff --git a/src/feature/control/control.h b/src/feature/control/control.h index 8180c4b603..08d8924e2d 100644 --- a/src/feature/control/control.h +++ b/src/feature/control/control.h @@ -51,17 +51,42 @@ typedef enum buildtimeout_set_event_t { typedef enum { BOOTSTRAP_STATUS_UNDEF=-1, BOOTSTRAP_STATUS_STARTING=0, - BOOTSTRAP_STATUS_CONN_DIR=5, - BOOTSTRAP_STATUS_HANDSHAKE_DIR=10, - BOOTSTRAP_STATUS_ONEHOP_CREATE=15, - BOOTSTRAP_STATUS_REQUESTING_STATUS=20, - BOOTSTRAP_STATUS_LOADING_STATUS=25, + + /* Initial connection to any relay */ + + BOOTSTRAP_STATUS_CONN_PT=1, + BOOTSTRAP_STATUS_CONN_DONE_PT=2, + BOOTSTRAP_STATUS_CONN_PROXY=3, + BOOTSTRAP_STATUS_CONN_DONE_PROXY=4, + BOOTSTRAP_STATUS_CONN=5, + BOOTSTRAP_STATUS_CONN_DONE=10, + BOOTSTRAP_STATUS_HANDSHAKE=14, + BOOTSTRAP_STATUS_HANDSHAKE_DONE=15, + + /* Loading directory info */ + + BOOTSTRAP_STATUS_ONEHOP_CREATE=20, + BOOTSTRAP_STATUS_REQUESTING_STATUS=25, + BOOTSTRAP_STATUS_LOADING_STATUS=30, BOOTSTRAP_STATUS_LOADING_KEYS=40, BOOTSTRAP_STATUS_REQUESTING_DESCRIPTORS=45, BOOTSTRAP_STATUS_LOADING_DESCRIPTORS=50, - BOOTSTRAP_STATUS_CONN_OR=80, - BOOTSTRAP_STATUS_HANDSHAKE_OR=85, - BOOTSTRAP_STATUS_CIRCUIT_CREATE=90, + BOOTSTRAP_STATUS_ENOUGH_DIRINFO=75, + + /* Connecting to a relay for AP circuits */ + + BOOTSTRAP_STATUS_AP_CONN_PT=76, + BOOTSTRAP_STATUS_AP_CONN_DONE_PT=77, + BOOTSTRAP_STATUS_AP_CONN_PROXY=78, + BOOTSTRAP_STATUS_AP_CONN_DONE_PROXY=79, + BOOTSTRAP_STATUS_AP_CONN=80, + BOOTSTRAP_STATUS_AP_CONN_DONE=85, + BOOTSTRAP_STATUS_AP_HANDSHAKE=89, + BOOTSTRAP_STATUS_AP_HANDSHAKE_DONE=90, + + /* Creating AP circuits */ + + BOOTSTRAP_STATUS_CIRCUIT_CREATE=95, BOOTSTRAP_STATUS_DONE=100 } bootstrap_status_t; diff --git a/src/feature/control/control_bootstrap.c b/src/feature/control/control_bootstrap.c index 0478f0783d..49e190dc51 100644 --- a/src/feature/control/control_bootstrap.c +++ b/src/feature/control/control_bootstrap.c @@ -33,9 +33,24 @@ static const struct { } boot_to_str_tab[] = { { BOOTSTRAP_STATUS_UNDEF, "undef", "Undefined" }, { BOOTSTRAP_STATUS_STARTING, "starting", "Starting" }, - { BOOTSTRAP_STATUS_CONN_DIR, "conn_dir", "Connecting to directory server" }, - { BOOTSTRAP_STATUS_HANDSHAKE_DIR, "handshake_dir", - "Finishing handshake with directory server" }, + + /* Initial connection to any relay */ + + { BOOTSTRAP_STATUS_CONN_PT, "conn_pt", "Connecting to pluggable transport" }, + { BOOTSTRAP_STATUS_CONN_DONE_PT, "conn_done_pt", + "Connected to pluggable transport" }, + { BOOTSTRAP_STATUS_CONN_PROXY, "conn_proxy", "Connecting to proxy" }, + { BOOTSTRAP_STATUS_CONN_DONE_PROXY, "conn_done_proxy", + "Connected to proxy" }, + { BOOTSTRAP_STATUS_CONN, "conn", "Connecting to a relay" }, + { BOOTSTRAP_STATUS_CONN_DONE, "conn_done", "Connected to a relay" }, + { BOOTSTRAP_STATUS_HANDSHAKE, "handshake", + "Handshaking with a relay" }, + { BOOTSTRAP_STATUS_HANDSHAKE_DONE, "handshake_done", + "Handshake with a relay done" }, + + /* Loading directory info */ + { BOOTSTRAP_STATUS_ONEHOP_CREATE, "onehop_create", "Establishing an encrypted directory connection" }, { BOOTSTRAP_STATUS_REQUESTING_STATUS, "requesting_status", @@ -48,9 +63,30 @@ static const struct { "Asking for relay descriptors" }, { BOOTSTRAP_STATUS_LOADING_DESCRIPTORS, "loading_descriptors", "Loading relay descriptors" }, - { BOOTSTRAP_STATUS_CONN_OR, "conn_or", "Connecting to the Tor network" }, - { BOOTSTRAP_STATUS_HANDSHAKE_OR, "handshake_or", - "Finishing handshake with first hop" }, + { BOOTSTRAP_STATUS_ENOUGH_DIRINFO, "enough_dirinfo", + "Loaded enough directory info to build circuits" }, + + /* Connecting to a relay for AP circuits */ + + { BOOTSTRAP_STATUS_AP_CONN_PT, "ap_conn_pt", + "Connecting to pluggable transport to build circuits" }, + { BOOTSTRAP_STATUS_AP_CONN_DONE_PT, "ap_conn_done_pt", + "Connected to pluggable transport to build circuits" }, + { BOOTSTRAP_STATUS_AP_CONN_PROXY, "ap_conn_proxy", + "Connecting to proxy " }, + { BOOTSTRAP_STATUS_AP_CONN_DONE_PROXY, "ap_conn_done_proxy", + "Connected to proxy to build circuits" }, + { BOOTSTRAP_STATUS_AP_CONN, "ap_conn", + "Connecting to a relay to build circuits" }, + { BOOTSTRAP_STATUS_AP_CONN_DONE, "ap_conn_done", + "Connected to a relay to build circuits" }, + { BOOTSTRAP_STATUS_AP_HANDSHAKE, "ap_handshake", + "Finishing handshake with a relay to build circuits" }, + { BOOTSTRAP_STATUS_AP_HANDSHAKE_DONE, "ap_handshake_done", + "Handshake fininshed with a relay to build circuits" }, + + /* Creating AP circuits */ + { BOOTSTRAP_STATUS_CIRCUIT_CREATE, "circuit_create", "Establishing a Tor circuit" }, { BOOTSTRAP_STATUS_DONE, "done", "Done" }, diff --git a/src/feature/nodelist/nodelist.c b/src/feature/nodelist/nodelist.c index e1d5e4d3fa..d94e73f48f 100644 --- a/src/feature/nodelist/nodelist.c +++ b/src/feature/nodelist/nodelist.c @@ -2546,7 +2546,7 @@ count_loading_descriptors_progress(void) if (fraction > 1.0) return 0; /* it's not the number of descriptors holding us back */ return BOOTSTRAP_STATUS_LOADING_DESCRIPTORS + (int) - (fraction*(BOOTSTRAP_STATUS_CONN_OR-1 - + (fraction*(BOOTSTRAP_STATUS_ENOUGH_DIRINFO-1 - BOOTSTRAP_STATUS_LOADING_DESCRIPTORS)); } @@ -2633,6 +2633,7 @@ update_router_have_minimum_dir_info(void) /* If paths have just become available in this update. */ if (res && !have_min_dir_info) { control_event_client_status(LOG_NOTICE, "ENOUGH_DIR_INFO"); + control_event_boot_dir(BOOTSTRAP_STATUS_ENOUGH_DIRINFO, 0); log_info(LD_DIR, "We now have enough directory information to build circuits."); } diff --git a/src/test/test_controller_events.c b/src/test/test_controller_events.c index 63013390e4..dc2bb77e9b 100644 --- a/src/test/test_controller_events.c +++ b/src/test/test_controller_events.c @@ -351,10 +351,10 @@ test_cntev_dirboot_defer_desc(void *arg) /* This event should get deferred */ control_event_boot_dir(BOOTSTRAP_STATUS_REQUESTING_DESCRIPTORS, 0); assert_bootmsg("0 TAG=starting"); - control_event_bootstrap(BOOTSTRAP_STATUS_CONN_DIR, 0); - assert_bootmsg("5 TAG=conn_dir"); - control_event_bootstrap(BOOTSTRAP_STATUS_HANDSHAKE_DIR, 0); - assert_bootmsg("10 TAG=handshake_dir"); + control_event_bootstrap(BOOTSTRAP_STATUS_CONN, 0); + assert_bootmsg("5 TAG=conn"); + control_event_bootstrap(BOOTSTRAP_STATUS_HANDSHAKE, 0); + assert_bootmsg("14 TAG=handshake"); /* The deferred event should appear */ control_event_boot_first_orconn(); assert_bootmsg("45 TAG=requesting_descriptors"); @@ -374,15 +374,15 @@ test_cntev_dirboot_defer_orconn(void *arg) control_event_bootstrap(BOOTSTRAP_STATUS_STARTING, 0); assert_bootmsg("0 TAG=starting"); /* This event should get deferred */ - control_event_boot_dir(BOOTSTRAP_STATUS_CONN_OR, 0); + control_event_boot_dir(BOOTSTRAP_STATUS_ENOUGH_DIRINFO, 0); assert_bootmsg("0 TAG=starting"); - control_event_bootstrap(BOOTSTRAP_STATUS_CONN_DIR, 0); - assert_bootmsg("5 TAG=conn_dir"); - control_event_bootstrap(BOOTSTRAP_STATUS_HANDSHAKE_DIR, 0); - assert_bootmsg("10 TAG=handshake_dir"); + control_event_bootstrap(BOOTSTRAP_STATUS_CONN, 0); + assert_bootmsg("5 TAG=conn"); + control_event_bootstrap(BOOTSTRAP_STATUS_HANDSHAKE, 0); + assert_bootmsg("14 TAG=handshake"); /* The deferred event should appear */ control_event_boot_first_orconn(); - assert_bootmsg("80 TAG=conn_or"); + assert_bootmsg("75 TAG=enough_dirinfo"); done: tor_free(saved_event_str); UNMOCK(queue_control_event_string); From f0f971409a4337db1b98f9821eed7dda2e5eb700 Mon Sep 17 00:00:00 2001 From: Taylor Yu Date: Thu, 20 Dec 2018 09:21:16 -0600 Subject: [PATCH 0351/2557] Add tests for bootstrap tracker Part of ticket 27617. --- src/test/include.am | 1 + src/test/test.c | 1 + src/test/test.h | 1 + src/test/test_btrack.c | 100 ++++++++++++++++++++ src/test/test_controller_events.c | 147 +++++++++++++++++++++++++++++- 5 files changed, 249 insertions(+), 1 deletion(-) create mode 100644 src/test/test_btrack.c diff --git a/src/test/include.am b/src/test/include.am index 648cf5a541..4725e8cbaa 100644 --- a/src/test/include.am +++ b/src/test/include.am @@ -94,6 +94,7 @@ src_test_test_SOURCES += \ src/test/test_address.c \ src/test/test_address_set.c \ src/test/test_bridges.c \ + src/test/test_btrack.c \ src/test/test_buffers.c \ src/test/test_bwmgt.c \ src/test/test_cell_formats.c \ diff --git a/src/test/test.c b/src/test/test.c index 85d41d9863..13e8c71709 100644 --- a/src/test/test.c +++ b/src/test/test.c @@ -857,6 +857,7 @@ struct testgroup_t testgroups[] = { { "consdiffmgr/", consdiffmgr_tests }, { "container/", container_tests }, { "control/", controller_tests }, + { "control/btrack/", btrack_tests }, { "control/event/", controller_event_tests }, { "crypto/", crypto_tests }, { "crypto/ope/", crypto_ope_tests }, diff --git a/src/test/test.h b/src/test/test.h index 1b10c3d12d..9f754469c8 100644 --- a/src/test/test.h +++ b/src/test/test.h @@ -180,6 +180,7 @@ extern struct testcase_t addr_tests[]; extern struct testcase_t address_set_tests[]; extern struct testcase_t address_tests[]; extern struct testcase_t bridges_tests[]; +extern struct testcase_t btrack_tests[]; extern struct testcase_t buffer_tests[]; extern struct testcase_t bwmgt_tests[]; extern struct testcase_t cell_format_tests[]; diff --git a/src/test/test_btrack.c b/src/test/test_btrack.c new file mode 100644 index 0000000000..7b5d108f98 --- /dev/null +++ b/src/test/test_btrack.c @@ -0,0 +1,100 @@ +/* Copyright (c) 2013-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "core/or/or.h" + +#include "test/test.h" +#include "test/log_test_helpers.h" + +#define OCIRC_EVENT_PRIVATE +#define ORCONN_EVENT_PRIVATE +#include "core/or/ocirc_event.h" +#include "core/or/orconn_event.h" + +static void +test_btrack_launch(void *arg) +{ + orconn_event_msg_t conn; + ocirc_event_msg_t circ; + + (void)arg; + conn.type = ORCONN_MSGTYPE_STATE; + conn.u.state.gid = 1; + conn.u.state.chan = 1; + conn.u.state.proxy_type = PROXY_NONE; + conn.u.state.state = OR_CONN_STATE_CONNECTING; + + setup_full_capture_of_logs(LOG_DEBUG); + orconn_event_publish(&conn); + expect_log_msg_containing("ORCONN gid=1 chan=1 proxy_type=0 state=1"); + expect_no_log_msg_containing("ORCONN BEST_"); + teardown_capture_of_logs(); + + circ.type = OCIRC_MSGTYPE_CHAN; + circ.u.chan.chan = 1; + circ.u.chan.onehop = true; + + setup_full_capture_of_logs(LOG_DEBUG); + ocirc_event_publish(&circ); + expect_log_msg_containing("ORCONN LAUNCH chan=1 onehop=1"); + expect_log_msg_containing("ORCONN BEST_ANY state -1->1 gid=1"); + teardown_capture_of_logs(); + + conn.u.state.gid = 2; + conn.u.state.chan = 2; + + setup_full_capture_of_logs(LOG_DEBUG); + orconn_event_publish(&conn); + expect_log_msg_containing("ORCONN gid=2 chan=2 proxy_type=0 state=1"); + expect_no_log_msg_containing("ORCONN BEST_"); + teardown_capture_of_logs(); + + circ.u.chan.chan = 2; + circ.u.chan.onehop = false; + + setup_full_capture_of_logs(LOG_DEBUG); + ocirc_event_publish(&circ); + expect_log_msg_containing("ORCONN LAUNCH chan=2 onehop=0"); + expect_log_msg_containing("ORCONN BEST_AP state -1->1 gid=2"); + teardown_capture_of_logs(); + + done: + ; +} + +static void +test_btrack_delete(void *arg) +{ + orconn_event_msg_t conn; + + (void)arg; + conn.type = ORCONN_MSGTYPE_STATE; + conn.u.state.gid = 1; + conn.u.state.chan = 1; + conn.u.state.proxy_type = PROXY_NONE; + conn.u.state.state = OR_CONN_STATE_CONNECTING; + + setup_full_capture_of_logs(LOG_DEBUG); + orconn_event_publish(&conn); + expect_log_msg_containing("ORCONN gid=1 chan=1 proxy_type=0"); + teardown_capture_of_logs(); + + conn.type = ORCONN_MSGTYPE_STATUS; + conn.u.status.gid = 1; + conn.u.status.status = OR_CONN_EVENT_CLOSED; + conn.u.status.reason = 0; + + setup_full_capture_of_logs(LOG_DEBUG); + orconn_event_publish(&conn); + expect_log_msg_containing("ORCONN DELETE gid=1 status=3 reason=0"); + teardown_capture_of_logs(); + + done: + ; +} + +struct testcase_t btrack_tests[] = { + { "launch", test_btrack_launch, TT_FORK, 0, NULL }, + { "delete", test_btrack_delete, TT_FORK, 0, NULL }, + END_OF_TESTCASES +}; diff --git a/src/test/test_controller_events.c b/src/test/test_controller_events.c index dc2bb77e9b..99e1eb7cb0 100644 --- a/src/test/test_controller_events.c +++ b/src/test/test_controller_events.c @@ -4,10 +4,14 @@ #define CONNECTION_PRIVATE #define TOR_CHANNEL_INTERNAL_ #define CONTROL_PRIVATE +#define OCIRC_EVENT_PRIVATE +#define ORCONN_EVENT_PRIVATE #include "core/or/or.h" #include "core/or/channel.h" #include "core/or/channeltls.h" #include "core/or/circuitlist.h" +#include "core/or/ocirc_event.h" +#include "core/or/orconn_event.h" #include "core/mainloop/connection.h" #include "feature/control/control.h" #include "test/test.h" @@ -388,7 +392,145 @@ test_cntev_dirboot_defer_orconn(void *arg) UNMOCK(queue_control_event_string); } -#define TEST(name, flags) \ +static void +setup_orconn_state(orconn_event_msg_t *msg, uint64_t gid, uint64_t chan, + int proxy_type) +{ + msg->type = ORCONN_MSGTYPE_STATE; + msg->u.state.gid = gid; + msg->u.state.chan = chan; + msg->u.state.proxy_type = proxy_type; +} + +static void +send_orconn_state(orconn_event_msg_t *msg, uint8_t state) +{ + msg->u.state.state = state; + orconn_event_publish(msg); +} + +static void +send_ocirc_chan(uint32_t gid, uint64_t chan, bool onehop) +{ + ocirc_event_msg_t msg; + + msg.type = OCIRC_MSGTYPE_CHAN; + msg.u.chan.gid = gid; + msg.u.chan.chan = chan; + msg.u.chan.onehop = onehop; + ocirc_event_publish(&msg); +} + +static void +test_cntev_orconn_state(void *arg) +{ + orconn_event_msg_t conn; + + (void)arg; + MOCK(queue_control_event_string, mock_queue_control_event_string); + control_testing_set_global_event_mask(EVENT_MASK_(EVENT_STATUS_CLIENT)); + setup_orconn_state(&conn, 1, 1, PROXY_NONE); + + send_orconn_state(&conn, OR_CONN_STATE_CONNECTING); + send_ocirc_chan(1, 1, true); + assert_bootmsg("5 TAG=conn"); + send_orconn_state(&conn, OR_CONN_STATE_TLS_HANDSHAKING); + assert_bootmsg("10 TAG=conn_done"); + send_orconn_state(&conn, OR_CONN_STATE_OR_HANDSHAKING_V3); + assert_bootmsg("14 TAG=handshake"); + send_orconn_state(&conn, OR_CONN_STATE_OPEN); + assert_bootmsg("15 TAG=handshake_done"); + + conn.u.state.gid = 2; + conn.u.state.chan = 2; + send_orconn_state(&conn, OR_CONN_STATE_CONNECTING); + /* It doesn't know it's an origin circuit yet */ + assert_bootmsg("15 TAG=handshake_done"); + send_ocirc_chan(2, 2, false); + assert_bootmsg("80 TAG=ap_conn"); + send_orconn_state(&conn, OR_CONN_STATE_TLS_HANDSHAKING); + assert_bootmsg("85 TAG=ap_conn_done"); + send_orconn_state(&conn, OR_CONN_STATE_OR_HANDSHAKING_V3); + assert_bootmsg("89 TAG=ap_handshake"); + send_orconn_state(&conn, OR_CONN_STATE_OPEN); + assert_bootmsg("90 TAG=ap_handshake_done"); + + done: + tor_free(saved_event_str); + UNMOCK(queue_control_event_string); +} + +static void +test_cntev_orconn_state_pt(void *arg) +{ + orconn_event_msg_t conn; + + (void)arg; + MOCK(queue_control_event_string, mock_queue_control_event_string); + control_testing_set_global_event_mask(EVENT_MASK_(EVENT_STATUS_CLIENT)); + setup_orconn_state(&conn, 1, 1, PROXY_PLUGGABLE); + send_ocirc_chan(1, 1, true); + + send_orconn_state(&conn, OR_CONN_STATE_CONNECTING); + assert_bootmsg("1 TAG=conn_pt"); + send_orconn_state(&conn, OR_CONN_STATE_PROXY_HANDSHAKING); + assert_bootmsg("2 TAG=conn_done_pt"); + send_orconn_state(&conn, OR_CONN_STATE_TLS_HANDSHAKING); + assert_bootmsg("10 TAG=conn_done"); + send_orconn_state(&conn, OR_CONN_STATE_OR_HANDSHAKING_V3); + assert_bootmsg("14 TAG=handshake"); + send_orconn_state(&conn, OR_CONN_STATE_OPEN); + assert_bootmsg("15 TAG=handshake_done"); + + send_ocirc_chan(2, 2, false); + conn.u.state.gid = 2; + conn.u.state.chan = 2; + send_orconn_state(&conn, OR_CONN_STATE_CONNECTING); + assert_bootmsg("76 TAG=ap_conn_pt"); + send_orconn_state(&conn, OR_CONN_STATE_PROXY_HANDSHAKING); + assert_bootmsg("77 TAG=ap_conn_done_pt"); + + done: + tor_free(saved_event_str); + UNMOCK(queue_control_event_string); +} + +static void +test_cntev_orconn_state_proxy(void *arg) +{ + orconn_event_msg_t conn; + + (void)arg; + MOCK(queue_control_event_string, mock_queue_control_event_string); + control_testing_set_global_event_mask(EVENT_MASK_(EVENT_STATUS_CLIENT)); + setup_orconn_state(&conn, 1, 1, PROXY_CONNECT); + send_ocirc_chan(1, 1, true); + + send_orconn_state(&conn, OR_CONN_STATE_CONNECTING); + assert_bootmsg("3 TAG=conn_proxy"); + send_orconn_state(&conn, OR_CONN_STATE_PROXY_HANDSHAKING); + assert_bootmsg("4 TAG=conn_done_proxy"); + send_orconn_state(&conn, OR_CONN_STATE_TLS_HANDSHAKING); + assert_bootmsg("10 TAG=conn_done"); + send_orconn_state(&conn, OR_CONN_STATE_OR_HANDSHAKING_V3); + assert_bootmsg("14 TAG=handshake"); + send_orconn_state(&conn, OR_CONN_STATE_OPEN); + assert_bootmsg("15 TAG=handshake_done"); + + send_ocirc_chan(2, 2, false); + conn.u.state.gid = 2; + conn.u.state.chan = 2; + send_orconn_state(&conn, OR_CONN_STATE_CONNECTING); + assert_bootmsg("78 TAG=ap_conn_proxy"); + send_orconn_state(&conn, OR_CONN_STATE_PROXY_HANDSHAKING); + assert_bootmsg("79 TAG=ap_conn_done_proxy"); + + done: + tor_free(saved_event_str); + UNMOCK(queue_control_event_string); +} + +#define TEST(name, flags) \ { #name, test_cntev_ ## name, flags, 0, NULL } struct testcase_t controller_event_tests[] = { @@ -398,5 +540,8 @@ struct testcase_t controller_event_tests[] = { TEST(event_mask, TT_FORK), TEST(dirboot_defer_desc, TT_FORK), TEST(dirboot_defer_orconn, TT_FORK), + TEST(orconn_state, TT_FORK), + TEST(orconn_state_pt, TT_FORK), + TEST(orconn_state_proxy, TT_FORK), END_OF_TESTCASES }; From 2100b35f0799276f9854dd625621deda33a9ecc3 Mon Sep 17 00:00:00 2001 From: Taylor Yu Date: Thu, 20 Dec 2018 15:50:34 -0600 Subject: [PATCH 0352/2557] changes file for ticket27167 --- changes/ticket27167 | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 changes/ticket27167 diff --git a/changes/ticket27167 b/changes/ticket27167 new file mode 100644 index 0000000000..81c66630c8 --- /dev/null +++ b/changes/ticket27167 @@ -0,0 +1,11 @@ + o Major features (bootstrap): + - Report the first connection to a relay as the earliest phases of + bootstrap progress, regardless of whether it's a connection for + building application circuits. This allows finer-grained + reporting of early progress than previously possible with the + improvements of ticket 27169. Closes tickets 27167 and 27103. + Addresses ticket 27308. + - Separately report the intermediate stage of having connected to + a proxy or pluggable transport, versus succesfully using that + proxy or pluggable transport to connect to a relay. Closes + tickets 27100 and 28884. From 8eadfad71daaad985ac1262436bc232d4b184d71 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Sun, 23 Dec 2018 19:58:25 +0200 Subject: [PATCH 0353/2557] Tweak ControlPort description in manpage --- changes/doc28805 | 4 ++++ doc/tor.1.txt | 5 +++-- 2 files changed, 7 insertions(+), 2 deletions(-) create mode 100644 changes/doc28805 diff --git a/changes/doc28805 b/changes/doc28805 new file mode 100644 index 0000000000..6c9fea44fa --- /dev/null +++ b/changes/doc28805 @@ -0,0 +1,4 @@ + o Documentation (manpage): + - Improve ControlPort description in tor manpage to mention that it + accepts address/port pair, and can be used multiple times. Closes ticket + 28805. diff --git a/doc/tor.1.txt b/doc/tor.1.txt index 4ff789a931..b058bebcb3 100644 --- a/doc/tor.1.txt +++ b/doc/tor.1.txt @@ -350,7 +350,7 @@ GENERAL OPTIONS all sockets will be set to this limit. Must be a value between 2048 and 262144, in 1024 byte increments. Default of 8192 is recommended. -[[ControlPort]] **ControlPort** __PORT__|**unix:**__path__|**auto** [__flags__]:: +[[ControlPort]] **ControlPort** \['address':]__port__|**unix:**__path__|**auto** [__flags__]:: If set, Tor will accept connections on this port and allow those connections to control the Tor process using the Tor Control Protocol (described in control-spec.txt in @@ -361,7 +361,8 @@ GENERAL OPTIONS methods means either method is sufficient to authenticate to Tor.) This option is required for many Tor controllers; most use the value of 9051. If a unix domain socket is used, you may quote the path using standard - C escape sequences. + C escape sequences. You can specify this directive multiple times, to + bind to multiple address/port pairs. Set it to "auto" to have Tor pick a port for you. (Default: 0) + + Recognized flags are... From c11247e957ae8e3fb253aac86196dcef13dac92c Mon Sep 17 00:00:00 2001 From: Kris Katterjohn Date: Sun, 23 Dec 2018 17:46:08 -0600 Subject: [PATCH 0354/2557] Fix a buffer overflow in setup_cfg() in src/test/test_voting_flags.c signed_descriptor_digest has a length of DIGEST_LEN but the memset used to fill it used DIGEST256_LEN. Signed-off-by: Kris Katterjohn --- src/test/test_voting_flags.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/test_voting_flags.c b/src/test/test_voting_flags.c index 0c4cedb516..c4c4c7354e 100644 --- a/src/test/test_voting_flags.c +++ b/src/test/test_voting_flags.c @@ -35,7 +35,7 @@ setup_cfg(flag_vote_test_cfg_t *c) strlcpy(c->expected.nickname, "testing100", sizeof(c->expected.nickname)); memset(c->ri.cache_info.identity_digest, 0xff, DIGEST_LEN); - memset(c->ri.cache_info.signed_descriptor_digest, 0xee, DIGEST256_LEN); + memset(c->ri.cache_info.signed_descriptor_digest, 0xee, DIGEST_LEN); c->ri.cache_info.published_on = c->now - 100; c->expected.published_on = c->now - 100; From 3ba7581129dcffb16cca2aa8cfc2bd0fa4c8dd06 Mon Sep 17 00:00:00 2001 From: Mike Perry Date: Tue, 23 Oct 2018 19:23:09 +0000 Subject: [PATCH 0355/2557] Provide a smartlist reverse-order traversal. We need this for padding negotiation so that we can have later machine revisions supercede earlier ones. Co-authored-by: George Kadianakis --- src/lib/smartlist_core/smartlist_foreach.h | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/lib/smartlist_core/smartlist_foreach.h b/src/lib/smartlist_core/smartlist_foreach.h index 54f08ac47d..14f2930c9f 100644 --- a/src/lib/smartlist_core/smartlist_foreach.h +++ b/src/lib/smartlist_core/smartlist_foreach.h @@ -83,6 +83,14 @@ ++var ## _sl_idx) { \ var = (sl)->list[var ## _sl_idx]; +#define SMARTLIST_FOREACH_REVERSE_BEGIN(sl, type, var) \ + STMT_BEGIN \ + int var ## _sl_idx, var ## _sl_len=(sl)->num_used; \ + type var; \ + for (var ## _sl_idx = var ## _sl_len-1; var ## _sl_idx >= 0; \ + --var ## _sl_idx) { \ + var = (sl)->list[var ## _sl_idx]; + #define SMARTLIST_FOREACH_END(var) \ var = NULL; \ (void) var ## _sl_idx; \ From 2a24e21fb07ade157f1226039d71713c0c5b47b7 Mon Sep 17 00:00:00 2001 From: Mike Perry Date: Tue, 23 Oct 2018 19:48:16 +0000 Subject: [PATCH 0356/2557] Circuit padding header. This is a good code review start point, to get an overview of the interfaces and types used in circuit padding. Co-authored-by: George Kadianakis --- src/core/or/circuitpadding.h | 685 +++++++++++++++++++++++++++++++++++ 1 file changed, 685 insertions(+) create mode 100644 src/core/or/circuitpadding.h diff --git a/src/core/or/circuitpadding.h b/src/core/or/circuitpadding.h new file mode 100644 index 0000000000..24034a4548 --- /dev/null +++ b/src/core/or/circuitpadding.h @@ -0,0 +1,685 @@ +/* + * Copyright (c) 2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file circuitpadding.h + * \brief Header file for circuitpadding.c. + **/ +#ifndef TOR_CIRCUITPADDING_H +#define TOR_CIRCUITPADDING_H + +#include "circpad_negotiation.h" +#include "lib/evloop/timers.h" + +typedef struct circuit_t circuit_t; +typedef struct origin_circuit_t origin_circuit_t; +typedef struct cell_t cell_t; + +/** + * Signed error return with the specific property that negative + * values mean error codes of various semantics, 0 means success, + * and positive values are unused. + * + * XXX: Tor uses this concept a lot but just calls it int. Should we move + * this somewhere centralized? Where? + */ +typedef int signed_error_t; + +/** + * These constants specify the types of events that can cause + * transitions between state machine states. + * + * Note that SENT and RECV are relative to this endpoint. For + * relays, SENT means packets destined towards the client and + * RECV means packets destined towards the relay. On the client, + * SENT means packets destined towards the relay, where as RECV + * means packets destined towards the client. + */ +typedef enum { + /* A non-padding cell was received. */ + CIRCPAD_EVENT_NONPADDING_RECV = 0, + /* A non-padding cell was sent. */ + CIRCPAD_EVENT_NONPADDING_SENT = 1, + /* A padding cell (RELAY_COMMAND_DROP) was sent. */ + CIRCPAD_EVENT_PADDING_SENT = 2, + /* A padding cell was received. */ + CIRCPAD_EVENT_PADDING_RECV = 3, + /* We tried to schedule padding but we ended up picking the infinity bin + * which means that padding was delayed infinitely */ + CIRCPAD_EVENT_INFINITY = 4, + /* All histogram bins are empty (we are out of tokens) */ + CIRCPAD_EVENT_BINS_EMPTY = 5, + /* just a counter of the events above */ + CIRCPAD_EVENT_LENGTH_COUNT = 6 +} circpad_event_t; +#define CIRCPAD_NUM_EVENTS ((int)CIRCPAD_EVENT_LENGTH_COUNT+1) + +/** Boolean type that says if we decided to transition states or not */ +typedef enum { + CIRCPAD_STATE_UNCHANGED = 0, + CIRCPAD_STATE_CHANGED = 1 +} circpad_decision_t; + +/** The type for the things in histogram bins (aka tokens) */ +typedef uint32_t circpad_hist_token_t; + +/** The type for histogram indexes (needs to be negative for errors) */ +typedef int8_t circpad_hist_index_t; + +/** The type for absolute time, from monotime_absolute_usec() */ +typedef uint64_t circpad_time_t; + +/** The type for timer delays, in microseconds */ +typedef uint32_t circpad_delay_t; + +/** + * An infinite padding cell delay means don't schedule any padding -- + * simply wait until a different event triggers a transition. + * + * This means that the maximum delay we can scedule is UINT32_MAX-1 + * microseconds, or about 4300 seconds (1.25 hours). + * XXX: Is this enough if we want to simulate light, intermittent + * activity on an onion service? + */ +#define CIRCPAD_DELAY_INFINITE (UINT32_MAX) + +/** + * Macro to clarify when we're checking the infinity bin. + * + * Works with either circpad_state_t or circpad_machineinfo_t + */ +#define CIRCPAD_INFINITY_BIN(mi) ((mi)->histogram_len-1) + +/** + * These constants form a bitfield that specifies when a state machine + * should be applied to a circuit. + * + * If any of these elements is set, then the circuit will be tested against + * that specific condition. If an element is unset, then we don't test it. + * (E.g. If neither NO_STREAMS or STREAMS are set, then we will not care + * whether a circuit has streams attached when we apply a state machine) + * + * The helper function circpad_circuit_state() converts circuit state + * flags into this more compact representation. + */ +typedef enum { + /* Only apply machine if the circuit is still building */ + CIRCPAD_CIRC_BUILDING = 1<<0, + /* Only apply machine if the circuit is open */ + CIRCPAD_CIRC_OPENED = 1<<1, + /* Only apply machine if the circuit has no attached streams */ + CIRCPAD_CIRC_NO_STREAMS = 1<<2, + /* Only apply machine if the circuit has attached streams */ + CIRCPAD_CIRC_STREAMS = 1<<3, + /* Only apply machine if the circuit still allows RELAY_EARLY cells */ + CIRCPAD_CIRC_HAS_RELAY_EARLY = 1<<4, + /* Only apply machine if the circuit has depleted its RELAY_EARLY cells + * allowance. */ + CIRCPAD_CIRC_HAS_NO_RELAY_EARLY = 1<<5 +} circpad_circuit_state_t; + +/** Bitmask that says "apply this machine to all states" */ +#define CIRCPAD_STATE_ALL \ + (CIRCPAD_CIRC_BUILDING|CIRCPAD_CIRC_OPENED| \ + CIRCPAD_CIRC_STREAMS|CIRCPAD_CIRC_NO_STREAMS| \ + CIRCPAD_CIRC_HAS_RELAY_EARLY|CIRCPAD_CIRC_HAS_NO_RELAY_EARLY) + +/** + * A compact circuit purpose bitfield mask that allows us to compactly + * specify which circuit purposes a machine should apply to. + * + * The helper function circpad_circ_purpose_to_mask() converts circuit + * purposes into bit positions in this bitmask. + */ +typedef uint32_t circpad_purpose_mask_t; + +/** Bitmask that says "apply this machine to all purposes". */ +#define CIRCPAD_PURPOSE_ALL (0xFFFFFFFF) + +/** + * This type specifies all of the conditions that must be met before + * a client decides to initiate padding on a circuit. + * + * A circuit must satisfy every sub-field in this type in order + * to be considered to match the conditions. + */ +typedef struct circpad_machine_conditions_t { + /** Only apply the machine *if* the circuit has at least this many hops */ + unsigned min_hops : 3; + + /** Only apply the machine *if* vanguards are enabled */ + unsigned requires_vanguards : 1; + + /** Only apply the machine *if* the circuit's state matches any of + * the bits set in this bitmask. */ + circpad_circuit_state_t state_mask; + + /** Only apply a machine *if* the circuit's purpose matches one + * of the bits set in this bitmask */ + circpad_purpose_mask_t purpose_mask; + +} circpad_machine_conditions_t; + +/** + * Token removal strategy options. + * + * The WTF-PAD histograms are meant to specify a target distribution to shape + * traffic towards. This is accomplished by removing tokens from the histogram + * when either padding or non-padding cells are sent. + * + * When we see a non-padding cell at a particular time since the last cell, you + * remove a token from the corresponding delay bin. These flags specify + * which bin to choose if that bin is already empty. + */ +typedef enum { + /** Don't remove any tokens */ + CIRCPAD_TOKEN_REMOVAL_NONE = 0, + /** + * Remove from the first non-zero higher bin index when current is zero. + * This is the recommended strategy from the Adaptive Padding paper. */ + CIRCPAD_TOKEN_REMOVAL_HIGHER = 1, + /** Remove from the first non-zero lower bin index when current is empty. */ + CIRCPAD_TOKEN_REMOVAL_LOWER = 2, + /** Remove from the closest non-zero bin index when current is empty. */ + CIRCPAD_TOKEN_REMOVAL_CLOSEST = 3, + /** Remove from the closest bin by time value (since bins are + * exponentially spaced). */ + CIRCPAD_TOKEN_REMOVAL_CLOSEST_USEC = 4, + /** Only remove from the exact bin corresponding to this delay. If + * the bin is 0, simply do nothing. Don't pick another bin. */ + CIRCPAD_TOKEN_REMOVAL_EXACT = 5 +} circpad_removal_t; + +/** + * Distribution types supported by circpad_distribution_sample(). + * + * These can be used instead of histograms for the inter-packet + * timing distribution, or to specify a distribution on the number + * of cells that can be sent while in a specific state of the state + * machine. */ +typedef enum { + CIRCPAD_DIST_NONE = 0, + CIRCPAD_DIST_UNIFORM = 1, + CIRCPAD_DIST_LOGISTIC = 2, + CIRCPAD_DIST_LOG_LOGISTIC = 3, + CIRCPAD_DIST_GEOMETRIC = 4, + CIRCPAD_DIST_WEIBULL = 5, + CIRCPAD_DIST_PARETO = 6 +} circpad_distribution_type_t; + +/** + * Distribution information. + * + * This type specifies a specific distribution above, as well as + * up to two parameters for that distribution. The specific + * per-distribution meaning of these parameters is specified + * in circpad_distribution_sample(). + */ +typedef struct circpad_distribution_t { + circpad_distribution_type_t type; + double param1; + double param2; +} circpad_distribution_t; + +/** State number type. Represents current state of state machine. */ +typedef uint16_t circpad_statenum_t; +#define CIRCPAD_STATENUM_MAX (UINT16_MAX) + +/** A histogram is used to sample padding delays given a machine state. This + * constant defines the maximum histogram width (i.e. the max number of bins) + * + * Each histogram bin is twice as large as the previous. Two exceptions: The + * first bin has zero width (which means that minimum delay is applied to the + * next padding cell), and the last bin (infinity bin) has infinite width + * (which means that the next padding cell will be delayed infinitely). */ +#define CIRCPAD_MAX_HISTOGRAM_LEN (sizeof(circpad_delay_t)*8 + 1) + +/** + * A state of a padding machine. The information here are immutable and + * represent the initial form of the state; it does not get updated as things + * happen. The mutable information that gets updated in runtime are carried in + * a circpad_machineinfo_t. + * + * This struct describes the histograms and parameters of a single + * state in the adaptive padding machine. Instances of this struct + * exist in global circpad machine definitions that come from torrc + * or the consensus. + */ +typedef struct circpad_state_t { + /** If a histogram is used for this state, this specifies the number of bins + * of this histogram. Histograms must have at least 2 bins. + * + * If a delay probability distribution is used for this state, this is set + * to 0. */ + circpad_hist_index_t histogram_len; + /** The histogram itself: an array of uint16s of tokens, whose + * widths are exponentially spaced, in microseconds */ + circpad_hist_token_t histogram[CIRCPAD_MAX_HISTOGRAM_LEN]; + /** Total number of tokens in this histogram. This is a constant and is *not* + * decremented every time we spend a token. It's used for initializing and + * refilling the histogram. */ + uint32_t histogram_total_tokens; + + /** Minimum padding delay of this state in microseconds. + * + * If histograms are used, this is the left (and right) bound of the first + * bin (since it has zero width). + * + * If a delay probability distribution is used, this represents the minimum + * delay we can sample from the distribution. + */ + circpad_delay_t start_usec; + + /** If histograms are used, this is the width of the whole histogram in + * microseconds, and it's used to calculate individual bin width. + * + * If a delay probability distribution is used, this is used as the max + * delay we can sample from the distribution. + */ + circpad_delay_t range_usec; + + /** + * Represents a delay probability distribution (aka IAT distribution). It's a + * parametrized way of encoding inter-packet delay information in + * microseconds. It can be used instead of histograms. + * + * If it is used, token_removal below must be set to + * CIRCPAD_TOKEN_REMOVAL_NONE. + * + * Start_usec, range_sec, and rtt_estimates are still applied to the + * results of sampling from this distribution (range_sec is used as a max). + */ + circpad_distribution_t iat_dist; + + /** + * The length dist is a parameterized way of encoding how long this + * state machine runs in terms of sent padding cells or all + * sent cells. Values are sampled from this distribution, clamped + * to max_len, and then start_len is added to that value. + * + * It may be specified instead of or in addition to + * the infinity bins and bins empty conditions. */ + circpad_distribution_t length_dist; + /** A minimum length value, added to the output of length_dist */ + uint16_t start_length; + /** A cap on the length value that can be sampled from the length_dist */ + uint64_t max_length; + + /** Should we decrement length when we see a nonpadding packet? + * XXX: Are there any machines that actually want to set this to 0? There may + * not be. OTOH, it's only a bit.. */ + unsigned length_includes_nonpadding : 1; + + /** + * This is an array that specifies the next state to transition to upon + * receipt an event matching the indicated array index. + * + * This aborts our scheduled packet and switches to the state + * corresponding to the index of the array. Tokens are filled upon + * this transition. + * + * States are allowed to transition to themselves, which means re-schedule + * a new padding timer. They are also allowed to temporarily "transition" + * to the "IGNORE" and "CANCEL" pseudo-states. See #defines below + * for details on state behavior and meaning. + */ + circpad_statenum_t next_state[CIRCPAD_NUM_EVENTS]; + + /** + * If true, estimate the RTT from this relay to the exit/website and add that + * to start_usec for use as the histogram bin 0 start delay. + * + * Right now this is only supported for relay-side state machines. + */ + unsigned use_rtt_estimate : 1; + + /** This specifies the token removal strategy to use upon padding and + * non-padding activity. */ + circpad_removal_t token_removal; +} circpad_state_t; + +/** + * The start state for this machine. + * + * In the original WTF-PAD, this is only used for transition to/from + * the burst state. All other fields are not used. But to simplify the + * code we've made it a first-class state. This has no performance + * consequences, but may make naive serialization of the state machine + * large, if we're not careful about how we represent empty fields. + */ +#define CIRCPAD_STATE_START 0 + +/** + * The burst state for this machine. + * + * In the original Adaptive Padding algorithm and in WTF-PAD + * (https://www.freehaven.net/anonbib/cache/ShWa-Timing06.pdf and + * https://www.cs.kau.se/pulls/hot/thebasketcase-wtfpad/), the burst + * state serves to detect bursts in traffic. This is done by using longer + * delays in its histogram, which represent the expected delays between + * bursts of packets in the target stream. If this delay expires without a + * real packet being sent, the burst state sends a padding packet and then + * immediately transitions to the gap state, which is used to generate + * a synthetic padding packet train. In this implementation, this transition + * needs to be explicitly specified in the burst state's transition events. + * + * Because of this flexibility, other padding mechanisms can transition + * between these two states arbitrarily, to encode other dynamics of + * target traffic. + */ +#define CIRCPAD_STATE_BURST 1 + +/** + * The gap state for this machine. + * + * In the original Adaptive Padding algorithm and in WTF-PAD, the gap + * state serves to simulate an artificial packet train composed of padding + * packets. It does this by specifying much lower inter-packet delays than + * the burst state, and transitioning back to itself after padding is sent + * if these timers expire before real traffic is sent. If real traffic is + * sent, it transitions back to the burst state. + * + * Again, in this implementation, these transitions must be specified + * explicitly, and other transitions are also permitted. + */ +#define CIRCPAD_STATE_GAP 2 + +/** + * End is a pseudo-state that causes the machine to go completely + * idle, and optionally get torn down (depending on the + * value of circpad_machine_t.should_negotiate_end) + * + * End MUST NOT occupy a slot in the machine state array. + */ +#define CIRCPAD_STATE_END CIRCPAD_STATENUM_MAX + +/** + * "Ignore" is a pseudo-state that means "do not react to this + * event". + * + * "Ignore" MUST NOT occupy a slot in the machine state array. + */ +#define CIRCPAD_STATE_IGNORE (CIRCPAD_STATENUM_MAX-1) + +/** + * "Cancel" is a pseudo-state that means "cancel pending timers, + * but remain in your current state". + * + * Cancel MUST NOT occupy a slot in the machine state array. + */ +#define CIRCPAD_STATE_CANCEL (CIRCPAD_STATENUM_MAX-2) + +/** + * Since we have 3 pseudo-states, the max state array length is + * up to one less than cancel's statenum. + */ +#define CIRCPAD_MAX_MACHINE_STATES (CIRCPAD_STATE_CANCEL-1) + +/** + * Mutable padding machine info. + * + * This structure contains mutable information about a padding + * machine. The mutable information must be kept separate because + * it exists per-circuit, where as the machines themselves are global. + * This separation is done to conserve space in the circuit structure. + * + * This is the per-circuit state that changes regarding the global state + * machine. Some parts of it are optional (ie NULL). + * + * XXX: Play with layout to minimize space on x64 Linux (most common relay). + */ +typedef struct circpad_machineinfo_t { + /** The callback pointer for the padding callbacks. + * + * These timers stick around the machineinfo until the machineinfo's circuit + * is closed, at which point the timer is cancelled. For this reason it's + * safe to assume that the machineinfo exists if this timer gets + * triggered. */ + tor_timer_t *padding_timer; + + /** The circuit for this machine */ + circuit_t *on_circ; + + /** A mutable copy of the histogram for the current state. + * NULL if remove_tokens is false for that state */ + circpad_hist_token_t *histogram; + /** Length of the above histogram. + * XXX: This field *could* be removed at the expense of added + * complexity+overhead for reaching back into the immutable machine + * state every time we need to inspect the histogram. It's only a byte, + * though, so it seemed worth it. + */ + circpad_hist_index_t histogram_len; + /** Remove token from this index upon sending padding */ + circpad_hist_index_t chosen_bin; + + /** Stop padding/transition if this many cells sent */ + uint64_t state_length; +#define CIRCPAD_STATE_LENGTH_INFINITE UINT64_MAX + + /** A scaled count of padding packets sent, used to limit padding overhead. + * When this reaches UINT16_MAX, we cut it and nonpadding_sent in half. */ + uint16_t padding_sent; + /** A scaled count of non-padding packets sent, used to limit padding + * overhead. When this reaches UINT16_MAX, we cut it and padding_sent in + * half. */ + uint16_t nonpadding_sent; + + /** + * EWMA estimate of the RTT of the circuit from this hop + * to the exit end, in microseconds. */ + circpad_delay_t rtt_estimate_usec; + + /** + * The last time we got an event relevant to estimating + * the RTT. Monotonic time in microseconds since system + * start. + */ + circpad_time_t last_received_time_usec; + + /** + * The time at which we scheduled a non-padding packet, + * or selected an infinite delay. + * + * Monotonic time in microseconds since system start. + * This is 0 if we haven't chosen a padding delay. + */ + circpad_time_t padding_scheduled_at_usec; + + /** What state is this machine in? */ + circpad_statenum_t current_state; + + /** + * True if we have scheduled a timer for padding. + * + * This is 1 if a timer is pending. It is 0 if + * no timer is scheduled. (It can be 0 even when + * padding_was_scheduled_at_usec is non-zero). + */ + unsigned is_padding_timer_scheduled : 1; + + /** + * If this is true, we have seen full duplex behavior. + * Stop updating the RTT. + */ + unsigned stop_rtt_update : 1; + +/** Max number of padding machines on each circuit. If changed, + * also ensure the machine_index bitwith supports the new size. */ +#define CIRCPAD_MAX_MACHINES (2) + /** Which padding machine index was this for. + * (make sure changes to the bitwidth can support the + * CIRCPAD_MAX_MACHINES define). */ + unsigned machine_index : 1; + +} circpad_machineinfo_t; + +/** Helper macro to get an actual state machine from a machineinfo */ +#define CIRCPAD_GET_MACHINE(machineinfo) \ + ((machineinfo)->on_circ->padding_machine[(machineinfo)->machine_index]) + +/** + * This specifies a particular padding machine to use after negotiation. + * + * The constants for machine_num_t are in trunnel. + * We want to be able to define extra numbers in the consensus/torrc, though. + */ +typedef uint8_t circpad_machine_num_t; + +/** Global state machine structure from the consensus */ +typedef struct circpad_machine_t { + /** Global machine number */ + circpad_machine_num_t machine_num; + + /** Which machine index slot should this machine go into in + * the array on the circuit_t */ + unsigned machine_index : 1; + + /** Send a padding negotiate to shut down machine at end state? */ + unsigned should_negotiate_end : 1; + + // These next three fields are origin machine-only... + /** Origin side or relay side */ + unsigned is_origin_side : 1; + + /** Which hop in the circuit should we send padding to/from? + * 1-indexed (ie: hop #1 is guard, #2 middle, #3 exit). */ + unsigned target_hopnum : 3; + + /** This machine only kills fascists if the following conditions are met. */ + circpad_machine_conditions_t conditions; + + /** How many padding cells can be sent before we apply overhead limits? + * XXX: Note that we can only allow up to 64k of padding cells on an + * otherwise quiet circuit. Is this enough? It's 33MB. */ + uint16_t allowed_padding_count; + + /** Padding percent cap: Stop padding if we exceed this percent overhead. + * 0 means no limit. Overhead is defined as percent of total traffic, so + * that we can use 0..100 here. This is the same definition as used in + * Prop#265. */ + uint8_t max_padding_percent; + + /** State array: indexed by circpad_statenum_t */ + circpad_state_t *states; + + /** + * Number of states this machine has (ie: length of the states array). + * XXX: This field is not needed other than for safety. */ + circpad_statenum_t num_states; +} circpad_machine_t; + +void circpad_new_consensus_params(const networkstatus_t *ns); + +/** + * The following are event call-in points that are of interest to + * the state machines. They are called during cell processing. */ +void circpad_deliver_unrecognized_cell_events(circuit_t *circ, + cell_direction_t dir); +void circpad_deliver_sent_relay_cell_events(circuit_t *circ, + uint8_t relay_command); +void circpad_deliver_recognized_relay_cell_events(circuit_t *circ, + uint8_t relay_command, + crypt_path_t *layer_hint); + +/** Cell events are delivered by the above delivery functions */ +void circpad_cell_event_nonpadding_sent(circuit_t *on_circ); +void circpad_cell_event_nonpadding_received(circuit_t *on_circ); +void circpad_cell_event_padding_sent(circuit_t *on_circ); +void circpad_cell_event_padding_received(circuit_t *on_circ); + +/** Internal events are events the machines send to themselves */ +circpad_decision_t circpad_internal_event_infinity(circpad_machineinfo_t *mi); +circpad_decision_t circpad_internal_event_bins_empty(circpad_machineinfo_t *); +circpad_decision_t circpad_internal_event_state_length_up( + circpad_machineinfo_t *); + +/** Machine creation events are events that cause us to set up or + * tear down padding state machines. */ +void circpad_machine_event_circ_added_hop(origin_circuit_t *on_circ); +void circpad_machine_event_circ_built(origin_circuit_t *circ); +void circpad_machine_event_circ_purpose_changed(origin_circuit_t *circ); +void circpad_machine_event_circ_has_streams(origin_circuit_t *circ); +void circpad_machine_event_circ_has_no_streams(origin_circuit_t *circ); +void circpad_machine_event_circ_has_no_relay_early(origin_circuit_t *circ); + +void circpad_machines_init(void); +void circpad_machines_free(void); + +void circpad_machine_states_init(circpad_machine_t *machine, + circpad_statenum_t num_states); + +void circpad_circuit_free_all_machineinfos(circuit_t *circ); + +bool circpad_padding_is_from_expected_hop(circuit_t *circ, + crypt_path_t *from_hop); + +/** Serializaton functions for writing to/from torrc and consensus */ +char *circpad_machine_to_string(const circpad_machine_t *machine); +const circpad_machine_t *circpad_string_to_machine(const char *str); + +/* Padding negotiation between client and middle */ +signed_error_t circpad_handle_padding_negotiate(circuit_t *circ, cell_t *cell); +signed_error_t circpad_handle_padding_negotiated(circuit_t *circ, cell_t *cell, + crypt_path_t *layer_hint); +signed_error_t circpad_negotiate_padding(origin_circuit_t *circ, + circpad_machine_num_t machine, + uint8_t target_hopnum, + uint8_t command); +bool circpad_padding_negotiated(circuit_t *circ, + circpad_machine_num_t machine, + uint8_t command, + uint8_t response); + +MOCK_DECL(circpad_decision_t, +circpad_machine_schedule_padding,(circpad_machineinfo_t *)); + +MOCK_DECL(circpad_decision_t, +circpad_machine_transition, (circpad_machineinfo_t *mi, + circpad_event_t event)); + +circpad_decision_t circpad_send_padding_cell_for_callback( + circpad_machineinfo_t *mi); + +#ifdef CIRCUITPADDING_PRIVATE +STATIC circpad_delay_t +circpad_machine_sample_delay(circpad_machineinfo_t *mi); + +STATIC bool +circpad_machine_reached_padding_limit(circpad_machineinfo_t *mi); + +STATIC +circpad_decision_t circpad_machine_remove_token(circpad_machineinfo_t *mi); + +STATIC circpad_delay_t +circpad_histogram_bin_to_usec(const circpad_machineinfo_t *mi, + circpad_hist_index_t bin); + +STATIC const circpad_state_t * +circpad_machine_current_state(const circpad_machineinfo_t *mi); + +STATIC circpad_hist_index_t circpad_histogram_usec_to_bin( + const circpad_machineinfo_t *mi, + circpad_delay_t us); + +STATIC circpad_machineinfo_t *circpad_circuit_machineinfo_new( + circuit_t *on_circ, + int machine_index); +STATIC void circpad_machine_remove_higher_token(circpad_machineinfo_t *mi, + circpad_delay_t target_bin_us); +STATIC void circpad_machine_remove_lower_token(circpad_machineinfo_t *mi, + circpad_delay_t target_bin_us); +STATIC void circpad_machine_remove_closest_token(circpad_machineinfo_t *mi, + circpad_delay_t target_bin_us, + bool use_usec); +STATIC void circpad_machine_setup_tokens(circpad_machineinfo_t *mi); + +#ifdef TOR_UNIT_TESTS +extern smartlist_t *origin_padding_machines; +extern smartlist_t *relay_padding_machines; +#endif + +#endif + +#endif From 2f7b5a2d4468532251dd7b177c02a9c192690174 Mon Sep 17 00:00:00 2001 From: Mike Perry Date: Tue, 23 Oct 2018 19:51:22 +0000 Subject: [PATCH 0357/2557] Hook up circuit padding to circuit_t. Co-authored-by: George Kadianakis --- src/core/or/circuit_st.h | 24 ++++++++++++++++++++++++ src/core/or/circuitlist.c | 4 ++++ src/core/or/origin_circuit_st.h | 4 ++++ 3 files changed, 32 insertions(+) diff --git a/src/core/or/circuit_st.h b/src/core/or/circuit_st.h index 2e33b37b01..0d9ad3cdd5 100644 --- a/src/core/or/circuit_st.h +++ b/src/core/or/circuit_st.h @@ -12,6 +12,11 @@ #include "core/or/cell_queue_st.h" struct hs_token_t; +typedef struct circpad_machine_t circpad_machine_t; +typedef struct circpad_machineinfo_t circpad_machineinfo_t; + +/** Number of padding state machines on a circuit. */ +#define CIRCPAD_MAX_MACHINES (2) /** "magic" value for an origin_circuit_t */ #define ORIGIN_CIRCUIT_MAGIC 0x35315243u @@ -177,6 +182,25 @@ struct circuit_t { /** Hashtable node: used to look up the circuit by its HS token using the HS circuitmap. */ HT_ENTRY(circuit_t) hs_circuitmap_node; + + /** Adaptive Padding state machines: these are immutable. The state machines + * that come from the consensus are saved to a global structure, to avoid + * per-circuit allocations. This merely points to the global copy. + * + * Each element of this array corresponds to a different padding machine, + * and we can have up to CIRCPAD_MAX_MACHINES such machines. */ + const circpad_machine_t *padding_machine[CIRCPAD_MAX_MACHINES]; + + /** Adaptive Padding machine info for above machines. This is the + * per-circuit mutable information, such as the current state and + * histogram token counts. Some of it is optional (aka NULL). + * If a machine is being shut down, these indexes can be NULL + * without the corresponding padding_machine being NULL, while we + * wait for the other end to respond to our shutdown request. + * + * Each element of this array corresponds to a different padding machine, + * and we can have up to CIRCPAD_MAX_MACHINES such machines. */ + circpad_machineinfo_t *padding_info[CIRCPAD_MAX_MACHINES]; }; #endif diff --git a/src/core/or/circuitlist.c b/src/core/or/circuitlist.c index c4b5f7ee3e..71f8becddc 100644 --- a/src/core/or/circuitlist.c +++ b/src/core/or/circuitlist.c @@ -62,6 +62,7 @@ #include "core/or/circuitlist.h" #include "core/or/circuituse.h" #include "core/or/circuitstats.h" +#include "core/or/circuitpadding.h" #include "core/mainloop/connection.h" #include "app/config/config.h" #include "core/or/connection_edge.h" @@ -1231,6 +1232,9 @@ circuit_free_(circuit_t *circ) CIRCUIT_IS_ORIGIN(circ) ? TO_ORIGIN_CIRCUIT(circ)->global_identifier : 0); + /* Free any circuit padding structures */ + circpad_circuit_free_all_machineinfos(circ); + if (should_free) { memwipe(mem, 0xAA, memlen); /* poison memory */ tor_free(mem); diff --git a/src/core/or/origin_circuit_st.h b/src/core/or/origin_circuit_st.h index 26cdf590f1..921076c1b9 100644 --- a/src/core/or/origin_circuit_st.h +++ b/src/core/or/origin_circuit_st.h @@ -161,6 +161,10 @@ struct origin_circuit_t { * connections to this circuit. */ unsigned int unusable_for_new_conns : 1; + /* If this flag is set (due to padding negotiation failure), we should + * not try to negotiate further circuit padding. */ + unsigned padding_negotiation_failed : 1; + /** * Tristate variable to guard against pathbias miscounting * due to circuit purpose transitions changing the decision From 70e9245f6feecceee96f0ea8d426e1a5a6fc9b8d Mon Sep 17 00:00:00 2001 From: Mike Perry Date: Tue, 23 Oct 2018 20:52:43 +0000 Subject: [PATCH 0358/2557] Initialize circuit padding machines and global state. Co-authored-by: George Kadianakis --- src/app/main/main.c | 10 ++++++++-- src/feature/nodelist/networkstatus.c | 2 ++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/app/main/main.c b/src/app/main/main.c index d71e43ec30..ba2dfebd77 100644 --- a/src/app/main/main.c +++ b/src/app/main/main.c @@ -22,6 +22,7 @@ #include "core/mainloop/netstatus.h" #include "core/or/channel.h" #include "core/or/channelpadding.h" +#include "core/or/circuitpadding.h" #include "core/or/channeltls.h" #include "core/or/circuitlist.h" #include "core/or/circuitmux_ewma.h" @@ -645,9 +646,13 @@ tor_init(int argc, char *argv[]) /* The options are now initialised */ const or_options_t *options = get_options(); - /* Initialize channelpadding parameters to defaults until we get - * a consensus */ + /* Initialize channelpadding and circpad parameters to defaults + * until we get a consensus */ channelpadding_new_consensus_params(NULL); + circpad_new_consensus_params(NULL); + + /* Initialize circuit padding to defaults+torrc until we get a consensus */ + circpad_machines_init(); /* Initialize predicted ports list after loading options */ predicted_ports_init(); @@ -766,6 +771,7 @@ tor_free_all(int postfork) dns_free_all(); clear_pending_onions(); circuit_free_all(); + circpad_machines_free(); entry_guards_free_all(); pt_free_all(); channel_tls_free_all(); diff --git a/src/feature/nodelist/networkstatus.c b/src/feature/nodelist/networkstatus.c index 65ea3cc491..3245c8ff16 100644 --- a/src/feature/nodelist/networkstatus.c +++ b/src/feature/nodelist/networkstatus.c @@ -44,6 +44,7 @@ #include "core/mainloop/netstatus.h" #include "core/or/channel.h" #include "core/or/channelpadding.h" +#include "core/or/circuitpadding.h" #include "core/or/circuitmux.h" #include "core/or/circuitmux_ewma.h" #include "core/or/circuitstats.h" @@ -2103,6 +2104,7 @@ networkstatus_set_current_consensus(const char *consensus, circuit_build_times_new_consensus_params( get_circuit_build_times_mutable(), c); channelpadding_new_consensus_params(c); + circpad_new_consensus_params(c); } /* Reset the failure count only if this consensus is actually valid. */ From 659a4f06d46a0e8e4f391eda3b6d86f2ab6e4db9 Mon Sep 17 00:00:00 2001 From: Mike Perry Date: Tue, 23 Oct 2018 20:55:10 +0000 Subject: [PATCH 0359/2557] Circuit padding ProtoVer plumbing. This helps us to determine if a middle node can pad to us or not. Co-authored-by: George Kadianakis --- src/core/or/or.h | 4 ++++ src/core/or/protover.c | 7 ++++++- src/core/or/protover.h | 1 + src/core/or/versions.c | 2 ++ src/feature/nodelist/nodelist.c | 2 +- src/rust/protover/protover.rs | 8 ++++++-- 6 files changed, 20 insertions(+), 4 deletions(-) diff --git a/src/core/or/or.h b/src/core/or/or.h index ca373d8ed5..f297bc9267 100644 --- a/src/core/or/or.h +++ b/src/core/or/or.h @@ -836,6 +836,10 @@ typedef struct protover_summary_flags_t { * service rendezvous point supporting version 3 as seen in proposal 224. * This requires HSRend=2. */ unsigned int supports_v3_rendezvous_point: 1; + + /** True iff this router has a protocol list that allows clients to + * negotiate link-level padding. Requires Padding>=1. */ + unsigned int supports_padding : 1; } protover_summary_flags_t; typedef struct routerinfo_t routerinfo_t; diff --git a/src/core/or/protover.c b/src/core/or/protover.c index e80fbfae81..c0c09c9d17 100644 --- a/src/core/or/protover.c +++ b/src/core/or/protover.c @@ -39,6 +39,9 @@ static int protocol_list_contains(const smartlist_t *protos, static const struct { protocol_type_t protover_type; const char *name; +/* If you add a new protocol here, you probably also want to add + * parsing for it in routerstatus_parse_entry_from_string() so that + * it is set in routerstatus_t */ } PROTOCOL_NAMES[] = { { PRT_LINK, "Link" }, { PRT_LINKAUTH, "LinkAuth" }, @@ -49,6 +52,7 @@ static const struct { { PRT_HSREND, "HSRend" }, { PRT_DESC, "Desc" }, { PRT_MICRODESC, "Microdesc"}, + { PRT_PADDING, "Padding"}, { PRT_CONS, "Cons" } }; @@ -396,7 +400,8 @@ protover_get_supported_protocols(void) "LinkAuth=3 " #endif "Microdesc=1-2 " - "Relay=1-2"; + "Relay=1-2 " + "Padding=1"; } /** The protocols from protover_get_supported_protocols(), as parsed into a diff --git a/src/core/or/protover.h b/src/core/or/protover.h index 7319d2f8c4..ffd4f2c18e 100644 --- a/src/core/or/protover.h +++ b/src/core/or/protover.h @@ -43,6 +43,7 @@ typedef enum protocol_type_t { PRT_DESC, PRT_MICRODESC, PRT_CONS, + PRT_PADDING, } protocol_type_t; bool protover_contains_long_protocol_names(const char *s); diff --git a/src/core/or/versions.c b/src/core/or/versions.c index 7bd1f5899f..736313a9cd 100644 --- a/src/core/or/versions.c +++ b/src/core/or/versions.c @@ -448,6 +448,8 @@ memoize_protover_summary(protover_summary_flags_t *out, out->supports_v3_rendezvous_point = protocol_list_supports_protocol(protocols, PRT_HSREND, PROTOVER_HS_RENDEZVOUS_POINT_V3); + out->supports_padding = + protocol_list_supports_protocol(protocols, PRT_PADDING, 1); protover_summary_flags_t *new_cached = tor_memdup(out, sizeof(*out)); cached = strmap_set(protover_summary_map, protocols, new_cached); diff --git a/src/feature/nodelist/nodelist.c b/src/feature/nodelist/nodelist.c index d94e73f48f..15b3f7b600 100644 --- a/src/feature/nodelist/nodelist.c +++ b/src/feature/nodelist/nodelist.c @@ -1106,7 +1106,7 @@ node_ed25519_id_matches(const node_t *node, const ed25519_public_key_t *id) /** Dummy object that should be unreturnable. Used to ensure that * node_get_protover_summary_flags() always returns non-NULL. */ static const protover_summary_flags_t zero_protover_flags = { - 0,0,0,0,0,0,0 + 0,0,0,0,0,0,0,0 }; /** Return the protover_summary_flags for a given node. */ diff --git a/src/rust/protover/protover.rs b/src/rust/protover/protover.rs index 8624afeafa..0b2a78c210 100644 --- a/src/rust/protover/protover.rs +++ b/src/rust/protover/protover.rs @@ -46,6 +46,7 @@ pub enum Protocol { LinkAuth, Microdesc, Relay, + Padding, } impl fmt::Display for Protocol { @@ -73,6 +74,7 @@ impl FromStr for Protocol { "LinkAuth" => Ok(Protocol::LinkAuth), "Microdesc" => Ok(Protocol::Microdesc), "Relay" => Ok(Protocol::Relay), + "Padding" => Ok(Protocol::Padding), _ => Err(ProtoverError::UnknownProtocol), } } @@ -163,7 +165,8 @@ pub(crate) fn get_supported_protocols_cstr() -> &'static CStr { Link=1-5 \ LinkAuth=3 \ Microdesc=1-2 \ - Relay=1-2" + Relay=1-2 \ + Padding=1" ) } else { cstr!( @@ -176,7 +179,8 @@ pub(crate) fn get_supported_protocols_cstr() -> &'static CStr { Link=1-5 \ LinkAuth=1,3 \ Microdesc=1-2 \ - Relay=1-2" + Relay=1-2 \ + Padding=1" ) } } From 4ca1df6b326f32fcc87d7d3fd6294221f6964235 Mon Sep 17 00:00:00 2001 From: Mike Perry Date: Tue, 23 Oct 2018 19:51:35 +0000 Subject: [PATCH 0360/2557] Add padding negotiation trunnel output. Co-authored-by: George Kadianakis --- src/trunnel/circpad_negotiation.c | 549 ++++++++++++++++++++++++ src/trunnel/circpad_negotiation.h | 195 +++++++++ src/trunnel/circpad_negotiation.trunnel | 44 ++ src/trunnel/include.am | 11 +- 4 files changed, 795 insertions(+), 4 deletions(-) create mode 100644 src/trunnel/circpad_negotiation.c create mode 100644 src/trunnel/circpad_negotiation.h create mode 100644 src/trunnel/circpad_negotiation.trunnel diff --git a/src/trunnel/circpad_negotiation.c b/src/trunnel/circpad_negotiation.c new file mode 100644 index 0000000000..236be06ada --- /dev/null +++ b/src/trunnel/circpad_negotiation.c @@ -0,0 +1,549 @@ +/* circpad_negotiation.c -- generated by Trunnel v1.5.2. + * https://gitweb.torproject.org/trunnel.git + * You probably shouldn't edit this file. + */ +#include +#include "trunnel-impl.h" + +#include "circpad_negotiation.h" + +#define TRUNNEL_SET_ERROR_CODE(obj) \ + do { \ + (obj)->trunnel_error_code_ = 1; \ + } while (0) + +#if defined(__COVERITY__) || defined(__clang_analyzer__) +/* If we're running a static analysis tool, we don't want it to complain + * that some of our remaining-bytes checks are dead-code. */ +int circpadnegotiation_deadcode_dummy__ = 0; +#define OR_DEADCODE_DUMMY || circpadnegotiation_deadcode_dummy__ +#else +#define OR_DEADCODE_DUMMY +#endif + +#define CHECK_REMAINING(nbytes, label) \ + do { \ + if (remaining < (nbytes) OR_DEADCODE_DUMMY) { \ + goto label; \ + } \ + } while (0) + +circpad_negotiate_t * +circpad_negotiate_new(void) +{ + circpad_negotiate_t *val = trunnel_calloc(1, sizeof(circpad_negotiate_t)); + if (NULL == val) + return NULL; + val->command = CIRCPAD_COMMAND_START; + return val; +} + +/** Release all storage held inside 'obj', but do not free 'obj'. + */ +static void +circpad_negotiate_clear(circpad_negotiate_t *obj) +{ + (void) obj; +} + +void +circpad_negotiate_free(circpad_negotiate_t *obj) +{ + if (obj == NULL) + return; + circpad_negotiate_clear(obj); + trunnel_memwipe(obj, sizeof(circpad_negotiate_t)); + trunnel_free_(obj); +} + +uint8_t +circpad_negotiate_get_version(const circpad_negotiate_t *inp) +{ + return inp->version; +} +int +circpad_negotiate_set_version(circpad_negotiate_t *inp, uint8_t val) +{ + if (! ((val == 0))) { + TRUNNEL_SET_ERROR_CODE(inp); + return -1; + } + inp->version = val; + return 0; +} +uint8_t +circpad_negotiate_get_command(const circpad_negotiate_t *inp) +{ + return inp->command; +} +int +circpad_negotiate_set_command(circpad_negotiate_t *inp, uint8_t val) +{ + if (! ((val == CIRCPAD_COMMAND_START || val == CIRCPAD_COMMAND_STOP))) { + TRUNNEL_SET_ERROR_CODE(inp); + return -1; + } + inp->command = val; + return 0; +} +uint8_t +circpad_negotiate_get_machine_type(const circpad_negotiate_t *inp) +{ + return inp->machine_type; +} +int +circpad_negotiate_set_machine_type(circpad_negotiate_t *inp, uint8_t val) +{ + inp->machine_type = val; + return 0; +} +uint8_t +circpad_negotiate_get_echo_request(const circpad_negotiate_t *inp) +{ + return inp->echo_request; +} +int +circpad_negotiate_set_echo_request(circpad_negotiate_t *inp, uint8_t val) +{ + if (! ((val == 0 || val == 1))) { + TRUNNEL_SET_ERROR_CODE(inp); + return -1; + } + inp->echo_request = val; + return 0; +} +const char * +circpad_negotiate_check(const circpad_negotiate_t *obj) +{ + if (obj == NULL) + return "Object was NULL"; + if (obj->trunnel_error_code_) + return "A set function failed on this object"; + if (! (obj->version == 0)) + return "Integer out of bounds"; + if (! (obj->command == CIRCPAD_COMMAND_START || obj->command == CIRCPAD_COMMAND_STOP)) + return "Integer out of bounds"; + if (! (obj->echo_request == 0 || obj->echo_request == 1)) + return "Integer out of bounds"; + return NULL; +} + +ssize_t +circpad_negotiate_encoded_len(const circpad_negotiate_t *obj) +{ + ssize_t result = 0; + + if (NULL != circpad_negotiate_check(obj)) + return -1; + + + /* Length of u8 version IN [0] */ + result += 1; + + /* Length of u8 command IN [CIRCPAD_COMMAND_START, CIRCPAD_COMMAND_STOP] */ + result += 1; + + /* Length of u8 machine_type */ + result += 1; + + /* Length of u8 echo_request IN [0, 1] */ + result += 1; + return result; +} +int +circpad_negotiate_clear_errors(circpad_negotiate_t *obj) +{ + int r = obj->trunnel_error_code_; + obj->trunnel_error_code_ = 0; + return r; +} +ssize_t +circpad_negotiate_encode(uint8_t *output, const size_t avail, const circpad_negotiate_t *obj) +{ + ssize_t result = 0; + size_t written = 0; + uint8_t *ptr = output; + const char *msg; +#ifdef TRUNNEL_CHECK_ENCODED_LEN + const ssize_t encoded_len = circpad_negotiate_encoded_len(obj); +#endif + + if (NULL != (msg = circpad_negotiate_check(obj))) + goto check_failed; + +#ifdef TRUNNEL_CHECK_ENCODED_LEN + trunnel_assert(encoded_len >= 0); +#endif + + /* Encode u8 version IN [0] */ + trunnel_assert(written <= avail); + if (avail - written < 1) + goto truncated; + trunnel_set_uint8(ptr, (obj->version)); + written += 1; ptr += 1; + + /* Encode u8 command IN [CIRCPAD_COMMAND_START, CIRCPAD_COMMAND_STOP] */ + trunnel_assert(written <= avail); + if (avail - written < 1) + goto truncated; + trunnel_set_uint8(ptr, (obj->command)); + written += 1; ptr += 1; + + /* Encode u8 machine_type */ + trunnel_assert(written <= avail); + if (avail - written < 1) + goto truncated; + trunnel_set_uint8(ptr, (obj->machine_type)); + written += 1; ptr += 1; + + /* Encode u8 echo_request IN [0, 1] */ + trunnel_assert(written <= avail); + if (avail - written < 1) + goto truncated; + trunnel_set_uint8(ptr, (obj->echo_request)); + written += 1; ptr += 1; + + + trunnel_assert(ptr == output + written); +#ifdef TRUNNEL_CHECK_ENCODED_LEN + { + trunnel_assert(encoded_len >= 0); + trunnel_assert((size_t)encoded_len == written); + } + +#endif + + return written; + + truncated: + result = -2; + goto fail; + check_failed: + (void)msg; + result = -1; + goto fail; + fail: + trunnel_assert(result < 0); + return result; +} + +/** As circpad_negotiate_parse(), but do not allocate the output + * object. + */ +static ssize_t +circpad_negotiate_parse_into(circpad_negotiate_t *obj, const uint8_t *input, const size_t len_in) +{ + const uint8_t *ptr = input; + size_t remaining = len_in; + ssize_t result = 0; + (void)result; + + /* Parse u8 version IN [0] */ + CHECK_REMAINING(1, truncated); + obj->version = (trunnel_get_uint8(ptr)); + remaining -= 1; ptr += 1; + if (! (obj->version == 0)) + goto fail; + + /* Parse u8 command IN [CIRCPAD_COMMAND_START, CIRCPAD_COMMAND_STOP] */ + CHECK_REMAINING(1, truncated); + obj->command = (trunnel_get_uint8(ptr)); + remaining -= 1; ptr += 1; + if (! (obj->command == CIRCPAD_COMMAND_START || obj->command == CIRCPAD_COMMAND_STOP)) + goto fail; + + /* Parse u8 machine_type */ + CHECK_REMAINING(1, truncated); + obj->machine_type = (trunnel_get_uint8(ptr)); + remaining -= 1; ptr += 1; + + /* Parse u8 echo_request IN [0, 1] */ + CHECK_REMAINING(1, truncated); + obj->echo_request = (trunnel_get_uint8(ptr)); + remaining -= 1; ptr += 1; + if (! (obj->echo_request == 0 || obj->echo_request == 1)) + goto fail; + trunnel_assert(ptr + remaining == input + len_in); + return len_in - remaining; + + truncated: + return -2; + fail: + result = -1; + return result; +} + +ssize_t +circpad_negotiate_parse(circpad_negotiate_t **output, const uint8_t *input, const size_t len_in) +{ + ssize_t result; + *output = circpad_negotiate_new(); + if (NULL == *output) + return -1; + result = circpad_negotiate_parse_into(*output, input, len_in); + if (result < 0) { + circpad_negotiate_free(*output); + *output = NULL; + } + return result; +} +circpad_negotiated_t * +circpad_negotiated_new(void) +{ + circpad_negotiated_t *val = trunnel_calloc(1, sizeof(circpad_negotiated_t)); + if (NULL == val) + return NULL; + val->command = CIRCPAD_COMMAND_START; + val->response = CIRCPAD_RESPONSE_ERR; + return val; +} + +/** Release all storage held inside 'obj', but do not free 'obj'. + */ +static void +circpad_negotiated_clear(circpad_negotiated_t *obj) +{ + (void) obj; +} + +void +circpad_negotiated_free(circpad_negotiated_t *obj) +{ + if (obj == NULL) + return; + circpad_negotiated_clear(obj); + trunnel_memwipe(obj, sizeof(circpad_negotiated_t)); + trunnel_free_(obj); +} + +uint8_t +circpad_negotiated_get_version(const circpad_negotiated_t *inp) +{ + return inp->version; +} +int +circpad_negotiated_set_version(circpad_negotiated_t *inp, uint8_t val) +{ + if (! ((val == 0))) { + TRUNNEL_SET_ERROR_CODE(inp); + return -1; + } + inp->version = val; + return 0; +} +uint8_t +circpad_negotiated_get_command(const circpad_negotiated_t *inp) +{ + return inp->command; +} +int +circpad_negotiated_set_command(circpad_negotiated_t *inp, uint8_t val) +{ + if (! ((val == CIRCPAD_COMMAND_START || val == CIRCPAD_COMMAND_STOP))) { + TRUNNEL_SET_ERROR_CODE(inp); + return -1; + } + inp->command = val; + return 0; +} +uint8_t +circpad_negotiated_get_response(const circpad_negotiated_t *inp) +{ + return inp->response; +} +int +circpad_negotiated_set_response(circpad_negotiated_t *inp, uint8_t val) +{ + if (! ((val == CIRCPAD_RESPONSE_ERR || val == CIRCPAD_RESPONSE_OK))) { + TRUNNEL_SET_ERROR_CODE(inp); + return -1; + } + inp->response = val; + return 0; +} +uint8_t +circpad_negotiated_get_machine_type(const circpad_negotiated_t *inp) +{ + return inp->machine_type; +} +int +circpad_negotiated_set_machine_type(circpad_negotiated_t *inp, uint8_t val) +{ + inp->machine_type = val; + return 0; +} +const char * +circpad_negotiated_check(const circpad_negotiated_t *obj) +{ + if (obj == NULL) + return "Object was NULL"; + if (obj->trunnel_error_code_) + return "A set function failed on this object"; + if (! (obj->version == 0)) + return "Integer out of bounds"; + if (! (obj->command == CIRCPAD_COMMAND_START || obj->command == CIRCPAD_COMMAND_STOP)) + return "Integer out of bounds"; + if (! (obj->response == CIRCPAD_RESPONSE_ERR || obj->response == CIRCPAD_RESPONSE_OK)) + return "Integer out of bounds"; + return NULL; +} + +ssize_t +circpad_negotiated_encoded_len(const circpad_negotiated_t *obj) +{ + ssize_t result = 0; + + if (NULL != circpad_negotiated_check(obj)) + return -1; + + + /* Length of u8 version IN [0] */ + result += 1; + + /* Length of u8 command IN [CIRCPAD_COMMAND_START, CIRCPAD_COMMAND_STOP] */ + result += 1; + + /* Length of u8 response IN [CIRCPAD_RESPONSE_ERR, CIRCPAD_RESPONSE_OK] */ + result += 1; + + /* Length of u8 machine_type */ + result += 1; + return result; +} +int +circpad_negotiated_clear_errors(circpad_negotiated_t *obj) +{ + int r = obj->trunnel_error_code_; + obj->trunnel_error_code_ = 0; + return r; +} +ssize_t +circpad_negotiated_encode(uint8_t *output, const size_t avail, const circpad_negotiated_t *obj) +{ + ssize_t result = 0; + size_t written = 0; + uint8_t *ptr = output; + const char *msg; +#ifdef TRUNNEL_CHECK_ENCODED_LEN + const ssize_t encoded_len = circpad_negotiated_encoded_len(obj); +#endif + + if (NULL != (msg = circpad_negotiated_check(obj))) + goto check_failed; + +#ifdef TRUNNEL_CHECK_ENCODED_LEN + trunnel_assert(encoded_len >= 0); +#endif + + /* Encode u8 version IN [0] */ + trunnel_assert(written <= avail); + if (avail - written < 1) + goto truncated; + trunnel_set_uint8(ptr, (obj->version)); + written += 1; ptr += 1; + + /* Encode u8 command IN [CIRCPAD_COMMAND_START, CIRCPAD_COMMAND_STOP] */ + trunnel_assert(written <= avail); + if (avail - written < 1) + goto truncated; + trunnel_set_uint8(ptr, (obj->command)); + written += 1; ptr += 1; + + /* Encode u8 response IN [CIRCPAD_RESPONSE_ERR, CIRCPAD_RESPONSE_OK] */ + trunnel_assert(written <= avail); + if (avail - written < 1) + goto truncated; + trunnel_set_uint8(ptr, (obj->response)); + written += 1; ptr += 1; + + /* Encode u8 machine_type */ + trunnel_assert(written <= avail); + if (avail - written < 1) + goto truncated; + trunnel_set_uint8(ptr, (obj->machine_type)); + written += 1; ptr += 1; + + + trunnel_assert(ptr == output + written); +#ifdef TRUNNEL_CHECK_ENCODED_LEN + { + trunnel_assert(encoded_len >= 0); + trunnel_assert((size_t)encoded_len == written); + } + +#endif + + return written; + + truncated: + result = -2; + goto fail; + check_failed: + (void)msg; + result = -1; + goto fail; + fail: + trunnel_assert(result < 0); + return result; +} + +/** As circpad_negotiated_parse(), but do not allocate the output + * object. + */ +static ssize_t +circpad_negotiated_parse_into(circpad_negotiated_t *obj, const uint8_t *input, const size_t len_in) +{ + const uint8_t *ptr = input; + size_t remaining = len_in; + ssize_t result = 0; + (void)result; + + /* Parse u8 version IN [0] */ + CHECK_REMAINING(1, truncated); + obj->version = (trunnel_get_uint8(ptr)); + remaining -= 1; ptr += 1; + if (! (obj->version == 0)) + goto fail; + + /* Parse u8 command IN [CIRCPAD_COMMAND_START, CIRCPAD_COMMAND_STOP] */ + CHECK_REMAINING(1, truncated); + obj->command = (trunnel_get_uint8(ptr)); + remaining -= 1; ptr += 1; + if (! (obj->command == CIRCPAD_COMMAND_START || obj->command == CIRCPAD_COMMAND_STOP)) + goto fail; + + /* Parse u8 response IN [CIRCPAD_RESPONSE_ERR, CIRCPAD_RESPONSE_OK] */ + CHECK_REMAINING(1, truncated); + obj->response = (trunnel_get_uint8(ptr)); + remaining -= 1; ptr += 1; + if (! (obj->response == CIRCPAD_RESPONSE_ERR || obj->response == CIRCPAD_RESPONSE_OK)) + goto fail; + + /* Parse u8 machine_type */ + CHECK_REMAINING(1, truncated); + obj->machine_type = (trunnel_get_uint8(ptr)); + remaining -= 1; ptr += 1; + trunnel_assert(ptr + remaining == input + len_in); + return len_in - remaining; + + truncated: + return -2; + fail: + result = -1; + return result; +} + +ssize_t +circpad_negotiated_parse(circpad_negotiated_t **output, const uint8_t *input, const size_t len_in) +{ + ssize_t result; + *output = circpad_negotiated_new(); + if (NULL == *output) + return -1; + result = circpad_negotiated_parse_into(*output, input, len_in); + if (result < 0) { + circpad_negotiated_free(*output); + *output = NULL; + } + return result; +} diff --git a/src/trunnel/circpad_negotiation.h b/src/trunnel/circpad_negotiation.h new file mode 100644 index 0000000000..d09080dc16 --- /dev/null +++ b/src/trunnel/circpad_negotiation.h @@ -0,0 +1,195 @@ +/* circpad_negotiation.h -- generated by Trunnel v1.5.2. + * https://gitweb.torproject.org/trunnel.git + * You probably shouldn't edit this file. + */ +#ifndef TRUNNEL_CIRCPAD_NEGOTIATION_H +#define TRUNNEL_CIRCPAD_NEGOTIATION_H + +#include +#include "trunnel.h" + +#define CIRCPAD_COMMAND_STOP 1 +#define CIRCPAD_COMMAND_START 2 +#define CIRCPAD_RESPONSE_OK 1 +#define CIRCPAD_RESPONSE_ERR 2 +#define CIRCPAD_MACHINE_CIRC_SETUP 1 +/** + * This command tells the relay to alter its min and max netflow + * timeout range values, and send padding at that rate (resuming + * if stopped). */ +#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_CIRCPAD_NEGOTIATE) +struct circpad_negotiate_st { + uint8_t version; + uint8_t command; + /** Machine type is left unbounded because we can specify + * new machines in the consensus */ + uint8_t machine_type; + /** If true, send a relay_drop reply.. */ + uint8_t echo_request; + uint8_t trunnel_error_code_; +}; +#endif +typedef struct circpad_negotiate_st circpad_negotiate_t; +/** + * This command tells the relay to alter its min and max netflow + * timeout range values, and send padding at that rate (resuming + * if stopped). */ +#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_CIRCPAD_NEGOTIATED) +struct circpad_negotiated_st { + uint8_t version; + uint8_t command; + uint8_t response; + /** Machine type is left unbounded because we can specify + * new machines in the consensus */ + uint8_t machine_type; + uint8_t trunnel_error_code_; +}; +#endif +typedef struct circpad_negotiated_st circpad_negotiated_t; +/** Return a newly allocated circpad_negotiate with all elements set + * to zero. + */ +circpad_negotiate_t *circpad_negotiate_new(void); +/** Release all storage held by the circpad_negotiate in 'victim'. (Do + * nothing if 'victim' is NULL.) + */ +void circpad_negotiate_free(circpad_negotiate_t *victim); +/** Try to parse a circpad_negotiate from the buffer in 'input', using + * up to 'len_in' bytes from the input buffer. On success, return the + * number of bytes consumed and set *output to the newly allocated + * circpad_negotiate_t. On failure, return -2 if the input appears + * truncated, and -1 if the input is otherwise invalid. + */ +ssize_t circpad_negotiate_parse(circpad_negotiate_t **output, const uint8_t *input, const size_t len_in); +/** Return the number of bytes we expect to need to encode the + * circpad_negotiate in 'obj'. On failure, return a negative value. + * Note that this value may be an overestimate, and can even be an + * underestimate for certain unencodeable objects. + */ +ssize_t circpad_negotiate_encoded_len(const circpad_negotiate_t *obj); +/** Try to encode the circpad_negotiate from 'input' into the buffer + * at 'output', using up to 'avail' bytes of the output buffer. On + * success, return the number of bytes used. On failure, return -2 if + * the buffer was not long enough, and -1 if the input was invalid. + */ +ssize_t circpad_negotiate_encode(uint8_t *output, size_t avail, const circpad_negotiate_t *input); +/** Check whether the internal state of the circpad_negotiate in 'obj' + * is consistent. Return NULL if it is, and a short message if it is + * not. + */ +const char *circpad_negotiate_check(const circpad_negotiate_t *obj); +/** Clear any errors that were set on the object 'obj' by its setter + * functions. Return true iff errors were cleared. + */ +int circpad_negotiate_clear_errors(circpad_negotiate_t *obj); +/** Return the value of the version field of the circpad_negotiate_t + * in 'inp' + */ +uint8_t circpad_negotiate_get_version(const circpad_negotiate_t *inp); +/** Set the value of the version field of the circpad_negotiate_t in + * 'inp' to 'val'. Return 0 on success; return -1 and set the error + * code on 'inp' on failure. + */ +int circpad_negotiate_set_version(circpad_negotiate_t *inp, uint8_t val); +/** Return the value of the command field of the circpad_negotiate_t + * in 'inp' + */ +uint8_t circpad_negotiate_get_command(const circpad_negotiate_t *inp); +/** Set the value of the command field of the circpad_negotiate_t in + * 'inp' to 'val'. Return 0 on success; return -1 and set the error + * code on 'inp' on failure. + */ +int circpad_negotiate_set_command(circpad_negotiate_t *inp, uint8_t val); +/** Return the value of the machine_type field of the + * circpad_negotiate_t in 'inp' + */ +uint8_t circpad_negotiate_get_machine_type(const circpad_negotiate_t *inp); +/** Set the value of the machine_type field of the circpad_negotiate_t + * in 'inp' to 'val'. Return 0 on success; return -1 and set the error + * code on 'inp' on failure. + */ +int circpad_negotiate_set_machine_type(circpad_negotiate_t *inp, uint8_t val); +/** Return the value of the echo_request field of the + * circpad_negotiate_t in 'inp' + */ +uint8_t circpad_negotiate_get_echo_request(const circpad_negotiate_t *inp); +/** Set the value of the echo_request field of the circpad_negotiate_t + * in 'inp' to 'val'. Return 0 on success; return -1 and set the error + * code on 'inp' on failure. + */ +int circpad_negotiate_set_echo_request(circpad_negotiate_t *inp, uint8_t val); +/** Return a newly allocated circpad_negotiated with all elements set + * to zero. + */ +circpad_negotiated_t *circpad_negotiated_new(void); +/** Release all storage held by the circpad_negotiated in 'victim'. + * (Do nothing if 'victim' is NULL.) + */ +void circpad_negotiated_free(circpad_negotiated_t *victim); +/** Try to parse a circpad_negotiated from the buffer in 'input', + * using up to 'len_in' bytes from the input buffer. On success, + * return the number of bytes consumed and set *output to the newly + * allocated circpad_negotiated_t. On failure, return -2 if the input + * appears truncated, and -1 if the input is otherwise invalid. + */ +ssize_t circpad_negotiated_parse(circpad_negotiated_t **output, const uint8_t *input, const size_t len_in); +/** Return the number of bytes we expect to need to encode the + * circpad_negotiated in 'obj'. On failure, return a negative value. + * Note that this value may be an overestimate, and can even be an + * underestimate for certain unencodeable objects. + */ +ssize_t circpad_negotiated_encoded_len(const circpad_negotiated_t *obj); +/** Try to encode the circpad_negotiated from 'input' into the buffer + * at 'output', using up to 'avail' bytes of the output buffer. On + * success, return the number of bytes used. On failure, return -2 if + * the buffer was not long enough, and -1 if the input was invalid. + */ +ssize_t circpad_negotiated_encode(uint8_t *output, size_t avail, const circpad_negotiated_t *input); +/** Check whether the internal state of the circpad_negotiated in + * 'obj' is consistent. Return NULL if it is, and a short message if + * it is not. + */ +const char *circpad_negotiated_check(const circpad_negotiated_t *obj); +/** Clear any errors that were set on the object 'obj' by its setter + * functions. Return true iff errors were cleared. + */ +int circpad_negotiated_clear_errors(circpad_negotiated_t *obj); +/** Return the value of the version field of the circpad_negotiated_t + * in 'inp' + */ +uint8_t circpad_negotiated_get_version(const circpad_negotiated_t *inp); +/** Set the value of the version field of the circpad_negotiated_t in + * 'inp' to 'val'. Return 0 on success; return -1 and set the error + * code on 'inp' on failure. + */ +int circpad_negotiated_set_version(circpad_negotiated_t *inp, uint8_t val); +/** Return the value of the command field of the circpad_negotiated_t + * in 'inp' + */ +uint8_t circpad_negotiated_get_command(const circpad_negotiated_t *inp); +/** Set the value of the command field of the circpad_negotiated_t in + * 'inp' to 'val'. Return 0 on success; return -1 and set the error + * code on 'inp' on failure. + */ +int circpad_negotiated_set_command(circpad_negotiated_t *inp, uint8_t val); +/** Return the value of the response field of the circpad_negotiated_t + * in 'inp' + */ +uint8_t circpad_negotiated_get_response(const circpad_negotiated_t *inp); +/** Set the value of the response field of the circpad_negotiated_t in + * 'inp' to 'val'. Return 0 on success; return -1 and set the error + * code on 'inp' on failure. + */ +int circpad_negotiated_set_response(circpad_negotiated_t *inp, uint8_t val); +/** Return the value of the machine_type field of the + * circpad_negotiated_t in 'inp' + */ +uint8_t circpad_negotiated_get_machine_type(const circpad_negotiated_t *inp); +/** Set the value of the machine_type field of the + * circpad_negotiated_t in 'inp' to 'val'. Return 0 on success; return + * -1 and set the error code on 'inp' on failure. + */ +int circpad_negotiated_set_machine_type(circpad_negotiated_t *inp, uint8_t val); + + +#endif diff --git a/src/trunnel/circpad_negotiation.trunnel b/src/trunnel/circpad_negotiation.trunnel new file mode 100644 index 0000000000..abbc929cc5 --- /dev/null +++ b/src/trunnel/circpad_negotiation.trunnel @@ -0,0 +1,44 @@ +/* These are the padding negotiation commands */ +const CIRCPAD_COMMAND_STOP = 1; +const CIRCPAD_COMMAND_START = 2; + +/* Responses to commands */ +const CIRCPAD_RESPONSE_OK = 1; +const CIRCPAD_RESPONSE_ERR = 2; + +/* Built-in machine types */ + +/* 1) Machine that obscures circuit setup */ +const CIRCPAD_MACHINE_CIRC_SETUP = 1; + +/** + * This command tells the relay to alter its min and max netflow + * timeout range values, and send padding at that rate (resuming + * if stopped). */ +struct circpad_negotiate { + u8 version IN [0]; + u8 command IN [CIRCPAD_COMMAND_START, CIRCPAD_COMMAND_STOP]; + + /** Machine type is left unbounded because we can specify + * new machines in the consensus */ + u8 machine_type; + + /** If true, send a relay_drop reply.. */ + // FIXME-MP-AP: Maybe we just say to transition to the first state + // here instead.. Also what about delay before responding? + u8 echo_request IN [0,1]; +}; + +/** + * This command tells the relay to alter its min and max netflow + * timeout range values, and send padding at that rate (resuming + * if stopped). */ +struct circpad_negotiated { + u8 version IN [0]; + u8 command IN [CIRCPAD_COMMAND_START, CIRCPAD_COMMAND_STOP]; + u8 response IN [CIRCPAD_RESPONSE_OK, CIRCPAD_RESPONSE_ERR]; + + /** Machine type is left unbounded because we can specify + * new machines in the consensus */ + u8 machine_type; +}; diff --git a/src/trunnel/include.am b/src/trunnel/include.am index b5db0609a8..4f4f1d3624 100644 --- a/src/trunnel/include.am +++ b/src/trunnel/include.am @@ -11,7 +11,8 @@ TRUNNELINPUTS = \ src/trunnel/link_handshake.trunnel \ src/trunnel/pwbox.trunnel \ src/trunnel/channelpadding_negotiation.trunnel \ - src/trunner/socks5.trunnel + src/trunnel/socks5.trunnel \ + src/trunnel/circpad_negotiation.trunnel TRUNNELSOURCES = \ src/ext/trunnel/trunnel.c \ @@ -23,8 +24,9 @@ TRUNNELSOURCES = \ src/trunnel/hs/cell_introduce1.c \ src/trunnel/hs/cell_rendezvous.c \ src/trunnel/channelpadding_negotiation.c \ - src/trunnel/socks5.c \ - src/trunnel/netinfo.c + src/trunnel/socks5.c \ + src/trunnel/netinfo.c \ + src/trunnel/circpad_negotiation.c TRUNNELHEADERS = \ src/ext/trunnel/trunnel.h \ @@ -39,7 +41,8 @@ TRUNNELHEADERS = \ src/trunnel/hs/cell_rendezvous.h \ src/trunnel/channelpadding_negotiation.h \ src/trunnel/socks5.h \ - src/trunnel/netinfo.h + src/trunnel/netinfo.h \ + src/trunnel/circpad_negotiation.h src_trunnel_libor_trunnel_a_SOURCES = $(TRUNNELSOURCES) src_trunnel_libor_trunnel_a_CPPFLAGS = \ From 43701e1ebe169a84fe98480bd0aaa09a9eebede6 Mon Sep 17 00:00:00 2001 From: Mike Perry Date: Tue, 23 Oct 2018 21:00:51 +0000 Subject: [PATCH 0361/2557] Circuit padding machine creation events. These event callbacks allow circuit padding to decide when to attempt to launch and negotiate new padding machines, and when to tear old ones down. Co-authored-by: George Kadianakis --- src/core/or/circuitbuild.c | 4 ++++ src/core/or/circuituse.c | 14 ++++++++++++++ src/core/or/connection_edge.c | 5 +++++ src/core/or/relay.c | 4 +++- 4 files changed, 26 insertions(+), 1 deletion(-) diff --git a/src/core/or/circuitbuild.c b/src/core/or/circuitbuild.c index b89ec09a99..2d8bc4d4ad 100644 --- a/src/core/or/circuitbuild.c +++ b/src/core/or/circuitbuild.c @@ -43,6 +43,7 @@ #include "core/or/circuitlist.h" #include "core/or/circuitstats.h" #include "core/or/circuituse.h" +#include "core/or/circuitpadding.h" #include "core/or/command.h" #include "core/or/connection_edge.h" #include "core/or/connection_or.h" @@ -950,12 +951,15 @@ circuit_send_next_onion_skin(origin_circuit_t *circ) crypt_path_t *hop = onion_next_hop_in_cpath(circ->cpath); circuit_build_times_handle_completed_hop(circ); + circpad_machine_event_circ_added_hop(circ); + if (hop) { /* Case two: we're on a hop after the first. */ return circuit_send_intermediate_onion_skin(circ, hop); } /* Case three: the circuit is finished. Do housekeeping tasks on it. */ + circpad_machine_event_circ_built(circ); return circuit_build_no_more_hops(circ); } diff --git a/src/core/or/circuituse.c b/src/core/or/circuituse.c index e230ad1005..f8298795e2 100644 --- a/src/core/or/circuituse.c +++ b/src/core/or/circuituse.c @@ -35,6 +35,7 @@ #include "core/or/circuitlist.h" #include "core/or/circuitstats.h" #include "core/or/circuituse.h" +#include "core/or/circuitpadding.h" #include "core/or/connection_edge.h" #include "core/or/policies.h" #include "feature/client/addressmap.h" @@ -1419,6 +1420,11 @@ circuit_detach_stream(circuit_t *circ, edge_connection_t *conn) if (circ->purpose == CIRCUIT_PURPOSE_S_REND_JOINED) { hs_dec_rdv_stream_counter(origin_circ); } + + /* If there are no more streams on this circ, tell circpad */ + if (!origin_circ->p_streams) + circpad_machine_event_circ_has_no_streams(origin_circ); + return; } } else { @@ -2587,6 +2593,12 @@ link_apconn_to_circ(entry_connection_t *apconn, origin_circuit_t *circ, /* add it into the linked list of streams on this circuit */ log_debug(LD_APP|LD_CIRC, "attaching new conn to circ. n_circ_id %u.", (unsigned)circ->base_.n_circ_id); + + /* If this is the first stream on this circuit, tell circpad + * that streams are attached */ + if (!circ->p_streams) + circpad_machine_event_circ_has_streams(circ); + /* reset it, so we can measure circ timeouts */ ENTRY_TO_CONN(apconn)->timestamp_last_read_allowed = time(NULL); ENTRY_TO_EDGE_CONN(apconn)->next_stream = circ->p_streams; @@ -3065,6 +3077,8 @@ circuit_change_purpose(circuit_t *circ, uint8_t new_purpose) if (CIRCUIT_IS_ORIGIN(circ)) { control_event_circuit_purpose_changed(TO_ORIGIN_CIRCUIT(circ), old_purpose); + + circpad_machine_event_circ_purpose_changed(TO_ORIGIN_CIRCUIT(circ)); } } diff --git a/src/core/or/connection_edge.c b/src/core/or/connection_edge.c index 9f76929e53..88be6040fd 100644 --- a/src/core/or/connection_edge.c +++ b/src/core/or/connection_edge.c @@ -67,6 +67,7 @@ #include "core/or/circuitbuild.h" #include "core/or/circuitlist.h" #include "core/or/circuituse.h" +#include "core/or/circuitpadding.h" #include "core/or/connection_edge.h" #include "core/or/connection_or.h" #include "core/or/policies.h" @@ -3694,6 +3695,10 @@ handle_hs_exit_conn(circuit_t *circ, edge_connection_t *conn) /* Link the circuit and the connection crypt path. */ conn->cpath_layer = origin_circ->cpath->prev; + /* If this is the first stream on this circuit, tell circpad */ + if (!origin_circ->p_streams) + circpad_machine_event_circ_has_streams(origin_circ); + /* Add it into the linked list of p_streams on this circuit */ conn->next_stream = origin_circ->p_streams; origin_circ->p_streams = conn; diff --git a/src/core/or/relay.c b/src/core/or/relay.c index 2e92f2a55d..679fc4433d 100644 --- a/src/core/or/relay.c +++ b/src/core/or/relay.c @@ -602,7 +602,9 @@ relay_send_command_from_edge_,(streamid_t stream_id, circuit_t *circ, * one of them. Don't worry about the conn protocol version: * append_cell_to_circuit_queue will fix it up. */ cell.command = CELL_RELAY_EARLY; - --origin_circ->remaining_relay_early_cells; + /* If we're out of relay early cells, tell circpad */ + if (--origin_circ->remaining_relay_early_cells == 0) + circpad_machine_event_circ_has_no_relay_early(origin_circ); log_debug(LD_OR, "Sending a RELAY_EARLY cell; %d remaining.", (int)origin_circ->remaining_relay_early_cells); /* Memorize the command that is sent as RELAY_EARLY cell; helps debug From 7be71903daff042e606e7a8445a6359100c9f8f5 Mon Sep 17 00:00:00 2001 From: Mike Perry Date: Tue, 23 Oct 2018 20:03:35 +0000 Subject: [PATCH 0362/2557] Circuit padding cell event callbacks. These callbacks allow the padding state machines to react to various types of sent and received relay cells. Co-authored-by: George Kadianakis --- src/core/or/or.h | 3 +++ src/core/or/relay.c | 25 ++++++++++++++++++------- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/src/core/or/or.h b/src/core/or/or.h index f297bc9267..bf5e3957ad 100644 --- a/src/core/or/or.h +++ b/src/core/or/or.h @@ -207,6 +207,9 @@ struct curve25519_public_key_t; #define RELAY_COMMAND_RENDEZVOUS_ESTABLISHED 39 #define RELAY_COMMAND_INTRODUCE_ACK 40 +#define RELAY_COMMAND_PADDING_NEGOTIATE 41 +#define RELAY_COMMAND_PADDING_NEGOTIATED 42 + /* Reasons why an OR connection is closed. */ #define END_OR_CONN_REASON_DONE 1 #define END_OR_CONN_REASON_REFUSED 2 /* connection refused */ diff --git a/src/core/or/relay.c b/src/core/or/relay.c index 679fc4433d..452777b2fc 100644 --- a/src/core/or/relay.c +++ b/src/core/or/relay.c @@ -55,6 +55,7 @@ #include "core/or/circuitbuild.h" #include "core/or/circuitlist.h" #include "core/or/circuituse.h" +#include "core/or/circuitpadding.h" #include "lib/compress/compress.h" #include "app/config/config.h" #include "core/mainloop/connection.h" @@ -80,7 +81,6 @@ #include "feature/nodelist/describe.h" #include "feature/nodelist/routerlist.h" #include "core/or/scheduler.h" -#include "feature/stats/rephist.h" #include "core/or/cell_st.h" #include "core/or/cell_queue_st.h" @@ -293,7 +293,9 @@ circuit_receive_relay_cell(cell_t *cell, circuit_t *circ, return 0; } - /* not recognized. pass it on. */ + /* not recognized. inform circpad and pass it on. */ + circpad_deliver_unrecognized_cell_events(circ, cell_direction); + if (cell_direction == CELL_DIRECTION_OUT) { cell->circ_id = circ->n_circ_id; /* switch it */ chan = circ->n_chan; @@ -524,6 +526,7 @@ relay_command_to_string(uint8_t command) case RELAY_COMMAND_INTRODUCE_ACK: return "INTRODUCE_ACK"; case RELAY_COMMAND_EXTEND2: return "EXTEND2"; case RELAY_COMMAND_EXTENDED2: return "EXTENDED2"; + case RELAY_COMMAND_PADDING_NEGOTIATE: return "PADDING_NEGOTIATE"; default: tor_snprintf(buf, sizeof(buf), "Unrecognized relay command %u", (unsigned)command); @@ -577,8 +580,8 @@ relay_send_command_from_edge_,(streamid_t stream_id, circuit_t *circ, log_debug(LD_OR,"delivering %d cell %s.", relay_command, cell_direction == CELL_DIRECTION_OUT ? "forward" : "backward"); - if (relay_command == RELAY_COMMAND_DROP) - rep_hist_padding_count_write(PADDING_TYPE_DROP); + /* Tell circpad we're sending a relay cell */ + circpad_deliver_sent_relay_cell_events(circ, relay_command); /* If we are sending an END cell and this circuit is used for a tunneled * directory request, advance its state. */ @@ -1483,9 +1486,11 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, } } + /* Tell circpad that we've recieved a recognized cell */ + circpad_deliver_recognized_relay_cell_events(circ, rh.command, layer_hint); + /* either conn is NULL, in which case we've got a control cell, or else * conn points to the recognized stream. */ - if (conn && !connection_state_is_open(TO_CONN(conn))) { if (conn->base_.type == CONN_TYPE_EXIT && (conn->base_.state == EXIT_CONN_STATE_CONNECTING || @@ -1506,8 +1511,14 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, switch (rh.command) { case RELAY_COMMAND_DROP: - rep_hist_padding_count_read(PADDING_TYPE_DROP); -// log_info(domain,"Got a relay-level padding cell. Dropping."); + /* Already examined in circpad_deliver_recognized_relay_cell_events */ + return 0; + case RELAY_COMMAND_PADDING_NEGOTIATE: + circpad_handle_padding_negotiate(circ, cell); + return 0; + case RELAY_COMMAND_PADDING_NEGOTIATED: + if (circpad_handle_padding_negotiated(circ, cell, layer_hint) == 0) + circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), rh.length); return 0; case RELAY_COMMAND_BEGIN: case RELAY_COMMAND_BEGIN_DIR: From 9aaf72ea58bd52d0af694411d16194ea15d612d1 Mon Sep 17 00:00:00 2001 From: Mike Perry Date: Tue, 23 Oct 2018 19:49:26 +0000 Subject: [PATCH 0363/2557] Circuit padding implementation. This implements all of the event handling, state machines, and padding decisions for circuit padding. I recommend reviewing this after you look at the call-in points into it from the rest of Tor. Co-authored-by: George Kadianakis --- src/core/include.am | 2 + src/core/or/circuitpadding.c | 2467 ++++++++++++++++++++++++++++++++++ 2 files changed, 2469 insertions(+) create mode 100644 src/core/or/circuitpadding.c diff --git a/src/core/include.am b/src/core/include.am index 5e69cb9ada..ae47c75e09 100644 --- a/src/core/include.am +++ b/src/core/include.am @@ -32,6 +32,7 @@ LIBTOR_APP_A_SOURCES = \ src/core/or/circuitlist.c \ src/core/or/circuitmux.c \ src/core/or/circuitmux_ewma.c \ + src/core/or/circuitpadding.c \ src/core/or/circuitstats.c \ src/core/or/circuituse.c \ src/core/or/command.c \ @@ -227,6 +228,7 @@ noinst_HEADERS += \ src/core/or/circuitmux.h \ src/core/or/circuitmux_ewma.h \ src/core/or/circuitstats.h \ + src/core/or/circuitpadding.h \ src/core/or/circuituse.h \ src/core/or/command.h \ src/core/or/connection_edge.h \ diff --git a/src/core/or/circuitpadding.c b/src/core/or/circuitpadding.c new file mode 100644 index 0000000000..9d65e2cf22 --- /dev/null +++ b/src/core/or/circuitpadding.c @@ -0,0 +1,2467 @@ +/* Copyright (c) 2017 The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include +#include "lib/math/fp.h" +#include "core/or/or.h" +#include "core/or/circuitpadding.h" +#include "core/or/circuitlist.h" +#include "core/or/circuituse.h" +#include "core/or/relay.h" +#include "feature/stats/rephist.h" +#include "feature/nodelist/networkstatus.h" + +#include "core/or/channel.h" + +#include "lib/time/compat_time.h" +#include "lib/crypt_ops/crypto_rand.h" + +#include "core/or/crypt_path_st.h" +#include "core/or/circuit_st.h" +#include "core/or/origin_circuit_st.h" +#include "feature/nodelist/routerstatus_st.h" +#include "feature/nodelist/node_st.h" +#include "core/or/cell_st.h" +#include "core/or/extend_info_st.h" +#include "core/crypto/relay_crypto.h" +#include "feature/nodelist/nodelist.h" + +#include "app/config/config.h" + +/* XXX: This is a dup of the constant in ./src/lib/time/tvdiff.c. + * Should/Do we have a header for time constants like this? */ +#define TOR_USEC_PER_SEC (1000000) + +static inline circpad_purpose_mask_t circpad_circ_purpose_to_mask(uint8_t + circ_purpose); +static inline circpad_circuit_state_t circpad_circuit_state( + origin_circuit_t *circ); +static void circpad_setup_machine_on_circ(circuit_t *on_circ, + const circpad_machine_t *machine); +static double circpad_distribution_sample(circpad_distribution_t dist); + +/** Cached consensus params */ +static uint8_t circpad_global_max_padding_percent; +static uint16_t circpad_global_allowed_cells; + +/** Global cell counts, for rate limiting */ +static uint64_t circpad_global_padding_sent; +static uint64_t circpad_global_nonpadding_sent; + +/** This is the list of circpad_machine_t's parsed from consensus and torrc + * that have origin_side == 1 (ie: are for client side) */ +STATIC smartlist_t *origin_padding_machines = NULL; + +/** This is the list of circpad_machine_t's parsed from consensus and torrc + * that have origin_side == 0 (ie: are for relay side) */ +STATIC smartlist_t *relay_padding_machines = NULL; + +/** Loop over the current padding state machines using loop_var as the + * loop variable. */ +#define FOR_EACH_CIRCUIT_MACHINE_BEGIN(loop_var) \ + STMT_BEGIN \ + for (int loop_var = 0; loop_var < CIRCPAD_MAX_MACHINES; loop_var++) { +#define FOR_EACH_CIRCUIT_MACHINE_END } STMT_END ; + +/** Loop over the current active padding state machines using loop_var + * as the loop variable. If a machine is not active, skip it. */ +#define FOR_EACH_ACTIVE_CIRCUIT_MACHINE_BEGIN(loop_var, circ) \ + FOR_EACH_CIRCUIT_MACHINE_BEGIN(loop_var) \ + if (!(circ)->padding_info[loop_var]) \ + continue; +#define FOR_EACH_ACTIVE_CIRCUIT_MACHINE_END } STMT_END ; + +/** + * Return a human-readable description for a circuit padding state. + */ +static const char * +circpad_state_to_string(circpad_statenum_t state) +{ + const char *descr; + + switch (state) { + case CIRCPAD_STATE_START: + descr = "START"; + break; + case CIRCPAD_STATE_BURST: + descr = "BURST"; + break; + case CIRCPAD_STATE_GAP: + descr = "GAP"; + break; + case CIRCPAD_STATE_END: + descr = "END"; + break; + default: + descr = "CUSTOM"; // XXX: Just return # in static char buf? + } + + return descr; +} + +/** + * Free the machineinfo at an index + */ +static void +circpad_circuit_machineinfo_free_idx(circuit_t *circ, int idx) +{ + if (circ->padding_info[idx]) { + tor_free(circ->padding_info[idx]->histogram); + timer_free(circ->padding_info[idx]->padding_timer); + tor_free(circ->padding_info[idx]); + } +} + +/** Free all the machineinfos in circ that match machine_num. */ +static void +free_circ_machineinfos_with_machine_num(circuit_t *circ, int machine_num) +{ + FOR_EACH_CIRCUIT_MACHINE_BEGIN(i) { + if (circ->padding_machine[i] && + circ->padding_machine[i]->machine_num == machine_num) { + circpad_circuit_machineinfo_free_idx(circ, i); + circ->padding_machine[i] = NULL; + } + } FOR_EACH_CIRCUIT_MACHINE_END; +} + +/** + * Free all padding machines and mutable info associated with circuit + */ +void +circpad_circuit_free_all_machineinfos(circuit_t *circ) +{ + FOR_EACH_CIRCUIT_MACHINE_BEGIN(i) { + circpad_circuit_machineinfo_free_idx(circ, i); + } FOR_EACH_CIRCUIT_MACHINE_END; +} + +/** + * Allocate a new mutable machineinfo structure. + */ +STATIC circpad_machineinfo_t * +circpad_circuit_machineinfo_new(circuit_t *on_circ, int machine_index) +{ + circpad_machineinfo_t *mi = tor_malloc_zero(sizeof(circpad_machineinfo_t)); + mi->machine_index = machine_index; + mi->on_circ = on_circ; + + return mi; +} + +/** + * Return the circpad_state_t for the current state based on the + * mutable info. + * + * This function returns NULL when the machine is in the end state or in an + * invalid state. + */ +STATIC const circpad_state_t * +circpad_machine_current_state(circpad_machineinfo_t *mi) +{ + const circpad_machine_t *machine = CIRCPAD_GET_MACHINE(mi); + + if (mi->current_state == CIRCPAD_STATE_END) { + return NULL; + } else if (BUG(mi->current_state >= machine->num_states)) { + log_fn(LOG_WARN,LD_CIRC, + "Invalid circuit padding state %d", + mi->current_state); + + return NULL; + } + + return &machine->states[mi->current_state]; +} + +/** + * Calculate the lower bound of a histogram bin. The upper bound + * is obtained by calling this function with bin+1, and subtracting 1. + * + * The 0th bin has a special value -- it only represents start_usec. + * This is so we can specify a probability on 0-delay values. + * + * After bin 0, bins are exponentially spaced, so that each subsequent + * bin is twice as large as the previous. This is done so that higher + * time resolution is given to lower time values. + * + * The infinity bin is a the last bin in the array (histogram_len-1). + * It has a usec value of CIRCPAD_DELAY_INFINITE (UINT32_MAX). + */ +STATIC circpad_delay_t +circpad_histogram_bin_to_usec(circpad_machineinfo_t *mi, + circpad_hist_index_t bin) +{ + const circpad_state_t *state = circpad_machine_current_state(mi); + circpad_delay_t start_usec; + + /* Our state should have been checked to be non-null by the caller + * (circpad_machine_remove_token()) */ + if (BUG(state == NULL)) { + return CIRCPAD_DELAY_INFINITE; + } + + if (state->use_rtt_estimate) + start_usec = mi->rtt_estimate_usec+state->start_usec; + else + start_usec = state->start_usec; + + if (bin >= CIRCPAD_INFINITY_BIN(state)) + return CIRCPAD_DELAY_INFINITE; + + if (bin == 0) + return start_usec; + + if (bin == 1) + return start_usec+1; + + /* The bin widths double every index, so that we can have more resolution + * for lower time values in the histogram. */ + const circpad_time_t bin_width_exponent = + 1 << (CIRCPAD_INFINITY_BIN(state) - bin); + return (circpad_delay_t)MIN(start_usec + + state->range_usec/bin_width_exponent, + CIRCPAD_DELAY_INFINITE); +} + +/** + * Return the bin that contains the usec argument. + * "Contains" is defined as us in [lower, upper). + * + * This function will never return the infinity bin (histogram_len-1), + * in order to simplify the rest of the code. + * + * This means that technically the last bin (histogram_len-2) + * has range [start_usec+range_usec, CIRCPAD_DELAY_INFINITE]. + */ +STATIC circpad_hist_index_t +circpad_histogram_usec_to_bin(const circpad_machineinfo_t *mi, + circpad_delay_t usec) +{ + const circpad_state_t *state = circpad_machine_current_state(mi); + circpad_delay_t start_usec; + int32_t bin; /* Larger than return type to properly clamp overflow */ + + /* Our state should have been checked to be non-null by the caller + * (circpad_machine_remove_token()) */ + if (BUG(state == NULL)) { + return 0; + } + + if (state->use_rtt_estimate) + start_usec = mi->rtt_estimate_usec+state->start_usec; + else + start_usec = state->start_usec; + + if (usec <= start_usec) + return 0; + + if (usec == start_usec+1) + return 1; + + const circpad_time_t histogram_range_usec = state->range_usec; + /* We need to find the bin corresponding to our position in the range. + * Since bins are exponentially spaced in powers of two, we need to + * take the log2 of our position in histogram_range_usec. However, + * since tor_log2() returns the floor(log2(u64)), we have to adjust + * it to behave like ceil(log2(u64)). This is verified in our tests + * to properly invert the operation done in + * circpad_histogram_bin_to_usec(). */ + bin = CIRCPAD_INFINITY_BIN(state) - + tor_log2(2*histogram_range_usec/(usec-start_usec+1)); + + /* Clamp the return value to account for timevals before the start + * of bin 0, or after the last bin. Don't return the infinity bin + * index. */ + bin = MIN(MAX(bin, 1), CIRCPAD_INFINITY_BIN(state)-1); + return bin; +} + +/** + * This function frees any token bins allocated from a previous state + * + * Called after a state transition, or if the bins are empty. + */ +STATIC void +circpad_machine_setup_tokens(circpad_machineinfo_t *mi) +{ + const circpad_state_t *state = circpad_machine_current_state(mi); + + /* If this state doesn't exist, or doesn't have token removal, + * free any previous state's histogram, and bail */ + if (!state || state->token_removal == CIRCPAD_TOKEN_REMOVAL_NONE) { + if (mi->histogram) { + tor_free(mi->histogram); + mi->histogram = NULL; + mi->histogram_len = 0; + } + return; + } + + /* Try to avoid re-mallocing if we don't really need to */ + if (!mi->histogram || (mi->histogram + && mi->histogram_len != state->histogram_len)) { + tor_free(mi->histogram); // null ok + mi->histogram = tor_malloc_zero(sizeof(circpad_hist_token_t) + *state->histogram_len); + } + mi->histogram_len = state->histogram_len; + + memcpy(mi->histogram, state->histogram, + sizeof(circpad_hist_token_t)*state->histogram_len); +} + +/** + * Choose a length for this state (in cells), if specified. + */ +static void +circpad_choose_state_length(circpad_machineinfo_t *mi) +{ + const circpad_state_t *state = circpad_machine_current_state(mi); + double length; + + if (!state || state->length_dist.type == CIRCPAD_DIST_NONE) { + mi->state_length = CIRCPAD_STATE_LENGTH_INFINITE; + return; + } + + length = circpad_distribution_sample(state->length_dist); + length = MAX(0, length); + length += state->start_length; + length = MIN(length, state->max_length); + + mi->state_length = clamp_double_to_int64(length); +} + +/** + * Sample a value from our iat_dist, and clamp it safely + * to circpad_delay_t. + */ +static circpad_delay_t +circpad_distribution_sample_iat_delay(const circpad_state_t *state, + circpad_delay_t start_usec) +{ + double val = circpad_distribution_sample(state->iat_dist); + /* These comparisons are safe, because the output is in the range + * [0, 2**32), and double has a precision of 53 bits. */ + val = MAX(0, val); + val = MIN(val, state->range_usec); + + /* This addition is exact: val is at most 2**32-1, start_usec + * is at most 2**32-1, and doubles have a precision of 53 bits. */ + val += start_usec; + + /* Clamp the distribution at infinite delay val */ + return (circpad_delay_t)MIN(tor_llround(val), CIRCPAD_DELAY_INFINITE); +} + +/** + * Sample an expected time-until-next-packet delay from the histogram. + * + * The bin is chosen with probability proportional to the number + * of tokens in each bin, and then a time value is chosen uniformly from + * that bin's [start,end) time range. + */ +static circpad_delay_t +circpad_machine_sample_delay(circpad_machineinfo_t *mi) +{ + const circpad_state_t *state = circpad_machine_current_state(mi); + const circpad_hist_token_t *histogram = NULL; + circpad_hist_index_t curr_bin = 0; + circpad_delay_t bin_start, bin_end; + circpad_delay_t start_usec; + /* These three must all be larger than circpad_hist_token_t, because + * we sum several circpad_hist_token_t values across the histogram */ + uint64_t curr_weight = 0; + uint64_t histogram_total_tokens = 0; + uint64_t bin_choice; + + tor_assert(state); + + if (state->use_rtt_estimate) + start_usec = mi->rtt_estimate_usec+state->start_usec; + else + start_usec = state->start_usec; + + if (state->iat_dist.type != CIRCPAD_DIST_NONE) { + /* Sample from a fixed IAT distribution and return */ + return circpad_distribution_sample_iat_delay(state, start_usec); + } else if (state->token_removal != CIRCPAD_TOKEN_REMOVAL_NONE) { + /* We have a mutable histogram. Do basic sanity check and apply: */ + if (BUG(!mi->histogram) || + BUG(mi->histogram_len != state->histogram_len)) { + return CIRCPAD_DELAY_INFINITE; + } + + histogram = mi->histogram; + for (circpad_hist_index_t b = 0; b < state->histogram_len; b++) + histogram_total_tokens += histogram[b]; + } else { + /* We have a histogram, but it's immutable */ + histogram = state->histogram; + histogram_total_tokens = state->histogram_total_tokens; + } + + bin_choice = crypto_rand_uint64(histogram_total_tokens); + + /* Skip all the initial zero bins */ + while (!histogram[curr_bin]) { + curr_bin++; + } + curr_weight = histogram[curr_bin]; + + // TODO: This is not constant-time. Pretty sure we don't + // really need it to be, though. + while (curr_weight < bin_choice) { + curr_bin++; + /* It should be impossible to run past the end of the histogram */ + if (BUG(curr_bin >= state->histogram_len)) { + return CIRCPAD_DELAY_INFINITE; + } + curr_weight += histogram[curr_bin]; + } + + /* Do some basic checking of the current bin we are in */ + if (BUG(curr_bin >= state->histogram_len) || + BUG(histogram[curr_bin] == 0)) { + return CIRCPAD_DELAY_INFINITE; + } + + // Store this index to remove the token upon callback. + if (state->token_removal != CIRCPAD_TOKEN_REMOVAL_NONE) { + mi->chosen_bin = curr_bin; + } + + if (curr_bin >= CIRCPAD_INFINITY_BIN(state)) { + if (state->token_removal != CIRCPAD_TOKEN_REMOVAL_NONE && + mi->histogram[curr_bin] > 0) { + mi->histogram[curr_bin]--; + } + + // Infinity: Don't send a padding packet. Wait for a real packet + // and then see if our bins are empty or what else we should do. + return CIRCPAD_DELAY_INFINITE; + } + + tor_assert(curr_bin < CIRCPAD_INFINITY_BIN(state)); + + bin_start = circpad_histogram_bin_to_usec(mi, curr_bin); + bin_end = circpad_histogram_bin_to_usec(mi, curr_bin+1); + + /* Truncate the high bin in case it's the infinity bin: + * Don't actually schedule an "infinite"-1 delay */ + bin_end = MIN(bin_end, start_usec+state->range_usec); + + // Sample uniformly between histogram[i] to histogram[i+1]-1, + // but no need to sample if they are the same timeval (aka bin 0 or bin 1). + if (bin_end <= bin_start+1) + return bin_start; + else + return (circpad_delay_t)crypto_rand_uint64_range(bin_start, bin_end); +} + +/** + * Sample a value from the specified probability distribution. + * + * This performs inverse transform sampling + * (https://en.wikipedia.org/wiki/Inverse_transform_sampling). + * + * XXX: These formulas were taken verbatim. Need a floating wizard + * to check them for catastropic cancellation and other issues (teor?). + * Also: is 32bits of double from [0.0,1.0) enough? + */ +static double +circpad_distribution_sample(circpad_distribution_t dist) +{ + double p = 0; + + switch (dist.type) { + case CIRCPAD_DIST_NONE: + return 0; + case CIRCPAD_DIST_UNIFORM: + p = crypto_rand_double(); + // param2 is upper bound, param1 is lower + /* The subtraction is exact as long as param2 and param1 are less than + * 2**53. The multiplication is accurate as long as (param2 - param1) + * is less than 2**52. (And when they are large, the low bits aren't + * important.) The result covers the full range of outputs, as long as + * p has a resolution of 1/2**32 or greater. */ + p *= (dist.param2 - dist.param1); + p += dist.param1; + return p; + case CIRCPAD_DIST_LOGISTIC: + p = crypto_rand_double(); + /* https://en.wikipedia.org/wiki/Logistic_distribution#Quantile_function + * param1 is Mu, param2 is s. */ + if (p <= 0.0) // Avoid log(0) + return 0; + return dist.param1 + dist.param2*tor_mathlog(p/(1.0-p)); + case CIRCPAD_DIST_LOG_LOGISTIC: + p = crypto_rand_double(); + /* https://en.wikipedia.org/wiki/Log-logistic_distribution#Quantiles + * param1 is Alpha, param2 is Beta */ + return dist.param1 * pow(p/(1.0-p), 1.0/dist.param2); + case CIRCPAD_DIST_GEOMETRIC: + p = crypto_rand_double(); + /* https://github.com/distributions-io/geometric-quantile/ + * param1 is 'p' (success probability) */ + return ceil(tor_mathlog(1.0-p)/tor_mathlog(1.0-dist.param1)); + case CIRCPAD_DIST_WEIBULL: + p = crypto_rand_double(); + /* https://en.wikipedia.org/wiki/Weibull_distribution \ + * #Cumulative_distribution_function + * param1 is k, param2 is Lambda */ + return dist.param2*pow(-tor_mathlog(1.0-p), 1.0/dist.param1); + case CIRCPAD_DIST_PARETO: + p = 1.0-crypto_rand_double(); // Pareto quantile needs (0,1] + + /* https://en.wikipedia.org/wiki/Generalized_Pareto_distribution \ + * #Generating_generalized_Pareto_random_variables + * param1 is Sigma, param2 is Xi + * Since it's piecewise, we must define it for 0 (or close to 0) */ + if (fabs(dist.param2) <= 1e-22) + return -dist.param1*tor_mathlog(p); + else + return dist.param1*(pow(p, -dist.param2) - 1.0)/dist.param2; + } + return 0; +} + +/** + * Find the index of the first bin whose upper bound is + * greater than the target, and that has tokens remaining. + */ +static circpad_hist_index_t +circpad_machine_first_higher_index(circpad_machineinfo_t *mi, + circpad_delay_t target_bin_usec) +{ + circpad_hist_index_t bin = circpad_histogram_usec_to_bin(mi, + target_bin_usec); + + /* Don't remove from the infinity bin */ + for (; bin < CIRCPAD_INFINITY_BIN(mi); bin++) { + if (mi->histogram[bin] && + circpad_histogram_bin_to_usec(mi, bin+1) > target_bin_usec) { + return bin; + } + } + + return mi->histogram_len; +} + +/** + * Find the index of the first bin whose lower bound is lower or equal to + * target_bin_usec, and that still has tokens remaining. + */ +static circpad_hist_index_t +circpad_machine_first_lower_index(circpad_machineinfo_t *mi, + circpad_delay_t target_bin_usec) +{ + circpad_hist_index_t bin = circpad_histogram_usec_to_bin(mi, + target_bin_usec); + + for (; bin >= 0; bin--) { + if (mi->histogram[bin] && + circpad_histogram_bin_to_usec(mi, bin) <= target_bin_usec) { + return bin; + } + } + + return -1; +} + +/** + * Remove a token from the first non-empty bin whose upper bound is + * greater than the target. + */ +STATIC void +circpad_machine_remove_higher_token(circpad_machineinfo_t *mi, + circpad_delay_t target_bin_usec) +{ + /* We need to remove the token from the first bin + * whose upper bound is greater than the target, and that + * has tokens remaining. */ + circpad_hist_index_t bin = circpad_machine_first_higher_index(mi, + target_bin_usec); + + if (bin >= 0 && bin < CIRCPAD_INFINITY_BIN(mi)) { + if (!BUG(mi->histogram[bin] == 0)) { + mi->histogram[bin]--; + } + } +} + +/** + * Remove a token from the first non-empty bin whose upper bound is + * lower than the target. + */ +STATIC void +circpad_machine_remove_lower_token(circpad_machineinfo_t *mi, + circpad_delay_t target_bin_usec) +{ + circpad_hist_index_t bin = circpad_machine_first_lower_index(mi, + target_bin_usec); + + if (bin >= 0 && bin < CIRCPAD_INFINITY_BIN(mi)) { + if (!BUG(mi->histogram[bin] == 0)) { + mi->histogram[bin]--; + } + } +} + +/* Helper macro: Ensure that the bin has tokens available, and BUG out of the + * function if it's not the case. */ +#define ENSURE_BIN_CAPACITY(bin_index) \ + if (BUG(mi->histogram[bin_index] == 0)) { \ + return; \ + } + +/** + * Remove a token from the closest non-empty bin to the target. + * + * If use_usec is true, measure "closest" in terms of bin start usec. + * If it is false, use bin index distance only. + */ +STATIC void +circpad_machine_remove_closest_token(circpad_machineinfo_t *mi, + circpad_delay_t target_bin_usec, + bool use_usec) +{ + circpad_hist_index_t lower, higher, current; + circpad_hist_index_t bin_to_remove = -1; + + lower = circpad_machine_first_lower_index(mi, target_bin_usec); + higher = circpad_machine_first_higher_index(mi, target_bin_usec); + current = circpad_histogram_usec_to_bin(mi, target_bin_usec); + + /* Sanity check the results */ + if (BUG(lower > current) || BUG(higher < current)) { + return; + } + + if (higher == mi->histogram_len && lower == -1) { + // Bins are empty + return; + } else if (higher == mi->histogram_len) { + /* All higher bins are empty */ + ENSURE_BIN_CAPACITY(lower); + mi->histogram[lower]--; + return; + } else if (lower == -1) { + /* All lower bins are empty */ + ENSURE_BIN_CAPACITY(higher); + mi->histogram[higher]--; + return; + } + + if (use_usec) { + /* Find the closest bin midpoint to the target */ + circpad_delay_t lower_usec = circpad_get_histogram_bin_midpoint(mi, lower); + circpad_delay_t higher_usec = + circpad_get_histogram_bin_midpoint(mi, higher); + + if (target_bin_usec < lower_usec) { + // Lower bin is closer + ENSURE_BIN_CAPACITY(lower); + bin_to_remove = lower; + } else if (target_bin_usec > higher_usec) { + // Higher bin is closer + ENSURE_BIN_CAPACITY(higher); + bin_to_remove = higher; + } else if (target_bin_usec-lower_usec > higher_usec-target_bin_usec) { + // Higher bin is closer + ENSURE_BIN_CAPACITY(higher); + bin_to_remove = higher; + } else { + // Lower bin is closer + ENSURE_BIN_CAPACITY(lower); + bin_to_remove = lower; + } + } else { + if (current - lower > higher - current) { + // Higher bin is closer + ENSURE_BIN_CAPACITY(higher); + mi->histogram[higher]--; + return; + } else { + // Lower bin is closer + ENSURE_BIN_CAPACITY(lower); + mi->histogram[lower]--; + return; + } + } +} + +#undef ENSURE_BIN_CAPACITY + +/** + * Remove a token from the exact bin corresponding to the target. + * + * If it is empty, do nothing. + */ +static void +circpad_machine_remove_exact(circpad_machineinfo_t *mi, + circpad_delay_t target_bin_usec) +{ + circpad_hist_index_t bin = circpad_histogram_usec_to_bin(mi, + target_bin_usec); + + if (mi->histogram[bin] > 0) + mi->histogram[bin]--; +} + +/** + * Check our state's cell limit count and tokens. + * + * Returns 1 if either limits are hit and we decide to change states, + * otherwise returns 0. + */ +static circpad_decision_t +check_machine_token_supply(circpad_machineinfo_t *mi) +{ + uint32_t histogram_total_tokens = 0; + + /* Check if bins empty. This requires summing up the current mutable + * machineinfo histogram token total and checking if it is zero. + * Machineinfo does not keep a running token count. We're assuming the + * extra space is not worth this short loop iteration. + * + * We also do not count infinity bin in histogram totals. + */ + if (mi->histogram_len && mi->histogram) { + for (circpad_hist_index_t b = 0; b < CIRCPAD_INFINITY_BIN(mi); b++) + histogram_total_tokens += mi->histogram[b]; + + /* If we change state, we're done */ + if (histogram_total_tokens == 0) { + if (circpad_internal_event_bins_empty(mi) == CIRCPAD_STATE_CHANGED) + return CIRCPAD_STATE_CHANGED; + } + } + + if (mi->state_length == 0) { + return circpad_internal_event_state_length_up(mi); + } + + return CIRCPAD_STATE_UNCHANGED; +} + +/** + * Remove a token from the bin corresponding to the delta since + * last packet. If that bin is empty, choose a token based on + * the specified removal strategy in the state machine. + * + * This function also updates and checks rate limit and state + * limit counters. + * + * Returns 1 if we transition states, 0 otherwise. + */ +circpad_decision_t +circpad_machine_remove_token(circpad_machineinfo_t *mi) +{ + const circpad_state_t *state = NULL; + circpad_time_t current_time; + circpad_delay_t target_bin_usec; + + /* Update non-padding counts for rate limiting: We scale at UINT16_MAX + * because we only use this for a percentile limit of 2 sig figs, and + * space is scare in the machineinfo struct. */ + mi->nonpadding_sent++; + if (mi->nonpadding_sent == UINT16_MAX) { + mi->padding_sent /= 2; + mi->nonpadding_sent /= 2; + } + + /* Dont remove any tokens if there was no padding scheduled */ + if (!mi->padding_scheduled_at_usec) { + return CIRCPAD_STATE_UNCHANGED; + } + + state = circpad_machine_current_state(mi); + current_time = monotime_absolute_usec(); + + /* If we have scheduled padding some time in the future, we want to see what + bin we are in at the current time */ + target_bin_usec = (circpad_delay_t) + MIN((current_time - mi->padding_scheduled_at_usec), + CIRCPAD_DELAY_INFINITE-1); + + /* We are treating this non-padding cell as a padding cell, so we cancel + padding timer, if present. */ + mi->padding_scheduled_at_usec = 0; + if (mi->is_padding_timer_scheduled) { + mi->is_padding_timer_scheduled = 0; + timer_disable(mi->padding_timer); + } + + /* If we are not in a padding state (like start or end), we're done */ + if (!state) + return CIRCPAD_STATE_UNCHANGED; + + /* If we're enforcing a state length on non-padding packets, + * decrement it */ + if (mi->state_length != CIRCPAD_STATE_LENGTH_INFINITE && + state->length_includes_nonpadding && + mi->state_length > 0) { + mi->state_length--; + } + + /* Perform the specified token removal strategy */ + switch (state->token_removal) { + case CIRCPAD_TOKEN_REMOVAL_NONE: + break; + case CIRCPAD_TOKEN_REMOVAL_CLOSEST_USEC: + circpad_machine_remove_closest_token(mi, target_bin_usec, 1); + break; + case CIRCPAD_TOKEN_REMOVAL_CLOSEST: + circpad_machine_remove_closest_token(mi, target_bin_usec, 0); + break; + case CIRCPAD_TOKEN_REMOVAL_LOWER: + circpad_machine_remove_lower_token(mi, target_bin_usec); + break; + case CIRCPAD_TOKEN_REMOVAL_HIGHER: + circpad_machine_remove_higher_token(mi, target_bin_usec); + break; + case CIRCPAD_TOKEN_REMOVAL_EXACT: + circpad_machine_remove_exact(mi, target_bin_usec); + break; + } + + /* Check our token and state length limits */ + return check_machine_token_supply(mi); +} + +/** + * Send a relay command with a relay cell payload on a circuit to + * the particular hopnum. + * + * Hopnum starts at 1 (1=guard, 2=middle, 3=exit, etc). + * + * Payload may be null. + * + * Returns negative on error, 0 on success. + */ +static signed_error_t +circpad_send_command_to_hop(origin_circuit_t *circ, uint8_t hopnum, + uint8_t relay_command, const uint8_t *payload, + ssize_t payload_len) +{ + crypt_path_t *target_hop = circuit_get_cpath_hop(circ, hopnum); + signed_error_t ret; + + /* Check that the cpath has the target hop */ + if (!target_hop) { + log_fn(LOG_WARN, LD_BUG, "Padding circuit %u has %d hops, not %d", + circ->global_identifier, circuit_get_cpath_len(circ), hopnum); + return -1; + } + + /* Check that the target hop is opened */ + if (target_hop->state != CPATH_STATE_OPEN) { + log_fn(LOG_WARN,LD_CIRC, + "Padding circuit %u has %d hops, not %d", + circ->global_identifier, + circuit_get_cpath_opened_len(circ), hopnum); + return -1; + } + + /* Send the drop command to the second hop */ + ret = relay_send_command_from_edge(0, TO_CIRCUIT(circ), relay_command, + (const char*)payload, payload_len, + target_hop); + return ret; +} + +/** + * Callback helper to send a padding cell. + * + * This helper is called after our histogram-sampled delay period passes + * without another packet being sent first. If a packet is sent before this + * callback happens, it is canceled. So when we're called here, send padding + * right away. + * + * If sending this padding cell forced us to transition states return + * CIRCPAD_STATE_CHANGED. Otherwise return CIRCPAD_STATE_UNCHANGED. + */ +circpad_decision_t +circpad_send_padding_cell_for_callback(circpad_machineinfo_t *mi) +{ + circuit_t *circ = mi->on_circ; + int machine_idx = mi->machine_index; + mi->padding_scheduled_at_usec = 0; + circpad_statenum_t state = mi->current_state; + + // Make sure circuit didn't close on us + if (mi->on_circ->marked_for_close) { + log_fn(LOG_INFO,LD_CIRC, + "Padding callback on a circuit marked for close. Ignoring."); + return CIRCPAD_STATE_CHANGED; + } + + /* If it's a histogram, reduce the token count */ + if (mi->histogram && mi->histogram_len) { + /* Basic sanity check on the histogram before removing anything */ + if (BUG(mi->chosen_bin >= mi->histogram_len) || + BUG(mi->histogram[mi->chosen_bin] == 0)) { + return CIRCPAD_STATE_CHANGED; + } + + mi->histogram[mi->chosen_bin]--; + } + + /* If we have a valid state length bound, consider it */ + if (mi->state_length != CIRCPAD_STATE_LENGTH_INFINITE && + !BUG(mi->state_length <= 0)) { + mi->state_length--; + } + + /* + * Update non-padding counts for rate limiting: We scale at UINT16_MAX + * because we only use this for a percentile limit of 2 sig figs, and + * space is scare in the machineinfo struct. + */ + mi->padding_sent++; + if (mi->padding_sent == UINT16_MAX) { + mi->padding_sent /= 2; + mi->nonpadding_sent /= 2; + } + circpad_global_padding_sent++; + + if (CIRCUIT_IS_ORIGIN(mi->on_circ)) { + circpad_send_command_to_hop(TO_ORIGIN_CIRCUIT(mi->on_circ), + CIRCPAD_GET_MACHINE(mi)->target_hopnum, + RELAY_COMMAND_DROP, NULL, 0); + log_fn(LOG_INFO,LD_CIRC, "Callback: Sending padding to origin circuit %u.", + TO_ORIGIN_CIRCUIT(mi->on_circ)->global_identifier); + } else { + // If we're a non-origin circ, we can just send from here as if we're the + // edge. + log_fn(LOG_INFO,LD_CIRC, + "Callback: Sending padding to non-origin circuit."); + relay_send_command_from_edge(0, mi->on_circ, RELAY_COMMAND_DROP, NULL, + 0, NULL); + } + + rep_hist_padding_count_write(PADDING_TYPE_DROP); + /* This is a padding cell sent from the client or from the middle node, + * (because it's invoked from circuitpadding.c) */ + circpad_cell_event_padding_sent(circ); + + /* The circpad_cell_event_padding_sent() could cause us to transition. + * Check that we still have a padding machineinfo, and then check our token + * supply. */ + if (circ->padding_info[machine_idx] != NULL) { + if (state != circ->padding_info[machine_idx]->current_state) + return CIRCPAD_STATE_CHANGED; + else + return check_machine_token_supply(circ->padding_info[machine_idx]); + } else { + return CIRCPAD_STATE_CHANGED; + } +} + +/** + * Tor-timer compatible callback that tells us to send a padding cell. + * + * Timers are associated with circpad_machineinfo_t's. When the machineinfo + * is freed on a circuit, the timers are cancelled. Since the lifetime + * of machineinfo is always longer than the timers, handles are not + * needed. + */ +static void +circpad_send_padding_callback(tor_timer_t *timer, void *args, + const struct monotime_t *time) +{ + circpad_machineinfo_t *mi = ((circpad_machineinfo_t*)args); + (void)timer; (void)time; + + if (mi && mi->on_circ) { + assert_circuit_ok(mi->on_circ); + circpad_send_padding_cell_for_callback(mi); + } else { + // This shouldn't happen (represents a timer leak) + log_fn(LOG_WARN,LD_CIRC, + "Circuit closed while waiting for padding timer."); + tor_fragile_assert(); + } + + // TODO-MP-AP: Unify this counter with channelpadding for rephist stats + //total_timers_pending--; +} + +/** + * Cache our consensus parameters upon consensus update. + */ +void +circpad_new_consensus_params(const networkstatus_t *ns) +{ + circpad_global_allowed_cells = + networkstatus_get_param(ns, "circpad_global_allowed_cells", + 0, 0, UINT16_MAX-1); + + circpad_global_max_padding_percent = + networkstatus_get_param(ns, "circpad_global_max_padding_pct", + 0, 0, 100); +} + +/** + * Check this machine against its padding limits, as well as global + * consensus limits. + * + * We have two limits: a percent and a cell count. The cell count + * limit must be reached before the percent is enforced (this is to + * optionally allow very light padding of things like circuit setup + * while there is no other traffic on the circuit). + * + * TODO: Don't apply limits to machines form torrc. + * + * Returns 1 if limits are set and we've hit them. Otherwise returns 0. + */ +static bool +circpad_machine_reached_padding_limit(circpad_machineinfo_t *mi) +{ + const circpad_machine_t *machine = CIRCPAD_GET_MACHINE(mi); + + /* If machine_padding_pct is non-zero, and we've sent more + * than the allowed count of padding cells, then check our + * percent limits for this machine. */ + if (machine->max_padding_percent && + mi->padding_sent >= machine->allowed_padding_count) { + uint32_t total_cells = mi->padding_sent + mi->nonpadding_sent; + + /* Check the percent */ + if ((100*(uint32_t)mi->padding_sent) / total_cells > + machine->max_padding_percent) { + return 1; // limit is reached. Stop. + } + } + + /* If circpad_max_global_padding_pct is non-zero, and we've + * sent more than the global padding cell limit, then check our + * gloabl tor process percentage limit on padding. */ + if (circpad_global_max_padding_percent && + circpad_global_padding_sent >= circpad_global_allowed_cells) { + uint64_t total_cells = circpad_global_padding_sent + + circpad_global_nonpadding_sent; + + /* Check the percent */ + if ((100*circpad_global_padding_sent) / total_cells > + circpad_global_max_padding_percent) { + return 1; // global limit reached. Stop. + } + } + + return 0; // All good! +} + +/** + * Schedule the next padding time according to the machineinfo on a + * circuit. + * + * The histograms represent inter-packet-delay. Whenever you get an packet + * event you should be scheduling your next timer (after cancelling any old + * ones and updating tokens accordingly). + * + * Returns 1 if we decide to transition states (due to infinity bin), + * 0 otherwise. + */ +circpad_decision_t +circpad_machine_schedule_padding(circpad_machineinfo_t *mi) +{ + circpad_delay_t in_usec = 0; + struct timeval timeout; + tor_assert(mi); + + // Don't pad in end (but also don't cancel any previously + // scheduled padding either). + if (mi->current_state == CIRCPAD_STATE_END) { + log_fn(LOG_INFO, LD_CIRC, "Padding end state"); + return CIRCPAD_STATE_UNCHANGED; + } + + /* Check our padding limits */ + if (circpad_machine_reached_padding_limit(mi)) { + if (CIRCUIT_IS_ORIGIN(mi->on_circ)) { + log_fn(LOG_INFO, LD_CIRC, + "Padding machine has reached padding limit on circuit %u", + TO_ORIGIN_CIRCUIT(mi->on_circ)->global_identifier); + } else { + log_fn(LOG_INFO, LD_CIRC, + "Padding machine has reached padding limit on circuit %"PRIu64 + ", %d", + mi->on_circ->n_chan ? mi->on_circ->n_chan->global_identifier : 0, + mi->on_circ->n_circ_id); + } + return CIRCPAD_STATE_UNCHANGED; + } + + if (mi->is_padding_timer_scheduled) { + /* Cancel current timer (if any) */ + timer_disable(mi->padding_timer); + mi->is_padding_timer_scheduled = 0; + } + + /* in_usec = in microseconds */ + in_usec = circpad_machine_sample_delay(mi); + mi->padding_scheduled_at_usec = monotime_absolute_usec(); + log_fn(LOG_INFO,LD_CIRC,"\tPadding in %u usec", in_usec); + + // Don't schedule if we have infinite delay. + if (in_usec == CIRCPAD_DELAY_INFINITE) { + return circpad_internal_event_infinity(mi); + } + + if (mi->state_length == 0) { + /* If we're at length 0, that means we hit 0 after sending + * a cell earlier, and emitted an event for it, but + * for whatever reason we did not decide to change states then. + * So maybe the machine is waiting for bins empty, or for an + * infinity event later? That would be a strange machine, + * but there's no reason to make it impossible. */ + return CIRCPAD_STATE_UNCHANGED; + } + + if (in_usec <= 0) { + return circpad_send_padding_cell_for_callback(mi); + } + + timeout.tv_sec = in_usec/TOR_USEC_PER_SEC; + timeout.tv_usec = (in_usec%TOR_USEC_PER_SEC); + + log_fn(LOG_INFO, LD_CIRC, "\tPadding in %u sec, %u usec", + (unsigned)timeout.tv_sec, (unsigned)timeout.tv_usec); + + if (mi->padding_timer) { + timer_set_cb(mi->padding_timer, circpad_send_padding_callback, mi); + } else { + mi->padding_timer = + timer_new(circpad_send_padding_callback, mi); + } + timer_schedule(mi->padding_timer, &timeout); + mi->is_padding_timer_scheduled = 1; + + // TODO-MP-AP: Unify with channelpadding counter + //rep_hist_padding_count_timers(++total_timers_pending); + + return CIRCPAD_STATE_UNCHANGED; +} + +/** + * If the machine transitioned to the END state, we need + * to check to see if it wants us to shut it down immediately. + * If it does, then we need to send the appropate negotation commands + * depending on which side it is. + * + * After this function is called, mi may point to freed memory. Do + * not access it. + */ +static void +circpad_machine_transitioned_to_end(circpad_machineinfo_t *mi) +{ + const circpad_machine_t *machine = CIRCPAD_GET_MACHINE(mi); + + /* + * We allow machines to shut down and delete themselves as opposed + * to just going back to START or waiting forever in END so that + * we can handle the case where this machine started while it was + * the only machine that matched conditions, but *since* then more + * "higher ranking" machines now match the conditions, and would + * be given a chance to take precidence over this one in + * circpad_add_matching_machines(). + * + * Returning to START or waiting forever in END would not give those + * other machines a chance to be launched, where as shutting down + * here does. + */ + if (machine->should_negotiate_end) { + circuit_t *on_circ = mi->on_circ; + if (machine->is_origin_side) { + /* We free the machine info here so that we can be replaced + * by a different machine. But we must leave the padding_machine + * in place to wait for the negotiated response */ + circpad_circuit_machineinfo_free_idx(on_circ, + machine->machine_index); + circpad_negotiate_padding(TO_ORIGIN_CIRCUIT(on_circ), + machine->machine_num, + machine->target_hopnum, + CIRCPAD_COMMAND_STOP); + } else { + circpad_circuit_machineinfo_free_idx(on_circ, + machine->machine_index); + circpad_padding_negotiated(on_circ, + machine->machine_num, + CIRCPAD_COMMAND_STOP, + CIRCPAD_RESPONSE_OK); + on_circ->padding_machine[machine->machine_index] = NULL; + } + } +} + +/** + * Generic state transition function for padding state machines. + * + * Given an event and our mutable machine info, decide if/how to + * transition to a different state, and perform actions accordingly. + * + * Returns 1 if we transition states, 0 otherwise. + */ +circpad_decision_t +circpad_machine_transition(circpad_machineinfo_t *mi, + circpad_event_t event) +{ + const circpad_state_t *state = + circpad_machine_current_state(mi); + + /* If state is null we are in the end state. */ + if (!state) { + /* If we in end state we don't pad no matter what. */ + return CIRCPAD_STATE_UNCHANGED; + } + + /* Check if this event is ignored or causes a cancel */ + if (state->next_state[event] == CIRCPAD_STATE_IGNORE) { + return CIRCPAD_STATE_UNCHANGED; + } else if (state->next_state[event] == CIRCPAD_STATE_CANCEL) { + /* Check cancel events and cancel any pending padding */ + mi->padding_scheduled_at_usec = 0; + if (mi->is_padding_timer_scheduled) { + mi->is_padding_timer_scheduled = 0; + /* Cancel current timer (if any) */ + timer_disable(mi->padding_timer); + } + return CIRCPAD_STATE_UNCHANGED; + } else { + circpad_statenum_t s = state->next_state[event]; + /* See if we need to transition to any other states based on this event. + * Whenever a transition happens, even to our own state, we schedule + * padding. + * + * So if a state only wants to schedule padding for an event, it specifies + * a transition to itself. All non-specified events are ignored. + */ + log_fn(LOG_INFO, LD_CIRC, + "Circpad machine %d transitioning from %s to %s", + mi->machine_index, circpad_state_to_string(mi->current_state), + circpad_state_to_string(s)); + + /* If this is not the same state, switch and init tokens, + * otherwise just reschedule padding. */ + if (mi->current_state != s) { + mi->current_state = s; + circpad_machine_setup_tokens(mi); + circpad_choose_state_length(mi); + + /* If we transition to the end state, check to see + * if this machine wants to be shut down at end */ + if (s == CIRCPAD_STATE_END) { + circpad_machine_transitioned_to_end(mi); + /* We transitioned but we don't pad in end. Also, mi + * may be freed. Returning STATE_CHANGED prevents us + * from accessing it in any callers of this function. */ + return CIRCPAD_STATE_CHANGED; + } + + /* We transitioned to a new state, schedule padding */ + circpad_machine_schedule_padding(mi); + return CIRCPAD_STATE_CHANGED; + } + + /* We transitioned back to the same state. Schedule padding, + * and inform if that causes a state transition. */ + return circpad_machine_schedule_padding(mi); + } + + return CIRCPAD_STATE_UNCHANGED; +} + +/** + * Estimate the circuit RTT from the current middle hop out to the + * end of the circuit. + * + * We estimate RTT by calculating the time between "receive" and + * "send" at a middle hop. This is because we "receive" a cell + * from the origin, and then relay it towards the exit before a + * response comes back. It is that response time from the exit side + * that we want to measure, so that we can make use of it for synthetic + * response delays. + */ +static void +circpad_estimate_circ_rtt_on_received(circuit_t *circ, + circpad_machineinfo_t *mi) +{ + /* Origin circuits don't estimate RTT. They could do it easily enough, + * but they have no reason to use it in any delay calculations. */ + if (CIRCUIT_IS_ORIGIN(circ) || mi->stop_rtt_update) + return; + + /* If we already have a last receieved packet time, that means we + * did not get a response before this packet. The RTT estimate + * only makes sense if we do not have multiple packets on the + * wire, so stop estimating if this is the second packet + * back to back. However, for the first set of back-to-back + * packets, we can wait until the very first response comes back + * to us, to measure that RTT (for the response to optimistic + * data, for example). Hence stop_rtt_update is only checked + * in this received side function, and not in send side below. + */ + if (mi->last_received_time_usec) { + /* We also allow multiple back-to-back packets if the circuit is not + * opened, to handle var cells. + * XXX: Will this work with out var cell plans? Maybe not, + * since we're opened at the middle hop as soon as we process + * one var extend2 :/ */ + if (circ->state == CIRCUIT_STATE_OPEN) { + log_fn(LOG_INFO, LD_CIRC, + "Stopping padding RTT estimation on circuit (%"PRIu64 + ", %d) after two back to back packets. Current RTT: %d", + circ->n_chan ? circ->n_chan->global_identifier : 0, + circ->n_circ_id, mi->rtt_estimate_usec); + mi->stop_rtt_update = 1; + } + } else { + mi->last_received_time_usec = monotime_absolute_usec(); + } +} + +/** + * Handles the "send" side of RTT calculation at middle nodes. + * + * This function calculates the RTT from the middle to the end + * of the circuit by subtracting the last received cell timestamp + * from the current time. It allows back-to-back cells until + * the circuit is opened, to allow for var cell handshakes. + * XXX: Check our var cell plans to make sure this will work. + */ +static void +circpad_estimate_circ_rtt_on_send(circuit_t *circ, + circpad_machineinfo_t *mi) +{ + /* Origin circuits don't estimate RTT. They could do it easily enough, + * but they have no reason to use it in any delay calculations. */ + if (CIRCUIT_IS_ORIGIN(circ)) + return; + + /* If last_received_time_usec is non-zero, we are waiting for a response + * from the exit side. Calculate the time delta and use it as RTT. */ + if (mi->last_received_time_usec) { + circpad_time_t rtt_time = monotime_absolute_usec() - + mi->last_received_time_usec; + + /* Reset the last RTT packet time, so we can tell if two cells + * arrive back to back */ + mi->last_received_time_usec = 0; + + /* Use INT32_MAX to ensure the addition doesn't overflow */ + if (rtt_time >= INT32_MAX) { + log_fn(LOG_WARN,LD_CIRC, + "Circuit padding RTT estimate overflowed: %"PRIu64 + " vs %"PRIu64, monotime_absolute_usec(), + mi->last_received_time_usec); + return; + } + + /* If the old RTT estimate is lower than this one, use this one, because + * the circuit is getting longer. If this estimate is somehow + * faster than the previous, then maybe that was network jitter. + * In that case, average them. */ + if (mi->rtt_estimate_usec < (circpad_delay_t)rtt_time) { + mi->rtt_estimate_usec = (circpad_delay_t)rtt_time; + } else { + mi->rtt_estimate_usec += (circpad_delay_t)rtt_time; + mi->rtt_estimate_usec /= 2; + } + } else if (circ->state == CIRCUIT_STATE_OPEN) { + /* If last_received_time_usec is zero, then we have gotten two cells back + * to back. Stop estimating RTT in this case. Note that we only + * stop RTT update if the circuit is opened, to allow for RTT estimates + * of var cells during circ setup. */ + mi->stop_rtt_update = 1; + + if (!mi->rtt_estimate_usec) { + log_fn(LOG_NOTICE, LD_CIRC, + "Got two cells back to back on a circuit before estimating RTT."); + } + } +} + +/** + * A "non-padding" cell has been sent from this endpoint. React + * according to any padding state machines on the circuit. + * + * For origin circuits, this means we sent a cell into the network. + * For middle relay circuits, this means we sent a cell towards the + * origin. + */ +void +circpad_cell_event_nonpadding_sent(circuit_t *on_circ) +{ + /* Update global cell count */ + circpad_global_nonpadding_sent++; + + /* If there are no machines then this loop should not iterate */ + FOR_EACH_ACTIVE_CIRCUIT_MACHINE_BEGIN(i, on_circ) { + /* First, update any RTT estimate */ + circpad_estimate_circ_rtt_on_send(on_circ, on_circ->padding_info[i]); + + /* Remove a token: this is the idea of adaptive padding, since we have an + * ideal distribution that we want our distribution to look like. */ + if (!circpad_machine_remove_token(on_circ->padding_info[i])) { + /* If removing a token did not cause a transition, check if + * non-padding sent event should */ + circpad_machine_transition(on_circ->padding_info[i], + CIRCPAD_EVENT_NONPADDING_SENT); + } + } FOR_EACH_ACTIVE_CIRCUIT_MACHINE_END; +} + +/** + * A "non-padding" cell has been received by this endpoint. React + * according to any padding state machines on the circuit. + * + * For origin circuits, this means we read a cell from the network. + * For middle relay circuits, this means we received a cell from the + * origin. + */ +void +circpad_cell_event_nonpadding_received(circuit_t *on_circ) +{ + FOR_EACH_ACTIVE_CIRCUIT_MACHINE_BEGIN(i, on_circ) { + /* First, update any RTT estimate */ + circpad_estimate_circ_rtt_on_received(on_circ, on_circ->padding_info[i]); + + circpad_machine_transition(on_circ->padding_info[i], + CIRCPAD_EVENT_NONPADDING_RECV); + } FOR_EACH_ACTIVE_CIRCUIT_MACHINE_END; +} + +/** + * A padding cell has been sent from this endpoint. React + * according to any padding state machines on the circuit. + * + * For origin circuits, this means we sent a cell into the network. + * For middle relay circuits, this means we sent a cell towards the + * origin. + */ +void +circpad_cell_event_padding_sent(circuit_t *on_circ) +{ + FOR_EACH_ACTIVE_CIRCUIT_MACHINE_BEGIN(i, on_circ) { + circpad_machine_transition(on_circ->padding_info[i], + CIRCPAD_EVENT_PADDING_SENT); + } FOR_EACH_ACTIVE_CIRCUIT_MACHINE_END; +} + +/** + * A padding cell has been received by this endpoint. React + * according to any padding state machines on the circuit. + * + * For origin circuits, this means we read a cell from the network. + * For middle relay circuits, this means we received a cell from the + * origin. + */ +void +circpad_cell_event_padding_received(circuit_t *on_circ) +{ + /* identical to padding sent */ + FOR_EACH_ACTIVE_CIRCUIT_MACHINE_BEGIN(i, on_circ) { + circpad_machine_transition(on_circ->padding_info[i], + CIRCPAD_EVENT_PADDING_RECV); + } FOR_EACH_ACTIVE_CIRCUIT_MACHINE_END; +} + +/** + * An "infinite" delay has ben chosen from one of our histograms. + * + * "Infinite" delays mean don't send padding -- but they can also + * mean transition to another state depending on the state machine + * definitions. Check the rules and react accordingly. + * + * Return 1 if we decide to transition, 0 otherwise. + */ +circpad_decision_t +circpad_internal_event_infinity(circpad_machineinfo_t *mi) +{ + return circpad_machine_transition(mi, CIRCPAD_EVENT_INFINITY); +} + +/** + * All of the bins of our current state's histogram's are empty. + * + * Check to see if this means transition to another state, and if + * not, refill the tokens. + * + * Return 1 if we decide to transition, 0 otherwise. + */ +circpad_decision_t +circpad_internal_event_bins_empty(circpad_machineinfo_t *mi) +{ + if (circpad_machine_transition(mi, CIRCPAD_EVENT_BINS_EMPTY) + == CIRCPAD_STATE_CHANGED) { + return CIRCPAD_STATE_CHANGED; + } else { + /* If we dont transition, then we refill the tokens */ + circpad_machine_setup_tokens(mi); + return CIRCPAD_STATE_UNCHANGED; + } +} + +/** + * This state has used up its cell count. Emit the event and + * see if we transition. + * + * Return 1 if we decide to transition, 0 otherwise. + */ +circpad_decision_t +circpad_internal_event_state_length_up(circpad_machineinfo_t *mi) +{ + return circpad_machine_transition(mi, CIRCPAD_EVENT_LENGTH_COUNT); +} + +/** + * Returns true if the circuit matches the conditions. + */ +static inline bool +circpad_machine_conditions_met(origin_circuit_t *circ, + const circpad_machine_t *machine) +{ + if (!(circpad_circ_purpose_to_mask(TO_CIRCUIT(circ)->purpose) + & machine->conditions.purpose_mask)) + return 0; + + if (machine->conditions.requires_vanguards) { + const or_options_t *options = get_options(); + + /* Pinned middles are effectively vanguards */ + if (!(options->HSLayer2Nodes || options->HSLayer3Nodes)) + return 0; + } + + /* We check for any bits set in the circuit state mask so that machines + * can say any of the following through their state bitmask: + * "I want to apply to circuits with either streams or no streams"; OR + * "I only want to apply to circuits with streams"; OR + * "I only want to apply to circuits without streams". */ + if (!(circpad_circuit_state(circ) & machine->conditions.state_mask)) + return 0; + + if (circuit_get_cpath_opened_len(circ) < machine->conditions.min_hops) + return 0; + + return 1; +} + +/** + * Returns a minimized representation of the circuit state. + * + * The padding code only cares if the circuit is building, + * opened, used for streams, and/or still has relay early cells. + * This returns a bitmask of all state properities that apply to + * this circuit. + */ +static inline +circpad_circuit_state_t +circpad_circuit_state(origin_circuit_t *circ) +{ + circpad_circuit_state_t retmask = 0; + + if (circ->p_streams) + retmask |= CIRCPAD_CIRC_STREAMS; + else + retmask |= CIRCPAD_CIRC_NO_STREAMS; + + /* We use has_opened to prevent cannibialized circs from flapping. */ + if (circ->has_opened) + retmask |= CIRCPAD_CIRC_OPENED; + else + retmask |= CIRCPAD_CIRC_BUILDING; + + if (circ->remaining_relay_early_cells > 0) + retmask |= CIRCPAD_CIRC_HAS_RELAY_EARLY; + else + retmask |= CIRCPAD_CIRC_HAS_NO_RELAY_EARLY; + + return retmask; +} + +/** + * Convert a normal circuit purpose into a bitmask that we can + * use for determining matching circuits. + */ +static inline +circpad_purpose_mask_t +circpad_circ_purpose_to_mask(uint8_t circ_purpose) +{ + /* Treat OR circ purposes as ignored. They should not be passed here*/ + if (BUG(circ_purpose <= CIRCUIT_PURPOSE_OR_MAX_)) { + return 0; + } + + /* Treat new client circuit purposes as "OMG ITS EVERYTHING". + * This also should not happen */ + if (BUG(circ_purpose - CIRCUIT_PURPOSE_OR_MAX_ - 1 > 32)) { + return CIRCPAD_PURPOSE_ALL; + } + + /* Convert the purpose to a bit position */ + return 1 << (circ_purpose - CIRCUIT_PURPOSE_OR_MAX_ - 1); +} + +/** + * Shut down any machines whose conditions no longer match + * the current circuit. + */ +static void +circpad_shutdown_old_machines(origin_circuit_t *on_circ) +{ + circuit_t *circ = TO_CIRCUIT(on_circ); + + FOR_EACH_ACTIVE_CIRCUIT_MACHINE_BEGIN(i, circ) { + if (!circpad_machine_conditions_met(on_circ, + circ->padding_machine[i])) { + // Clear machineinfo (frees timers) + circpad_circuit_machineinfo_free_idx(circ, i); + // Send padding negotiate stop + circpad_negotiate_padding(on_circ, + circ->padding_machine[i]->machine_num, + circ->padding_machine[i]->target_hopnum, + CIRCPAD_COMMAND_STOP); + } + } FOR_EACH_ACTIVE_CIRCUIT_MACHINE_END; +} + +/** + * Negotiate new machines that would apply to this circuit. + * + * This function checks to see if we have any free machine indexes, + * and for each free machine index, it initializes the most recently + * added origin-side padding machine that matches the target machine + * index and circuit conditions, and negotiates it with the appropriate + * middle relay. + */ +static void +circpad_add_matching_machines(origin_circuit_t *on_circ) +{ + circuit_t *circ = TO_CIRCUIT(on_circ); + +#ifdef TOR_UNIT_TESTS + /* Tests don't have to init our padding machines */ + if (!origin_padding_machines) + return; +#endif + + /* If padding negotiation failed before, do not try again */ + if (on_circ->padding_negotiation_failed) + return; + + FOR_EACH_CIRCUIT_MACHINE_BEGIN(i) { + /* If there is a padding machine info, this index is occupied. + * No need to check conditions for this index. */ + if (circ->padding_info[i]) + continue; + + /* We have a free machine index. Check the origin padding + * machines in reverse order, so that more recently added + * machines take priority over older ones. */ + SMARTLIST_FOREACH_REVERSE_BEGIN(origin_padding_machines, + circpad_machine_t *, + machine) { + /* Machine definitions have a specific target machine index. + * This is so event ordering is deterministic with respect + * to which machine gets events first when there are two + * machines installed on a circuit. Make sure we only + * add this machine if its target machine index is free. */ + if (machine->machine_index == i && + circpad_machine_conditions_met(on_circ, machine)) { + + // We can only replace this machine if the target hopnum + // is the same, otherwise we'll get invalid data + if (circ->padding_machine[i]) { + if (circ->padding_machine[i]->target_hopnum != + machine->target_hopnum) + continue; + /* Replace it. (Don't free - is global). */ + circ->padding_machine[i] = NULL; + } + + /* Set up the machine immediately so that the slot is occupied. + * We will tear it down on error return, or if there is an error + * response from the relay. */ + circpad_setup_machine_on_circ(circ, machine); + if (circpad_negotiate_padding(on_circ, machine->machine_num, + machine->target_hopnum, + CIRCPAD_COMMAND_START) < 0) { + circpad_circuit_machineinfo_free_idx(circ, i); + circ->padding_machine[i] = NULL; + on_circ->padding_negotiation_failed = 1; + } else { + /* Success. Don't try any more machines */ + return; + } + } + } SMARTLIST_FOREACH_END(machine); + } FOR_EACH_CIRCUIT_MACHINE_END; +} + +/** + * Event that tells us we added a hop to an origin circuit. + * + * This event is used to decide if we should create a padding machine + * on a circuit. + */ +void +circpad_machine_event_circ_added_hop(origin_circuit_t *on_circ) +{ + /* Since our padding conditions do not specify a max_hops, + * all we can do is add machines here */ + circpad_add_matching_machines(on_circ); +} + +/** + * Event that tells us that an origin circuit is now built. + * + * Shut down any machines that only applied to un-built circuits. + * Activate any new ones. + */ +void +circpad_machine_event_circ_built(origin_circuit_t *circ) +{ + circpad_shutdown_old_machines(circ); + circpad_add_matching_machines(circ); +} + +/** + * Circpad purpose changed event. + * + * Shut down any machines that don't apply to our circ purpose. + * Activate any new ones that do. + */ +void +circpad_machine_event_circ_purpose_changed(origin_circuit_t *circ) +{ + circpad_shutdown_old_machines(circ); + circpad_add_matching_machines(circ); +} + +/** + * Event that tells us that an origin circuit is out of RELAY_EARLY + * cells. + * + * Shut down any machines that only applied to RELAY_EARLY circuits. + * Activate any new ones. + */ +void +circpad_machine_event_circ_has_no_relay_early(origin_circuit_t *circ) +{ + circpad_shutdown_old_machines(circ); + circpad_add_matching_machines(circ); +} + +/** + * Streams attached event. + * + * Called from link_apconn_to_circ() and handle_hs_exit_conn() + * + * Shut down any machines that only applied to machines without + * streams. Activate any new ones. + */ +void +circpad_machine_event_circ_has_streams(origin_circuit_t *circ) +{ + circpad_shutdown_old_machines(circ); + circpad_add_matching_machines(circ); +} + +/** + * Streams detached event. + * + * Called from circuit_detach_stream() + * + * Shut down any machines that only applied to machines without + * streams. Activate any new ones. + */ +void +circpad_machine_event_circ_has_no_streams(origin_circuit_t *circ) +{ + circpad_shutdown_old_machines(circ); + circpad_add_matching_machines(circ); +} + +/** + * Verify that padding is coming from the expected hop. + * + * Returns true if from_hop matches the target hop from + * one of our padding machines. + * + * Returns false if we're not an origin circuit, or if from_hop + * does not match one of the padding machines. + */ +bool +circpad_padding_is_from_expected_hop(circuit_t *circ, + crypt_path_t *from_hop) +{ + crypt_path_t *target_hop = NULL; + if (!CIRCUIT_IS_ORIGIN(circ)) + return 0; + + FOR_EACH_CIRCUIT_MACHINE_BEGIN(i) { + /* We have to check padding_machine and not padding_info/active + * machines here because padding may arrive after we shut down a + * machine. The info is gone, but the padding_machine waits + * for the padding_negotiated response to come back. */ + if (!circ->padding_machine[i]) + continue; + + target_hop = circuit_get_cpath_hop(TO_ORIGIN_CIRCUIT(circ), + circ->padding_machine[i]->target_hopnum); + + if (target_hop == from_hop) + return 1; + } FOR_EACH_CIRCUIT_MACHINE_END; + + return 0; +} + +/** + * Deliver circpad events for an "unrecognized cell". + * + * Unrecognized cells are sent to relays and are forwarded + * onto the next hop of their circuits. Unrecognized cells + * are by definition not padding. We need to tell relay-side + * state machines that a non-padding cell was sent or received, + * depending on the direction, so they can update their histograms + * and decide to pad or not. + */ +void +circpad_deliver_unrecognized_cell_events(circuit_t *circ, + cell_direction_t dir) +{ + // We should never see unrecognized cells at origin. + // Our caller emits a warn when this happens. + if (CIRCUIT_IS_ORIGIN(circ)) { + return; + } + + if (dir == CELL_DIRECTION_OUT) { + /* When direction is out (away from origin), then we received non-padding + cell coming from the origin to us. */ + circpad_cell_event_nonpadding_received(circ); + } else if (dir == CELL_DIRECTION_IN) { + /* It's in and not origin, so the cell is going away from us. + * So we are relaying a non-padding cell towards the origin. */ + circpad_cell_event_nonpadding_sent(circ); + } +} + +/** + * Deliver circpad events for "recognized" relay cells. + * + * Recognized cells are destined for this hop, either client or middle. + * Check if this is a padding cell or not, and send the appropiate + * received event. + */ +void +circpad_deliver_recognized_relay_cell_events(circuit_t *circ, + uint8_t relay_command, + crypt_path_t *layer_hint) +{ + /* Padding negotiate cells are ignored by the state machines + * for simplicity. */ + if (relay_command == RELAY_COMMAND_PADDING_NEGOTIATE || + relay_command == RELAY_COMMAND_PADDING_NEGOTIATED) { + return; + } + + if (relay_command == RELAY_COMMAND_DROP) { + rep_hist_padding_count_read(PADDING_TYPE_DROP); + + if (CIRCUIT_IS_ORIGIN(circ)) { + if (circpad_padding_is_from_expected_hop(circ, layer_hint)) { + circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), 0); + } else { + /* This is unexpected padding. Ignore it for now. */ + return; + } + } + + /* The cell should be recognized by now, which means that we are on the + destination, which means that we received a padding cell. We might be + the client or the Middle node, still, because leaky-pipe. */ + circpad_cell_event_padding_received(circ); + log_fn(LOG_INFO, LD_CIRC, "Got padding cell on %s circuit %u.", + CIRCUIT_IS_ORIGIN(circ) ? "origin" : "non-origin", + CIRCUIT_IS_ORIGIN(circ) ? + TO_ORIGIN_CIRCUIT(circ)->global_identifier : 0); + } else { + /* We received a non-padding cell on the edge */ + circpad_cell_event_nonpadding_received(circ); + } +} + +/** + * Deliver circpad events for relay cells sent from us. + * + * If this is a padding cell, update our padding stats + * and deliver the event. Otherwise just deliver the event. + */ +void +circpad_deliver_sent_relay_cell_events(circuit_t *circ, + uint8_t relay_command) +{ + /* Padding negotiate cells are ignored by the state machines + * for simplicity. */ + if (relay_command == RELAY_COMMAND_PADDING_NEGOTIATE || + relay_command == RELAY_COMMAND_PADDING_NEGOTIATED) { + return; + } + + /* RELAY_COMMAND_DROP is the multi-hop (aka circuit-level) padding cell in + * tor. (CELL_PADDING is a channel-level padding cell, which is not relayed + * or processed here) */ + if (relay_command == RELAY_COMMAND_DROP) { + /* Optimization: The event for RELAY_COMMAND_DROP is sent directly + * from circpad_send_padding_cell_for_callback(). This is to avoid + * putting a cell_t and a relay_header_t on the stack repeatedly + * if we decide to send a long train of padidng cells back-to-back + * with 0 delay. So we do nothing here. */ + return; + } else { + /* This is a non-padding cell sent from the client or from + * this node. */ + circpad_cell_event_nonpadding_sent(circ); + } +} + +/** + * Initialize the states array for a circpad machine. + */ +void +circpad_machine_states_init(circpad_machine_t *machine, + circpad_statenum_t num_states) +{ + if (BUG(num_states > CIRCPAD_MAX_MACHINE_STATES)) { + num_states = CIRCPAD_MAX_MACHINE_STATES; + } + + machine->num_states = num_states; + machine->states = tor_malloc_zero(sizeof(circpad_state_t)*num_states); + + /* Initialize the default next state for all events to + * "ignore" -- if events aren't specified, they are ignored. */ + for (circpad_statenum_t s = 0; s < num_states; s++) { + for (int e = 0; e < CIRCPAD_NUM_EVENTS; e++) { + machine->states[s].next_state[e] = CIRCPAD_STATE_IGNORE; + } + } +} + +static void +circpad_setup_machine_on_circ(circuit_t *on_circ, + const circpad_machine_t *machine) +{ + if (CIRCUIT_IS_ORIGIN(on_circ) && !machine->is_origin_side) { + log_fn(LOG_WARN, LD_BUG, + "Can't set up non-origin machine on origin circuit!"); + return; + } + + if (!CIRCUIT_IS_ORIGIN(on_circ) && machine->is_origin_side) { + log_fn(LOG_WARN, LD_BUG, + "Can't set up origin machine on non-origin circuit!"); + return; + } + + tor_assert_nonfatal(on_circ->padding_machine[machine->machine_index] + == NULL); + tor_assert_nonfatal(on_circ->padding_info[machine->machine_index] == NULL); + + on_circ->padding_info[machine->machine_index] = + circpad_circuit_machineinfo_new(on_circ, machine->machine_index); + on_circ->padding_machine[machine->machine_index] = machine; +} + +static void +circpad_circ_client_machine_init(void) +{ + circpad_machine_t *circ_client_machine + = tor_malloc_zero(sizeof(circpad_machine_t)); + + // XXX: Better conditions for merge.. Or disable this machine in + // merge? + circ_client_machine->conditions.min_hops = 2; + circ_client_machine->conditions.state_mask = + CIRCPAD_CIRC_BUILDING|CIRCPAD_CIRC_OPENED|CIRCPAD_CIRC_HAS_RELAY_EARLY; + circ_client_machine->conditions.purpose_mask = CIRCPAD_PURPOSE_ALL; + + circ_client_machine->target_hopnum = 2; + circ_client_machine->is_origin_side = 1; + + /* Start, gap, burst */ + circpad_machine_states_init(circ_client_machine, 3); + + circ_client_machine->states[CIRCPAD_STATE_START]. + next_state[CIRCPAD_EVENT_NONPADDING_RECV] = CIRCPAD_STATE_BURST; + + circ_client_machine->states[CIRCPAD_STATE_BURST]. + next_state[CIRCPAD_EVENT_NONPADDING_RECV] = CIRCPAD_STATE_BURST; + circ_client_machine->states[CIRCPAD_STATE_BURST]. + next_state[CIRCPAD_EVENT_PADDING_RECV] = CIRCPAD_STATE_BURST; + + /* If we are in burst state, and we send a non-padding cell, then we cancel + the timer for the next padding cell: + We dont want to send fake extends when actual extends are going on */ + circ_client_machine->states[CIRCPAD_STATE_BURST]. + next_state[CIRCPAD_EVENT_NONPADDING_SENT] = CIRCPAD_STATE_CANCEL; + + circ_client_machine->states[CIRCPAD_STATE_BURST]. + next_state[CIRCPAD_EVENT_BINS_EMPTY] = CIRCPAD_STATE_END; + + circ_client_machine->states[CIRCPAD_STATE_BURST].token_removal = + CIRCPAD_TOKEN_REMOVAL_CLOSEST; + + // FIXME: Tune this histogram + circ_client_machine->states[CIRCPAD_STATE_BURST].histogram_len = 2; + circ_client_machine->states[CIRCPAD_STATE_BURST].start_usec = 500; + circ_client_machine->states[CIRCPAD_STATE_BURST].range_usec = 1000000; + /* We have 5 tokens in the histogram, which means that all circuits will look + * like they have 7 hops (since we start this machine after the second hop, + * and tokens are decremented for any valid hops, and fake extends are + * used after that -- 2+5==7). */ + circ_client_machine->states[CIRCPAD_STATE_BURST].histogram[0] = 5; + circ_client_machine->states[CIRCPAD_STATE_BURST].histogram_total_tokens = 5; + + circ_client_machine->machine_num = smartlist_len(origin_padding_machines); + smartlist_add(origin_padding_machines, circ_client_machine); +} + +static void +circpad_circ_responder_machine_init(void) +{ + circpad_machine_t *circ_responder_machine + = tor_malloc_zero(sizeof(circpad_machine_t)); + + /* Shut down the machine after we've sent enough packets */ + circ_responder_machine->should_negotiate_end = 1; + + /* The relay-side doesn't care what hopnum it is, but for consistency, + * let's match the client */ + circ_responder_machine->target_hopnum = 2; + circ_responder_machine->is_origin_side = 0; + + /* Start, gap, burst */ + circpad_machine_states_init(circ_responder_machine, 3); + + /* This is the settings of the state machine. In the future we are gonna + serialize this into the consensus or the torrc */ + + /* We transition to the burst state on padding receive and on non-padding + * recieve */ + circ_responder_machine->states[CIRCPAD_STATE_START]. + next_state[CIRCPAD_EVENT_PADDING_RECV] = CIRCPAD_STATE_BURST; + circ_responder_machine->states[CIRCPAD_STATE_START]. + next_state[CIRCPAD_EVENT_NONPADDING_RECV] = CIRCPAD_STATE_BURST; + + /* Inside the burst state we _stay_ in the burst state when a non-padding + * is sent */ + circ_responder_machine->states[CIRCPAD_STATE_BURST]. + next_state[CIRCPAD_EVENT_NONPADDING_SENT] = CIRCPAD_STATE_BURST; + + /* Inside the burst state we transition to the gap state when we receive a + * padding cell */ + circ_responder_machine->states[CIRCPAD_STATE_BURST]. + next_state[CIRCPAD_EVENT_PADDING_RECV] = CIRCPAD_STATE_GAP; + + /* These describe the padding charasteristics when in burst state */ + + /* use_rtt_estimate tries to estimate how long padding cells take to go from + C->M, and uses that as what as the base of the histogram */ + circ_responder_machine->states[CIRCPAD_STATE_BURST].use_rtt_estimate = 1; + /* The histogram is 2 bins: an empty one, and infinity */ + circ_responder_machine->states[CIRCPAD_STATE_BURST].histogram_len = 2; + circ_responder_machine->states[CIRCPAD_STATE_BURST].start_usec = 5000; + circ_responder_machine->states[CIRCPAD_STATE_BURST].range_usec = 1000000; + /* During burst state we wait forever for padding to arrive. + + We are waiting for a padding cell from the client to come in, so that we + respond, and we immitate how extend looks like */ + circ_responder_machine->states[CIRCPAD_STATE_BURST].histogram[0] = 0; + // Only infinity bin: + circ_responder_machine->states[CIRCPAD_STATE_BURST].histogram[1] = 1; + circ_responder_machine->states[CIRCPAD_STATE_BURST]. + histogram_total_tokens = 1; + + /* From the gap state, we _stay_ in the gap state, when we receive padding + * or non padding */ + circ_responder_machine->states[CIRCPAD_STATE_GAP]. + next_state[CIRCPAD_EVENT_PADDING_RECV] = CIRCPAD_STATE_GAP; + circ_responder_machine->states[CIRCPAD_STATE_GAP]. + next_state[CIRCPAD_EVENT_NONPADDING_RECV] = CIRCPAD_STATE_GAP; + + /* And from the gap state, we go to the end, when the bins are empty or a + * non-padding cell is sent */ + circ_responder_machine->states[CIRCPAD_STATE_GAP]. + next_state[CIRCPAD_EVENT_BINS_EMPTY] = CIRCPAD_STATE_END; + circ_responder_machine->states[CIRCPAD_STATE_GAP]. + next_state[CIRCPAD_EVENT_NONPADDING_SENT] = CIRCPAD_STATE_END; + + // FIXME: Tune this histogram + + /* The gap state is the delay you wait after you receive a padding cell + before you send a padding response */ + circ_responder_machine->states[CIRCPAD_STATE_GAP].use_rtt_estimate = 1; + circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram_len = 6; + circ_responder_machine->states[CIRCPAD_STATE_GAP].start_usec = 5000; + circ_responder_machine->states[CIRCPAD_STATE_GAP].range_usec = 1000000; + circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram[0] = 0; + circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram[1] = 1; + circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram[2] = 2; + circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram[3] = 2; + circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram[4] = 1; + /* Total number of tokens */ + circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram_total_tokens = 6; + circ_responder_machine->states[CIRCPAD_STATE_GAP].token_removal = + CIRCPAD_TOKEN_REMOVAL_CLOSEST_USEC; + + circ_responder_machine->machine_num = smartlist_len(relay_padding_machines); + smartlist_add(relay_padding_machines, circ_responder_machine); +} + +/** + * Initialize all of our padding machines. + * + * This is called at startup. It sets up some global machines, and then + * loads some from torrc, and from the tor consensus. + */ +void +circpad_machines_init(void) +{ + tor_assert_nonfatal(origin_padding_machines == NULL); + tor_assert_nonfatal(relay_padding_machines == NULL); + + origin_padding_machines = smartlist_new(); + relay_padding_machines = smartlist_new(); + + // TODO: Parse machines from consensus and torrc + + circpad_circ_client_machine_init(); + circpad_circ_responder_machine_init(); +} + +/** + * Free our padding machines + */ +void +circpad_machines_free(void) +{ + if (origin_padding_machines) { + SMARTLIST_FOREACH(origin_padding_machines, + circpad_machine_t *, + m, tor_free(m->states); tor_free(m)); + smartlist_free(origin_padding_machines); + } + + if (relay_padding_machines) { + SMARTLIST_FOREACH(relay_padding_machines, + circpad_machine_t *, + m, tor_free(m->states); tor_free(m)); + smartlist_free(relay_padding_machines); + } +} + +/** + * Check the Protover info to see if a node supports padding. + */ +static bool +circpad_node_supports_padding(const node_t *node) +{ + if (node->rs) { + log_fn(LOG_INFO, LD_CIRC, "Checking padding: %s", + node->rs->pv.supports_padding ? "supported" : "unsupported"); + return node->rs->pv.supports_padding; + } + + log_fn(LOG_INFO, LD_CIRC, "Empty routerstatus in padding check"); + return 0; +} + +/** + * Get a node_t for the nth hop in our circuit, starting from 1. + * + * Returns node_t from the consensus for that hop, if it is opened. + * Otherwise returns NULL. + */ +static const node_t * +circuit_get_nth_node(origin_circuit_t *circ, int hop) +{ + crypt_path_t *iter = circuit_get_cpath_hop(circ, hop); + + if (!iter || iter->state != CPATH_STATE_OPEN) + return NULL; + + return node_get_by_id(iter->extend_info->identity_digest); +} + +/** + * Return true if a particular circuit supports padding + * at the desired hop. + */ +static bool +circpad_circuit_supports_padding(origin_circuit_t *circ, + int target_hopnum) +{ + const node_t *hop; + + if (!(hop = circuit_get_nth_node(circ, target_hopnum))) { + return 0; + } + + return circpad_node_supports_padding(hop); +} + +/** + * Try to negotiate padding. + * + * Returns -1 on error, 0 on success. + */ +signed_error_t +circpad_negotiate_padding(origin_circuit_t *circ, + circpad_machine_num_t machine, + uint8_t target_hopnum, + uint8_t command) +{ + circpad_negotiate_t type; + cell_t cell; + ssize_t len; + + /* Check that the target hop lists support for padding in + * its ProtoVer fields */ + if (!circpad_circuit_supports_padding(circ, target_hopnum)) { + return -1; + } + + memset(&cell, 0, sizeof(cell_t)); + memset(&type, 0, sizeof(circpad_negotiate_t)); + // This gets reset to RELAY_EARLY appropriately by + // relay_send_command_from_edge_. At least, it looks that way. + // QQQ-MP-AP: Verify that. + cell.command = CELL_RELAY; + + circpad_negotiate_set_command(&type, command); + circpad_negotiate_set_version(&type, 0); + circpad_negotiate_set_machine_type(&type, machine); + + if ((len = circpad_negotiate_encode(cell.payload, CELL_PAYLOAD_SIZE, + &type)) < 0) + return -1; + + log_fn(LOG_INFO,LD_CIRC, "Negotiating padding on circuit %u", + circ->global_identifier); + + return circpad_send_command_to_hop(circ, target_hopnum, + RELAY_COMMAND_PADDING_NEGOTIATE, + cell.payload, len); +} + +/** + * Try to negotiate padding. + * + * Returns 1 if successful (or already set up), 0 otherwise. + */ +bool +circpad_padding_negotiated(circuit_t *circ, + circpad_machine_num_t machine, + uint8_t command, + uint8_t response) +{ + circpad_negotiated_t type; + cell_t cell; + ssize_t len; + + memset(&cell, 0, sizeof(cell_t)); + memset(&type, 0, sizeof(circpad_negotiated_t)); + // This gets reset to RELAY_EARLY appropriately by + // relay_send_command_from_edge_. At least, it looks that way. + // QQQ-MP-AP: Verify that. + cell.command = CELL_RELAY; + + circpad_negotiated_set_command(&type, command); + circpad_negotiated_set_response(&type, response); + circpad_negotiated_set_version(&type, 0); + circpad_negotiated_set_machine_type(&type, machine); + + if ((len = circpad_negotiated_encode(cell.payload, CELL_PAYLOAD_SIZE, + &type)) < 0) + return 0; + + /* Use relay_send because we're from the middle to the origin. We don't + * need to specify a target hop or layer_hint. */ + return relay_send_command_from_edge(0, circ, + RELAY_COMMAND_PADDING_NEGOTIATED, + (void*)cell.payload, + (size_t)len, NULL) == 0; +} + +/** + * Parse and react to a padding_negotiate cell. + * + * This is called at the middle node upon receipt of the client's choice of + * state machine, so that it can use the requested state machine index, if + * it is available. + * + * Returns -1 on error, 0 on success. + */ +signed_error_t +circpad_handle_padding_negotiate(circuit_t *circ, cell_t *cell) +{ + int retval = 0; + circpad_negotiate_t *negotiate; + + if (CIRCUIT_IS_ORIGIN(circ)) { + log_fn(LOG_WARN, LD_PROTOCOL, + "Padding negotiate cell unsupported at origin."); + return -1; + } + + if (circpad_negotiate_parse(&negotiate, cell->payload+RELAY_HEADER_SIZE, + CELL_PAYLOAD_SIZE-RELAY_HEADER_SIZE) < 0) { + log_fn(LOG_WARN, LD_CIRC, + "Received malformed PADDING_NEGOTIATE cell; dropping."); + return -1; + } + + if (negotiate->command == CIRCPAD_COMMAND_STOP) { + /* Free the machine corresponding to this machine type */ + free_circ_machineinfos_with_machine_num(circ, negotiate->machine_type); + log_fn(LOG_WARN, LD_CIRC, + "Received circuit padding stop command for unknown machine."); + goto err; + } else if (negotiate->command == CIRCPAD_COMMAND_START) { + SMARTLIST_FOREACH_BEGIN(relay_padding_machines, + const circpad_machine_t *, m) { + if (m->machine_num == negotiate->machine_type) { + circpad_setup_machine_on_circ(circ, m); + goto done; + } + } SMARTLIST_FOREACH_END(m); + } + + err: + retval = -1; + + done: + circpad_padding_negotiated(circ, negotiate->machine_type, + negotiate->command, + (retval == 0) ? CIRCPAD_RESPONSE_OK : CIRCPAD_RESPONSE_ERR); + circpad_negotiate_free(negotiate); + + return retval; +} + +/** + * Parse and react to a padding_negotiated cell. + * + * This is called at the origin upon receipt of the middle's response + * to our choice of state machine. + * + * Returns -1 on error, 0 on success. + */ +signed_error_t +circpad_handle_padding_negotiated(circuit_t *circ, cell_t *cell, + crypt_path_t *layer_hint) +{ + circpad_negotiated_t *negotiated; + + if (!CIRCUIT_IS_ORIGIN(circ)) { + log_fn(LOG_WARN, LD_PROTOCOL, + "Padding negotiated cell unsupported at non-origin."); + return -1; + } + + /* Verify this came from the expected hop */ + if (!circpad_padding_is_from_expected_hop(circ, layer_hint)) { + log_fn(LOG_WARN, LD_PROTOCOL, + "Padding negotiated cell from wrong hop!"); + return -1; + } + + if (circpad_negotiated_parse(&negotiated, cell->payload+RELAY_HEADER_SIZE, + CELL_PAYLOAD_SIZE-RELAY_HEADER_SIZE) < 0) { + log_fn(LOG_WARN, LD_CIRC, + "Received malformed PADDING_NEGOTIATED cell; " + "dropping."); + return -1; + } + + if (negotiated->command == CIRCPAD_COMMAND_STOP) { + /* There may not be a padding_info here if we shut down the + * machine in circpad_shutdown_old_machines(). Or, if + * circpad_add_matching_matchines() added a new machine, + * there may be a padding_machine for a different machine num + * than this response. */ + free_circ_machineinfos_with_machine_num(circ, negotiated->machine_type); + } else if (negotiated->command == CIRCPAD_COMMAND_START && + negotiated->response == CIRCPAD_RESPONSE_ERR) { + // This can happen due to consensus drift.. free the machines + // and be sad + free_circ_machineinfos_with_machine_num(circ, negotiated->machine_type); + TO_ORIGIN_CIRCUIT(circ)->padding_negotiation_failed = 1; + log_fn(LOG_INFO, LD_CIRC, + "Middle node did not accept our padding request."); + } + + circpad_negotiated_free(negotiated); + return 0; +} + +/* Serialization */ +// TODO: Should we use keyword=value here? Are there helpers for that? +#if 0 +static void +circpad_state_serialize(const circpad_state_t *state, + smartlist_t *chunks) +{ + smartlist_add_asprintf(chunks, " %u", state->histogram[0]); + for (int i = 1; i < state->histogram_len; i++) { + smartlist_add_asprintf(chunks, ",%u", + state->histogram[i]); + } + + smartlist_add_asprintf(chunks, " 0x%x", + state->transition_cancel_events); + + for (int i = 0; i < CIRCPAD_NUM_STATES; i++) { + smartlist_add_asprintf(chunks, ",0x%x", + state->transition_events[i]); + } + + smartlist_add_asprintf(chunks, " %u %u", + state->use_rtt_estimate, + state->token_removal); +} + +char * +circpad_machine_to_string(const circpad_machine_t *machine) +{ + smartlist_t *chunks = smartlist_new(); + char *out; + (void)machine; + + circpad_state_serialize(&machine->start, chunks); + circpad_state_serialize(&machine->gap, chunks); + circpad_state_serialize(&machine->burst, chunks); + + out = smartlist_join_strings(chunks, "", 0, NULL); + + SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp)); + smartlist_free(chunks); + return out; +} + +// XXX: Writeme +const circpad_machine_t * +circpad_string_to_machine(const char *str) +{ + (void)str; + return NULL; +} + +#endif From d62340018c8d363ea67ef01dc4a740e47fce2a10 Mon Sep 17 00:00:00 2001 From: Mike Perry Date: Thu, 23 Aug 2018 20:31:16 +0000 Subject: [PATCH 0364/2557] Add relay crypto mock points for tests. Co-authored-by: George Kadianakis --- src/core/or/relay.c | 6 +++--- src/core/or/relay.h | 5 +++++ 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/core/or/relay.c b/src/core/or/relay.c index 452777b2fc..9c0f3bbbe3 100644 --- a/src/core/or/relay.c +++ b/src/core/or/relay.c @@ -355,11 +355,11 @@ circuit_receive_relay_cell(cell_t *cell, circuit_t *circ, * - Encrypt it to the right layer * - Append it to the appropriate cell_queue on circ. */ -static int -circuit_package_relay_cell(cell_t *cell, circuit_t *circ, +MOCK_IMPL(int, +circuit_package_relay_cell, (cell_t *cell, circuit_t *circ, cell_direction_t cell_direction, crypt_path_t *layer_hint, streamid_t on_stream, - const char *filename, int lineno) + const char *filename, int lineno)) { channel_t *chan; /* where to send the cell */ diff --git a/src/core/or/relay.h b/src/core/or/relay.h index db7f17b96c..e84727e373 100644 --- a/src/core/or/relay.h +++ b/src/core/or/relay.h @@ -78,6 +78,11 @@ void destroy_cell_queue_append(destroy_cell_queue_t *queue, void channel_unlink_all_circuits(channel_t *chan, smartlist_t *detached_out); MOCK_DECL(int, channel_flush_from_first_active_circuit, (channel_t *chan, int max)); +MOCK_DECL(int, circuit_package_relay_cell, (cell_t *cell, circuit_t *circ, + cell_direction_t cell_direction, + crypt_path_t *layer_hint, streamid_t on_stream, + const char *filename, int lineno)); + void update_circuit_on_cmux_(circuit_t *circ, cell_direction_t direction, const char *file, int lineno); #define update_circuit_on_cmux(circ, direction) \ From a336d816a68e5eaddd9d80f7179699274b367a1d Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Wed, 2 Jan 2019 15:19:12 +0200 Subject: [PATCH 0365/2557] Circuit padding tests. Co-authored-by: George Kadianakis --- src/core/or/circuitpadding.c | 55 +- src/core/or/circuitpadding.h | 5 + src/lib/smartlist_core/smartlist_foreach.h | 5 + src/lib/time/compat_time.c | 4 +- src/lib/time/compat_time.h | 2 +- src/test/Makefile.nmake | 1 + src/test/include.am | 1 + src/test/test.c | 1 + src/test/test.h | 1 + src/test/test_circuitpadding.c | 2358 ++++++++++++++++++++ src/test/test_containers.c | 25 + 11 files changed, 2438 insertions(+), 20 deletions(-) create mode 100644 src/test/test_circuitpadding.c diff --git a/src/core/or/circuitpadding.c b/src/core/or/circuitpadding.c index 9d65e2cf22..6cfbf4ba56 100644 --- a/src/core/or/circuitpadding.c +++ b/src/core/or/circuitpadding.c @@ -157,7 +157,7 @@ circpad_circuit_machineinfo_new(circuit_t *on_circ, int machine_index) * invalid state. */ STATIC const circpad_state_t * -circpad_machine_current_state(circpad_machineinfo_t *mi) +circpad_machine_current_state(const circpad_machineinfo_t *mi) { const circpad_machine_t *machine = CIRCPAD_GET_MACHINE(mi); @@ -189,7 +189,7 @@ circpad_machine_current_state(circpad_machineinfo_t *mi) * It has a usec value of CIRCPAD_DELAY_INFINITE (UINT32_MAX). */ STATIC circpad_delay_t -circpad_histogram_bin_to_usec(circpad_machineinfo_t *mi, +circpad_histogram_bin_to_usec(const circpad_machineinfo_t *mi, circpad_hist_index_t bin) { const circpad_state_t *state = circpad_machine_current_state(mi); @@ -224,6 +224,18 @@ circpad_histogram_bin_to_usec(circpad_machineinfo_t *mi, CIRCPAD_DELAY_INFINITE); } +/** Return the midpoint of the histogram bin bin_index. */ +static circpad_delay_t +circpad_get_histogram_bin_midpoint(const circpad_machineinfo_t *mi, + int bin_index) +{ + circpad_delay_t left_bound = circpad_histogram_bin_to_usec(mi, bin_index); + circpad_delay_t right_bound = + circpad_histogram_bin_to_usec(mi, bin_index+1)-1; + + return left_bound + (right_bound - left_bound)/2; +} + /** * Return the bin that contains the usec argument. * "Contains" is defined as us in [lower, upper). @@ -446,6 +458,8 @@ circpad_machine_sample_delay(circpad_machineinfo_t *mi) tor_assert(curr_bin < CIRCPAD_INFINITY_BIN(state)); bin_start = circpad_histogram_bin_to_usec(mi, curr_bin); + /* We don't need to reduct 1 from the upper bound because the random range + * function below samples from [bin_start, bin_end) */ bin_end = circpad_histogram_bin_to_usec(mi, curr_bin+1); /* Truncate the high bin in case it's the infinity bin: @@ -532,7 +546,7 @@ circpad_distribution_sample(circpad_distribution_t dist) * greater than the target, and that has tokens remaining. */ static circpad_hist_index_t -circpad_machine_first_higher_index(circpad_machineinfo_t *mi, +circpad_machine_first_higher_index(const circpad_machineinfo_t *mi, circpad_delay_t target_bin_usec) { circpad_hist_index_t bin = circpad_histogram_usec_to_bin(mi, @@ -554,7 +568,7 @@ circpad_machine_first_higher_index(circpad_machineinfo_t *mi, * target_bin_usec, and that still has tokens remaining. */ static circpad_hist_index_t -circpad_machine_first_lower_index(circpad_machineinfo_t *mi, +circpad_machine_first_lower_index(const circpad_machineinfo_t *mi, circpad_delay_t target_bin_usec) { circpad_hist_index_t bin = circpad_histogram_usec_to_bin(mi, @@ -619,7 +633,9 @@ circpad_machine_remove_lower_token(circpad_machineinfo_t *mi, /** * Remove a token from the closest non-empty bin to the target. * - * If use_usec is true, measure "closest" in terms of bin start usec. + * If use_usec is true, measure "closest" in terms of the next closest bin + * midpoint. + * * If it is false, use bin index distance only. */ STATIC void @@ -639,8 +655,9 @@ circpad_machine_remove_closest_token(circpad_machineinfo_t *mi, return; } + /* Take care of edge cases first */ if (higher == mi->histogram_len && lower == -1) { - // Bins are empty + /* All bins are empty */ return; } else if (higher == mi->histogram_len) { /* All higher bins are empty */ @@ -654,6 +671,7 @@ circpad_machine_remove_closest_token(circpad_machineinfo_t *mi, return; } + /* Now handle the intermediate cases */ if (use_usec) { /* Find the closest bin midpoint to the target */ circpad_delay_t lower_usec = circpad_get_histogram_bin_midpoint(mi, lower); @@ -677,6 +695,9 @@ circpad_machine_remove_closest_token(circpad_machineinfo_t *mi, ENSURE_BIN_CAPACITY(lower); bin_to_remove = lower; } + mi->histogram[bin_to_remove]--; + log_debug(LD_GENERAL, "Removing token from bin %d", bin_to_remove); + return; } else { if (current - lower > higher - current) { // Higher bin is closer @@ -756,7 +777,7 @@ check_machine_token_supply(circpad_machineinfo_t *mi) * * Returns 1 if we transition states, 0 otherwise. */ -circpad_decision_t +STATIC circpad_decision_t circpad_machine_remove_token(circpad_machineinfo_t *mi) { const circpad_state_t *state = NULL; @@ -841,10 +862,10 @@ circpad_machine_remove_token(circpad_machineinfo_t *mi) * * Returns negative on error, 0 on success. */ -static signed_error_t -circpad_send_command_to_hop(origin_circuit_t *circ, uint8_t hopnum, - uint8_t relay_command, const uint8_t *payload, - ssize_t payload_len) +MOCK_IMPL(STATIC signed_error_t, +circpad_send_command_to_hop,(origin_circuit_t *circ, uint8_t hopnum, + uint8_t relay_command, const uint8_t *payload, + ssize_t payload_len)) { crypt_path_t *target_hop = circuit_get_cpath_hop(circ, hopnum); signed_error_t ret; @@ -1017,7 +1038,7 @@ circpad_new_consensus_params(const networkstatus_t *ns) * * Returns 1 if limits are set and we've hit them. Otherwise returns 0. */ -static bool +STATIC bool circpad_machine_reached_padding_limit(circpad_machineinfo_t *mi) { const circpad_machine_t *machine = CIRCPAD_GET_MACHINE(mi); @@ -1025,7 +1046,7 @@ circpad_machine_reached_padding_limit(circpad_machineinfo_t *mi) /* If machine_padding_pct is non-zero, and we've sent more * than the allowed count of padding cells, then check our * percent limits for this machine. */ - if (machine->max_padding_percent && + if (machine->max_padding_percent && mi->padding_sent >= machine->allowed_padding_count) { uint32_t total_cells = mi->padding_sent + mi->nonpadding_sent; @@ -1046,7 +1067,7 @@ circpad_machine_reached_padding_limit(circpad_machineinfo_t *mi) /* Check the percent */ if ((100*circpad_global_padding_sent) / total_cells > - circpad_global_max_padding_percent) { + circpad_global_max_padding_percent) { return 1; // global limit reached. Stop. } } @@ -1205,9 +1226,9 @@ circpad_machine_transitioned_to_end(circpad_machineinfo_t *mi) * * Returns 1 if we transition states, 0 otherwise. */ -circpad_decision_t -circpad_machine_transition(circpad_machineinfo_t *mi, - circpad_event_t event) +MOCK_IMPL(circpad_decision_t, +circpad_machine_transition,(circpad_machineinfo_t *mi, + circpad_event_t event)) { const circpad_state_t *state = circpad_machine_current_state(mi); diff --git a/src/core/or/circuitpadding.h b/src/core/or/circuitpadding.h index 24034a4548..4680c6be43 100644 --- a/src/core/or/circuitpadding.h +++ b/src/core/or/circuitpadding.h @@ -675,6 +675,11 @@ STATIC void circpad_machine_remove_closest_token(circpad_machineinfo_t *mi, bool use_usec); STATIC void circpad_machine_setup_tokens(circpad_machineinfo_t *mi); +MOCK_DECL(STATIC signed_error_t, +circpad_send_command_to_hop,(origin_circuit_t *circ, uint8_t hopnum, + uint8_t relay_command, const uint8_t *payload, + ssize_t payload_len)); + #ifdef TOR_UNIT_TESTS extern smartlist_t *origin_padding_machines; extern smartlist_t *relay_padding_machines; diff --git a/src/lib/smartlist_core/smartlist_foreach.h b/src/lib/smartlist_core/smartlist_foreach.h index 14f2930c9f..c9afebd6a2 100644 --- a/src/lib/smartlist_core/smartlist_foreach.h +++ b/src/lib/smartlist_core/smartlist_foreach.h @@ -83,6 +83,11 @@ ++var ## _sl_idx) { \ var = (sl)->list[var ## _sl_idx]; +/** Iterates over the items in smartlist sl in reverse order, similar to + * SMARTLIST_FOREACH_BEGIN + * + * NOTE: This macro is incompatible with SMARTLIST_DEL_CURRENT. + */ #define SMARTLIST_FOREACH_REVERSE_BEGIN(sl, type, var) \ STMT_BEGIN \ int var ## _sl_idx, var ## _sl_len=(sl)->num_used; \ diff --git a/src/lib/time/compat_time.c b/src/lib/time/compat_time.c index f1ddb4fdc4..387b0fad22 100644 --- a/src/lib/time/compat_time.c +++ b/src/lib/time/compat_time.c @@ -787,8 +787,8 @@ monotime_absolute_nsec(void) return monotime_diff_nsec(&initialized_at, &now); } -uint64_t -monotime_absolute_usec(void) +MOCK_IMPL(uint64_t, +monotime_absolute_usec,(void)) { return monotime_absolute_nsec() / 1000; } diff --git a/src/lib/time/compat_time.h b/src/lib/time/compat_time.h index 44fab62de5..bf1bd28801 100644 --- a/src/lib/time/compat_time.h +++ b/src/lib/time/compat_time.h @@ -103,7 +103,7 @@ uint64_t monotime_absolute_nsec(void); /** * Return the number of microseconds since the timer system was initialized. */ -uint64_t monotime_absolute_usec(void); +MOCK_DECL(uint64_t, monotime_absolute_usec,(void)); /** * Return the number of milliseconds since the timer system was initialized. */ diff --git a/src/test/Makefile.nmake b/src/test/Makefile.nmake index aa16a22b52..ca6a84cf8a 100644 --- a/src/test/Makefile.nmake +++ b/src/test/Makefile.nmake @@ -19,6 +19,7 @@ TEST_OBJECTS = test.obj test_addr.obj test_channel.obj test_channeltls.obj \ test_cell_formats.obj test_relay.obj test_replay.obj \ test_channelpadding.obj \ test_circuitstats.obj \ + test_circuitpadding.obj \ test_scheduler.obj test_introduce.obj test_hs.obj tinytest.obj tinytest.obj: ..\ext\tinytest.c diff --git a/src/test/include.am b/src/test/include.am index 4725e8cbaa..4da0b84392 100644 --- a/src/test/include.am +++ b/src/test/include.am @@ -101,6 +101,7 @@ src_test_test_SOURCES += \ src/test/test_cell_queue.c \ src/test/test_channel.c \ src/test/test_channelpadding.c \ + src/test/test_circuitpadding.c \ src/test/test_channeltls.c \ src/test/test_checkdir.c \ src/test/test_circuitlist.c \ diff --git a/src/test/test.c b/src/test/test.c index 13e8c71709..a0a138b03d 100644 --- a/src/test/test.c +++ b/src/test/test.c @@ -845,6 +845,7 @@ struct testgroup_t testgroups[] = { { "channeltls/", channeltls_tests }, { "checkdir/", checkdir_tests }, { "circuitbuild/", circuitbuild_tests }, + { "circuitpadding/", circuitpadding_tests }, { "circuitlist/", circuitlist_tests }, { "circuitmux/", circuitmux_tests }, { "circuitstats/", circuitstats_tests }, diff --git a/src/test/test.h b/src/test/test.h index 9f754469c8..9f6eb0a7e6 100644 --- a/src/test/test.h +++ b/src/test/test.h @@ -187,6 +187,7 @@ extern struct testcase_t cell_format_tests[]; extern struct testcase_t cell_queue_tests[]; extern struct testcase_t channel_tests[]; extern struct testcase_t channelpadding_tests[]; +extern struct testcase_t circuitpadding_tests[]; extern struct testcase_t channeltls_tests[]; extern struct testcase_t checkdir_tests[]; extern struct testcase_t circuitbuild_tests[]; diff --git a/src/test/test_circuitpadding.c b/src/test/test_circuitpadding.c new file mode 100644 index 0000000000..78f93f7b24 --- /dev/null +++ b/src/test/test_circuitpadding.c @@ -0,0 +1,2358 @@ +#define TOR_CHANNEL_INTERNAL_ +#define TOR_TIMERS_PRIVATE +#define CIRCUITPADDING_PRIVATE +#define NETWORKSTATUS_PRIVATE + +#include "core/or/or.h" +#include "test.h" +#include "lib/testsupport/testsupport.h" +#include "core/or/connection_or.h" +#include "core/or/channel.h" +#include "core/or/channeltls.h" +#include +#include "lib/evloop/compat_libevent.h" +#include "lib/time/compat_time.h" +#include "core/or/relay.h" +#include "core/or/circuitlist.h" +#include "core/or/circuitbuild.h" +#include "core/or/circuitpadding.h" +#include "core/crypto/relay_crypto.h" +#include "core/or/protover.h" +#include "feature/nodelist/nodelist.h" +#include "lib/evloop/compat_libevent.h" +#include "app/config/config.h" + +#include "feature/nodelist/routerstatus_st.h" +#include "feature/nodelist/networkstatus_st.h" +#include "feature/nodelist/node_st.h" +#include "core/or/cell_st.h" +#include "core/or/crypt_path_st.h" +#include "core/or/or_circuit_st.h" +#include "core/or/origin_circuit_st.h" + +extern smartlist_t *connection_array; + +#define USEC_PER_SEC (1000000) +#define NSEC_PER_USEC (1000) +#define NSEC_PER_MSEC (1000*1000) + +circid_t get_unique_circ_id_by_chan(channel_t *chan); +void helper_create_basic_machine(void); +static void helper_create_conditional_machines(void); + +static or_circuit_t * new_fake_orcirc(channel_t *nchan, channel_t *pchan); +channel_t *new_fake_channel(void); +void test_circuitpadding_negotiation(void *arg); +void test_circuitpadding_wronghop(void *arg); +void test_circuitpadding_conditions(void *arg); + +void test_circuitpadding_serialize(void *arg); +void test_circuitpadding_rtt(void *arg); +void test_circuitpadding_tokens(void *arg); +void test_circuitpadding_circuitsetup_machine(void *arg); + +static void +simulate_single_hop_extend(circuit_t *client, circuit_t *mid_relay, + int padding); +void free_fake_orcirc(circuit_t *circ); +void free_fake_origin_circuit(origin_circuit_t *circ); + +static int deliver_negotiated = 1; +static int64_t curr_mocked_time; + +static node_t padding_node; +static node_t non_padding_node; + +static channel_t dummy_channel; +static circpad_machine_t circ_client_machine; + +static void +timers_advance_and_run(int64_t msec_update) +{ + curr_mocked_time += msec_update*NSEC_PER_MSEC; + monotime_coarse_set_mock_time_nsec(curr_mocked_time); + monotime_set_mock_time_nsec(curr_mocked_time); + timers_run_pending(); +} + +static void +nodes_init(void) +{ + padding_node.rs = tor_malloc_zero(sizeof(routerstatus_t)); + padding_node.rs->pv.supports_padding = 1; + + non_padding_node.rs = tor_malloc_zero(sizeof(routerstatus_t)); + non_padding_node.rs->pv.supports_padding = 0; +} + +static void +nodes_free(void) +{ + tor_free(padding_node.rs); + + tor_free(non_padding_node.rs); +} + +static const node_t * +node_get_by_id_mock(const char *identity_digest) +{ + if (identity_digest[0] == 1) { + return &padding_node; + } else if (identity_digest[0] == 0) { + return &non_padding_node; + } + + return NULL; +} + +static or_circuit_t * +new_fake_orcirc(channel_t *nchan, channel_t *pchan) +{ + or_circuit_t *orcirc = NULL; + circuit_t *circ = NULL; + crypt_path_t tmp_cpath; + char whatevs_key[CPATH_KEY_MATERIAL_LEN]; + + orcirc = tor_malloc_zero(sizeof(*orcirc)); + circ = &(orcirc->base_); + circ->magic = OR_CIRCUIT_MAGIC; + + //circ->n_chan = nchan; + circ->n_circ_id = get_unique_circ_id_by_chan(nchan); + circ->n_mux = NULL; /* ?? */ + cell_queue_init(&(circ->n_chan_cells)); + circ->n_hop = NULL; + circ->streams_blocked_on_n_chan = 0; + circ->streams_blocked_on_p_chan = 0; + circ->n_delete_pending = 0; + circ->p_delete_pending = 0; + circ->received_destroy = 0; + circ->state = CIRCUIT_STATE_OPEN; + circ->purpose = CIRCUIT_PURPOSE_OR; + circ->package_window = CIRCWINDOW_START_MAX; + circ->deliver_window = CIRCWINDOW_START_MAX; + circ->n_chan_create_cell = NULL; + + //orcirc->p_chan = pchan; + orcirc->p_circ_id = get_unique_circ_id_by_chan(pchan); + cell_queue_init(&(orcirc->p_chan_cells)); + + circuit_set_p_circid_chan(orcirc, orcirc->p_circ_id, pchan); + circuit_set_n_circid_chan(circ, circ->n_circ_id, nchan); + + memset(&tmp_cpath, 0, sizeof(tmp_cpath)); + if (circuit_init_cpath_crypto(&tmp_cpath, whatevs_key, + sizeof(whatevs_key), 0, 0)<0) { + log_warn(LD_BUG,"Circuit initialization failed"); + return NULL; + } + orcirc->crypto = tmp_cpath.crypto; + + return orcirc; +} + +void +free_fake_orcirc(circuit_t *circ) +{ + or_circuit_t *orcirc = TO_OR_CIRCUIT(circ); + + relay_crypto_clear(&orcirc->crypto); + + circpad_circuit_free_all_machineinfos(circ); + tor_free(circ); +} + +void +free_fake_origin_circuit(origin_circuit_t *circ) +{ + circpad_circuit_free_all_machineinfos(TO_CIRCUIT(circ)); + circuit_clear_cpath(circ); + tor_free(circ); +} + +void dummy_nop_timer(void); + +//static int dont_stop_libevent = 0; + +static circuit_t *client_side; +static circuit_t *relay_side; + +static int n_client_cells = 0; +static int n_relay_cells = 0; + +static int +circuit_package_relay_cell_mock(cell_t *cell, circuit_t *circ, + cell_direction_t cell_direction, + crypt_path_t *layer_hint, streamid_t on_stream, + const char *filename, int lineno); + +static void +circuitmux_attach_circuit_mock(circuitmux_t *cmux, circuit_t *circ, + cell_direction_t direction); + +static void +circuitmux_attach_circuit_mock(circuitmux_t *cmux, circuit_t *circ, + cell_direction_t direction) +{ + (void)cmux; + (void)circ; + (void)direction; + + return; +} + +static int +circuit_package_relay_cell_mock(cell_t *cell, circuit_t *circ, + cell_direction_t cell_direction, + crypt_path_t *layer_hint, streamid_t on_stream, + const char *filename, int lineno) +{ + (void)cell; (void)on_stream; (void)filename; (void)lineno; + + if (circ == client_side) { + if (cell->payload[0] == RELAY_COMMAND_PADDING_NEGOTIATE) { + // Deliver to relay + circpad_handle_padding_negotiate(relay_side, cell); + } else { + + int is_target_hop = circpad_padding_is_from_expected_hop(circ, + layer_hint); + tt_int_op(cell_direction, OP_EQ, CELL_DIRECTION_OUT); + tt_int_op(is_target_hop, OP_EQ, 1); + + // No need to pretend a padding cell was sent: This event is + // now emitted internally when the circuitpadding code sends them. + //circpad_cell_event_padding_sent(client_side); + + // Receive padding cell at middle + circpad_deliver_recognized_relay_cell_events(relay_side, + cell->payload[0], NULL); + } + n_client_cells++; + } else if (circ == relay_side) { + tt_int_op(cell_direction, OP_EQ, CELL_DIRECTION_IN); + + if (cell->payload[0] == RELAY_COMMAND_PADDING_NEGOTIATED) { + // XXX: blah need right layer_hint.. + if (deliver_negotiated) + circpad_handle_padding_negotiated(client_side, cell, + TO_ORIGIN_CIRCUIT(client_side) + ->cpath->next); + } else if (cell->payload[0] == RELAY_COMMAND_PADDING_NEGOTIATE) { + circpad_handle_padding_negotiate(client_side, cell); + } else { + // No need to pretend a padding cell was sent: This event is + // now emitted internally when the circuitpadding code sends them. + //circpad_cell_event_padding_sent(relay_side); + + // Receive padding cell at client + circpad_deliver_recognized_relay_cell_events(client_side, + cell->payload[0], + TO_ORIGIN_CIRCUIT(client_side)->cpath->next); + } + + n_relay_cells++; + } + + done: + timers_advance_and_run(1); + return 0; +} + +// Test reading and writing padding to strings (or options_t + consensus) +void +test_circuitpadding_serialize(void *arg) +{ + (void)arg; +} + +static signed_error_t +circpad_send_command_to_hop_mock(origin_circuit_t *circ, uint8_t hopnum, + uint8_t relay_command, const uint8_t *payload, + ssize_t payload_len) +{ + (void) circ; + (void) hopnum; + (void) relay_command; + (void) payload; + (void) payload_len; + return 0; +} + +void +test_circuitpadding_rtt(void *arg) +{ + /* Test Plan: + * + * 1. Test RTT measurement server side + * a. test usage of measured RTT + * 2. Test termination of RTT measurement + * a. test non-update of RTT + * 3. Test client side circuit and non-application of RTT.. + */ + circpad_delay_t rtt_estimate; + (void)arg; + + MOCK(circuitmux_attach_circuit, circuitmux_attach_circuit_mock); + MOCK(circpad_send_command_to_hop, circpad_send_command_to_hop_mock); + + dummy_channel.cmux = circuitmux_alloc(); + relay_side = TO_CIRCUIT(new_fake_orcirc(&dummy_channel, &dummy_channel)); + client_side = TO_CIRCUIT(origin_circuit_new()); + relay_side->purpose = CIRCUIT_PURPOSE_OR; + client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL; + + monotime_init(); + monotime_enable_test_mocking(); + monotime_set_mock_time_nsec(1*NSEC_PER_USEC); + monotime_coarse_set_mock_time_nsec(1*NSEC_PER_USEC); + curr_mocked_time = 1*NSEC_PER_USEC; + + timers_initialize(); + circpad_machines_init(); + helper_create_basic_machine(); + + MOCK(circuit_package_relay_cell, + circuit_package_relay_cell_mock); + + client_side->padding_machine[0] = &circ_client_machine; + client_side->padding_info[0] = circpad_circuit_machineinfo_new(client_side, + 0); + + relay_side->padding_machine[0] = &circ_client_machine; + relay_side->padding_info[0] = circpad_circuit_machineinfo_new(client_side,0); + + /* Test 1: Test measuring RTT */ + circpad_cell_event_nonpadding_received((circuit_t*)relay_side); + tt_int_op(relay_side->padding_info[0]->last_received_time_usec, OP_NE, 0); + + timers_advance_and_run(20); + + circpad_cell_event_nonpadding_sent((circuit_t*)relay_side); + tt_int_op(relay_side->padding_info[0]->last_received_time_usec, OP_EQ, 0); + + tt_int_op(relay_side->padding_info[0]->rtt_estimate_usec, OP_GE, 19000); + tt_int_op(relay_side->padding_info[0]->rtt_estimate_usec, OP_LE, 30000); + tt_int_op(circpad_histogram_bin_to_usec(relay_side->padding_info[0], 0), + OP_EQ, + relay_side->padding_info[0]->rtt_estimate_usec+ + circpad_machine_current_state( + relay_side->padding_info[0])->start_usec); + + circpad_cell_event_nonpadding_received((circuit_t*)relay_side); + circpad_cell_event_nonpadding_received((circuit_t*)relay_side); + tt_int_op(relay_side->padding_info[0]->last_received_time_usec, OP_NE, 0); + timers_advance_and_run(20); + circpad_cell_event_nonpadding_sent((circuit_t*)relay_side); + circpad_cell_event_nonpadding_sent((circuit_t*)relay_side); + tt_int_op(relay_side->padding_info[0]->last_received_time_usec, OP_EQ, 0); + + tt_int_op(relay_side->padding_info[0]->rtt_estimate_usec, OP_GE, 20000); + tt_int_op(relay_side->padding_info[0]->rtt_estimate_usec, OP_LE, 21000); + tt_int_op(circpad_histogram_bin_to_usec(relay_side->padding_info[0], 0), + OP_EQ, + relay_side->padding_info[0]->rtt_estimate_usec+ + circpad_machine_current_state( + relay_side->padding_info[0])->start_usec); + + /* Test 2: Termination of RTT measurement (from the previous test) */ + tt_int_op(relay_side->padding_info[0]->stop_rtt_update, OP_EQ, 1); + rtt_estimate = relay_side->padding_info[0]->rtt_estimate_usec; + + circpad_cell_event_nonpadding_received((circuit_t*)relay_side); + timers_advance_and_run(4); + circpad_cell_event_nonpadding_sent((circuit_t*)relay_side); + + tt_int_op(relay_side->padding_info[0]->rtt_estimate_usec, OP_EQ, + rtt_estimate); + tt_int_op(relay_side->padding_info[0]->last_received_time_usec, OP_EQ, 0); + tt_int_op(relay_side->padding_info[0]->stop_rtt_update, OP_EQ, 1); + tt_int_op(circpad_histogram_bin_to_usec(relay_side->padding_info[0], 0), + OP_EQ, + relay_side->padding_info[0]->rtt_estimate_usec+ + circpad_machine_current_state( + relay_side->padding_info[0])->start_usec); + + /* Test 3: Make sure client side machine properly ignores RTT */ + circpad_cell_event_nonpadding_received((circuit_t*)client_side); + tt_int_op(client_side->padding_info[0]->last_received_time_usec, OP_EQ, 0); + + timers_advance_and_run(20); + circpad_cell_event_nonpadding_sent((circuit_t*)client_side); + tt_int_op(client_side->padding_info[0]->last_received_time_usec, OP_EQ, 0); + + tt_int_op(client_side->padding_info[0]->rtt_estimate_usec, OP_EQ, 0); + tt_int_op(circpad_histogram_bin_to_usec(client_side->padding_info[0], 0), + OP_NE, client_side->padding_info[0]->rtt_estimate_usec); + tt_int_op(circpad_histogram_bin_to_usec(client_side->padding_info[0], 0), + OP_EQ, + circpad_machine_current_state( + client_side->padding_info[0])->start_usec); + done: + free_fake_orcirc(relay_side); + circuitmux_detach_all_circuits(dummy_channel.cmux, NULL); + circuitmux_free(dummy_channel.cmux); + timers_shutdown(); + monotime_disable_test_mocking(); + UNMOCK(circuit_package_relay_cell); + UNMOCK(circuitmux_attach_circuit); + tor_free(circ_client_machine.states); + + return; +} + +void +helper_create_basic_machine(void) +{ + /* Start, burst */ + circpad_machine_states_init(&circ_client_machine, 2); + + circ_client_machine.states[CIRCPAD_STATE_START]. + next_state[CIRCPAD_EVENT_NONPADDING_RECV] = CIRCPAD_STATE_BURST; + + circ_client_machine.states[CIRCPAD_STATE_BURST]. + next_state[CIRCPAD_EVENT_PADDING_RECV] = CIRCPAD_STATE_BURST; + circ_client_machine.states[CIRCPAD_STATE_BURST]. + next_state[CIRCPAD_EVENT_NONPADDING_RECV] = CIRCPAD_STATE_BURST; + + circ_client_machine.states[CIRCPAD_STATE_BURST]. + next_state[CIRCPAD_EVENT_NONPADDING_SENT] = CIRCPAD_STATE_CANCEL; + + // FIXME: Is this what we want? + circ_client_machine.states[CIRCPAD_STATE_BURST].token_removal = + CIRCPAD_TOKEN_REMOVAL_HIGHER; + + // FIXME: Tune this histogram + circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_len = 5; + circ_client_machine.states[CIRCPAD_STATE_BURST].start_usec = 500; + circ_client_machine.states[CIRCPAD_STATE_BURST].range_usec = 1000000; + circ_client_machine.states[CIRCPAD_STATE_BURST].histogram[0] = 1; + circ_client_machine.states[CIRCPAD_STATE_BURST].histogram[1] = 0; + circ_client_machine.states[CIRCPAD_STATE_BURST].histogram[2] = 2; + circ_client_machine.states[CIRCPAD_STATE_BURST].histogram[3] = 2; + circ_client_machine.states[CIRCPAD_STATE_BURST].histogram[4] = 2; + circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_total_tokens = 7; + circ_client_machine.states[CIRCPAD_STATE_BURST].use_rtt_estimate = 1; + + return; +} + +#define BIG_HISTOGRAM_LEN 10 + +/** Setup a machine with a big histogram */ +static void +helper_create_machine_with_big_histogram(circpad_removal_t removal_strategy) +{ + const int tokens_per_bin = 2; + + /* Start, burst */ + circpad_machine_states_init(&circ_client_machine, 2); + + circpad_state_t *burst_state = + &circ_client_machine.states[CIRCPAD_STATE_BURST]; + + circ_client_machine.states[CIRCPAD_STATE_START]. + next_state[CIRCPAD_EVENT_NONPADDING_RECV] = CIRCPAD_STATE_BURST; + + burst_state->next_state[CIRCPAD_EVENT_PADDING_RECV] = CIRCPAD_STATE_BURST; + burst_state->next_state[CIRCPAD_EVENT_NONPADDING_RECV] =CIRCPAD_STATE_BURST; + + burst_state->next_state[CIRCPAD_EVENT_NONPADDING_SENT] =CIRCPAD_STATE_CANCEL; + + burst_state->token_removal = CIRCPAD_TOKEN_REMOVAL_HIGHER; + + burst_state->histogram_len = BIG_HISTOGRAM_LEN; + burst_state->start_usec = 0; + burst_state->range_usec = 1000; + + int n_tokens = 0; + for (int i = 0; i < BIG_HISTOGRAM_LEN ; i++) { + burst_state->histogram[i] = tokens_per_bin; + n_tokens += tokens_per_bin; + } + + burst_state->histogram_total_tokens = n_tokens; + burst_state->length_dist.type = CIRCPAD_DIST_UNIFORM; + burst_state->length_dist.param1 = n_tokens; + burst_state->length_dist.param2 = n_tokens; + burst_state->max_length = n_tokens; + burst_state->length_includes_nonpadding = 1; + burst_state->use_rtt_estimate = 0; + burst_state->token_removal = removal_strategy; +} + +static circpad_decision_t +circpad_machine_schedule_padding_mock(circpad_machineinfo_t *mi) +{ + (void)mi; + return 0; +} + +static uint64_t +mock_monotime_absolute_usec(void) +{ + return 100; +} + +/** Test higher token removal strategy by bin */ +static void +test_circuitpadding_token_removal_higher(void *arg) +{ + circpad_machineinfo_t *mi; + (void)arg; + + /* Mock it up */ + MOCK(monotime_absolute_usec, mock_monotime_absolute_usec); + MOCK(circpad_machine_schedule_padding,circpad_machine_schedule_padding_mock); + + /* Setup test environment (time etc.) */ + client_side = (circuit_t *)origin_circuit_new(); + client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL; + monotime_enable_test_mocking(); + + /* Create test machine */ + helper_create_machine_with_big_histogram(CIRCPAD_TOKEN_REMOVAL_HIGHER); + client_side->padding_machine[0] = &circ_client_machine; + client_side->padding_info[0] = + circpad_circuit_machineinfo_new(client_side, 0); + + /* move the machine to the right state */ + circpad_cell_event_nonpadding_received((circuit_t*)client_side); + tt_int_op(client_side->padding_info[0]->current_state, OP_EQ, + CIRCPAD_STATE_BURST); + + /* Get the machine and setup tokens */ + mi = client_side->padding_info[0]; + tt_assert(mi); + + /*************************************************************************/ + + uint64_t current_time = monotime_absolute_usec(); + + /* Test left boundaries of each histogram bin: */ + const circpad_delay_t bin_left_bounds[] = + {0, 1, 7, 15, 31, 62, 125, 250, 500, CIRCPAD_DELAY_INFINITE}; + for (int i = 0; i < BIG_HISTOGRAM_LEN ; i++) { + tt_uint_op(bin_left_bounds[i], OP_EQ, + circpad_histogram_bin_to_usec(mi, i)); + } + + /* Check that all bins have two tokens right now */ + for (int i = 0; i < BIG_HISTOGRAM_LEN ; i++) { + tt_int_op(mi->histogram[i], OP_EQ, 2); + } + + /* This is the right order to remove tokens from this histogram. That is, we + * first remove tokens from the 4th bin since 57 usec is nearest to the 4th + * bin midpoint (31 + (62-31)/2 == 46). Then we remove from the 3rd bin for + * the same reason, then from the 5th, etc. */ + const int bin_removal_order[] = {4, 5, 6, 7, 8}; + unsigned i; + + /* Remove all tokens from all bins apart from the infinity bin */ + for (i = 0; i < sizeof(bin_removal_order)/sizeof(int) ; i++) { + int bin_to_remove = bin_removal_order[i]; + log_debug(LD_GENERAL, "Testing that %d attempt removes %d bin", + i, bin_to_remove); + + tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 2); + + mi->padding_scheduled_at_usec = current_time - 57; + circpad_machine_remove_token(mi); + + tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 1); + + mi->padding_scheduled_at_usec = current_time - 57; + circpad_machine_remove_token(mi); + + /* Test that we cleaned out this bin. Don't do this in the case of the last + bin since the tokens will get refilled */ + if (i != BIG_HISTOGRAM_LEN - 2) { + tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 0); + } + } + + /* Check that all lowe bins are not touched */ + for (i=0; i < 4 ; i++) { + tt_int_op(mi->histogram[i], OP_EQ, 2); + } + + /* Test below the lowest bin, for coverage */ + tt_int_op(client_side->padding_info[0]->current_state, OP_EQ, + CIRCPAD_STATE_BURST); + circ_client_machine.states[CIRCPAD_STATE_BURST].start_usec = 100; + mi->padding_scheduled_at_usec = current_time - 1; + circpad_machine_remove_token(mi); + tt_int_op(mi->histogram[0], OP_EQ, 1); + + done: + free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side)); + monotime_disable_test_mocking(); + tor_free(circ_client_machine.states); +} + +/** Test lower token removal strategy by bin */ +static void +test_circuitpadding_token_removal_lower(void *arg) +{ + circpad_machineinfo_t *mi; + (void)arg; + + /* Mock it up */ + MOCK(monotime_absolute_usec, mock_monotime_absolute_usec); + MOCK(circpad_machine_schedule_padding,circpad_machine_schedule_padding_mock); + + /* Setup test environment (time etc.) */ + client_side = (circuit_t *)origin_circuit_new(); + client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL; + monotime_enable_test_mocking(); + + /* Create test machine */ + helper_create_machine_with_big_histogram(CIRCPAD_TOKEN_REMOVAL_LOWER); + client_side->padding_machine[0] = &circ_client_machine; + client_side->padding_info[0] = + circpad_circuit_machineinfo_new(client_side, 0); + + /* move the machine to the right state */ + circpad_cell_event_nonpadding_received((circuit_t*)client_side); + tt_int_op(client_side->padding_info[0]->current_state, OP_EQ, + CIRCPAD_STATE_BURST); + + /* Get the machine and setup tokens */ + mi = client_side->padding_info[0]; + tt_assert(mi); + + /*************************************************************************/ + + uint64_t current_time = monotime_absolute_usec(); + + /* Test left boundaries of each histogram bin: */ + const circpad_delay_t bin_left_bounds[] = + {0, 1, 7, 15, 31, 62, 125, 250, 500, CIRCPAD_DELAY_INFINITE}; + for (int i = 0; i < BIG_HISTOGRAM_LEN ; i++) { + tt_uint_op(bin_left_bounds[i], OP_EQ, + circpad_histogram_bin_to_usec(mi, i)); + } + + /* Check that all bins have two tokens right now */ + for (int i = 0; i < BIG_HISTOGRAM_LEN ; i++) { + tt_int_op(mi->histogram[i], OP_EQ, 2); + } + + /* This is the right order to remove tokens from this histogram. That is, we + * first remove tokens from the 4th bin since 57 usec is nearest to the 4th + * bin midpoint (31 + (62-31)/2 == 46). Then we remove from the 3rd bin for + * the same reason, then from the 5th, etc. */ + const int bin_removal_order[] = {4, 3, 2, 1, 0}; + unsigned i; + + /* Remove all tokens from all bins apart from the infinity bin */ + for (i = 0; i < sizeof(bin_removal_order)/sizeof(int) ; i++) { + int bin_to_remove = bin_removal_order[i]; + log_debug(LD_GENERAL, "Testing that %d attempt removes %d bin", + i, bin_to_remove); + + tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 2); + + mi->padding_scheduled_at_usec = current_time - 57; + circpad_machine_remove_token(mi); + + tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 1); + + mi->padding_scheduled_at_usec = current_time - 57; + circpad_machine_remove_token(mi); + + /* Test that we cleaned out this bin. Don't do this in the case of the last + bin since the tokens will get refilled */ + if (i != BIG_HISTOGRAM_LEN - 2) { + tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 0); + } + } + + /* Check that all higher bins are untouched */ + for (i = 5; i < BIG_HISTOGRAM_LEN ; i++) { + tt_int_op(mi->histogram[i], OP_EQ, 2); + } + + /* Test above the highest bin, for coverage */ + tt_int_op(client_side->padding_info[0]->current_state, OP_EQ, + CIRCPAD_STATE_BURST); + circ_client_machine.states[CIRCPAD_STATE_BURST].start_usec = 100; + mi->padding_scheduled_at_usec = current_time - 29202; + circpad_machine_remove_token(mi); + tt_int_op(mi->histogram[BIG_HISTOGRAM_LEN-2], OP_EQ, 1); + + done: + free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side)); + monotime_disable_test_mocking(); + tor_free(circ_client_machine.states); +} + +/** Test closest token removal strategy by bin */ +static void +test_circuitpadding_closest_token_removal(void *arg) +{ + circpad_machineinfo_t *mi; + (void)arg; + + /* Mock it up */ + MOCK(monotime_absolute_usec, mock_monotime_absolute_usec); + MOCK(circpad_machine_schedule_padding,circpad_machine_schedule_padding_mock); + + /* Setup test environment (time etc.) */ + client_side = (circuit_t *)origin_circuit_new(); + client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL; + monotime_enable_test_mocking(); + + /* Create test machine */ + helper_create_machine_with_big_histogram(CIRCPAD_TOKEN_REMOVAL_CLOSEST); + client_side->padding_machine[0] = &circ_client_machine; + client_side->padding_info[0] = + circpad_circuit_machineinfo_new(client_side, 0); + + /* move the machine to the right state */ + circpad_cell_event_nonpadding_received((circuit_t*)client_side); + tt_int_op(client_side->padding_info[0]->current_state, OP_EQ, + CIRCPAD_STATE_BURST); + + /* Get the machine and setup tokens */ + mi = client_side->padding_info[0]; + tt_assert(mi); + + /*************************************************************************/ + + uint64_t current_time = monotime_absolute_usec(); + + /* Test left boundaries of each histogram bin: */ + const circpad_delay_t bin_left_bounds[] = + {0, 1, 7, 15, 31, 62, 125, 250, 500, CIRCPAD_DELAY_INFINITE}; + for (int i = 0; i < BIG_HISTOGRAM_LEN ; i++) { + tt_uint_op(bin_left_bounds[i], OP_EQ, + circpad_histogram_bin_to_usec(mi, i)); + } + + /* Check that all bins have two tokens right now */ + for (int i = 0; i < BIG_HISTOGRAM_LEN ; i++) { + tt_int_op(mi->histogram[i], OP_EQ, 2); + } + + /* This is the right order to remove tokens from this histogram. That is, we + * first remove tokens from the 4th bin since 57 usec is nearest to the 4th + * bin midpoint (31 + (62-31)/2 == 46). Then we remove from the 3rd bin for + * the same reason, then from the 5th, etc. */ + const int bin_removal_order[] = {4, 3, 5, 2, 6, 1, 7, 0, 8, 9}; + + /* Remove all tokens from all bins apart from the infinity bin */ + for (int i = 0; i < BIG_HISTOGRAM_LEN-1 ; i++) { + int bin_to_remove = bin_removal_order[i]; + log_debug(LD_GENERAL, "Testing that %d attempt removes %d bin", + i, bin_to_remove); + + tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 2); + + mi->padding_scheduled_at_usec = current_time - 57; + circpad_machine_remove_token(mi); + + tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 1); + + mi->padding_scheduled_at_usec = current_time - 57; + circpad_machine_remove_token(mi); + + /* Test that we cleaned out this bin. Don't do this in the case of the last + bin since the tokens will get refilled */ + if (i != BIG_HISTOGRAM_LEN - 2) { + tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 0); + } + } + + /* Check that all bins have been refilled */ + for (int i = 0; i < BIG_HISTOGRAM_LEN ; i++) { + tt_int_op(mi->histogram[i], OP_EQ, 2); + } + + /* Test below the lowest bin, for coverage */ + tt_int_op(client_side->padding_info[0]->current_state, OP_EQ, + CIRCPAD_STATE_BURST); + circ_client_machine.states[CIRCPAD_STATE_BURST].start_usec = 100; + mi->padding_scheduled_at_usec = current_time - 102; + mi->histogram[0] = 0; + circpad_machine_remove_token(mi); + tt_int_op(mi->histogram[1], OP_EQ, 1); + + /* Test above the highest bin, for coverage */ + tt_int_op(client_side->padding_info[0]->current_state, OP_EQ, + CIRCPAD_STATE_BURST); + circ_client_machine.states[CIRCPAD_STATE_BURST].start_usec = 100; + mi->padding_scheduled_at_usec = current_time - 29202; + circpad_machine_remove_token(mi); + tt_int_op(mi->histogram[BIG_HISTOGRAM_LEN-2], OP_EQ, 1); + + done: + free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side)); + monotime_disable_test_mocking(); + tor_free(circ_client_machine.states); +} + +/** Test closest token removal strategy with usec */ +static void +test_circuitpadding_closest_token_removal_usec(void *arg) +{ + circpad_machineinfo_t *mi; + (void)arg; + + /* Mock it up */ + MOCK(monotime_absolute_usec, mock_monotime_absolute_usec); + MOCK(circpad_machine_schedule_padding,circpad_machine_schedule_padding_mock); + + /* Setup test environment (time etc.) */ + client_side = (circuit_t *)origin_circuit_new(); + client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL; + monotime_enable_test_mocking(); + + /* Create test machine */ + helper_create_machine_with_big_histogram(CIRCPAD_TOKEN_REMOVAL_CLOSEST_USEC); + client_side->padding_machine[0] = &circ_client_machine; + client_side->padding_info[0] = + circpad_circuit_machineinfo_new(client_side, 0); + + /* move the machine to the right state */ + circpad_cell_event_nonpadding_received((circuit_t*)client_side); + tt_int_op(client_side->padding_info[0]->current_state, OP_EQ, + CIRCPAD_STATE_BURST); + + /* Get the machine and setup tokens */ + mi = client_side->padding_info[0]; + tt_assert(mi); + + /*************************************************************************/ + + uint64_t current_time = monotime_absolute_usec(); + + /* Test left boundaries of each histogram bin: */ + const circpad_delay_t bin_left_bounds[] = + {0, 1, 7, 15, 31, 62, 125, 250, 500, CIRCPAD_DELAY_INFINITE}; + for (int i = 0; i < BIG_HISTOGRAM_LEN ; i++) { + tt_uint_op(bin_left_bounds[i], OP_EQ, + circpad_histogram_bin_to_usec(mi, i)); + } + + /* XXX we want to test remove_token_exact and + circpad_machine_remove_closest_token() with usec */ + + /* Check that all bins have two tokens right now */ + for (int i = 0; i < BIG_HISTOGRAM_LEN ; i++) { + tt_int_op(mi->histogram[i], OP_EQ, 2); + } + + /* This is the right order to remove tokens from this histogram. That is, we + * first remove tokens from the 4th bin since 57 usec is nearest to the 4th + * bin midpoint (31 + (62-31)/2 == 46). Then we remove from the 3rd bin for + * the same reason, then from the 5th, etc. */ + const int bin_removal_order[] = {4, 3, 5, 2, 1, 0, 6, 7, 8, 9}; + + /* Remove all tokens from all bins apart from the infinity bin */ + for (int i = 0; i < BIG_HISTOGRAM_LEN-1 ; i++) { + int bin_to_remove = bin_removal_order[i]; + log_debug(LD_GENERAL, "Testing that %d attempt removes %d bin", + i, bin_to_remove); + + tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 2); + + mi->padding_scheduled_at_usec = current_time - 57; + circpad_machine_remove_token(mi); + + tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 1); + + mi->padding_scheduled_at_usec = current_time - 57; + circpad_machine_remove_token(mi); + + /* Test that we cleaned out this bin. Don't do this in the case of the last + bin since the tokens will get refilled */ + if (i != BIG_HISTOGRAM_LEN - 2) { + tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 0); + } + } + + /* Check that all bins have been refilled */ + for (int i = 0; i < BIG_HISTOGRAM_LEN ; i++) { + tt_int_op(mi->histogram[i], OP_EQ, 2); + } + + /* Test below the lowest bin, for coverage */ + tt_int_op(client_side->padding_info[0]->current_state, OP_EQ, + CIRCPAD_STATE_BURST); + circ_client_machine.states[CIRCPAD_STATE_BURST].start_usec = 100; + mi->padding_scheduled_at_usec = current_time - 102; + mi->histogram[0] = 0; + circpad_machine_remove_token(mi); + tt_int_op(mi->histogram[1], OP_EQ, 1); + + /* Test above the highest bin, for coverage */ + tt_int_op(client_side->padding_info[0]->current_state, OP_EQ, + CIRCPAD_STATE_BURST); + circ_client_machine.states[CIRCPAD_STATE_BURST].start_usec = 100; + mi->padding_scheduled_at_usec = current_time - 29202; + circpad_machine_remove_token(mi); + tt_int_op(mi->histogram[BIG_HISTOGRAM_LEN-2], OP_EQ, 1); + + done: + free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side)); + monotime_disable_test_mocking(); + tor_free(circ_client_machine.states); +} + +/** Test closest token removal strategy with usec */ +static void +test_circuitpadding_token_removal_exact(void *arg) +{ + circpad_machineinfo_t *mi; + (void)arg; + + /* Mock it up */ + MOCK(monotime_absolute_usec, mock_monotime_absolute_usec); + MOCK(circpad_machine_schedule_padding,circpad_machine_schedule_padding_mock); + + /* Setup test environment (time etc.) */ + client_side = (circuit_t *)origin_circuit_new(); + client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL; + monotime_enable_test_mocking(); + + /* Create test machine */ + helper_create_machine_with_big_histogram(CIRCPAD_TOKEN_REMOVAL_EXACT); + client_side->padding_machine[0] = &circ_client_machine; + client_side->padding_info[0] = + circpad_circuit_machineinfo_new(client_side, 0); + + /* move the machine to the right state */ + circpad_cell_event_nonpadding_received((circuit_t*)client_side); + tt_int_op(client_side->padding_info[0]->current_state, OP_EQ, + CIRCPAD_STATE_BURST); + + /* Get the machine and setup tokens */ + mi = client_side->padding_info[0]; + tt_assert(mi); + + /**********************************************************************/ + uint64_t current_time = monotime_absolute_usec(); + + /* Ensure that we will clear out bin #4 with this usec */ + mi->padding_scheduled_at_usec = current_time - 57; + tt_int_op(mi->histogram[4], OP_EQ, 2); + circpad_machine_remove_token(mi); + mi->padding_scheduled_at_usec = current_time - 57; + tt_int_op(mi->histogram[4], OP_EQ, 1); + circpad_machine_remove_token(mi); + tt_int_op(mi->histogram[4], OP_EQ, 0); + + /* Ensure that we will not remove any other tokens even tho we try to, since + * this is what the exact strategy dictates */ + mi->padding_scheduled_at_usec = current_time - 57; + circpad_machine_remove_token(mi); + for (int i = 0; i < BIG_HISTOGRAM_LEN ; i++) { + if (i != 4) { + tt_int_op(mi->histogram[i], OP_EQ, 2); + } + } + + done: + free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side)); + monotime_disable_test_mocking(); + tor_free(circ_client_machine.states); +} + +#undef BIG_HISTOGRAM_LEN + +void +test_circuitpadding_tokens(void *arg) +{ + const circpad_state_t *state; + circpad_machineinfo_t *mi; + (void)arg; + + /** Test plan: + * + * 1. Test symmetry between bin_to_usec and usec_to_bin + * a. Test conversion + * b. Test edge transitions (lower, upper) + * 2. Test remove higher on an empty bin + * a. Normal bin + * b. Infinity bin + * c. Bin 0 + * d. No higher + * 3. Test remove lower + * a. Normal bin + * b. Bin 0 + * c. No lower + * 4. Test remove closest + * a. Closest lower + * b. Closest higher + * c. Closest 0 + * d. Closest Infinity + */ + client_side = TO_CIRCUIT(origin_circuit_new()); + client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL; + + monotime_init(); + monotime_enable_test_mocking(); + monotime_set_mock_time_nsec(1*NSEC_PER_USEC); + monotime_coarse_set_mock_time_nsec(1*NSEC_PER_USEC); + curr_mocked_time = 1*NSEC_PER_USEC; + + timers_initialize(); + + helper_create_basic_machine(); + client_side->padding_machine[0] = &circ_client_machine; + client_side->padding_info[0] = circpad_circuit_machineinfo_new(client_side, + 0); + + mi = client_side->padding_info[0]; + + // Pretend a non-padding cell was sent + // XXX: This messes us up.. Padding gets scheduled.. + circpad_cell_event_nonpadding_sent((circuit_t*)client_side); + circpad_cell_event_nonpadding_received((circuit_t*)client_side); + /* We have to save the infinity bin because one inf delay + * could have been chosen when we transition to burst */ + circpad_hist_token_t inf_bin = mi->histogram[4]; + + tt_int_op(client_side->padding_info[0]->current_state, OP_EQ, + CIRCPAD_STATE_BURST); + + state = circpad_machine_current_state(client_side->padding_info[0]); + + // Test 0: convert bin->usec->bin + // Bin 0+1 have different semantics + for (int bin = 0; bin < 2; bin++) { + circpad_delay_t usec = + circpad_histogram_bin_to_usec(client_side->padding_info[0], bin); + int bin2 = circpad_histogram_usec_to_bin(client_side->padding_info[0], + usec); + tt_int_op(bin, OP_EQ, bin2); + } + for (int bin = 2; bin < state->histogram_len-1; bin++) { + circpad_delay_t usec = + circpad_histogram_bin_to_usec(client_side->padding_info[0], bin); + int bin2 = circpad_histogram_usec_to_bin(client_side->padding_info[0], + usec); + tt_int_op(bin, OP_EQ, bin2); + /* Verify we round down */ + bin2 = circpad_histogram_usec_to_bin(client_side->padding_info[0], + usec+3); + tt_int_op(bin, OP_EQ, bin2); + + bin2 = circpad_histogram_usec_to_bin(client_side->padding_info[0], + usec-1); + tt_int_op(bin, OP_EQ, bin2+1); + } + + // Test 1: converting usec->bin->usec->bin + // Bin 0+1 have different semantics. + for (circpad_delay_t i = 0; i <= state->start_usec+1; i++) { + int bin = circpad_histogram_usec_to_bin(client_side->padding_info[0], + i); + circpad_delay_t usec = + circpad_histogram_bin_to_usec(client_side->padding_info[0], bin); + int bin2 = circpad_histogram_usec_to_bin(client_side->padding_info[0], + usec); + tt_int_op(bin, OP_EQ, bin2); + tt_int_op(i, OP_LE, usec); + } + for (circpad_delay_t i = state->start_usec+1; + i <= state->start_usec + state->range_usec; i++) { + int bin = circpad_histogram_usec_to_bin(client_side->padding_info[0], + i); + circpad_delay_t usec = + circpad_histogram_bin_to_usec(client_side->padding_info[0], bin); + int bin2 = circpad_histogram_usec_to_bin(client_side->padding_info[0], + usec); + tt_int_op(bin, OP_EQ, bin2); + tt_int_op(i, OP_GE, usec); + } + + /* 2.a. Normal higher bin */ + { + tt_int_op(mi->histogram[2], OP_EQ, 2); + tt_int_op(mi->histogram[3], OP_EQ, 2); + circpad_machine_remove_higher_token(mi, + circpad_histogram_bin_to_usec(mi, 2)+1); + tt_int_op(mi->histogram[3], OP_EQ, 2); + tt_int_op(mi->histogram[2], OP_EQ, 1); + + circpad_machine_remove_higher_token(mi, + circpad_histogram_bin_to_usec(mi, 2)+1); + tt_int_op(mi->histogram[2], OP_EQ, 0); + + tt_int_op(mi->histogram[3], OP_EQ, 2); + circpad_machine_remove_higher_token(mi, + circpad_histogram_bin_to_usec(mi, 2)+1); + circpad_machine_remove_higher_token(mi, + circpad_histogram_bin_to_usec(mi, 2)+1); + tt_int_op(mi->histogram[3], OP_EQ, 0); + circpad_machine_remove_higher_token(mi, + circpad_histogram_bin_to_usec(mi, 2)+1); + tt_int_op(mi->histogram[3], OP_EQ, 0); + } + + /* 2.b. Higher Infinity bin */ + { + tt_int_op(mi->histogram[4], OP_EQ, inf_bin); + circpad_machine_remove_higher_token(mi, + circpad_histogram_bin_to_usec(mi, 2)+1); + tt_int_op(mi->histogram[4], OP_EQ, inf_bin); + + /* Test past the infinity bin */ + circpad_machine_remove_higher_token(mi, + circpad_histogram_bin_to_usec(mi, 5)+1000000); + + tt_int_op(mi->histogram[4], OP_EQ, inf_bin); + } + + /* 2.c. Bin 0 */ + { + tt_int_op(mi->histogram[0], OP_EQ, 1); + circpad_machine_remove_higher_token(mi, + state->start_usec/2); + tt_int_op(mi->histogram[0], OP_EQ, 0); + } + + /* Drain the infinity bin and cause a refill */ + while (inf_bin != 0) { + tt_int_op(mi->histogram[4], OP_EQ, inf_bin); + circpad_cell_event_nonpadding_received((circuit_t*)client_side); + inf_bin--; + } + + circpad_cell_event_nonpadding_sent((circuit_t*)client_side); + + // We should have refilled here. + tt_int_op(mi->histogram[4], OP_EQ, 2); + + /* 3.a. Bin 0 */ + { + tt_int_op(mi->histogram[0], OP_EQ, 1); + circpad_machine_remove_higher_token(mi, + state->start_usec/2); + tt_int_op(mi->histogram[0], OP_EQ, 0); + } + + /* 3.b. Test remove lower normal bin */ + { + tt_int_op(mi->histogram[3], OP_EQ, 2); + circpad_machine_remove_lower_token(mi, + circpad_histogram_bin_to_usec(mi, 3)+1); + circpad_machine_remove_lower_token(mi, + circpad_histogram_bin_to_usec(mi, 3)+1); + tt_int_op(mi->histogram[3], OP_EQ, 0); + tt_int_op(mi->histogram[2], OP_EQ, 2); + circpad_machine_remove_lower_token(mi, + circpad_histogram_bin_to_usec(mi, 3)+1); + circpad_machine_remove_lower_token(mi, + circpad_histogram_bin_to_usec(mi, 3)+1); + /* 3.c. No lower */ + circpad_machine_remove_lower_token(mi, + circpad_histogram_bin_to_usec(mi, 3)+1); + tt_int_op(mi->histogram[2], OP_EQ, 0); + } + + /* 4. Test remove closest + * a. Closest lower + * b. Closest higher + * c. Closest 0 + * d. Closest Infinity + */ + circpad_machine_setup_tokens(mi); + tt_int_op(mi->histogram[2], OP_EQ, 2); + circpad_machine_remove_closest_token(mi, + circpad_histogram_bin_to_usec(mi, 2)+1, 0); + circpad_machine_remove_closest_token(mi, + circpad_histogram_bin_to_usec(mi, 2)+1, 0); + tt_int_op(mi->histogram[2], OP_EQ, 0); + tt_int_op(mi->histogram[3], OP_EQ, 2); + circpad_machine_remove_closest_token(mi, + circpad_histogram_bin_to_usec(mi, 2)+1, 0); + circpad_machine_remove_closest_token(mi, + circpad_histogram_bin_to_usec(mi, 2)+1, 0); + tt_int_op(mi->histogram[3], OP_EQ, 0); + tt_int_op(mi->histogram[0], OP_EQ, 1); + circpad_machine_remove_closest_token(mi, + circpad_histogram_bin_to_usec(mi, 2)+1, 0); + tt_int_op(mi->histogram[0], OP_EQ, 0); + tt_int_op(mi->histogram[4], OP_EQ, 2); + circpad_machine_remove_closest_token(mi, + circpad_histogram_bin_to_usec(mi, 2)+1, 0); + tt_int_op(mi->histogram[4], OP_EQ, 2); + + /* 5. Test remove closest usec + * a. Closest 0 + * b. Closest lower (below midpoint) + * c. Closest higher (above midpoint) + * d. Closest Infinity + */ + circpad_machine_setup_tokens(mi); + + tt_int_op(mi->histogram[0], OP_EQ, 1); + circpad_machine_remove_closest_token(mi, + circpad_histogram_bin_to_usec(mi, 0)/3, 1); + tt_int_op(mi->histogram[0], OP_EQ, 0); + tt_int_op(mi->histogram[2], OP_EQ, 2); + circpad_machine_remove_closest_token(mi, + circpad_histogram_bin_to_usec(mi, 0)/3, 1); + circpad_machine_remove_closest_token(mi, + circpad_histogram_bin_to_usec(mi, 0)/3, 1); + tt_int_op(mi->histogram[2], OP_EQ, 0); + tt_int_op(mi->histogram[3], OP_EQ, 2); + circpad_machine_remove_closest_token(mi, + circpad_histogram_bin_to_usec(mi, 4), 1); + circpad_machine_remove_closest_token(mi, + circpad_histogram_bin_to_usec(mi, 4), 1); + tt_int_op(mi->histogram[3], OP_EQ, 0); + tt_int_op(mi->histogram[4], OP_EQ, 2); + circpad_machine_remove_closest_token(mi, + circpad_histogram_bin_to_usec(mi, 4), 1); + circpad_machine_remove_closest_token(mi, + circpad_histogram_bin_to_usec(mi, 4), 1); + tt_int_op(mi->histogram[4], OP_EQ, 2); + + // XXX: Need more coverage of the actual usec branches + + done: + free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side)); + monotime_disable_test_mocking(); + tor_free(circ_client_machine.states); +} + +void +test_circuitpadding_wronghop(void *arg) +{ + /** + * Test plan: + * 1. Padding sent from hop 1 and 3 to client + * 2. Send negotiated from hop 1 and 3 to client + * 3. Garbled negotiated cell + * 4. Padding negotiate sent to client + * 5. Send negotiate stop command for unknown machine + * 6. Send negotiated to relay + * 7. Garbled padding negotiate cell + */ + (void)arg; + uint32_t read_bw = 0, overhead_bw = 0; + cell_t cell; + signed_error_t ret; + origin_circuit_t *orig_client; + + MOCK(circuitmux_attach_circuit, circuitmux_attach_circuit_mock); + + client_side = (circuit_t *)origin_circuit_new(); + dummy_channel.cmux = circuitmux_alloc(); + relay_side = (circuit_t *)new_fake_orcirc(&dummy_channel, + &dummy_channel); + orig_client = TO_ORIGIN_CIRCUIT(client_side); + + relay_side->purpose = CIRCUIT_PURPOSE_OR; + client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL; + nodes_init(); + + monotime_init(); + monotime_enable_test_mocking(); + monotime_set_mock_time_nsec(1*TOR_NSEC_PER_USEC); + monotime_coarse_set_mock_time_nsec(1*TOR_NSEC_PER_USEC); + curr_mocked_time = 1*TOR_NSEC_PER_USEC; + + timers_initialize(); + circpad_machines_init(); + + MOCK(node_get_by_id, + node_get_by_id_mock); + + MOCK(circuit_package_relay_cell, + circuit_package_relay_cell_mock); + + /* Build three hops */ + simulate_single_hop_extend(client_side, relay_side, 1); + simulate_single_hop_extend(client_side, relay_side, 1); + simulate_single_hop_extend(client_side, relay_side, 1); + + /* verify padding was negotiated */ + tt_ptr_op(relay_side->padding_machine[0], OP_NE, NULL); + tt_ptr_op(relay_side->padding_info[0], OP_NE, NULL); + + /* verify echo was sent */ + tt_int_op(n_relay_cells, OP_EQ, 1); + tt_int_op(n_client_cells, OP_EQ, 1); + + read_bw = orig_client->n_delivered_read_circ_bw; + overhead_bw = orig_client->n_overhead_read_circ_bw; + + /* 1. Test padding from first and third hop */ + circpad_deliver_recognized_relay_cell_events(client_side, + RELAY_COMMAND_DROP, + TO_ORIGIN_CIRCUIT(client_side)->cpath); + tt_int_op(read_bw, OP_EQ, + orig_client->n_delivered_read_circ_bw); + tt_int_op(overhead_bw, OP_EQ, + orig_client->n_overhead_read_circ_bw); + + circpad_deliver_recognized_relay_cell_events(client_side, + RELAY_COMMAND_DROP, + TO_ORIGIN_CIRCUIT(client_side)->cpath->next->next); + tt_int_op(read_bw, OP_EQ, + orig_client->n_delivered_read_circ_bw); + tt_int_op(overhead_bw, OP_EQ, + orig_client->n_overhead_read_circ_bw); + + circpad_deliver_recognized_relay_cell_events(client_side, + RELAY_COMMAND_DROP, + TO_ORIGIN_CIRCUIT(client_side)->cpath->next); + tt_int_op(read_bw, OP_EQ, + orig_client->n_delivered_read_circ_bw); + tt_int_op(overhead_bw, OP_LT, + orig_client->n_overhead_read_circ_bw); + + /* 2. Test padding negotiated not handled from hops 1,3 */ + ret = circpad_handle_padding_negotiated(client_side, &cell, + TO_ORIGIN_CIRCUIT(client_side)->cpath); + tt_int_op(ret, OP_EQ, -1); + + ret = circpad_handle_padding_negotiated(client_side, &cell, + TO_ORIGIN_CIRCUIT(client_side)->cpath->next->next); + tt_int_op(ret, OP_EQ, -1); + + /* 3. Garbled negotiated cell */ + memset(&cell, 255, sizeof(cell)); + ret = circpad_handle_padding_negotiated(client_side, &cell, + TO_ORIGIN_CIRCUIT(client_side)->cpath->next); + tt_int_op(ret, OP_EQ, -1); + + /* 4. Test that negotiate is dropped at origin */ + read_bw = orig_client->n_delivered_read_circ_bw; + overhead_bw = orig_client->n_overhead_read_circ_bw; + relay_send_command_from_edge(0, relay_side, + RELAY_COMMAND_PADDING_NEGOTIATE, + (void*)cell.payload, + (size_t)3, NULL); + tt_int_op(read_bw, OP_EQ, + orig_client->n_delivered_read_circ_bw); + tt_int_op(overhead_bw, OP_EQ, + orig_client->n_overhead_read_circ_bw); + + tt_int_op(n_relay_cells, OP_EQ, 2); + tt_int_op(n_client_cells, OP_EQ, 1); + + /* 5. Test that asking to stop the wrong machine does nothing */ + circpad_negotiate_padding(TO_ORIGIN_CIRCUIT(client_side), + 255, 2, CIRCPAD_COMMAND_STOP); + tt_ptr_op(client_side->padding_machine[0], OP_NE, NULL); + tt_ptr_op(client_side->padding_info[0], OP_NE, NULL); + tt_ptr_op(relay_side->padding_machine[0], OP_NE, NULL); + tt_ptr_op(relay_side->padding_info[0], OP_NE, NULL); + tt_int_op(n_relay_cells, OP_EQ, 3); + tt_int_op(n_client_cells, OP_EQ, 2); + + /* 6. Sending negotiated command to relay does nothing */ + ret = circpad_handle_padding_negotiated(relay_side, &cell, NULL); + tt_int_op(ret, OP_EQ, -1); + + /* 7. Test garbled negotated cell (bad command 255) */ + memset(&cell, 0, sizeof(cell)); + ret = circpad_handle_padding_negotiate(relay_side, &cell); + tt_int_op(ret, OP_EQ, -1); + tt_int_op(n_client_cells, OP_EQ, 2); + + /* Test 2: Test no padding */ + free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side)); + free_fake_orcirc(relay_side); + + client_side = (circuit_t *)origin_circuit_new(); + relay_side = (circuit_t *)new_fake_orcirc(&dummy_channel, + &dummy_channel); + relay_side->purpose = CIRCUIT_PURPOSE_OR; + client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL; + + simulate_single_hop_extend(client_side, relay_side, 1); + simulate_single_hop_extend(client_side, relay_side, 0); + + /* verify no padding was negotiated */ + tt_ptr_op(relay_side->padding_machine[0], OP_EQ, NULL); + tt_ptr_op(client_side->padding_machine[0], OP_EQ, NULL); + tt_int_op(n_relay_cells, OP_EQ, 3); + tt_int_op(n_client_cells, OP_EQ, 2); + + /* verify no echo was sent */ + tt_int_op(n_relay_cells, OP_EQ, 3); + tt_int_op(n_client_cells, OP_EQ, 2); + + /* Finish circuit */ + simulate_single_hop_extend(client_side, relay_side, 1); + + /* Spoof padding negotiated on circuit with no padding */ + circpad_padding_negotiated(relay_side, + CIRCPAD_MACHINE_CIRC_SETUP, + CIRCPAD_COMMAND_START, + CIRCPAD_RESPONSE_OK); + + /* verify no padding was negotiated */ + tt_ptr_op(relay_side->padding_machine[0], OP_EQ, NULL); + tt_ptr_op(client_side->padding_machine[0], OP_EQ, NULL); + + circpad_padding_negotiated(relay_side, + CIRCPAD_MACHINE_CIRC_SETUP, + CIRCPAD_COMMAND_START, + CIRCPAD_RESPONSE_ERR); + + /* verify no padding was negotiated */ + tt_ptr_op(relay_side->padding_machine[0], OP_EQ, NULL); + tt_ptr_op(client_side->padding_machine[0], OP_EQ, NULL); + + done: + free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side)); + free_fake_orcirc(relay_side); + circuitmux_detach_all_circuits(dummy_channel.cmux, NULL); + circuitmux_free(dummy_channel.cmux); + monotime_disable_test_mocking(); + UNMOCK(node_get_by_id); + UNMOCK(circuit_package_relay_cell); + UNMOCK(circuitmux_attach_circuit); + nodes_free(); +} + +void +test_circuitpadding_negotiation(void *arg) +{ + /** + * Test plan: + * 1. Test circuit where padding is supported by middle + * a. Make sure padding negotiation is sent + * b. Test padding negotiation delivery and parsing + * 2. Test circuit where padding is unsupported by middle + * a. Make sure padding negotiation is not sent + * 3. Test failure to negotiate a machine due to desync. + */ + (void)arg; + + MOCK(circuitmux_attach_circuit, circuitmux_attach_circuit_mock); + + client_side = TO_CIRCUIT(origin_circuit_new()); + dummy_channel.cmux = circuitmux_alloc(); + relay_side = TO_CIRCUIT(new_fake_orcirc(&dummy_channel, &dummy_channel)); + + relay_side->purpose = CIRCUIT_PURPOSE_OR; + client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL; + nodes_init(); + + monotime_init(); + monotime_enable_test_mocking(); + monotime_set_mock_time_nsec(1*NSEC_PER_USEC); + monotime_coarse_set_mock_time_nsec(1*NSEC_PER_USEC); + curr_mocked_time = 1*NSEC_PER_USEC; + + timers_initialize(); + circpad_machines_init(); + + MOCK(node_get_by_id, + node_get_by_id_mock); + + MOCK(circuit_package_relay_cell, + circuit_package_relay_cell_mock); + + /* Build two hops */ + simulate_single_hop_extend(client_side, relay_side, 1); + simulate_single_hop_extend(client_side, relay_side, 1); + + /* verify padding was negotiated */ + tt_ptr_op(relay_side->padding_machine[0], OP_NE, NULL); + tt_ptr_op(relay_side->padding_info[0], OP_NE, NULL); + + /* verify echo was sent */ + tt_int_op(n_relay_cells, OP_EQ, 1); + tt_int_op(n_client_cells, OP_EQ, 1); + + /* Finish circuit */ + simulate_single_hop_extend(client_side, relay_side, 1); + + /* Test 2: Test no padding */ + free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side)); + free_fake_orcirc(relay_side); + + client_side = TO_CIRCUIT(origin_circuit_new()); + relay_side = TO_CIRCUIT(new_fake_orcirc(&dummy_channel, &dummy_channel)); + relay_side->purpose = CIRCUIT_PURPOSE_OR; + client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL; + + simulate_single_hop_extend(client_side, relay_side, 1); + simulate_single_hop_extend(client_side, relay_side, 0); + + /* verify no padding was negotiated */ + tt_ptr_op(client_side->padding_machine[0], OP_EQ, NULL); + tt_ptr_op(relay_side->padding_machine[0], OP_EQ, NULL); + tt_int_op(n_relay_cells, OP_EQ, 1); + tt_int_op(n_client_cells, OP_EQ, 1); + + /* verify no echo was sent */ + tt_int_op(n_relay_cells, OP_EQ, 1); + tt_int_op(n_client_cells, OP_EQ, 1); + + /* Finish circuit */ + simulate_single_hop_extend(client_side, relay_side, 1); + + /* Force negotiate padding. */ + circpad_negotiate_padding(TO_ORIGIN_CIRCUIT(client_side), + CIRCPAD_MACHINE_CIRC_SETUP, + 2, CIRCPAD_COMMAND_START); + + /* verify no padding was negotiated */ + tt_ptr_op(relay_side->padding_machine[0], OP_EQ, NULL); + tt_ptr_op(client_side->padding_machine[0], OP_EQ, NULL); + + /* verify no echo was sent */ + tt_int_op(n_relay_cells, OP_EQ, 1); + tt_int_op(n_client_cells, OP_EQ, 1); + + /* 3. Test failure to negotiate a machine due to desync */ + free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side)); + free_fake_orcirc(relay_side); + + client_side = TO_CIRCUIT(origin_circuit_new()); + relay_side = TO_CIRCUIT(new_fake_orcirc(&dummy_channel, &dummy_channel)); + relay_side->purpose = CIRCUIT_PURPOSE_OR; + client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL; + + SMARTLIST_FOREACH(relay_padding_machines, + circpad_machine_t *, + m, tor_free(m->states); tor_free(m)); + smartlist_free(relay_padding_machines); + relay_padding_machines = smartlist_new(); + + simulate_single_hop_extend(client_side, relay_side, 1); + simulate_single_hop_extend(client_side, relay_side, 1); + + /* verify echo was sent */ + tt_int_op(n_client_cells, OP_EQ, 2); + tt_int_op(n_relay_cells, OP_EQ, 2); + + /* verify no padding was negotiated */ + tt_ptr_op(client_side->padding_info[0], OP_EQ, NULL); + tt_ptr_op(client_side->padding_machine[0], OP_EQ, NULL); + tt_ptr_op(relay_side->padding_machine[0], OP_EQ, NULL); + tt_ptr_op(relay_side->padding_info[0], OP_EQ, NULL); + + done: + free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side)); + free_fake_orcirc(relay_side); + circuitmux_detach_all_circuits(dummy_channel.cmux, NULL); + circuitmux_free(dummy_channel.cmux); + monotime_disable_test_mocking(); + UNMOCK(node_get_by_id); + UNMOCK(circuit_package_relay_cell); + UNMOCK(circuitmux_attach_circuit); + nodes_free(); +} + +static void +simulate_single_hop_extend(circuit_t *client, circuit_t *mid_relay, + int padding) +{ + char whatevs_key[CPATH_KEY_MATERIAL_LEN]; + char digest[DIGEST_LEN]; + tor_addr_t addr; + + // Pretend a non-padding cell was sent + circpad_cell_event_nonpadding_sent((circuit_t*)client); + + // Receive extend cell at middle + circpad_cell_event_nonpadding_received((circuit_t*)mid_relay); + + // Advance time a tiny bit so we can calculate an RTT + curr_mocked_time += 10 * NSEC_PER_MSEC; + monotime_coarse_set_mock_time_nsec(curr_mocked_time); + monotime_set_mock_time_nsec(curr_mocked_time); + + // Receive extended cell at middle + circpad_cell_event_nonpadding_sent((circuit_t*)mid_relay); + + // Receive extended cell at first hop + circpad_cell_event_nonpadding_received((circuit_t*)client); + + // Add a hop to cpath + crypt_path_t *hop = tor_malloc_zero(sizeof(crypt_path_t)); + onion_append_to_cpath(&TO_ORIGIN_CIRCUIT(client)->cpath, hop); + + hop->magic = CRYPT_PATH_MAGIC; + hop->state = CPATH_STATE_OPEN; + + // add an extend info to indicate if this node supports padding or not. + // (set the first byte of the digest for our mocked node_get_by_id) + digest[0] = padding; + + hop->extend_info = extend_info_new( + padding ? "padding" : "non-padding", + digest, NULL, NULL, NULL, + &addr, padding); + + circuit_init_cpath_crypto(hop, whatevs_key, sizeof(whatevs_key), 0, 0); + + hop->package_window = circuit_initial_package_window(); + hop->deliver_window = CIRCWINDOW_START; + + // Signal that the hop was added + circpad_machine_event_circ_added_hop(TO_ORIGIN_CIRCUIT(client)); +} + +static circpad_machine_t * +helper_create_conditional_machine(void) +{ + circpad_machine_t *ret = tor_malloc_zero(sizeof(circpad_machine_t)); + + /* Start, burst */ + circpad_machine_states_init(ret, 2); + + ret->states[CIRCPAD_STATE_START]. + next_state[CIRCPAD_EVENT_PADDING_SENT] = CIRCPAD_STATE_BURST; + + ret->states[CIRCPAD_STATE_BURST]. + next_state[CIRCPAD_EVENT_PADDING_SENT] = CIRCPAD_STATE_BURST; + + ret->states[CIRCPAD_STATE_BURST]. + next_state[CIRCPAD_EVENT_LENGTH_COUNT] = CIRCPAD_STATE_END; + + ret->states[CIRCPAD_STATE_BURST].token_removal = + CIRCPAD_TOKEN_REMOVAL_NONE; + + ret->states[CIRCPAD_STATE_BURST].histogram_len = 3; + ret->states[CIRCPAD_STATE_BURST].start_usec = 0; + ret->states[CIRCPAD_STATE_BURST].range_usec = 1000000; + ret->states[CIRCPAD_STATE_BURST].histogram[0] = 6; + ret->states[CIRCPAD_STATE_BURST].histogram[1] = 0; + ret->states[CIRCPAD_STATE_BURST].histogram[1] = 0; + ret->states[CIRCPAD_STATE_BURST].histogram_total_tokens = 6; + ret->states[CIRCPAD_STATE_BURST].use_rtt_estimate = 0; + ret->states[CIRCPAD_STATE_BURST].length_includes_nonpadding = 1; + + return ret; +} + +static void +helper_create_conditional_machines(void) +{ + circpad_machine_t *add = helper_create_conditional_machine(); + origin_padding_machines = smartlist_new(); + relay_padding_machines = smartlist_new(); + + add->machine_num = 2; + add->is_origin_side = 1; + add->should_negotiate_end = 1; + add->target_hopnum = 2; + + /* Let's have this one end after 4 packets */ + add->states[CIRCPAD_STATE_BURST].length_dist.type = CIRCPAD_DIST_UNIFORM; + add->states[CIRCPAD_STATE_BURST].length_dist.param1 = 4; + add->states[CIRCPAD_STATE_BURST].length_dist.param2 = 4; + add->states[CIRCPAD_STATE_BURST].max_length = 4; + + add->conditions.requires_vanguards = 0; + add->conditions.min_hops = 2; + add->conditions.state_mask = CIRCPAD_CIRC_BUILDING| + CIRCPAD_CIRC_NO_STREAMS|CIRCPAD_CIRC_HAS_RELAY_EARLY; + add->conditions.purpose_mask = CIRCPAD_PURPOSE_ALL; + + smartlist_add(origin_padding_machines, add); + + add = helper_create_conditional_machine(); + add->machine_num = 3; + add->is_origin_side = 1; + add->should_negotiate_end = 1; + add->target_hopnum = 2; + + /* Let's have this one end after 4 packets */ + add->states[CIRCPAD_STATE_BURST].length_dist.type = CIRCPAD_DIST_UNIFORM; + add->states[CIRCPAD_STATE_BURST].length_dist.param1 = 4; + add->states[CIRCPAD_STATE_BURST].length_dist.param2 = 4; + add->states[CIRCPAD_STATE_BURST].max_length = 4; + + add->conditions.requires_vanguards = 1; + add->conditions.min_hops = 3; + add->conditions.state_mask = CIRCPAD_CIRC_OPENED| + CIRCPAD_CIRC_STREAMS|CIRCPAD_CIRC_HAS_NO_RELAY_EARLY; + add->conditions.purpose_mask = CIRCPAD_PURPOSE_ALL; + smartlist_add(origin_padding_machines, add); + + add = helper_create_conditional_machine(); + add->machine_num = 2; + smartlist_add(relay_padding_machines, add); + + add = helper_create_conditional_machine(); + add->machine_num = 3; + smartlist_add(relay_padding_machines, add); +} + +void +test_circuitpadding_conditions(void *arg) +{ + /** + * Test plan: + * 0. Make a few origin and client machines with diff conditions + * * vanguards, purposes, has_opened circs, no relay early + * * Client side should_negotiate_end + * * Length limits + * 1. Test STATE_END transitions + * 2. Test new machine after end with same conditions + * 3. Test new machine due to changed conditions + * * Esp: built event, no relay early, no streams + * XXX: Diff test: + * 1. Test STATE_END with pending timers + * 2. Test marking a circuit before padding callback fires + * 3. Test freeing a circuit before padding callback fires + */ + (void)arg; + MOCK(circuitmux_attach_circuit, circuitmux_attach_circuit_mock); + + nodes_init(); + dummy_channel.cmux = circuitmux_alloc(); + relay_side = (circuit_t *)new_fake_orcirc(&dummy_channel, + &dummy_channel); + client_side = (circuit_t *)origin_circuit_new(); + relay_side->purpose = CIRCUIT_PURPOSE_OR; + client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL; + + monotime_init(); + monotime_enable_test_mocking(); + monotime_set_mock_time_nsec(1*NSEC_PER_USEC); + monotime_coarse_set_mock_time_nsec(1*NSEC_PER_USEC); + curr_mocked_time = 1*NSEC_PER_USEC; + + timers_initialize(); + helper_create_conditional_machines(); + + MOCK(circuit_package_relay_cell, + circuit_package_relay_cell_mock); + MOCK(node_get_by_id, + node_get_by_id_mock); + + /* Simulate extend. This should result in the original machine getting + * added, since the circuit is not built */ + simulate_single_hop_extend(client_side, relay_side, 1); + simulate_single_hop_extend(client_side, relay_side, 1); + + /* Verify that machine #2 is added */ + tt_int_op(client_side->padding_machine[0]->machine_num, OP_EQ, 2); + tt_int_op(relay_side->padding_machine[0]->machine_num, OP_EQ, 2); + + /* Deliver a padding cell to the client, to trigger burst state */ + circpad_cell_event_padding_sent(client_side); + + /* This should have trigger length shutdown condition on client.. */ + tt_ptr_op(client_side->padding_info[0], OP_EQ, NULL); + tt_ptr_op(client_side->padding_machine[0], OP_EQ, NULL); + + /* Verify machine is gone from both sides */ + tt_ptr_op(relay_side->padding_info[0], OP_EQ, NULL); + tt_ptr_op(relay_side->padding_machine[0], OP_EQ, NULL); + + /* Send another event.. verify machine gets re-added properly + * (test race with shutdown) */ + simulate_single_hop_extend(client_side, relay_side, 1); + tt_int_op(client_side->padding_machine[0]->machine_num, OP_EQ, 2); + tt_int_op(relay_side->padding_machine[0]->machine_num, OP_EQ, 2); + + TO_ORIGIN_CIRCUIT(client_side)->p_streams = 0; + circpad_machine_event_circ_has_no_streams(TO_ORIGIN_CIRCUIT(client_side)); + tt_int_op(client_side->padding_machine[0]->machine_num, OP_EQ, 2); + tt_int_op(relay_side->padding_machine[0]->machine_num, OP_EQ, 2); + + /* Now make the circuit opened and send built event */ + TO_ORIGIN_CIRCUIT(client_side)->has_opened = 1; + circpad_machine_event_circ_built(TO_ORIGIN_CIRCUIT(client_side)); + tt_int_op(client_side->padding_machine[0]->machine_num, OP_EQ, 2); + tt_int_op(relay_side->padding_machine[0]->machine_num, OP_EQ, 2); + + TO_ORIGIN_CIRCUIT(client_side)->remaining_relay_early_cells = 0; + circpad_machine_event_circ_has_no_relay_early( + TO_ORIGIN_CIRCUIT(client_side)); + tt_int_op(client_side->padding_machine[0]->machine_num, OP_EQ, 2); + tt_int_op(relay_side->padding_machine[0]->machine_num, OP_EQ, 2); + + get_options_mutable()->HSLayer2Nodes = (void*)1; + TO_ORIGIN_CIRCUIT(client_side)->p_streams = (void*)1; + circpad_machine_event_circ_has_streams(TO_ORIGIN_CIRCUIT(client_side)); + + /* Verify different machine is added */ + tt_int_op(client_side->padding_machine[0]->machine_num, OP_EQ, 3); + tt_int_op(relay_side->padding_machine[0]->machine_num, OP_EQ, 3); + + /* Hold off on negotiated */ + deliver_negotiated = 0; + + /* Deliver a padding cell to the client, to trigger burst state */ + circpad_cell_event_padding_sent(client_side); + + /* This should have trigger length shutdown condition on client + * but not the response for the padding machine */ + tt_ptr_op(client_side->padding_info[0], OP_EQ, NULL); + tt_ptr_op(client_side->padding_machine[0], OP_NE, NULL); + + /* Verify machine is gone from the relay (but negotiated not back yet */ + tt_ptr_op(relay_side->padding_info[0], OP_EQ, NULL); + tt_ptr_op(relay_side->padding_machine[0], OP_EQ, NULL); + + /* Add another hop and verify it's back */ + simulate_single_hop_extend(client_side, relay_side, 1); + + tt_int_op(client_side->padding_machine[0]->machine_num, OP_EQ, 3); + tt_int_op(relay_side->padding_machine[0]->machine_num, OP_EQ, 3); + + tt_ptr_op(client_side->padding_info[0], OP_NE, NULL); + tt_ptr_op(relay_side->padding_info[0], OP_NE, NULL); + + done: + /* XXX: Free everything */ + return; +} + +void +test_circuitpadding_circuitsetup_machine(void *arg) +{ + /** + * Test case plan: + * + * 1. Simulate a normal circuit setup pattern + * a. Application traffic + * + * FIXME: This should focus more on exercising the machine + * features rather than actual traffic patterns. For example, + * test cancellation and bins empty/refill + */ + (void)arg; + + MOCK(circuitmux_attach_circuit, circuitmux_attach_circuit_mock); + + dummy_channel.cmux = circuitmux_alloc(); + client_side = TO_CIRCUIT(origin_circuit_new()); + relay_side = TO_CIRCUIT(new_fake_orcirc(&dummy_channel, &dummy_channel)); + + relay_side->purpose = CIRCUIT_PURPOSE_OR; + client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL; + + nodes_init(); + + monotime_init(); + monotime_enable_test_mocking(); + monotime_set_mock_time_nsec(1*NSEC_PER_USEC); + monotime_coarse_set_mock_time_nsec(1*NSEC_PER_USEC); + curr_mocked_time = 1*NSEC_PER_USEC; + + timers_initialize(); + circpad_machines_init(); + + MOCK(circuit_package_relay_cell, + circuit_package_relay_cell_mock); + MOCK(node_get_by_id, + node_get_by_id_mock); + + /* Test case #1: Build a 3 hop circuit, then wait and let pad */ + simulate_single_hop_extend(client_side, relay_side, 1); + simulate_single_hop_extend(client_side, relay_side, 1); + simulate_single_hop_extend(client_side, relay_side, 1); + + tt_int_op(n_client_cells, OP_EQ, 1); + tt_int_op(n_relay_cells, OP_EQ, 1); + tt_int_op(client_side->padding_info[0]->current_state, OP_EQ, + CIRCPAD_STATE_BURST); + tt_int_op(relay_side->padding_info[0]->current_state, OP_EQ, + CIRCPAD_STATE_BURST); + + tt_int_op(client_side->padding_info[0]->padding_scheduled_at_usec, + OP_NE, 0); + tt_int_op(relay_side->padding_info[0]->is_padding_timer_scheduled, + OP_EQ, 0); + timers_advance_and_run(2000); + tt_int_op(n_client_cells, OP_EQ, 2); + tt_int_op(n_relay_cells, OP_EQ, 1); + + tt_int_op(relay_side->padding_info[0]->current_state, OP_EQ, + CIRCPAD_STATE_GAP); + + tt_int_op(client_side->padding_info[0]->padding_scheduled_at_usec, + OP_EQ, 0); + tt_int_op(relay_side->padding_info[0]->padding_scheduled_at_usec, + OP_NE, 0); + timers_advance_and_run(5000); + tt_int_op(n_client_cells, OP_EQ, 2); + tt_int_op(n_relay_cells, OP_EQ, 2); + + tt_int_op(client_side->padding_info[0]->padding_scheduled_at_usec, + OP_NE, 0); + tt_int_op(relay_side->padding_info[0]->padding_scheduled_at_usec, + OP_EQ, 0); + timers_advance_and_run(2000); + tt_int_op(n_client_cells, OP_EQ, 3); + tt_int_op(n_relay_cells, OP_EQ, 2); + + tt_int_op(client_side->padding_info[0]->padding_scheduled_at_usec, + OP_EQ, 0); + tt_int_op(relay_side->padding_info[0]->padding_scheduled_at_usec, + OP_NE, 0); + timers_advance_and_run(5000); + tt_int_op(n_client_cells, OP_EQ, 3); + tt_int_op(n_relay_cells, OP_EQ, 3); + + tt_int_op(client_side->padding_info[0]->padding_scheduled_at_usec, + OP_NE, 0); + tt_int_op(relay_side->padding_info[0]->padding_scheduled_at_usec, + OP_EQ, 0); + timers_advance_and_run(2000); + tt_int_op(n_client_cells, OP_EQ, 4); + tt_int_op(n_relay_cells, OP_EQ, 3); + + tt_int_op(client_side->padding_info[0]->padding_scheduled_at_usec, + OP_EQ, 0); + tt_int_op(relay_side->padding_info[0]->padding_scheduled_at_usec, + OP_NE, 0); + timers_advance_and_run(5000); + tt_int_op(n_client_cells, OP_EQ, 4); + tt_int_op(n_relay_cells, OP_EQ, 4); + + tt_int_op(client_side->padding_info[0]->padding_scheduled_at_usec, + OP_NE, 0); + tt_int_op(relay_side->padding_info[0]->padding_scheduled_at_usec, + OP_EQ, 0); + timers_advance_and_run(2000); + tt_int_op(n_client_cells, OP_EQ, 5); + tt_int_op(n_relay_cells, OP_EQ, 4); + + tt_int_op(client_side->padding_info[0]->padding_scheduled_at_usec, + OP_EQ, 0); + tt_int_op(relay_side->padding_info[0]->padding_scheduled_at_usec, + OP_NE, 0); + timers_advance_and_run(5000); + tt_int_op(n_client_cells, OP_EQ, 5); + tt_int_op(n_relay_cells, OP_EQ, 5); + + tt_int_op(client_side->padding_info[0]->padding_scheduled_at_usec, + OP_NE, 0); + tt_int_op(relay_side->padding_info[0]->padding_scheduled_at_usec, + OP_EQ, 0); + timers_advance_and_run(2000); + tt_int_op(n_client_cells, OP_EQ, 6); + tt_int_op(n_relay_cells, OP_EQ, 5); + + tt_int_op(client_side->padding_info[0]->padding_scheduled_at_usec, + OP_EQ, 0); + tt_int_op(relay_side->padding_info[0]->padding_scheduled_at_usec, + OP_NE, 0); + timers_advance_and_run(5000); + tt_int_op(n_client_cells, OP_EQ, 6); + tt_int_op(n_relay_cells, OP_EQ, 6); + + tt_int_op(client_side->padding_info[0]->current_state, + OP_EQ, CIRCPAD_STATE_END); + tt_int_op(client_side->padding_info[0]->padding_scheduled_at_usec, + OP_EQ, 0); + tt_int_op(relay_side->padding_info[0]->current_state, + OP_EQ, CIRCPAD_STATE_GAP); + tt_int_op(relay_side->padding_info[0]->padding_scheduled_at_usec, + OP_EQ, 0); + + /* Verify we can't schedule padding in END state */ + circpad_decision_t ret = + circpad_machine_schedule_padding(client_side->padding_info[0]); + tt_int_op(ret, OP_EQ, CIRCPAD_STATE_UNCHANGED); + + /* Simulate application traffic */ + circpad_cell_event_nonpadding_sent(client_side); + circpad_deliver_unrecognized_cell_events(relay_side, CELL_DIRECTION_OUT); + circpad_deliver_unrecognized_cell_events(relay_side, CELL_DIRECTION_IN); + circpad_deliver_recognized_relay_cell_events(client_side, RELAY_COMMAND_DATA, + TO_ORIGIN_CIRCUIT(client_side)->cpath->next); + + tt_ptr_op(client_side->padding_info[0], OP_EQ, NULL); + tt_ptr_op(client_side->padding_machine[0], OP_EQ, NULL); + + tt_ptr_op(relay_side->padding_info[0], OP_EQ, NULL); + tt_ptr_op(relay_side->padding_machine[0], OP_EQ, NULL); + tt_int_op(n_client_cells, OP_EQ, 6); + tt_int_op(n_relay_cells, OP_EQ, 7); + + // Test timer cancellation + simulate_single_hop_extend(client_side, relay_side, 1); + simulate_single_hop_extend(client_side, relay_side, 1); + timers_advance_and_run(5000); + circpad_cell_event_padding_received(client_side); + + tt_int_op(client_side->padding_info[0]->current_state, OP_EQ, + CIRCPAD_STATE_BURST); + tt_int_op(relay_side->padding_info[0]->current_state, OP_EQ, + CIRCPAD_STATE_GAP); + + tt_int_op(n_client_cells, OP_EQ, 8); + tt_int_op(n_relay_cells, OP_EQ, 8); + tt_int_op(client_side->padding_info[0]->padding_scheduled_at_usec, + OP_NE, 0); + tt_int_op(relay_side->padding_info[0]->padding_scheduled_at_usec, + OP_NE, 0); + + /* Test timer cancel due to state rules */ + circpad_cell_event_nonpadding_sent(client_side); + tt_int_op(client_side->padding_info[0]->padding_scheduled_at_usec, + OP_EQ, 0); + circpad_cell_event_padding_received(client_side); + tt_int_op(client_side->padding_info[0]->padding_scheduled_at_usec, + OP_NE, 0); + + /* Simulate application traffic to cancel timer */ + circpad_cell_event_nonpadding_sent(client_side); + circpad_deliver_unrecognized_cell_events(relay_side, CELL_DIRECTION_OUT); + circpad_deliver_unrecognized_cell_events(relay_side, CELL_DIRECTION_IN); + circpad_deliver_recognized_relay_cell_events(client_side, RELAY_COMMAND_DATA, + TO_ORIGIN_CIRCUIT(client_side)->cpath->next); + + tt_ptr_op(client_side->padding_info[0], OP_EQ, NULL); + tt_ptr_op(client_side->padding_machine[0], OP_EQ, NULL); + + tt_ptr_op(relay_side->padding_info[0], OP_EQ, NULL); + tt_ptr_op(relay_side->padding_machine[0], OP_EQ, NULL); + + /* No cells sent, except negotiate end from relay */ + tt_int_op(n_client_cells, OP_EQ, 8); + tt_int_op(n_relay_cells, OP_EQ, 9); + + /* Test mark for close and free */ + simulate_single_hop_extend(client_side, relay_side, 1); + simulate_single_hop_extend(client_side, relay_side, 1); + timers_advance_and_run(5000); + circpad_cell_event_padding_received(client_side); + + tt_int_op(n_client_cells, OP_EQ, 10); + tt_int_op(n_relay_cells, OP_EQ, 10); + + tt_int_op(client_side->padding_info[0]->current_state, OP_EQ, + CIRCPAD_STATE_BURST); + tt_int_op(relay_side->padding_info[0]->current_state, OP_EQ, + CIRCPAD_STATE_GAP); + + tt_int_op(client_side->padding_info[0]->padding_scheduled_at_usec, + OP_NE, 0); + tt_int_op(relay_side->padding_info[0]->padding_scheduled_at_usec, + OP_NE, 0); + circuit_mark_for_close(client_side, END_CIRC_REASON_FLAG_REMOTE); + free_fake_orcirc(relay_side); + timers_advance_and_run(5000); + + /* No cells sent */ + tt_int_op(n_client_cells, OP_EQ, 10); + tt_int_op(n_relay_cells, OP_EQ, 10); + + done: + free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side)); + + circuitmux_detach_all_circuits(dummy_channel.cmux, NULL); + circuitmux_free(dummy_channel.cmux); + timers_shutdown(); + monotime_disable_test_mocking(); + UNMOCK(circuit_package_relay_cell); + UNMOCK(circuitmux_attach_circuit); + + return; +} + +/** Helper function: Initializes a padding machine where every state uses the + * uniform probability distribution. */ +static void +helper_circpad_circ_distribution_machine_setup(int min, int max) +{ + circpad_machine_states_init(&circ_client_machine, 7); + + circpad_state_t *zero_st = &circ_client_machine.states[0]; + zero_st->next_state[CIRCPAD_EVENT_NONPADDING_RECV] = 1; + zero_st->iat_dist.type = CIRCPAD_DIST_UNIFORM; + zero_st->iat_dist.param1 = min; + zero_st->iat_dist.param2 = max; + zero_st->start_usec = min; + zero_st->range_usec = max; + + circpad_state_t *first_st = &circ_client_machine.states[1]; + first_st->next_state[CIRCPAD_EVENT_NONPADDING_RECV] = 2; + first_st->iat_dist.type = CIRCPAD_DIST_LOGISTIC; + first_st->iat_dist.param1 = min; + first_st->iat_dist.param2 = max; + first_st->start_usec = min; + first_st->range_usec = max; + + circpad_state_t *second_st = &circ_client_machine.states[2]; + second_st->next_state[CIRCPAD_EVENT_NONPADDING_RECV] = 3; + second_st->iat_dist.type = CIRCPAD_DIST_LOG_LOGISTIC; + second_st->iat_dist.param1 = min; + second_st->iat_dist.param2 = max; + second_st->start_usec = min; + second_st->range_usec = max; + + circpad_state_t *third_st = &circ_client_machine.states[3]; + third_st->next_state[CIRCPAD_EVENT_NONPADDING_RECV] = 4; + third_st->iat_dist.type = CIRCPAD_DIST_GEOMETRIC; + third_st->iat_dist.param1 = min; + third_st->iat_dist.param2 = max; + third_st->start_usec = min; + third_st->range_usec = max; + + circpad_state_t *fourth_st = &circ_client_machine.states[4]; + fourth_st->next_state[CIRCPAD_EVENT_NONPADDING_RECV] = 5; + fourth_st->iat_dist.type = CIRCPAD_DIST_WEIBULL; + fourth_st->iat_dist.param1 = min; + fourth_st->iat_dist.param2 = max; + fourth_st->start_usec = min; + fourth_st->range_usec = max; + + circpad_state_t *fifth_st = &circ_client_machine.states[5]; + fifth_st->next_state[CIRCPAD_EVENT_NONPADDING_RECV] = 6; + fifth_st->iat_dist.type = CIRCPAD_DIST_PARETO; + fifth_st->iat_dist.param1 = min; + fifth_st->iat_dist.param2 = max; + fifth_st->start_usec = min; + fifth_st->range_usec = max; +} + +/** Simple test that the padding delays sampled from a uniform distribution + * actually faill within the uniform distribution range. */ +/* TODO: Upgrade this test so that each state tests a different prob + * distribution */ +static void +test_circuitpadding_sample_distribution(void *arg) +{ + circpad_machineinfo_t *mi; + int n_samples; + int n_states; + + (void) arg; + + /* mock this function so that we dont actually schedule any padding */ + MOCK(circpad_machine_schedule_padding, + circpad_machine_schedule_padding_mock); + + /* Initialize a machine with multiple probability distributions that should + * return values between 0 and 5 */ + circpad_machines_init(); + helper_circpad_circ_distribution_machine_setup(0, 10); + + /* Initialize machine and circuits */ + client_side = TO_CIRCUIT(origin_circuit_new()); + client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL; + client_side->padding_machine[0] = &circ_client_machine; + client_side->padding_info[0] = + circpad_circuit_machineinfo_new(client_side, 0); + mi = client_side->padding_info[0]; + + /* For every state, sample a bunch of values from the distribution and ensure + * they fall within range. */ + for (n_states = 0 ; n_states < 6; n_states++) { + /* Make sure we in the right state */ + tt_int_op(client_side->padding_info[0]->current_state, OP_EQ, n_states); + + for (n_samples = 0; n_samples < 100; n_samples++) { + circpad_delay_t delay = circpad_machine_sample_delay(mi); + tt_int_op(delay, OP_GE, 0); + tt_int_op(delay, OP_LE, 10); + } + + /* send a non-padding cell to move to the next machine state */ + circpad_cell_event_nonpadding_received((circuit_t*)client_side); + } + + done: + free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side)); + UNMOCK(circpad_machine_schedule_padding); +} + +static circpad_decision_t +circpad_machine_transition_mock(circpad_machineinfo_t *mi, + circpad_event_t event) +{ + (void) mi; + (void) event; + + return CIRCPAD_STATE_UNCHANGED; +} + +/* Test per-machine padding rate limits */ +static void +test_circuitpadding_machine_rate_limiting(void *arg) +{ + (void) arg; + bool retval; + circpad_machineinfo_t *mi; + int i; + + /* Ignore machine transitions for the purposes of this function, we only + * really care about padding counts */ + MOCK(circpad_machine_transition, circpad_machine_transition_mock); + MOCK(circpad_send_command_to_hop, circpad_send_command_to_hop_mock); + + /* Setup machine and circuits */ + client_side = TO_CIRCUIT(origin_circuit_new()); + client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL; + helper_create_basic_machine(); + client_side->padding_machine[0] = &circ_client_machine; + client_side->padding_info[0] = + circpad_circuit_machineinfo_new(client_side, 0); + mi = client_side->padding_info[0]; + /* Set up the machine info so that we can get through the basic functions */ + mi->state_length = CIRCPAD_STATE_LENGTH_INFINITE; + + /* First we are going to test the per-machine rate limits */ + circ_client_machine.max_padding_percent = 50; + circ_client_machine.allowed_padding_count = 100; + + /* Check padding limit, should be fine since we haven't sent anything yet. */ + retval = circpad_machine_reached_padding_limit(mi); + tt_int_op(retval, OP_EQ, 0); + + /* Send 99 padding cells which is below circpad_global_allowed_cells=100, so + * the rate limit will not trigger */ + for (i=0;i<99;i++) { + circpad_send_padding_cell_for_callback(mi); + } + retval = circpad_machine_reached_padding_limit(mi); + tt_int_op(retval, OP_EQ, 0); + + /* Now send another padding cell to pass circpad_global_allowed_cells=100, + and see that the limit will trigger */ + circpad_send_padding_cell_for_callback(mi); + retval = circpad_machine_reached_padding_limit(mi); + tt_int_op(retval, OP_EQ, 1); + + retval = circpad_machine_schedule_padding(mi); + tt_int_op(retval, OP_EQ, CIRCPAD_STATE_UNCHANGED); + + /* Cover wrap */ + for (;ipadding_sent, OP_EQ, UINT16_MAX/2+1); + + tt_ptr_op(client_side->padding_info[0], OP_EQ, mi); + for (i=0;inonpadding_sent, OP_EQ, UINT16_MAX/2); + tt_int_op(mi->padding_sent, OP_EQ, UINT16_MAX/4+1); + + done: + free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side)); +} + +/* Test global padding rate limits */ +static void +test_circuitpadding_global_rate_limiting(void *arg) +{ + (void) arg; + bool retval; + circpad_machineinfo_t *mi; + int i; + + /* Ignore machine transitions for the purposes of this function, we only + * really care about padding counts */ + MOCK(circpad_machine_transition, circpad_machine_transition_mock); + MOCK(circuitmux_attach_circuit, circuitmux_attach_circuit_mock); + MOCK(circuit_package_relay_cell, + circuit_package_relay_cell_mock); + MOCK(monotime_absolute_usec, mock_monotime_absolute_usec); + + monotime_init(); + monotime_enable_test_mocking(); + monotime_set_mock_time_nsec(1*TOR_NSEC_PER_USEC); + monotime_coarse_set_mock_time_nsec(1*TOR_NSEC_PER_USEC); + curr_mocked_time = 1*TOR_NSEC_PER_USEC; + timers_initialize(); + + client_side = (circuit_t *)origin_circuit_new(); + client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL; + dummy_channel.cmux = circuitmux_alloc(); + + /* Setup machine and circuits */ + relay_side = (circuit_t *)new_fake_orcirc(&dummy_channel, &dummy_channel); + relay_side->purpose = CIRCUIT_PURPOSE_OR; + helper_create_basic_machine(); + relay_side->padding_machine[0] = &circ_client_machine; + relay_side->padding_info[0] = + circpad_circuit_machineinfo_new(relay_side, 0); + mi = relay_side->padding_info[0]; + /* Set up the machine info so that we can get through the basic functions */ + mi->state_length = CIRCPAD_STATE_LENGTH_INFINITE; + + simulate_single_hop_extend(client_side, relay_side, 1); + simulate_single_hop_extend(client_side, relay_side, 1); + + /* Now test the global limits by setting up the consensus */ + networkstatus_t vote1; + vote1.net_params = smartlist_new(); + smartlist_split_string(vote1.net_params, + "circpad_global_allowed_cells=100 circpad_global_max_padding_pct=50", + NULL, 0, 0); + /* Register global limits with the padding subsystem */ + circpad_new_consensus_params(&vote1); + + /* Check padding limit, should be fine since we haven't sent anything yet. */ + retval = circpad_machine_reached_padding_limit(mi); + tt_int_op(retval, OP_EQ, 0); + + /* Send 99 padding cells which is below circpad_global_allowed_cells=100, so + * the rate limit will not trigger */ + for (i=0;i<99;i++) { + circpad_send_padding_cell_for_callback(mi); + } + retval = circpad_machine_reached_padding_limit(mi); + tt_int_op(retval, OP_EQ, 0); + + /* Now send another padding cell to pass circpad_global_allowed_cells=100, + and see that the limit will trigger */ + circpad_send_padding_cell_for_callback(mi); + retval = circpad_machine_reached_padding_limit(mi); + tt_int_op(retval, OP_EQ, 1); + + retval = circpad_machine_schedule_padding(mi); + tt_int_op(retval, OP_EQ, CIRCPAD_STATE_UNCHANGED); + + /* Now send 92 non-padding cells to get near the + * circpad_global_max_padding_pct=50 limit; in particular with 96 non-padding + * cells, the padding traffic is still 51% of total traffic so limit should + * trigger */ + for (i=0;i<92;i++) { + circpad_cell_event_nonpadding_sent(relay_side); + } + retval = circpad_machine_reached_padding_limit(mi); + tt_int_op(retval, OP_EQ, 1); + + /* Send another non-padding cell to bring the padding traffic to 50% of total + * traffic and get past the limit */ + circpad_cell_event_nonpadding_sent(relay_side); + retval = circpad_machine_reached_padding_limit(mi); + tt_int_op(retval, OP_EQ, 0); + + done: + free_fake_orcirc(relay_side); + circuitmux_detach_all_circuits(dummy_channel.cmux, NULL); + circuitmux_free(dummy_channel.cmux); + SMARTLIST_FOREACH(vote1.net_params, char *, cp, tor_free(cp)); + smartlist_free(vote1.net_params); +} + +#define TEST_CIRCUITPADDING(name, flags) \ + { #name, test_##name, (flags), NULL, NULL } + +struct testcase_t circuitpadding_tests[] = { + //TEST_CIRCUITPADDING(circuitpadding_circuitsetup_machine, 0), + TEST_CIRCUITPADDING(circuitpadding_tokens, TT_FORK), + TEST_CIRCUITPADDING(circuitpadding_negotiation, TT_FORK), + TEST_CIRCUITPADDING(circuitpadding_wronghop, TT_FORK), + TEST_CIRCUITPADDING(circuitpadding_circuitsetup_machine, TT_FORK), + TEST_CIRCUITPADDING(circuitpadding_conditions, TT_FORK), + TEST_CIRCUITPADDING(circuitpadding_rtt, TT_FORK), + TEST_CIRCUITPADDING(circuitpadding_sample_distribution, TT_FORK), + TEST_CIRCUITPADDING(circuitpadding_machine_rate_limiting, TT_FORK), + TEST_CIRCUITPADDING(circuitpadding_global_rate_limiting, TT_FORK), + TEST_CIRCUITPADDING(circuitpadding_token_removal_lower, TT_FORK), + TEST_CIRCUITPADDING(circuitpadding_token_removal_higher, TT_FORK), + TEST_CIRCUITPADDING(circuitpadding_closest_token_removal, TT_FORK), + TEST_CIRCUITPADDING(circuitpadding_closest_token_removal_usec, TT_FORK), + TEST_CIRCUITPADDING(circuitpadding_token_removal_exact, TT_FORK), + END_OF_TESTCASES +}; + diff --git a/src/test/test_containers.c b/src/test/test_containers.c index 717eb0892a..ad0edf4aa3 100644 --- a/src/test/test_containers.c +++ b/src/test/test_containers.c @@ -96,6 +96,30 @@ test_container_smartlist_basic(void *arg) tor_free(v555); } +/** Test SMARTLIST_FOREACH_REVERSE_BEGIN loop macro */ +static void +test_container_smartlist_foreach_reverse(void *arg) +{ + smartlist_t *sl = smartlist_new(); + int i; + + (void) arg; + + /* Add integers to smartlist in increasing order */ + for (i=0;i<100;i++) { + smartlist_add(sl, (void*)(uintptr_t)i); + } + + /* Pop them out in reverse and test their value */ + SMARTLIST_FOREACH_REVERSE_BEGIN(sl, void*, k) { + i--; + tt_ptr_op(k, OP_EQ, (void*)(uintptr_t)i); + } SMARTLIST_FOREACH_END(k); + + done: + smartlist_free(sl); +} + /** Run unit tests for smartlist-of-strings functionality. */ static void test_container_smartlist_strings(void *arg) @@ -1281,6 +1305,7 @@ test_container_smartlist_strings_eq(void *arg) struct testcase_t container_tests[] = { CONTAINER_LEGACY(smartlist_basic), CONTAINER_LEGACY(smartlist_strings), + CONTAINER_LEGACY(smartlist_foreach_reverse), CONTAINER_LEGACY(smartlist_overlap), CONTAINER_LEGACY(smartlist_digests), CONTAINER_LEGACY(smartlist_join), From 8ad497bb578b13c66489843905764a60545e6388 Mon Sep 17 00:00:00 2001 From: Mike Perry Date: Tue, 23 Oct 2018 21:02:31 +0000 Subject: [PATCH 0366/2557] Config option to specify specific MiddleNodes. Hope is this will make it easier to test on the live tor network. Does not need to be merged if we don't want to, but will come in handy for researchers. Co-authored-by: George Kadianakis --- doc/tor.1.txt | 30 +++++++++++++++++++++++------- src/app/config/config.c | 6 ++++++ src/app/config/or_options_st.h | 3 +++ src/core/or/circuitbuild.c | 19 ++++++++++++++++++- src/feature/nodelist/nodelist.c | 2 +- src/feature/nodelist/routerlist.c | 2 ++ 6 files changed, 53 insertions(+), 9 deletions(-) diff --git a/doc/tor.1.txt b/doc/tor.1.txt index 4ff789a931..455356163c 100644 --- a/doc/tor.1.txt +++ b/doc/tor.1.txt @@ -1020,6 +1020,21 @@ The following options are useful only for clients (that is, if The .exit address notation, if enabled via MapAddress, overrides this option. +[[MiddleNodes]] **MiddleNodes** __node__,__node__,__...__:: + A list of identity fingerprints and country codes of nodes + to use for "middle" hops in your normal circuits. + Normal circuits include all circuits except for direct connections + to directory servers. Middle hops are all hops other than exit and entry. + ++ + The HSLayer2Node and HSLayer3Node options override this option for onion + service circuits, if they are set. The vanguards addon will read this + option, and if set, it will set HSLayer2Nodes and HSLayer3Nodes to nodes + from this set. ++ + The ExcludeNodes option overrides this option: any node listed in both + MiddleNodes and ExcludeNodes is treated as excluded. See + the **ExcludeNodes** option for more information on how to specify nodes. + [[EntryNodes]] **EntryNodes** __node__,__node__,__...__:: A list of identity fingerprints and country codes of nodes to use for the first hop in your normal circuits. @@ -1036,13 +1051,14 @@ The following options are useful only for clients (that is, if If StrictNodes is set to 1, Tor will treat solely the ExcludeNodes option as a requirement to follow for all the circuits you generate, even if doing so will break functionality for you (StrictNodes applies to neither - ExcludeExitNodes nor to ExitNodes). If StrictNodes is set to 0, Tor will - still try to avoid nodes in the ExcludeNodes list, but it will err on the - side of avoiding unexpected errors. Specifically, StrictNodes 0 tells Tor - that it is okay to use an excluded node when it is *necessary* to perform - relay reachability self-tests, connect to a hidden service, provide a - hidden service to a client, fulfill a .exit request, upload directory - information, or download directory information. (Default: 0) + ExcludeExitNodes nor to ExitNodes, nor to MiddleNodes). If StrictNodes + is set to 0, Tor will still try to avoid nodes in the ExcludeNodes list, + but it will err on the side of avoiding unexpected errors. + Specifically, StrictNodes 0 tells Tor that it is okay to use an excluded + node when it is *necessary* to perform relay reachability self-tests, + connect to a hidden service, provide a hidden service to a client, + fulfill a .exit request, upload directory information, or download + directory information. (Default: 0) [[FascistFirewall]] **FascistFirewall** **0**|**1**:: If 1, Tor will only create outgoing connections to ORs running on ports diff --git a/src/app/config/config.c b/src/app/config/config.c index 32b487dd24..728b7ff65f 100644 --- a/src/app/config/config.c +++ b/src/app/config/config.c @@ -418,6 +418,10 @@ static config_var_t option_vars_[] = { V(ExcludeExitNodes, ROUTERSET, NULL), OBSOLETE("ExcludeSingleHopRelays"), V(ExitNodes, ROUTERSET, NULL), + /* Researchers need a way to tell their clients to use specific + * middles that they also control, to allow safe live-network + * experimentation with new padding machines. */ + V(MiddleNodes, ROUTERSET, NULL), V(ExitPolicy, LINELIST, NULL), V(ExitPolicyRejectPrivate, BOOL, "1"), V(ExitPolicyRejectLocalInterfaces, BOOL, "0"), @@ -1690,6 +1694,7 @@ options_need_geoip_info(const or_options_t *options, const char **reason_out) int routerset_usage = routerset_needs_geoip(options->EntryNodes) || routerset_needs_geoip(options->ExitNodes) || + routerset_needs_geoip(options->MiddleNodes) || routerset_needs_geoip(options->ExcludeExitNodes) || routerset_needs_geoip(options->ExcludeNodes) || routerset_needs_geoip(options->HSLayer2Nodes) || @@ -2129,6 +2134,7 @@ options_act(const or_options_t *old_options) options->HSLayer2Nodes) || !routerset_equal(old_options->HSLayer3Nodes, options->HSLayer3Nodes) || + !routerset_equal(old_options->MiddleNodes, options->MiddleNodes) || options->StrictNodes != old_options->StrictNodes) { log_info(LD_CIRC, "Changed to using entry guards or bridges, or changed " diff --git a/src/app/config/or_options_st.h b/src/app/config/or_options_st.h index c2bc1079a5..63a17c9771 100644 --- a/src/app/config/or_options_st.h +++ b/src/app/config/or_options_st.h @@ -72,6 +72,9 @@ struct or_options_t { routerset_t *ExitNodes; /**< Structure containing nicknames, digests, * country codes and IP address patterns of ORs to * consider as exits. */ + routerset_t *MiddleNodes; /**< Structure containing nicknames, digests, + * country codes and IP address patterns of ORs to + * consider as middles. */ routerset_t *EntryNodes;/**< Structure containing nicknames, digests, * country codes and IP address patterns of ORs to * consider as entry points. */ diff --git a/src/core/or/circuitbuild.c b/src/core/or/circuitbuild.c index 2d8bc4d4ad..22e4cf96d8 100644 --- a/src/core/or/circuitbuild.c +++ b/src/core/or/circuitbuild.c @@ -2610,7 +2610,24 @@ choose_good_middle_server(uint8_t purpose, return choice; } - choice = router_choose_random_node(excluded, options->ExcludeNodes, flags); + if (options->MiddleNodes) { + smartlist_t *sl = smartlist_new(); + routerset_get_all_nodes(sl, options->MiddleNodes, + options->ExcludeNodes, 1); + + smartlist_subtract(sl, excluded); + + choice = node_sl_choose_by_bandwidth(sl, WEIGHT_FOR_MID); + smartlist_free(sl); + if (choice) { + log_fn(LOG_INFO, LD_CIRC, "Chose fixed middle node: %s", + hex_str(choice->identity, DIGEST_LEN)); + } else { + log_fn(LOG_NOTICE, LD_CIRC, "Restricted middle not available"); + } + } else { + choice = router_choose_random_node(excluded, options->ExcludeNodes, flags); + } smartlist_free(excluded); return choice; } diff --git a/src/feature/nodelist/nodelist.c b/src/feature/nodelist/nodelist.c index 15b3f7b600..33601fe1fa 100644 --- a/src/feature/nodelist/nodelist.c +++ b/src/feature/nodelist/nodelist.c @@ -2350,7 +2350,7 @@ compute_frac_paths_available(const networkstatus_t *consensus, const int authdir = authdir_mode_v3(options); count_usable_descriptors(num_present_out, num_usable_out, - mid, consensus, now, NULL, + mid, consensus, now, options->MiddleNodes, USABLE_DESCRIPTOR_ALL); log_debug(LD_NET, "%s: %d present, %d usable", diff --git a/src/feature/nodelist/routerlist.c b/src/feature/nodelist/routerlist.c index b4d56459df..c8a658414b 100644 --- a/src/feature/nodelist/routerlist.c +++ b/src/feature/nodelist/routerlist.c @@ -3221,6 +3221,8 @@ refresh_all_country_info(void) routerset_refresh_countries(options->EntryNodes); if (options->ExitNodes) routerset_refresh_countries(options->ExitNodes); + if (options->MiddleNodes) + routerset_refresh_countries(options->MiddleNodes); if (options->ExcludeNodes) routerset_refresh_countries(options->ExcludeNodes); if (options->ExcludeExitNodes) From 2ccf3268375cd46e8c948e94ba58e0d2f03fe722 Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Tue, 27 Nov 2018 01:56:23 +0200 Subject: [PATCH 0367/2557] Implement and test probability distributions used by WTF-PAD. This project introduces the prob_distr.c subsystem which implements all the probability distributions that WTF-PAD needs. It also adds unittests for all of them. Code and tests courtesy of Riastradh. Co-authored-by: Taylor R Campbell Co-authored-by: Mike Perry --- src/core/or/circuitpadding.c | 8 +- src/lib/crypt_ops/crypto_rand.c | 11 + src/lib/crypt_ops/crypto_rand.h | 1 + src/lib/math/.may_include | 2 + src/lib/math/fp.c | 25 + src/lib/math/fp.h | 1 + src/lib/math/include.am | 6 +- src/lib/math/prob_distr.c | 1628 +++++++++++++++++++++++++++++++ src/lib/math/prob_distr.h | 156 +++ src/test/include.am | 2 + src/test/prob_distr_mpfr_ref.c | 64 ++ src/test/test.c | 1 + src/test/test.h | 2 + src/test/test_prob_distr.c | 1414 +++++++++++++++++++++++++++ src/test/test_slow.c | 1 + 15 files changed, 3316 insertions(+), 6 deletions(-) create mode 100644 src/lib/math/prob_distr.c create mode 100644 src/lib/math/prob_distr.h create mode 100644 src/test/prob_distr_mpfr_ref.c create mode 100644 src/test/test_prob_distr.c diff --git a/src/core/or/circuitpadding.c b/src/core/or/circuitpadding.c index 6cfbf4ba56..a9d927619d 100644 --- a/src/core/or/circuitpadding.c +++ b/src/core/or/circuitpadding.c @@ -516,10 +516,10 @@ circpad_distribution_sample(circpad_distribution_t dist) * param1 is Alpha, param2 is Beta */ return dist.param1 * pow(p/(1.0-p), 1.0/dist.param2); case CIRCPAD_DIST_GEOMETRIC: - p = crypto_rand_double(); - /* https://github.com/distributions-io/geometric-quantile/ - * param1 is 'p' (success probability) */ - return ceil(tor_mathlog(1.0-p)/tor_mathlog(1.0-dist.param1)); + { + /* param1 is 'p' (success probability) */ + return geometric_sample(dist.param1); + } case CIRCPAD_DIST_WEIBULL: p = crypto_rand_double(); /* https://en.wikipedia.org/wiki/Weibull_distribution \ diff --git a/src/lib/crypt_ops/crypto_rand.c b/src/lib/crypt_ops/crypto_rand.c index cffd0610f3..7a2c417e5a 100644 --- a/src/lib/crypt_ops/crypto_rand.c +++ b/src/lib/crypt_ops/crypto_rand.c @@ -528,6 +528,17 @@ crypto_rand_unmocked(char *to, size_t n) #endif } +/** + * Draw an unsigned 32-bit integer uniformly at random. + */ +uint32_t +crypto_rand_uint32(void) +{ + uint32_t rand; + crypto_rand((void*)&rand, sizeof(rand)); + return rand; +} + /** * Return a pseudorandom integer, chosen uniformly from the values * between 0 and max-1 inclusive. max must be between 1 and diff --git a/src/lib/crypt_ops/crypto_rand.h b/src/lib/crypt_ops/crypto_rand.h index 0c538d81ac..61fd82c806 100644 --- a/src/lib/crypt_ops/crypto_rand.h +++ b/src/lib/crypt_ops/crypto_rand.h @@ -27,6 +27,7 @@ int crypto_rand_int(unsigned int max); int crypto_rand_int_range(unsigned int min, unsigned int max); uint64_t crypto_rand_uint64_range(uint64_t min, uint64_t max); time_t crypto_rand_time_range(time_t min, time_t max); +uint32_t crypto_rand_uint32(void); uint64_t crypto_rand_uint64(uint64_t max); double crypto_rand_double(void); struct tor_weak_rng_t; diff --git a/src/lib/math/.may_include b/src/lib/math/.may_include index 1fd26864dc..f8bc264a5f 100644 --- a/src/lib/math/.may_include +++ b/src/lib/math/.may_include @@ -3,3 +3,5 @@ orconfig.h lib/cc/*.h lib/log/*.h lib/math/*.h +lib/testsupport/*.h +lib/crypt_ops/*.h diff --git a/src/lib/math/fp.c b/src/lib/math/fp.c index d5989db637..57082fa468 100644 --- a/src/lib/math/fp.c +++ b/src/lib/math/fp.c @@ -117,3 +117,28 @@ ENABLE_GCC_WARNING(double-promotion) ENABLE_GCC_WARNING(float-conversion) #endif } + +/* isinf() wrapper for tor */ +int +tor_isinf(double x) +{ + /* Same as above, work around the "double promotion" warnings */ +#if defined(MINGW_ANY) && GCC_VERSION >= 409 +#define PROBLEMATIC_FLOAT_CONVERSION_WARNING +DISABLE_GCC_WARNING(float-conversion) +#endif /* defined(MINGW_ANY) && GCC_VERSION >= 409 */ +#if defined(__clang__) +#if __has_warning("-Wdouble-promotion") +#define PROBLEMATIC_DOUBLE_PROMOTION_WARNING +DISABLE_GCC_WARNING(double-promotion) +#endif +#endif /* defined(__clang__) */ + return isinf(x); +#ifdef PROBLEMATIC_DOUBLE_PROMOTION_WARNING +ENABLE_GCC_WARNING(double-promotion) +#endif +#ifdef PROBLEMATIC_FLOAT_CONVERSION_WARNING +ENABLE_GCC_WARNING(float-conversion) +#endif +} + diff --git a/src/lib/math/fp.h b/src/lib/math/fp.h index e27b8f8d80..ddf3ed24d6 100644 --- a/src/lib/math/fp.h +++ b/src/lib/math/fp.h @@ -19,5 +19,6 @@ double tor_mathlog(double d) ATTR_CONST; long tor_lround(double d) ATTR_CONST; int64_t tor_llround(double d) ATTR_CONST; int64_t clamp_double_to_int64(double number); +int tor_isinf(double x); #endif diff --git a/src/lib/math/include.am b/src/lib/math/include.am index b088b3f3cc..6d65ce90a7 100644 --- a/src/lib/math/include.am +++ b/src/lib/math/include.am @@ -7,7 +7,8 @@ endif src_lib_libtor_math_a_SOURCES = \ src/lib/math/fp.c \ - src/lib/math/laplace.c + src/lib/math/laplace.c \ + src/lib/math/prob_distr.c src_lib_libtor_math_testing_a_SOURCES = \ @@ -17,4 +18,5 @@ src_lib_libtor_math_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) noinst_HEADERS += \ src/lib/math/fp.h \ - src/lib/math/laplace.h + src/lib/math/laplace.h \ + src/lib/math/prob_distr.h diff --git a/src/lib/math/prob_distr.c b/src/lib/math/prob_distr.c new file mode 100644 index 0000000000..832d3b4d96 --- /dev/null +++ b/src/lib/math/prob_distr.c @@ -0,0 +1,1628 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file prob_distr.c + * + * \brief + * Implements various probability distributions. + * Almost all code is courtesy of Riastradh. + * + * \details + * Here are some details that might help you understand this file: + * + * - Throughout this file, `eps' means the largest relative error of a + * correctly rounded floating-point operation, which in binary64 + * floating-point arithmetic is 2^-53. Here the relative error of a + * true value x from a computed value y is |x - y|/|x|. This + * definition of epsilon is conventional for numerical analysts when + * writing error analyses. (If your libm doesn't provide correctly + * rounded exp and log, their relative error is usually below 2*2^-53 + * and probably closer to 1.1*2^-53 instead.) + * + * The C constant DBL_EPSILON is actually twice this, and should + * perhaps rather be named ulp(1) -- that is, it is the distance from + * 1 to the next greater floating-point number, which is usually of + * more interest to programmers and hardware engineers. + * + * Since this file is concerned mainly with error bounds rather than + * with low-level bit-hacking of floating-point numbers, we adopt the + * numerical analysts' definition in the comments, though we do use + * DBL_EPSILON in a handful of places where it is convenient to use + * some function of eps = DBL_EPSILON/2 in a case analysis. + * + * - In various functions (e.g. sample_log_logistic()) we jump through hoops so + * that we can use reals closer to 0 than closer to 1, since we achieve much + * greater accuracy for floating point numbers near 0. In particular, we can + * represent differences as small as 10^-300 for numbers near 0, but of no + * less than 10^-16 for numbers near 1. + **/ + +#define PROB_DISTR_PRIVATE + +#include "orconfig.h" + +#include "lib/math/prob_distr.h" + +#include "lib/crypt_ops/crypto_rand.h" +#include "lib/cc/ctassert.h" + +#include +#include +#include + +/** Validators for downcasting macros below */ +#define validate_container_of(PTR, TYPE, FIELD) \ + (0 * sizeof((PTR) - &((TYPE *)(((char *)(PTR)) - \ + offsetof(TYPE, FIELD)))->FIELD)) +#define validate_const_container_of(PTR, TYPE, FIELD) \ + (0 * sizeof((PTR) - &((const TYPE *)(((const char *)(PTR)) - \ + offsetof(TYPE, FIELD)))->FIELD)) +/** Downcasting macro */ +#define container_of(PTR, TYPE, FIELD) \ + ((TYPE *)(((char *)(PTR)) - offsetof(TYPE, FIELD)) \ + + validate_container_of(PTR, TYPE, FIELD)) +/** Constified downcasting macro */ +#define const_container_of(PTR, TYPE, FIELD) \ + ((const TYPE *)(((const char *)(PTR)) - offsetof(TYPE, FIELD)) \ + + validate_const_container_of(PTR, TYPE, FIELD)) + +/** + * Count number of one bits in 32-bit word. + */ +static unsigned +bitcount32(uint32_t x) +{ + + /* Count two-bit groups. */ + x -= (x >> 1) & UINT32_C(0x55555555); + + /* Count four-bit groups. */ + x = ((x >> 2) & UINT32_C(0x33333333)) + (x & UINT32_C(0x33333333)); + + /* Count eight-bit groups. */ + x = (x + (x >> 4)) & UINT32_C(0x0f0f0f0f); + + /* Sum all eight-bit groups, and extract the sum. */ + return (x * UINT32_C(0x01010101)) >> 24; +} + +/** + * Count leading zeros in 32-bit word. + */ +static unsigned +clz32(uint32_t x) +{ + + /* Round up to a power of two. */ + x |= x >> 1; + x |= x >> 2; + x |= x >> 4; + x |= x >> 8; + x |= x >> 16; + + /* Subtract count of one bits from 32. */ + return (32 - bitcount32(x)); +} + +/* + * Some lemmas that will be used throughout this file to prove various error + * bounds: + * + * Lemma 1. If |d| <= 1/2, then 1/(1 + d) <= 2. + * + * Proof. If 0 <= d <= 1/2, then 1 + d >= 1, so that 1/(1 + d) <= 1. + * If -1/2 <= d <= 0, then 1 + d >= 1/2, so that 1/(1 + d) <= 2. QED. + * + * Lemma 2. If b = a*(1 + d)/(1 + d') for |d'| < 1/2 and nonzero a, b, + * then b = a*(1 + e) for |e| <= 2|d' - d|. + * + * Proof. |a - b|/|a| + * = |a - a*(1 + d)/(1 + d')|/|a| + * = |1 - (1 + d)/(1 + d')| + * = |(1 + d' - 1 - d)/(1 + d')| + * = |(d' - d)/(1 + d')| + * <= 2|d' - d|, by Lemma 1, + * + * QED. + * + * Lemma 3. For |d|, |d'| < 1/4, + * + * |log((1 + d)/(1 + d'))| <= 4|d - d'|. + * + * Proof. Write + * + * log((1 + d)/(1 + d')) + * = log(1 + (1 + d)/(1 + d') - 1) + * = log(1 + (1 + d - 1 - d')/(1 + d') + * = log(1 + (d - d')/(1 + d')). + * + * By Lemma 1, |(d - d')/(1 + d')| < 2|d' - d| < 1, so the Taylor + * series of log(1 + x) converges absolutely for (d - d')/(1 + d'), + * and thus we have + * + * |log(1 + (d - d')/(1 + d'))| + * = |\sum_{n=1}^\infty ((d - d')/(1 + d'))^n/n| + * <= \sum_{n=1}^\infty |(d - d')/(1 + d')|^n/n + * <= \sum_{n=1}^\infty |2(d' - d)|^n/n + * <= \sum_{n=1}^\infty |2(d' - d)|^n + * = 1/(1 - |2(d' - d)|) + * <= 4|d' - d|, + * + * QED. + * + * Lemma 4. If 1/e <= 1 + x <= e, then + * + * log(1 + (1 + d) x) = (1 + d') log(1 + x) + * + * for |d'| < 8|d|. + * + * Proof. Write + * + * log(1 + (1 + d) x) + * = log(1 + x + x*d) + * = log((1 + x) (1 + x + x*d)/(1 + x)) + * = log(1 + x) + log((1 + x + x*d)/(1 + x)) + * = log(1 + x) (1 + log((1 + x + x*d)/(1 + x))/log(1 + x)). + * + * The relative error is bounded by + * + * |log((1 + x + x*d)/(1 + x))/log(1 + x)| + * <= 4|x + x*d - x|/|log(1 + x)|, by Lemma 3, + * = 4|x*d|/|log(1 + x)| + * < 8|d|, + * + * since in this range 0 < 1 - 1/e < x/log(1 + x) <= e - 1 < 2. QED. + */ + +/** + * Compute the logistic function: f(x) = 1/(1 + e^{-x}) = e^x/(1 + e^x). + * Maps a log-odds-space probability in [-\infty, +\infty] into a direct-space + * probability in [0,1]. Inverse of logit. + * + * Ill-conditioned for large x; the identity logistic(-x) = 1 - + * logistic(x) and the function logistichalf(x) = logistic(x) - 1/2 may + * help to rearrange a computation. + * + * This implementation gives relative error bounded by 7 eps. + */ +STATIC double +logistic(double x) +{ + if (x <= log(DBL_EPSILON/2)) { + /* + * If x <= log(DBL_EPSILON/2) = log(eps), then e^x <= eps. In this case + * we will approximate the logistic() function with e^x because the + * relative error is less than eps. Here is a calculation of the + * relative error between the logistic() function and e^x and a proof + * that it's less than eps: + * + * |e^x - e^x/(1 + e^x)|/|e^x/(1 + e^x)| + * <= |1 - 1/(1 + e^x)|*|1 + e^x| + * = |e^x/(1 + e^x)|*|1 + e^x| + * = |e^x| + * <= eps. + */ + return exp(x); /* return e^x */ + } else if (x <= -log(DBL_EPSILON/2)) { + /* + * e^{-x} > 0, so 1 + e^{-x} > 1, and 0 < 1/(1 + + * e^{-x}) < 1; further, since e^{-x} < 1 + e^{-x}, we + * also have 0 < 1/(1 + e^{-x}) < 1. Thus, if exp has + * relative error d0, + has relative error d1, and / + * has relative error d2, then we get + * + * (1 + d2)/[(1 + (1 + d0) e^{-x})(1 + d1)] + * = (1 + d0)/[1 + e^{-x} + d0 e^{-x} + * + d1 + d1 e^{-x} + d0 d1 e^{-x}] + * = (1 + d0)/[(1 + e^{-x}) + * * (1 + d0 e^{-x}/(1 + e^{-x}) + * + d1/(1 + e^{-x}) + * + d0 d1 e^{-x}/(1 + e^{-x}))]. + * = (1 + d0)/[(1 + e^{-x})(1 + d')] + * = [1/(1 + e^{-x})] (1 + d0)/(1 + d') + * + * where + * + * d' = d0 e^{-x}/(1 + e^{-x}) + * + d1/(1 + e^{-x}) + * + d0 d1 e^{-x}/(1 + e^{-x}). + * + * By Lemma 2 this relative error is bounded by + * + * 2|d0 - d'| + * = 2|d0 - d0 e^{-x}/(1 + e^{-x}) + * - d1/(1 + e^{-x}) + * - d0 d1 e^{-x}/(1 + e^{-x})| + * <= 2|d0| + 2|d0 e^{-x}/(1 + e^{-x})| + * + 2|d1/(1 + e^{-x})| + * + 2|d0 d1 e^{-x}/(1 + e^{-x})| + * <= 2|d0| + 2|d0| + 2|d1| + 2|d0 d1| + * <= 4|d0| + 2|d1| + 2|d0 d1| + * <= 6 eps + 2 eps^2. + */ + return 1/(1 + exp(-x)); + } else { + /* + * e^{-x} <= eps, so the relative error of 1 from 1/(1 + * + e^{-x}) is + * + * |1/(1 + e^{-x}) - 1|/|1/(1 + e^{-x})| + * = |e^{-x}/(1 + e^{-x})|/|1/(1 + e^{-x})| + * = |e^{-x}| + * <= eps. + * + * This computation avoids an intermediate overflow + * exception, although the effect on the result is + * harmless. + * + * XXX Should maybe raise inexact here. + */ + return 1; + } +} + +/** + * Compute the logit function: log p/(1 - p). Defined on [0,1]. Maps + * a direct-space probability in [0,1] to a log-odds-space probability + * in [-\infty, +\infty]. Inverse of logistic. + * + * Ill-conditioned near 1/2 and 1; the identity logit(1 - p) = + * -logit(p) and the function logithalf(p0) = logit(1/2 + p0) may help + * to rearrange a computation for p in [1/(1 + e), 1 - 1/(1 + e)]. + * + * This implementation gives relative error bounded by 10 eps. + */ +STATIC double +logit(double p) +{ + + /* logistic(-1) <= p <= logistic(+1) */ + if (1/(1 + exp(1)) <= p && p <= 1/(1 + exp(-1))) { + /* + * For inputs near 1/2, we want to compute log1p(near + * 0) rather than log(near 1), so write this as: + * + * log(p/(1 - p)) = -log((1 - p)/p) + * = -log(1 + (1 - p)/p - 1) + * = -log(1 + (1 - p - p)/p) + * = -log(1 + (1 - 2p)/p). + * + * Since p = 2p/2 <= 1 <= 2*2p = 4p, the floating-point + * evaluation of 1 - 2p is exact; the only error arises + * from division and log1p. First, note that if + * logistic(-1) <= p <= logistic(+1), (1 - 2p)/p lies + * in the bounds of Lemma 4. + * + * If division has relative error d0 and log1p has + * relative error d1, the outcome is + * + * -(1 + d1) log(1 + (1 - 2p) (1 + d0)/p) + * = -(1 + d1) (1 + d') log(1 + (1 - 2p)/p) + * = -(1 + d1 + d' + d1 d') log(1 + (1 - 2p)/p). + * + * where |d'| < 8|d0| by Lemma 4. The relative error + * is then bounded by + * + * |d1 + d' + d1 d'| + * <= |d1| + 8|d0| + 8|d1 d0| + * <= 9 eps + 8 eps^2. + */ + return -log1p((1 - 2*p)/p); + } else { + /* + * For inputs near 0, although 1 - p may be rounded to + * 1, it doesn't matter much because the magnitude of + * the result is so much larger. For inputs near 1, we + * can compute 1 - p exactly, although the precision on + * the input is limited so we won't ever get more than + * about 700 for the output. + * + * If - has relative error d0, / has relative error d1, + * and log has relative error d2, then + * + * (1 + d2) log((1 + d0) p/[(1 - p)(1 + d1)]) + * = (1 + d2) [log(p/(1 - p)) + log((1 + d0)/(1 + d1))] + * = log(p/(1 - p)) + d2 log(p/(1 - p)) + * + (1 + d2) log((1 + d0)/(1 + d1)) + * = log(p/(1 - p))*[1 + d2 + + * + (1 + d2) log((1 + d0)/(1 + d1))/log(p/(1 - p))] + * + * Since 0 <= p < logistic(-1) or logistic(+1) < p <= + * 1, we have |log(p/(1 - p))| > 1. Hence this error + * is bounded by + * + * |d2 + (1 + d2) log((1 + d0)/(1 + d1))/log(p/(1 - p))| + * <= |d2| + |(1 + d2) log((1 + d0)/(1 + d1)) + * / log(p/(1 - p))| + * <= |d2| + |(1 + d2) log((1 + d0)/(1 + d1))| + * <= |d2| + 4|(1 + d2) (d0 - d1)|, by Lemma 3, + * <= |d2| + 4|d0 - d1 + d2 d0 - d1 d0| + * <= |d2| + 4|d0| + 4|d1| + 4|d2 d0| + 4|d1 d0| + * <= 9 eps + 8 eps^2. + */ + return log(p/(1 - p)); + } +} + +/** + * Compute the logit function, translated in input by 1/2: logithalf(p) + * = logit(1/2 + p). Defined on [-1/2, 1/2]. Inverse of logistichalf. + * + * Ill-conditioned near +/-1/2. If |p0| > 1/2 - 1/(1 + e), it may be + * better to compute 1/2 + p0 or -1/2 - p0 and to use logit instead. + * This implementation gives relative error bounded by 34 eps. + */ +STATIC double +logithalf(double p0) +{ + + if (fabs(p0) <= 0.5 - 1/(1 + exp(1))) { + /* + * logit(1/2 + p0) + * = log((1/2 + p0)/(1 - (1/2 + p0))) + * = log((1/2 + p0)/(1/2 - p0)) + * = log(1 + (1/2 + p0)/(1/2 - p0) - 1) + * = log(1 + (1/2 + p0 - (1/2 - p0))/(1/2 - p0)) + * = log(1 + (1/2 + p0 - 1/2 + p0)/(1/2 - p0)) + * = log(1 + 2 p0/(1/2 - p0)) + * + * If the error of subtraction is d0, the error of + * division is d1, and the error of log1p is d2, then + * what we compute is + * + * (1 + d2) log(1 + (1 + d1) 2 p0/[(1 + d0) (1/2 - p0)]) + * = (1 + d2) log(1 + (1 + d') 2 p0/(1/2 - p0)) + * = (1 + d2) (1 + d'') log(1 + 2 p0/(1/2 - p0)) + * = (1 + d2 + d'' + d2 d'') log(1 + 2 p0/(1/2 - p0)), + * + * where |d'| < 2|d0 - d1| <= 4 eps by Lemma 2, and + * |d''| < 8|d'| < 32 eps by Lemma 4 since + * + * 1/e <= 1 + 2*p0/(1/2 - p0) <= e + * + * when |p0| <= 1/2 - 1/(1 + e). Hence the relative + * error is bounded by + * + * |d2 + d'' + d2 d''| + * <= |d2| + |d''| + |d2 d''| + * <= |d1| + 32 |d0| + 32 |d1 d0| + * <= 33 eps + 32 eps^2. + */ + return log1p(2*p0/(0.5 - p0)); + } else { + /* + * We have a choice of computing logit(1/2 + p0) or + * -logit(1 - (1/2 + p0)) = -logit(1/2 - p0). It + * doesn't matter which way we do this: either way, + * since 1/2 p0 <= 1/2 <= 2 p0, the sum and difference + * are computed exactly. So let's do the one that + * skips the final negation. + * + * The result is + * + * (1 + d1) log((1 + d0) (1/2 + p0)/[(1 + d2) (1/2 - p0)]) + * = (1 + d1) (1 + log((1 + d0)/(1 + d2)) + * / log((1/2 + p0)/(1/2 - p0))) + * * log((1/2 + p0)/(1/2 - p0)) + * = (1 + d') log((1/2 + p0)/(1/2 - p0)) + * = (1 + d') logit(1/2 + p0) + * + * where + * + * d' = d1 + log((1 + d0)/(1 + d2))/logit(1/2 + p0) + * + d1 log((1 + d0)/(1 + d2))/logit(1/2 + p0). + * + * For |p| > 1/2 - 1/(1 + e), logit(1/2 + p0) > 1. + * Provided |d0|, |d2| < 1/4, by Lemma 3 we have + * + * |log((1 + d0)/(1 + d2))| <= 4|d0 - d2|. + * + * Hence the relative error is bounded by + * + * |d'| <= |d1| + 4|d0 - d2| + 4|d1| |d0 - d2| + * <= |d1| + 4|d0| + 4|d2| + 4|d1 d0| + 4|d1 d2| + * <= 9 eps + 8 eps^2. + */ + return log((0.5 + p0)/(0.5 - p0)); + } +} + +/* + * The following random_uniform_01 is tailored for IEEE 754 binary64 + * floating-point or smaller. It can be adapted to larger + * floating-point formats like i387 80-bit or IEEE 754 binary128, but + * it may require sampling more bits. + */ +CTASSERT(FLT_RADIX == 2); +CTASSERT(-DBL_MIN_EXP <= 1021); +CTASSERT(DBL_MANT_DIG <= 53); + +/** + * Draw a floating-point number in [0, 1] with uniform distribution. + * + * Note that the probability of returning 0 is less than 2^-1074, so + * callers need not check for it. However, callers that cannot handle + * rounding to 1 must deal with that, because it occurs with + * probability 2^-54, which is small but nonnegligible. + */ +STATIC double +random_uniform_01(void) +{ + uint32_t z, x, hi, lo; + double s; + + /* + * Draw an exponent, geometrically distributed, but give up if + * we get a run of more than 1088 zeros, which really means the + * system is broken. + */ + z = 0; + while ((x = crypto_rand_uint32()) == 0) { + if (z >= 1088) + /* Your bit sampler is broken. Go home. */ + return 0; + z += 32; + } + z += clz32(x); + + /* + * Pick 32-bit halves of an odd normalized significand. + * Picking it odd breaks ties in the subsequent rounding, which + * occur only with measure zero in the uniform distribution on + * [0, 1]. + */ + hi = crypto_rand_uint32() | UINT32_C(0x80000000); + lo = crypto_rand_uint32() | UINT32_C(0x00000001); + + /* Round to nearest scaled significand in [2^63, 2^64]. */ + s = hi*(double)4294967296 + lo; + + /* Rescale into [1/2, 1] and apply exponent in one swell foop. */ + return s * ldexp(1, -(64 + z)); +} + +/*******************************************************************/ + +/* Functions for specific probability distributions start here: */ + +/* + * Logistic(mu, sigma) distribution, supported on (-\infty,+\infty) + * + * This is the uniform distribution on [0,1] mapped into log-odds + * space, scaled by sigma and translated by mu. + * + * pdf(x) = e^{-(x - mu)/sigma} sigma (1 + e^{-(x - mu)/sigma})^2 + * cdf(x) = 1/(1 + e^{-(x - mu)/sigma}) = logistic((x - mu)/sigma) + * sf(x) = 1 - cdf(x) = 1 - logistic((x - mu)/sigma = logistic(-(x - mu)/sigma) + * icdf(p) = mu + sigma log p/(1 - p) = mu + sigma logit(p) + * isf(p) = mu + sigma log (1 - p)/p = mu - sigma logit(p) + */ + +/** + * Compute the CDF of the Logistic(mu, sigma) distribution: the + * logistic function. Well-conditioned for negative inputs and small + * positive inputs; ill-conditioned for large positive inputs. + */ +STATIC double +cdf_logistic(double x, double mu, double sigma) +{ + return logistic((x - mu)/sigma); +} + +/** + * Compute the SF of the Logistic(mu, sigma) distribution: the logistic + * function reflected over the y axis. Well-conditioned for positive + * inputs and small negative inputs; ill-conditioned for large negative + * inputs. + */ +STATIC double +sf_logistic(double x, double mu, double sigma) +{ + return logistic(-(x - mu)/sigma); +} + +/** + * Compute the inverse of the CDF of the Logistic(mu, sigma) + * distribution: the logit function. Well-conditioned near 0; + * ill-conditioned near 1/2 and 1. + */ +STATIC double +icdf_logistic(double p, double mu, double sigma) +{ + return mu + sigma*logit(p); +} + +/** + * Compute the inverse of the SF of the Logistic(mu, sigma) + * distribution: the -logit function. Well-conditioned near 0; + * ill-conditioned near 1/2 and 1. + */ +STATIC double +isf_logistic(double p, double mu, double sigma) +{ + return mu - sigma*logit(p); +} + +/* + * LogLogistic(alpha, beta) distribution, supported on (0, +\infty). + * + * This is the uniform distribution on [0,1] mapped into odds space, + * scaled by positive alpha and shaped by positive beta. + * + * Equivalent to computing exp of a Logistic(log alpha, 1/beta) sample. + * (Name arises because the pdf has LogLogistic(x; alpha, beta) = + * Logistic(log x; log alpha, 1/beta) and mathematicians got their + * covariance contravariant.) + * + * pdf(x) = (beta/alpha) (x/alpha)^{beta - 1}/(1 + (x/alpha)^beta)^2 + * = (1/e^mu sigma) (x/e^mu)^{1/sigma - 1} / + * (1 + (x/e^mu)^{1/sigma})^2 + * cdf(x) = 1/(1 + (x/alpha)^-beta) = 1/(1 + (x/e^mu)^{-1/sigma}) + * = 1/(1 + (e^{log x}/e^mu)^{-1/sigma}) + * = 1/(1 + (e^{log x - mu})^{-1/sigma}) + * = 1/(1 + e^{-(log x - mu)/sigma}) + * = logistic((log x - mu)/sigma) + * = logistic((log x - log alpha)/(1/beta)) + * sf(x) = 1 - 1/(1 + (x/alpha)^-beta) + * = (x/alpha)^-beta/(1 + (x/alpha)^-beta) + * = 1/((x/alpha)^beta + 1) + * = 1/(1 + (x/alpha)^beta) + * icdf(p) = alpha (p/(1 - p))^{1/beta} + * = alpha e^{logit(p)/beta} + * = e^{mu + sigma logit(p)} + * isf(p) = alpha ((1 - p)/p)^{1/beta} + * = alpha e^{-logit(p)/beta} + * = e^{mu - sigma logit(p)} + */ + +/** + * Compute the CDF of the LogLogistic(alpha, beta) distribution. + * Well-conditioned for all x and alpha, and the condition number + * + * -beta/[1 + (x/alpha)^{-beta}] + * + * grows linearly with beta. + * + * Loosely, the relative error of this implementation is bounded by + * + * 4 eps + 2 eps^2 + O(beta eps), + * + * so don't bother trying this for beta anywhere near as large as + * 1/eps, around which point it levels off at 1. + */ +STATIC double +cdf_log_logistic(double x, double alpha, double beta) +{ + /* + * Let d0 be the error of x/alpha; d1, of pow; d2, of +; and + * d3, of the final quotient. The exponentiation gives + * + * ((1 + d0) x/alpha)^{-beta} + * = (x/alpha)^{-beta} (1 + d0)^{-beta} + * = (x/alpha)^{-beta} (1 + (1 + d0)^{-beta} - 1) + * = (x/alpha)^{-beta} (1 + d') + * + * where d' = (1 + d0)^{-beta} - 1. If y = (x/alpha)^{-beta}, + * the denominator is + * + * (1 + d2) (1 + (1 + d1) (1 + d') y) + * = (1 + d2) (1 + y + (d1 + d' + d1 d') y) + * = 1 + y + (1 + d2) (d1 + d' + d1 d') y + * = (1 + y) (1 + (1 + d2) (d1 + d' + d1 d') y/(1 + y)) + * = (1 + y) (1 + d''), + * + * where d'' = (1 + d2) (d1 + d' + d1 d') y/(1 + y). The + * final result is + * + * (1 + d3) / [(1 + d2) (1 + d'') (1 + y)] + * = (1 + d''') / (1 + y) + * + * for |d'''| <= 2|d3 - d''| by Lemma 2 as long as |d''| < 1/2 + * (which may not be the case for very large beta). This + * relative error is therefore bounded by + * + * |d'''| + * <= 2|d3 - d''| + * <= 2|d3| + 2|(1 + d2) (d1 + d' + d1 d') y/(1 + y)| + * <= 2|d3| + 2|(1 + d2) (d1 + d' + d1 d')| + * = 2|d3| + 2|d1 + d' + d1 d' + d2 d1 + d2 d' + d2 d1 d'| + * <= 2|d3| + 2|d1| + 2|d'| + 2|d1 d'| + 2|d2 d1| + 2|d2 d'| + * + 2|d2 d1 d'| + * <= 4 eps + 2 eps^2 + (2 + 2 eps + 2 eps^2) |d'|. + * + * Roughly, |d'| = |(1 + d0)^{-beta} - 1| grows like beta eps, + * until it levels off at 1. + */ + return 1/(1 + pow(x/alpha, -beta)); +} + +/** + * Compute the SF of the LogLogistic(alpha, beta) distribution. + * Well-conditioned for all x and alpha, and the condition number + * + * beta/[1 + (x/alpha)^beta] + * + * grows linearly with beta. + * + * Loosely, the relative error of this implementation is bounded by + * + * 4 eps + 2 eps^2 + O(beta eps) + * + * so don't bother trying this for beta anywhere near as large as + * 1/eps, beyond which point it grows unbounded. + */ +STATIC double +sf_log_logistic(double x, double alpha, double beta) +{ + /* + * The error analysis here is essentially the same as in + * cdf_log_logistic, except that rather than levelling off at + * 1, |(1 + d0)^beta - 1| grows unbounded. + */ + return 1/(1 + pow(x/alpha, beta)); +} + +/** + * Compute the inverse of the CDF of the LogLogistic(alpha, beta) + * distribution. Ill-conditioned for p near 1 and beta near 0 with + * condition number 1/[beta (1 - p)]. + */ +STATIC double +icdf_log_logistic(double p, double alpha, double beta) +{ + return alpha*pow(p/(1 - p), 1/beta); +} + +/** + * Compute the inverse of the SF of the LogLogistic(alpha, beta) + * distribution. Ill-conditioned for p near 1 and for large beta, with + * condition number -1/[beta (1 - p)]. + */ +STATIC double +isf_log_logistic(double p, double alpha, double beta) +{ + return alpha*pow((1 - p)/p, 1/beta); +} + +/* + * Weibull(lambda, k) distribution, supported on (0, +\infty). + * + * pdf(x) = (k/lambda) (x/lambda)^{k - 1} e^{-(x/lambda)^k} + * cdf(x) = 1 - e^{-(x/lambda)^k} + * icdf(p) = lambda * (-log (1 - p))^{1/k} + * sf(x) = e^{-(x/lambda)^k} + * isf(p) = lambda * (-log p)^{1/k} + */ + +/** + * Compute the CDF of the Weibull(lambda, k) distribution. + * Well-conditioned for small x and k, and for large lambda -- + * condition number + * + * -k (x/lambda)^k exp(-(x/lambda)^k)/[exp(-(x/lambda)^k) - 1] + * + * grows linearly with k, x^k, and lambda^{-k}. + */ +STATIC double +cdf_weibull(double x, double lambda, double k) +{ + return -expm1(-pow(x/lambda, k)); +} + +/** + * Compute the SF of the Weibull(lambda, k) distribution. + * Well-conditioned for small x and k, and for large lambda -- + * condition number + * + * -k (x/lambda)^k + * + * grows linearly with k, x^k, and lambda^{-k}. + */ +STATIC double +sf_weibull(double x, double lambda, double k) +{ + return exp(-pow(x/lambda, k)); +} + +/** + * Compute the inverse of the CDF of the Weibull(lambda, k) + * distribution. Ill-conditioned for p near 1, and for k near 0; + * condition number is + * + * (p/(1 - p))/(k log(1 - p)). + */ +STATIC double +icdf_weibull(double p, double lambda, double k) +{ + return lambda*pow(-log1p(-p), 1/k); +} + +/** + * Compute the inverse of the SF of the Weibull(lambda, k) + * distribution. Ill-conditioned for p near 0, and for k near 0; + * condition number is + * + * 1/(k log(p)). + */ +STATIC double +isf_weibull(double p, double lambda, double k) +{ + return lambda*pow(-log(p), 1/k); +} + +/* + * GeneralizedPareto(mu, sigma, xi), supported on (mu, +\infty) for + * nonnegative xi, or (mu, mu - sigma/xi) for negative xi. + * + * Samples: + * = mu - sigma log U, if xi = 0; + * = mu + sigma (U^{-xi} - 1)/xi = mu + sigma*expm1(-xi log U)/xi, if xi =/= 0, + * where U is uniform on (0,1]. + * = mu + sigma (e^{xi X} - 1)/xi, + * where X has standard exponential distribution. + * + * pdf(x) = sigma^{-1} (1 + xi (x - mu)/sigma)^{-(1 + 1/xi)} + * cdf(x) = 1 - (1 + xi (x - mu)/sigma)^{-1/xi} + * = 1 - e^{-log(1 + xi (x - mu)/sigma)/xi} + * --> 1 - e^{-(x - mu)/sigma} as xi --> 0 + * sf(x) = (1 + xi (x - mu)/sigma)^{-1/xi} + * --> e^{-(x - mu)/sigma} as xi --> 0 + * icdf(p) = mu + sigma*(p^{-xi} - 1)/xi + * = mu + sigma*expm1(-xi log p)/xi + * --> mu + sigma*log p as xi --> 0 + * isf(p) = mu + sigma*((1 - p)^{xi} - 1)/xi + * = mu + sigma*expm1(-xi log1p(-p))/xi + * --> mu + sigma*log1p(-p) as xi --> 0 + */ + +/** + * Compute the CDF of the GeneralizedPareto(mu, sigma, xi) + * distribution. Well-conditioned everywhere. For standard + * distribution (mu=0, sigma=1), condition number + * + * (x/(1 + x xi)) / ((1 + x xi)^{1/xi} - 1) + * + * is bounded by 1, attained only at x = 0. + */ +STATIC double +cdf_genpareto(double x, double mu, double sigma, double xi) +{ + double x_0 = (x - mu)/sigma; + + /* + * log(1 + xi x_0)/xi + * = (-1/xi) \sum_{n=1}^\infty (-xi x_0)^n/n + * = (-1/xi) (-xi x_0 + \sum_{n=2}^\infty (-xi x_0)^n/n) + * = x_0 - (1/xi) \sum_{n=2}^\infty (-xi x_0)^n/n + * = x_0 - x_0 \sum_{n=2}^\infty (-xi x_0)^{n-1}/n + * = x_0 (1 - d), + * + * where d = \sum_{n=2}^\infty (-xi x_0)^{n-1}/n. If |xi| < + * eps/4|x_0|, then + * + * |d| <= \sum_{n=2}^\infty (eps/4)^{n-1}/n + * <= \sum_{n=2}^\infty (eps/4)^{n-1} + * = \sum_{n=1}^\infty (eps/4)^n + * = (eps/4) \sum_{n=0}^\infty (eps/4)^n + * = (eps/4)/(1 - eps/4) + * < eps/2 + * + * for any 0 < eps < 2. Thus, the relative error of x_0 from + * log(1 + xi x_0)/xi is bounded by eps. + */ + if (fabs(xi) < 1e-17/x_0) + return -expm1(-x_0); + else + return -expm1(-log1p(xi*x_0)/xi); +} + +/** + * Compute the SF of the GeneralizedPareto(mu, sigma, xi) distribution. + * For standard distribution (mu=0, sigma=1), ill-conditioned for xi + * near 0; condition number + * + * -x (1 + x xi)^{(-1 - xi)/xi}/(1 + x xi)^{-1/xi} + * = -x (1 + x xi)^{-1/xi - 1}/(1 + x xi)^{-1/xi} + * = -(x/(1 + x xi)) (1 + x xi)^{-1/xi}/(1 + x xi)^{-1/xi} + * = -x/(1 + x xi) + * + * is bounded by 1/xi. + */ +STATIC double +sf_genpareto(double x, double mu, double sigma, double xi) +{ + double x_0 = (x - mu)/sigma; + + if (fabs(xi) < 1e-17/x_0) + return exp(-x_0); + else + return exp(-log1p(xi*x_0)/xi); +} + +/** + * Compute the inverse of the CDF of the GeneralizedPareto(mu, sigma, + * xi) distribution. Ill-conditioned for p near 1; condition number is + * + * xi (p/(1 - p))/(1 - (1 - p)^xi) + */ +STATIC double +icdf_genpareto(double p, double mu, double sigma, double xi) +{ + /* + * To compute f(xi) = (U^{-xi} - 1)/xi = (e^{-xi log U} - 1)/xi + * for xi near zero (note f(xi) --> -log U as xi --> 0), write + * the absolutely convergent Taylor expansion + * + * f(xi) = (1/xi)*(-xi log U + \sum_{n=2}^\infty (-xi log U)^n/n! + * = -log U + (1/xi)*\sum_{n=2}^\infty (-xi log U)^n/n! + * = -log U + \sum_{n=2}^\infty xi^{n-1} (-log U)^n/n! + * = -log U - log U \sum_{n=2}^\infty (-xi log U)^{n-1}/n! + * = -log U (1 + \sum_{n=2}^\infty (-xi log U)^{n-1}/n!). + * + * Let d = \sum_{n=2}^\infty (-xi log U)^{n-1}/n!. What do we + * lose if we discard it and use -log U as an approximation to + * f(xi)? If |xi| < eps/-4log U, then + * + * |d| <= \sum_{n=2}^\infty |xi log U|^{n-1}/n! + * <= \sum_{n=2}^\infty (eps/4)^{n-1}/n! + * <= \sum_{n=1}^\infty (eps/4)^n + * = (eps/4) \sum_{n=0}^\infty (eps/4)^n + * = (eps/4)/(1 - eps/4) + * < eps/2, + * + * for any 0 < eps < 2. Hence, as long as |xi| < eps/-2log U, + * f(xi) = -log U (1 + d) for |d| <= eps/2. |d| is the + * relative error of f(xi) from -log U; from this bound, the + * relative error of -log U from f(xi) is at most (eps/2)/(1 - + * eps/2) = eps/2 + (eps/2)^2 + (eps/2)^3 + ... < eps for 0 < + * eps < 1. Since -log U < 1000 for all U in (0, 1] in + * binary64 floating-point, we can safely cut xi off at 1e-20 < + * eps/4000 and attain <1ulp error from series truncation. + */ + if (fabs(xi) <= 1e-20) + return mu - sigma*log1p(-p); + else + return mu + sigma*expm1(-xi*log1p(-p))/xi; +} + +/** + * Compute the inverse of the SF of the GeneralizedPareto(mu, sigma, + * xi) distribution. Ill-conditioned for p near 1; conditon number is + * + * -xi/(1 - p^{-xi}) + */ +STATIC double +isf_genpareto(double p, double mu, double sigma, double xi) +{ + if (fabs(xi) <= 1e-20) + return mu - sigma*log(p); + else + return mu + sigma*expm1(-xi*log(p))/xi; +} + +/*******************************************************************/ + +/** + * Deterministic samplers, parametrized by uniform integer and (0,1] + * samples. No guarantees are made about _which_ mapping from the + * integer and (0,1] samples these use; all that is guaranteed is the + * distribution of the outputs conditioned on a uniform distribution on + * the inputs. The automatic tests in test_prob_distr.c double-check + * the particular mappings we use. + * + * Beware: Unlike random_uniform_01(), these are not guaranteed to be + * supported on all possible outputs. See Ilya Mironov, `On the + * Significance of the Least Significant Bits for Differential + * Privacy', for an example of what can go wrong if you try to use + * these to conceal information from an adversary but you expose the + * specific full-precision floating-point values. + * + * Note: None of these samplers use rejection sampling; they are all + * essentially inverse-CDF transforms with tweaks. If you were to add, + * say, a Gamma sampler with the Marsaglia-Tsang method, you would have + * to parametrize it by a potentially infinite stream of uniform (and + * perhaps normal) samples rather than a fixed number, which doesn't + * make for quite as nice automatic testing as for these. + */ + +/** + * Deterministically sample from the interval [a, b], indexed by a + * uniform random floating-point number p0 in (0, 1]. + * + * Note that even if p0 is nonzero, the result may be equal to a, if + * ulp(a)/2 is nonnegligible, e.g. if a = 1. For maximum resolution, + * arrange |a| <= |b|. + */ +STATIC double +sample_uniform_interval(double p0, double a, double b) +{ + /* + * XXX Prove that the distribution is, in fact, uniform on + * [a,b], particularly around p0 = 1, or at least has very + * small deviation from uniform, quantified appropriately + * (e.g., like in Monahan 1984, or by KL divergence). It + * almost certainly does but it would be nice to quantify the + * error. + */ + if ((a <= 0 && 0 <= b) || (b <= 0 && 0 <= a)) { + /* + * When ab < 0, (1 - t) a + t b is monotonic, since for + * a <= b it is a sum of nondecreasing functions of t, + * and for b <= a, of nonincreasing functions of t. + * Further, clearly at 0 and 1 it attains a and b, + * respectively. Hence it is bounded within [a, b]. + */ + return (1 - p0)*a + p0*b; + } else { + /* + * a + (b - a) t is monotonic -- it is obviously a + * nondecreasing function of t for a <= b. Further, it + * attains a at 0, and while it may overshoot b at 1, + * we have a + * + * Theorem. If 0 <= t < 1, then the floating-point + * evaluation of a + (b - a) t is bounded in [a, b]. + * + * Lemma 1. If 0 <= t < 1 is a floating-point number, + * then for any normal floating-point number x except + * the smallest in magnitude, |round(x*t)| < |x|. + * + * Proof. WLOG, assume x >= 0. Since the rounding + * function and t |---> x*t are nondecreasing, their + * composition t |---> round(x*t) is also + * nondecreasing, so it suffices to consider the + * largest floating-point number below 1, in particular + * t = 1 - ulp(1)/2. + * + * Case I: If x is a power of two, then the next + * floating-point number below x is x - ulp(x)/2 = x - + * x*ulp(1)/2 = x*(1 - ulp(1)/2) = x*t, so, since x*t + * is a floating-point number, multiplication is exact, + * and thus round(x*t) = x*t < x. + * + * Case II: If x is not a power of two, then the + * greatest lower bound of real numbers rounded to x is + * x - ulp(x)/2 = x - ulp(T(x))/2 = x - T(x)*ulp(1)/2, + * where T(X) is the largest power of two below x. + * Anything below this bound is rounded to a + * floating-point number smaller than x, and x*t = x*(1 + * - ulp(1)/2) = x - x*ulp(1)/2 < x - T(x)*ulp(1)/2 + * since T(x) < x, so round(x*t) < x*t < x. QED. + * + * Lemma 2. If x and y are subnormal, then round(x + + * y) = x + y. + * + * Proof. It is a matter of adding the significands, + * since if we treat subnormals as having an implicit + * zero bit before the `binary' point, their exponents + * are all the same. There is at most one carry/borrow + * bit, which can always be acommodated either in a + * subnormal, or, at largest, in the implicit one bit + * of a normal. + * + * Lemma 3. Let x and y be floating-point numbers. If + * round(x - y) is subnormal or zero, then it is equal + * to x - y. + * + * Proof. Case I (equal): round(x - y) = 0 iff x = y; + * hence if round(x - y) = 0, then round(x - y) = 0 = x + * - y. + * + * Case II (subnormal/subnormal): If x and y are both + * subnormal, this follows directly from Lemma 2. + * + * Case IIIa (normal/subnormal): If x is normal and y + * is subnormal, then x and y must share sign, or else + * x - y would be larger than x and thus rounded to + * normal. If s is the smallest normal positive + * floating-point number, |x| < 2s since by + * construction 2s - |y| is normal for all subnormal y. + * This means that x and y must have the same exponent, + * so the difference is the difference of significands, + * which is exact. + * + * Case IIIb (subnormal/normal): Same as case IIIa for + * -(y - x). + * + * Case IV (normal/normal): If x and y are both normal, + * then they must share sign, or else x - y would be + * larger than x and thus rounded to normal. Note that + * |y| < 2|x|, for if |y| >= 2|x|, then |x| - |y| <= + * -|x| but -|x| is normal like x. Also, |x|/2 < |y|: + * if |x|/2 is subnormal, it must hold because y is + * normal; if |x|/2 is normal, then |x|/2 >= s, so + * since |x| - |y| < s, + * + * |x|/2 = |x| - |x|/2 <= |x| - s <= |y|; + * + * that is, |x|/2 < |y| < 2|x|, so by the Sterbenz + * lemma, round(x - y) = x - y. QED. + * + * Proof of theorem. WLOG, assume 0 <= a <= b. Since + * round(a + round(round(b - a)*t) is nondecreasing in + * t and attains a at 0, the lower end of the bound is + * trivial; we must show the upper end of the bound + * strictly. It suffices to show this for the largest + * floating-point number below 1, namely 1 - ulp(1)/2. + * + * Case I: round(b - a) is normal. Then it is at most + * the smallest floating-point number above b - a. By + * Lemma 1, round(round(b - a)*t) < round(b - a). + * Since the inequality is strict, and since + * round(round(b - a)*t) is a floating-point number + * below round(b - a), and since there are no + * floating-point numbers between b - a and round(b - + * a), we must have round(round(b - a)*t) < b - a. + * Then since y |---> round(a + y) is nondecreasing, we + * must have + * + * round(a + round(round(b - a)*t)) + * <= round(a + (b - a)) + * = round(b) = b. + * + * Case II: round(b - a) is subnormal. In this case, + * Lemma 1 falls apart -- we are not guaranteed the + * strict inequality. However, by Lemma 3, the + * difference is exact: round(b - a) = b - a. Thus, + * + * round(a + round(round(b - a)*t)) + * <= round(a + round((b - a)*t)) + * <= round(a + (b - a)) + * = round(b) + * = b, + * + * QED. + */ + + /* p0 is restricted to [0,1], but we use >= to silence -Wfloat-equal. */ + if (p0 >= 1) + return b; + return a + (b - a)*p0; + } +} + +/** + * Deterministically sample from the standard logistic distribution, + * indexed by a uniform random 32-bit integer s and uniform random + * floating-point numbers t and p0 in (0, 1]. + */ +STATIC double +sample_logistic(uint32_t s, double t, double p0) +{ + double sign = (s & 1) ? -1 : +1; + double r; + + /* + * We carve up the interval (0, 1) into subregions to compute + * the inverse CDF precisely: + * + * A = (0, 1/(1 + e)] ---> (-\infty, -1] + * B = [1/(1 + e), 1/2] ---> [-1, 0] + * C = [1/2, 1 - 1/(1 + e)] ---> [0, 1] + * D = [1 - 1/(1 + e), 1) ---> [1, +\infty) + * + * Cases D and C are mirror images of cases A and B, + * respectively, so we choose between them by the sign chosen + * by a fair coin toss. We choose between cases A and B by a + * coin toss weighted by + * + * 2/(1 + e) = 1 - [1/2 - 1/(1 + e)]/(1/2): + * + * if it comes up heads, scale p0 into a uniform (0, 1/(1 + e)] + * sample p; if it comes up tails, scale p0 into a uniform (0, + * 1/2 - 1/(1 + e)] sample and compute the inverse CDF of p = + * 1/2 - p0. + */ + if (t <= 2/(1 + exp(1))) { + /* p uniform in (0, 1/(1 + e)], represented by p. */ + p0 /= 1 + exp(1); + r = logit(p0); + } else { + /* + * p uniform in [1/(1 + e), 1/2), actually represented + * by p0 = 1/2 - p uniform in (0, 1/2 - 1/(1 + e)], so + * that p = 1/2 - p. + */ + p0 *= 0.5 - 1/(1 + exp(1)); + r = logithalf(p0); + } + + /* + * We have chosen from the negative half of the standard + * logistic distribution, which is symmetric with the positive + * half. Now use the sign to choose uniformly between them. + */ + return sign*r; +} + +/** + * Deterministically sample from the logistic distribution scaled by + * sigma and translated by mu. + */ +static double +sample_logistic_locscale(uint32_t s, double t, double p0, double mu, + double sigma) +{ + + return mu + sigma*sample_logistic(s, t, p0); +} + +/** + * Deterministically sample from the standard log-logistic + * distribution, indexed by a uniform random 32-bit integer s and a + * uniform random floating-point number p0 in (0, 1]. + */ +STATIC double +sample_log_logistic(uint32_t s, double p0) +{ + + /* + * Carve up the interval (0, 1) into (0, 1/2] and [1/2, 1); the + * condition numbers of the icdf and the isf coincide at 1/2. + */ + p0 *= 0.5; + if ((s & 1) == 0) { + /* p = p0 in (0, 1/2] */ + return p0/(1 - p0); + } else { + /* p = 1 - p0 in [1/2, 1) */ + return (1 - p0)/p0; + } +} + +/** + * Deterministically sample from the log-logistic distribution with + * scale alpha and shape beta. + */ +static double +sample_log_logistic_scaleshape(uint32_t s, double p0, double alpha, + double beta) +{ + double x = sample_log_logistic(s, p0); + + return alpha*pow(x, 1/beta); +} + +/** + * Deterministically sample from the standard exponential distribution, + * indexed by a uniform random 32-bit integer s and a uniform random + * floating-point number p0 in (0, 1]. + */ +static double +sample_exponential(uint32_t s, double p0) +{ + /* + * We would like to evaluate log(p) for p near 0, and log1p(-p) + * for p near 1. Simply carve the interval into (0, 1/2] and + * [1/2, 1) by a fair coin toss. + */ + p0 *= 0.5; + if ((s & 1) == 0) + /* p = p0 in (0, 1/2] */ + return -log(p0); + else + /* p = 1 - p0 in [1/2, 1) */ + return -log1p(-p0); +} + +/** + * Deterministically sample from a Weibull distribution with scale + * lambda and shape k -- just an exponential with a shape parameter in + * addition to a scale parameter. (Yes, lambda really is the scale, + * _not_ the rate.) + */ +STATIC double +sample_weibull(uint32_t s, double p0, double lambda, double k) +{ + + return lambda*pow(sample_exponential(s, p0), 1/k); +} + +/** + * Deterministically sample from the generalized Pareto distribution + * with shape xi, indexed by a uniform random 32-bit integer s and a + * uniform random floating-point number p0 in (0, 1]. + */ +STATIC double +sample_genpareto(uint32_t s, double p0, double xi) +{ + double x = sample_exponential(s, p0); + + /* + * Write f(xi) = (e^{xi x} - 1)/xi for xi near zero as the + * absolutely convergent Taylor series + * + * f(x) = (1/xi) (xi x + \sum_{n=2}^\infty (xi x)^n/n!) + * = x + (1/xi) \sum_{n=2}^\inty (xi x)^n/n! + * = x + \sum_{n=2}^\infty xi^{n-1} x^n/n! + * = x + x \sum_{n=2}^\infty (xi x)^{n-1}/n! + * = x (1 + \sum_{n=2}^\infty (xi x)^{n-1}/n!). + * + * d = \sum_{n=2}^\infty (xi x)^{n-1}/n! is the relative error + * of f(x) from x. If |xi| < eps/4x, then + * + * |d| <= \sum_{n=2}^\infty |xi x|^{n-1}/n! + * <= \sum_{n=2}^\infty (eps/4)^{n-1}/n! + * <= \sum_{n=1}^\infty (eps/4) + * = (eps/4) \sum_{n=0}^\infty (eps/4)^n + * = (eps/4)/(1 - eps/4) + * < eps/2, + * + * for any 0 < eps < 2. Hence, as long as |xi| < eps/2x, f(xi) + * = x (1 + d) for |d| <= eps/2, so x = f(xi) (1 + d') for |d'| + * <= eps. What bound should we use for x? + * + * - If x is exponentially distributed, x > 200 with + * probability below e^{-200} << 2^{-256}, i.e. never. + * + * - If x is computed by -log(U) for U in (0, 1], x is + * guaranteed to be below 1000 in IEEE 754 binary64 + * floating-point. + * + * We can safely cut xi off at 1e-20 < eps/4000 and attain an + * error bounded by 0.5 ulp for this expression. + */ + return (fabs(xi) < 1e-20 ? x : expm1(xi*x)/xi); +} + +/** + * Deterministically sample from a generalized Pareto distribution with + * shape xi, scaled by sigma and translated by mu. + */ +static double +sample_genpareto_locscale(uint32_t s, double p0, double mu, double sigma, + double xi) +{ + + return mu + sigma*sample_genpareto(s, p0, xi); +} + +/** + * Deterministically sample from the geometric distribution with + * per-trial success probability p. + * + * XXX Quantify the error (KL divergence?) of this + * ceiling-of-exponential sampler from a true geometric distribution, + * which we could get by rejection sampling. Relevant papers: + * + * John F. Monahan, `Accuracy in Random Number Generation', + * Mathematics of Computation 45(172), October 1984, pp. 559--568. +*https://pdfs.semanticscholar.org/aca6/74b96da1df77b2224e8cfc5dd6d61a471632.pdf + * + * Karl Bringmann and Tobias Friedrich, `Exact and Efficient + * Generation of Geometric Random Variates and Random Graphs', in + * Proceedings of the 40th International Colloaquium on Automata, + * Languages, and Programming -- ICALP 2013, Springer LNCS 7965, + * pp.267--278. + * https://doi.org/10.1007/978-3-642-39206-1_23 + * https://people.mpi-inf.mpg.de/~kbringma/paper/2013ICALP-1.pdf + */ +static double +sample_geometric(uint32_t s, double p0, double p) +{ + double x = sample_exponential(s, p0); + + /* This is actually a check against 1, but we do >= so that the compiler + does not raise a -Wfloat-equal */ + if (p >= 1) + return 1; + + return (-x/log1p(-p)); +} + +/*******************************************************************/ + +/** Public API for probability distributions: + * + * For each probability distribution we define each public functions + * (sample/cdf/sf/icdf/isf) as part of its dist_ops structure. + */ + +/** Functions for uniform distribution */ +const struct dist_ops uniform_ops = { + .name = "uniform", + .sample = uniform_sample, + .cdf = uniform_cdf, + .sf = uniform_sf, + .icdf = uniform_icdf, + .isf = uniform_isf, +}; + +double +uniform_sample(const struct dist *dist) +{ + const struct uniform *U = const_container_of(dist, struct uniform, + base); + double p0 = random_uniform_01(); + + return sample_uniform_interval(p0, U->a, U->b); +} + +double +uniform_cdf(const struct dist *dist, double x) +{ + const struct uniform *U = const_container_of(dist, struct uniform, + base); + + if (x < U->a) + return 0; + else if (x < U->b) + return (x - U->a)/(U->b - U->a); + else + return 1; +} + +double +uniform_sf(const struct dist *dist, double x) +{ + const struct uniform *U = const_container_of(dist, struct uniform, + base); + + if (x > U->b) + return 0; + else if (x > U->a) + return (U->b - x)/(U->b - U->a); + else + return 1; +} + +double +uniform_icdf(const struct dist *dist, double p) +{ + const struct uniform *U = const_container_of(dist, struct uniform, + base); + double w = U->b - U->a; + + return (p < 0.5 ? (U->a + w*p) : (U->b - w*(1 - p))); +} + +double +uniform_isf(const struct dist *dist, double p) +{ + const struct uniform *U = const_container_of(dist, struct uniform, + base); + double w = U->b - U->a; + + return (p < 0.5 ? (U->b - w*p) : (U->a + w*(1 - p))); +} + +/** Functions for logistic distribution: */ +const struct dist_ops logistic_ops = { + .name = "logistic", + .sample = logistic_sample, + .cdf = logistic_cdf, + .sf = logistic_sf, + .icdf = logistic_icdf, + .isf = logistic_isf, +}; + +double +logistic_sample(const struct dist *dist) +{ + const struct logistic *L = const_container_of(dist, struct logistic, + base); + uint32_t s = crypto_rand_uint32(); + double t = random_uniform_01(); + double p0 = random_uniform_01(); + + return sample_logistic_locscale(s, t, p0, L->mu, L->sigma); +} + +double +logistic_cdf(const struct dist *dist, double x) +{ + const struct logistic *L = const_container_of(dist, struct logistic, + base); + + return cdf_logistic(x, L->mu, L->sigma); +} + +double +logistic_sf(const struct dist *dist, double x) +{ + const struct logistic *L = const_container_of(dist, struct logistic, + base); + + return sf_logistic(x, L->mu, L->sigma); +} + +double +logistic_icdf(const struct dist *dist, double p) +{ + const struct logistic *L = const_container_of(dist, struct logistic, + base); + + return icdf_logistic(p, L->mu, L->sigma); +} + +double +logistic_isf(const struct dist *dist, double p) +{ + const struct logistic *L = const_container_of(dist, struct logistic, + base); + + return isf_logistic(p, L->mu, L->sigma); +} + +/** Functions for log-logistic distribution: */ +const struct dist_ops log_logistic_ops = { + .name = "log logistic", + .sample = log_logistic_sample, + .cdf = log_logistic_cdf, + .sf = log_logistic_sf, + .icdf = log_logistic_icdf, + .isf = log_logistic_isf, +}; + +double +log_logistic_sample(const struct dist *dist) +{ + const struct log_logistic *LL = const_container_of(dist, struct + log_logistic, base); + uint32_t s = crypto_rand_uint32(); + double p0 = random_uniform_01(); + + return sample_log_logistic_scaleshape(s, p0, LL->alpha, LL->beta); +} + +double +log_logistic_cdf(const struct dist *dist, double x) +{ + const struct log_logistic *LL = const_container_of(dist, + struct log_logistic, base); + + return cdf_log_logistic(x, LL->alpha, LL->beta); +} + +double +log_logistic_sf(const struct dist *dist, double x) +{ + const struct log_logistic *LL = const_container_of(dist, + struct log_logistic, base); + + return sf_log_logistic(x, LL->alpha, LL->beta); +} + +double +log_logistic_icdf(const struct dist *dist, double p) +{ + const struct log_logistic *LL = const_container_of(dist, + struct log_logistic, base); + + return icdf_log_logistic(p, LL->alpha, LL->beta); +} + +double +log_logistic_isf(const struct dist *dist, double p) +{ + const struct log_logistic *LL = const_container_of(dist, + struct log_logistic, base); + + return isf_log_logistic(p, LL->alpha, LL->beta); +} + +/** Functions for Weibull distribution */ +const struct dist_ops weibull_ops = { + .name = "Weibull", + .sample = weibull_sample, + .cdf = weibull_cdf, + .sf = weibull_sf, + .icdf = weibull_icdf, + .isf = weibull_isf, +}; + +double +weibull_sample(const struct dist *dist) +{ + const struct weibull *W = const_container_of(dist, struct weibull, + base); + uint32_t s = crypto_rand_uint32(); + double p0 = random_uniform_01(); + + return sample_weibull(s, p0, W->lambda, W->k); +} + +double +weibull_cdf(const struct dist *dist, double x) +{ + const struct weibull *W = const_container_of(dist, struct weibull, + base); + + return cdf_weibull(x, W->lambda, W->k); +} + +double +weibull_sf(const struct dist *dist, double x) +{ + const struct weibull *W = const_container_of(dist, struct weibull, + base); + + return sf_weibull(x, W->lambda, W->k); +} + +double +weibull_icdf(const struct dist *dist, double p) +{ + const struct weibull *W = const_container_of(dist, struct weibull, + base); + + return icdf_weibull(p, W->lambda, W->k); +} + +double +weibull_isf(const struct dist *dist, double p) +{ + const struct weibull *W = const_container_of(dist, struct weibull, + base); + + return isf_weibull(p, W->lambda, W->k); +} + +/** Functions for generalized Pareto distributions */ +const struct dist_ops genpareto_ops = { + .name = "generalized Pareto", + .sample = genpareto_sample, + .cdf = genpareto_cdf, + .sf = genpareto_sf, + .icdf = genpareto_icdf, + .isf = genpareto_isf, +}; + +double +genpareto_sample(const struct dist *dist) +{ + const struct genpareto *GP = const_container_of(dist, struct genpareto, + base); + uint32_t s = crypto_rand_uint32(); + double p0 = random_uniform_01(); + + return sample_genpareto_locscale(s, p0, GP->mu, GP->sigma, GP->xi); +} + +double +genpareto_cdf(const struct dist *dist, double x) +{ + const struct genpareto *GP = const_container_of(dist, struct genpareto, + base); + + return cdf_genpareto(x, GP->mu, GP->sigma, GP->xi); +} + +double +genpareto_sf(const struct dist *dist, double x) +{ + const struct genpareto *GP = const_container_of(dist, struct genpareto, + base); + + return sf_genpareto(x, GP->mu, GP->sigma, GP->xi); +} + +double +genpareto_icdf(const struct dist *dist, double p) +{ + const struct genpareto *GP = const_container_of(dist, struct genpareto, + base); + + return icdf_genpareto(p, GP->mu, GP->sigma, GP->xi); +} + +double +genpareto_isf(const struct dist *dist, double p) +{ + const struct genpareto *GP = const_container_of(dist, struct genpareto, + base); + + return isf_genpareto(p, GP->mu, GP->sigma, GP->xi); +} + +/* Deterministically sample from the geometric distribution with + * per-trial success probability p. */ +double +geometric_sample(double p) +{ + uint32_t s = crypto_rand_uint32(); + double p0 = random_uniform_01(); + return sample_geometric(s, p0, p); +} + diff --git a/src/lib/math/prob_distr.h b/src/lib/math/prob_distr.h new file mode 100644 index 0000000000..c2fd6c74b3 --- /dev/null +++ b/src/lib/math/prob_distr.h @@ -0,0 +1,156 @@ + +/** + * \file prob_distr.h + * + * \brief Header for prob_distr.c + **/ + +#ifndef TOR_PROB_DISTR_H +#define TOR_PROB_DISTR_H + +#include "lib/cc/compat_compiler.h" +#include "lib/cc/torint.h" +#include "lib/testsupport/testsupport.h" + +/** + * Container for distribution parameters for sampling, CDF, &c. + */ +struct dist { + const struct dist_ops *ops; +}; + +#define DIST_BASE(OPS) { .ops = (OPS) } + +struct dist_ops { + const char *name; + double (*sample)(const struct dist *); + double (*cdf)(const struct dist *, double x); + double (*sf)(const struct dist *, double x); + double (*icdf)(const struct dist *, double p); + double (*isf)(const struct dist *, double p); +}; + +/* Geometric distribution */ + +double geometric_sample(double p); + +/* Pareto distribution */ + +struct genpareto { + struct dist base; + double mu; + double sigma; + double xi; +}; + +double genpareto_sample(const struct dist *dist); +double genpareto_cdf(const struct dist *dist, double x); +double genpareto_sf(const struct dist *dist, double x); +double genpareto_icdf(const struct dist *dist, double p); +double genpareto_isf(const struct dist *dist, double p); + +extern const struct dist_ops genpareto_ops; + +/* Weibull distribution */ + +struct weibull { + struct dist base; + double lambda; + double k; +}; + +double weibull_sample(const struct dist *dist); +double weibull_cdf(const struct dist *dist, double x); +double weibull_sf(const struct dist *dist, double x); +double weibull_icdf(const struct dist *dist, double p); +double weibull_isf(const struct dist *dist, double p); + +extern const struct dist_ops weibull_ops; + +/* Log-logistic distribution */ + +struct log_logistic { + struct dist base; + double alpha; + double beta; +}; + +double log_logistic_sample(const struct dist *dist); +double log_logistic_cdf(const struct dist *dist, double x); +double log_logistic_sf(const struct dist *dist, double x); +double log_logistic_icdf(const struct dist *dist, double p); +double log_logistic_isf(const struct dist *dist, double p); + +extern const struct dist_ops log_logistic_ops; + +/* Logistic distribution */ + +struct logistic { + struct dist base; + double mu; + double sigma; +}; + +double logistic_sample(const struct dist *dist); +double logistic_cdf(const struct dist *dist, double x); +double logistic_sf(const struct dist *dist, double x); +double logistic_icdf(const struct dist *dist, double p); +double logistic_isf(const struct dist *dist, double p); + +extern const struct dist_ops logistic_ops; + +/* Uniform distribution */ + +struct uniform { + struct dist base; + double a; + double b; +}; + +double uniform_sample(const struct dist *dist); +double uniform_cdf(const struct dist *dist, double x); +double uniform_sf(const struct dist *dist, double x); +double uniform_icdf(const struct dist *dist, double p); +double uniform_isf(const struct dist *dist, double p); + +extern const struct dist_ops uniform_ops; + +/** Only by unittests */ + +#ifdef PROB_DISTR_PRIVATE + +STATIC double logithalf(double p0); +STATIC double logit(double p); + +STATIC double random_uniform_01(void); + +STATIC double logistic(double x); +STATIC double cdf_logistic(double x, double mu, double sigma); +STATIC double sf_logistic(double x, double mu, double sigma); +STATIC double icdf_logistic(double p, double mu, double sigma); +STATIC double isf_logistic(double p, double mu, double sigma); +STATIC double sample_logistic(uint32_t s, double t, double p0); + +STATIC double cdf_log_logistic(double x, double alpha, double beta); +STATIC double sf_log_logistic(double x, double alpha, double beta); +STATIC double icdf_log_logistic(double p, double alpha, double beta); +STATIC double isf_log_logistic(double p, double alpha, double beta); +STATIC double sample_log_logistic(uint32_t s, double p0); + +STATIC double cdf_weibull(double x, double lambda, double k); +STATIC double sf_weibull(double x, double lambda, double k); +STATIC double icdf_weibull(double p, double lambda, double k); +STATIC double isf_weibull(double p, double lambda, double k); +STATIC double sample_weibull(uint32_t s, double p0, double lambda, double k); + +STATIC double sample_uniform_interval(double p0, double a, double b); + +STATIC double cdf_genpareto(double x, double mu, double sigma, double xi); +STATIC double sf_genpareto(double x, double mu, double sigma, double xi); +STATIC double icdf_genpareto(double p, double mu, double sigma, double xi); +STATIC double isf_genpareto(double p, double mu, double sigma, double xi); +STATIC double sample_genpareto(uint32_t s, double p0, double xi); + +#endif + +#endif diff --git a/src/test/include.am b/src/test/include.am index 4da0b84392..b276500fd5 100644 --- a/src/test/include.am +++ b/src/test/include.am @@ -157,6 +157,7 @@ src_test_test_SOURCES += \ src/test/test_periodic_event.c \ src/test/test_policy.c \ src/test/test_process.c \ + src/test/test_prob_distr.c \ src/test/test_procmon.c \ src/test/test_proto_http.c \ src/test/test_proto_misc.c \ @@ -207,6 +208,7 @@ src_test_test_slow_SOURCES += \ src/test/test_slow.c \ src/test/test_crypto_slow.c \ src/test/test_process_slow.c \ + src/test/test_prob_distr.c \ src/test/testing_common.c \ src/test/testing_rsakeys.c \ src/ext/tinytest.c diff --git a/src/test/prob_distr_mpfr_ref.c b/src/test/prob_distr_mpfr_ref.c new file mode 100644 index 0000000000..4e64d731cd --- /dev/null +++ b/src/test/prob_distr_mpfr_ref.c @@ -0,0 +1,64 @@ +/* Copyright 2012-2018, The Tor Project, Inc + * See LICENSE for licensing information */ + +/** prob_distr_mpfr_ref.c + * + * Example reference file for GNU MPFR vectors tested in test_prob_distr.c . + * Code by Riastradh. + */ + +#include +#include +#include +#include + +/* Must come after so we get mpfr_printf. */ +#include + +/* gcc -o mpfr prob_distr_mpfr_ref.c -lmpfr -lm */ + +/* Computes logit(p) for p = .49999 */ +int +main(void) +{ + mpfr_t p, q, r; + mpfr_init(p); + mpfr_set_prec(p, 200); + mpfr_init(q); + mpfr_set_prec(q, 200); + mpfr_init(r); + mpfr_set_prec(r, 200); + mpfr_set_d(p, .49999, MPFR_RNDN); + mpfr_set_d(q, 1, MPFR_RNDN); + /* r := q - p = 1 - p */ + mpfr_sub(r, q, p, MPFR_RNDN); + /* q := p/r = p/(1 - p) */ + mpfr_div(q, p, r, MPFR_RNDN); + /* r := log(q) = log(p/(1 - p)) */ + mpfr_log(r, q, MPFR_RNDN); + mpfr_printf("mpfr 200-bit\t%.128Rg\n", r); + + /* + * Print a double approximation to logit three different ways. All + * three agree bit for bit on the libms I tried, with the nextafter + * adjustment (which is well within the 10 eps relative error bound + * advertised). Apparently I must have used the Goldberg expression + * for what I wrote down in the test case. + */ + printf("mpfr 53-bit\t%.17g\n", nextafter(mpfr_get_d(r, MPFR_RNDN), 0), 0); + volatile double p0 = .49999; + printf("log1p\t\t%.17g\n", nextafter(-log1p((1 - 2*p0)/p0), 0)); + volatile double x = (1 - 2*p0)/p0; + volatile double xp1 = x + 1; + printf("Goldberg\t%.17g\n", -x*log(xp1)/(xp1 - 1)); + + /* + * Print a bad approximation, using the naive expression, to see a + * lot of wrong digits, far beyond the 10 eps relative error attained + * by -log1p((1 - 2*p)/p). + */ + printf("naive\t\t%.17g\n", log(p0/(1 - p0))); + + fflush(stdout); + return ferror(stdout); +} diff --git a/src/test/test.c b/src/test/test.c index a0a138b03d..902565dfbe 100644 --- a/src/test/test.c +++ b/src/test/test.c @@ -901,6 +901,7 @@ struct testgroup_t testgroups[] = { { "parsecommon/", parsecommon_tests }, { "periodic-event/" , periodic_event_tests }, { "policy/" , policy_tests }, + { "prob_distr/", prob_distr_tests }, { "procmon/", procmon_tests }, { "process/", process_tests }, { "proto/http/", proto_http_tests }, diff --git a/src/test/test.h b/src/test/test.h index 9f6eb0a7e6..39953e9f7e 100644 --- a/src/test/test.h +++ b/src/test/test.h @@ -243,6 +243,8 @@ extern struct testcase_t parsecommon_tests[]; extern struct testcase_t pem_tests[]; extern struct testcase_t periodic_event_tests[]; extern struct testcase_t policy_tests[]; +extern struct testcase_t prob_distr_tests[]; +extern struct testcase_t slow_stochastic_prob_distr_tests[]; extern struct testcase_t procmon_tests[]; extern struct testcase_t process_tests[]; extern struct testcase_t proto_http_tests[]; diff --git a/src/test/test_prob_distr.c b/src/test/test_prob_distr.c new file mode 100644 index 0000000000..bf0f9e059d --- /dev/null +++ b/src/test/test_prob_distr.c @@ -0,0 +1,1414 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file test_prob_distr.c + * \brief Test probability distributions. + * \detail + * + * For each probability distribution we do two kinds of tests: + * + * a) We do numerical deterministic testing of their cdf/icdf/sf/isf functions + * and the various relationships between them for each distribution. We also + * do deterministic tests on their sampling functions. Test vectors for + * these tests were computed from alternative implementations and were + * eyeballed to make sure they make sense + * (e.g. src/test/prob_distr_mpfr_ref.c computes logit(p) using GNU mpfr + * with 200-bit precision and is then tested in test_logit_logistic()). + * + * b) We do stochastic hypothesis testing (G-test) to ensure that sampling from + * the given distributions is distributed properly. The stochastic tests are + * slow and their false positive rate is not well suited for CI, so they are + * currently disabled-by-default and put into 'tests-slow'. + */ + +#define PROB_DISTR_PRIVATE + +#include "orconfig.h" + +#include "test/test.h" + +#include "core/or/or.h" + +#include "lib/math/prob_distr.h" +#include "lib/math/fp.h" +#include "lib/crypt_ops/crypto_rand.h" + +#include +#include +#include +#include +#include +#include +#include + +/** + * Return floor(d) converted to size_t, as a workaround for complaints + * under -Wbad-function-cast for (size_t)floor(d). + */ +static size_t +floor_to_size_t(double d) +{ + double integral_d = floor(d); + return (size_t)integral_d; +} + +/** + * Return ceil(d) converted to size_t, as a workaround for complaints + * under -Wbad-function-cast for (size_t)ceil(d). + */ +static size_t +ceil_to_size_t(double d) +{ + double integral_d = ceil(d); + return (size_t)integral_d; +} + +/* + * Geometric(p) distribution, supported on {1, 2, 3, ...}. + * + * Compute the probability mass function Geom(n; p) of the number of + * trials before the first success when success has probability p. + */ +static double +logpmf_geometric(unsigned n, double p) +{ + /* This is actually a check against 1, but we do >= so that the compiler + does not raise a -Wfloat-equal */ + if (p >= 1) { + if (n == 1) + return 0; + else + return -HUGE_VAL; + } + return (n - 1)*log1p(-p) + log(p); +} + +/** + * Compute the logistic function, translated in output by 1/2: + * logistichalf(x) = logistic(x) - 1/2. Well-conditioned on the entire + * real plane, with maximum condition number 1 at 0. + * + * This implementation gives relative error bounded by 5 eps. + */ +static double +logistichalf(double x) +{ + /* + * Rewrite this with the identity + * + * 1/(1 + e^{-x}) - 1/2 + * = (1 - 1/2 - e^{-x}/2)/(1 + e^{-x}) + * = (1/2 - e^{-x}/2)/(1 + e^{-x}) + * = (1 - e^{-x})/[2 (1 + e^{-x})] + * = -(e^{-x} - 1)/[2 (1 + e^{-x})], + * + * which we can evaluate by -expm1(-x)/[2 (1 + exp(-x))]. + * + * Suppose exp has error d0, + has error d1, expm1 has error + * d2, and / has error d3, so we evaluate + * + * -(1 + d2) (1 + d3) (e^{-x} - 1) + * / [2 (1 + d1) (1 + (1 + d0) e^{-x})]. + * + * In the denominator, + * + * 1 + (1 + d0) e^{-x} + * = 1 + e^{-x} + d0 e^{-x} + * = (1 + e^{-x}) (1 + d0 e^{-x}/(1 + e^{-x})), + * + * so the relative error of the numerator is + * + * d' = d2 + d3 + d2 d3, + * and of the denominator, + * d'' = d1 + d0 e^{-x}/(1 + e^{-x}) + d0 d1 e^{-x}/(1 + e^{-x}) + * = d1 + d0 L(-x) + d0 d1 L(-x), + * + * where L(-x) is logistic(-x). By Lemma 1 the relative error + * of the quotient is bounded by + * + * 2|d2 + d3 + d2 d3 - d1 - d0 L(x) + d0 d1 L(x)|, + * + * Since 0 < L(x) < 1, this is bounded by + * + * 2|d2| + 2|d3| + 2|d2 d3| + 2|d1| + 2|d0| + 2|d0 d1| + * <= 4 eps + 2 eps^2. + */ + if (x < log(DBL_EPSILON/8)) { + /* + * Avoid overflow in e^{-x}. When x < log(eps/4), we + * we further have x < logit(eps/4), so that + * logistic(x) < eps/4. Hence the relative error of + * logistic(x) - 1/2 from -1/2 is bounded by eps/2, and + * so the relative error of -1/2 from logistic(x) - 1/2 + * is bounded by eps. + */ + return -0.5; + } else { + return -expm1(-x)/(2*(1 + exp(-x))); + } +} + +/** + * Compute the log of the sum of the exps. Caller should arrange the + * array in descending order to minimize error because I don't want to + * deal with using temporary space and the one caller in this file + * arranges that anyway. + * + * Warning: This implementation does not handle infinite or NaN inputs + * sensibly, because I don't need that here at the moment. (NaN, or + * -inf and +inf together, should yield NaN; +inf and finite should + * yield +inf; otherwise all -inf should be ignored because exp(-inf) = + * 0.) + */ +static double +logsumexp(double *A, size_t n) +{ + double maximum, sum; + size_t i; + + if (n == 0) + return log(0); + + maximum = A[0]; + for (i = 1; i < n; i++) { + if (A[i] > maximum) + maximum = A[i]; + } + + sum = 0; + for (i = n; i --> 0;) + sum += exp(A[i] - maximum); + + return log(sum) + maximum; +} + +/** + * Compute log(1 - e^x). Defined only for negative x so that e^x < 1. + * This is the complement of a probability in log space. + */ +static double +log1mexp(double x) +{ + + /* + * We want to compute log on [0, 1/2) but log1p on [1/2, +inf), + * so partition x at -log(2) = log(1/2). + */ + if (-log(2) < x) + return log(-expm1(x)); + else + return log1p(-exp(x)); +} + +/* + * Tests of numerical errors in computing logit, logistic, and the + * various cdfs, sfs, icdfs, and isfs. + */ + +#define arraycount(A) (sizeof(A)/sizeof(A[0])) + +/** Return relative error between actual and expected. + * Special cases: If expected is zero or infinite, return 1 if + * actual is equal to expected and 0 if not, since the + * usual notion of relative error is undefined but we only use this + * for testing relerr(e, a) <= bound. If either is NaN, return NaN, + * which has the property that NaN <= bound is false no matter what + * bound is. + * + * Beware: if you test !(relerr(e, a) > bound), then then the result + * is true when a is NaN because NaN > bound is false too. See + * CHECK_RELERR for correct use to decide when to report failure. + */ +static double +relerr(double expected, double actual) +{ + /* + * To silence -Wfloat-equal, we have to test for equality using + * inequalities: we have (fabs(expected) <= 0) iff (expected == 0), + * and (actual <= expected && actual >= expected) iff actual == + * expected whether expected is zero or infinite. + */ + if (fabs(expected) <= 0 || tor_isinf(expected)) { + if (actual <= expected && actual >= expected) + return 0; + else + return 1; + } else { + return fabs((expected - actual)/expected); + } +} + +/** Check that relative error of expected and actual is within + * relerr_bound. Caller must arrange to have i and relerr_bound in + * scope. */ +#define CHECK_RELERR(expected, actual) do { \ + double check_expected = (expected); \ + double check_actual = (actual); \ + const char *str_expected = #expected; \ + const char *str_actual = #actual; \ + double check_relerr = relerr(expected, actual); \ + if (!(relerr(check_expected, check_actual) <= relerr_bound)) { \ + log_warn(LD_GENERAL, "%s:%d: case %u: relerr(%s=%.17e, %s=%.17e)" \ + " = %.17e > %.17e\n", \ + __func__, __LINE__, (unsigned) i, \ + str_expected, check_expected, \ + str_actual, check_actual, \ + check_relerr, relerr_bound); \ + ok = false; \ + } \ +} while (0) + +/* Check that a <= b. + * Caller must arrange to have i in scope. */ +#define CHECK_LE(a, b) do { \ + double check_a = (a); \ + double check_b = (b); \ + const char *str_a = #a; \ + const char *str_b = #b; \ + if (!(check_a <= check_b)) { \ + log_warn(LD_GENERAL, "%s:%d: case %u: %s=%.17e > %s=%.17e\n", \ + __func__, __LINE__, (unsigned) i, \ + str_a, check_a, str_b, check_b); \ + ok = false; \ + } \ +} while (0) + +/** + * Test the logit and logistic functions. Confirm that they agree with + * the cdf, sf, icdf, and isf of the standard Logistic distribution. + * Confirm that the sampler for the standard logistic distribution maps + * [0, 1] into the right subinterval for the inverse transform, for + * this implementation. + */ +static void +test_logit_logistic(void *arg) +{ + (void) arg; + + static const struct { + double x; /* x = logit(p) */ + double p; /* p = logistic(x) */ + double phalf; /* p - 1/2 = logistic(x) - 1/2 */ + } cases[] = { + { -HUGE_VAL, 0, -0.5 }, + { -1000, 0, -0.5 }, + { -710, 4.47628622567513e-309, -0.5 }, + { -708, 3.307553003638408e-308, -0.5 }, + { -2, .11920292202211755, -.3807970779778824 }, + { -1.0000001, .2689414017088022, -.23105859829119776 }, + { -1, .2689414213699951, -.23105857863000487 }, + { -0.9999999, .26894144103118883, -.2310585589688111 }, + /* see src/test/prob_distr_mpfr_ref.c for computation */ + { -4.000000000537333e-5, .49999, -1.0000000000010001e-5 }, + { -4.000000000533334e-5, .49999, -.00001 }, + { -4.000000108916878e-9, .499999999, -1.0000000272292198e-9 }, + { -4e-9, .499999999, -1e-9 }, + { -4e-16, .5, -1e-16 }, + { -4e-300, .5, -1e-300 }, + { 0, .5, 0 }, + { 4e-300, .5, 1e-300 }, + { 4e-16, .5, 1e-16 }, + { 3.999999886872274e-9, .500000001, 9.999999717180685e-10 }, + { 4e-9, .500000001, 1e-9 }, + { 4.0000000005333336e-5, .50001, .00001 }, + { 8.000042667076272e-3, .502, .002 }, + { 0.9999999, .7310585589688111, .2310585589688111 }, + { 1, .7310585786300049, .23105857863000487 }, + { 1.0000001, .7310585982911977, .23105859829119774 }, + { 2, .8807970779778823, .3807970779778824 }, + { 708, 1, .5 }, + { 710, 1, .5 }, + { 1000, 1, .5 }, + { HUGE_VAL, 1, .5 }, + }; + double relerr_bound = 3e-15; /* >10eps */ + size_t i; + bool ok = true; + + for (i = 0; i < arraycount(cases); i++) { + double x = cases[i].x; + double p = cases[i].p; + double phalf = cases[i].phalf; + + /* + * cdf is logistic, icdf is logit, and symmetry for + * sf/isf. + */ + CHECK_RELERR(logistic(x), cdf_logistic(x, 0, 1)); + CHECK_RELERR(logistic(-x), sf_logistic(x, 0, 1)); + CHECK_RELERR(logit(p), icdf_logistic(p, 0, 1)); + CHECK_RELERR(-logit(p), isf_logistic(p, 0, 1)); + + CHECK_RELERR(cdf_logistic(x, 0, 1), cdf_logistic(x*2, 0, 2)); + CHECK_RELERR(sf_logistic(x, 0, 1), sf_logistic(x*2, 0, 2)); + CHECK_RELERR(icdf_logistic(p, 0, 1), icdf_logistic(p, 0, 2)/2); + CHECK_RELERR(isf_logistic(p, 0, 1), isf_logistic(p, 0, 2)/2); + + CHECK_RELERR(cdf_logistic(x, 0, 1), cdf_logistic(x/2, 0, .5)); + CHECK_RELERR(sf_logistic(x, 0, 1), sf_logistic(x/2, 0, .5)); + CHECK_RELERR(icdf_logistic(p, 0, 1), icdf_logistic(p, 0,.5)*2); + CHECK_RELERR(isf_logistic(p, 0, 1), isf_logistic(p, 0, .5)*2); + + CHECK_RELERR(cdf_logistic(x, 0, 1), cdf_logistic(x*2 + 1, 1, 2)); + CHECK_RELERR(sf_logistic(x, 0, 1), sf_logistic(x*2 + 1, 1, 2)); + + /* + * For p near 0 and p near 1/2, the arithmetic of + * translating by 1 loses precision. + */ + if (fabs(p) > DBL_EPSILON && fabs(p) < 0.4) { + CHECK_RELERR(icdf_logistic(p, 0, 1), + (icdf_logistic(p, 1, 2) - 1)/2); + CHECK_RELERR(isf_logistic(p, 0, 1), + (isf_logistic(p, 1, 2) - 1)/2); + } + + CHECK_RELERR(p, logistic(x)); + CHECK_RELERR(phalf, logistichalf(x)); + + /* + * On the interior floating-point numbers, either logit or + * logithalf had better give the correct answer. + * + * For probabilities near 0, we can get much finer resolution with + * logit, and for probabilities near 1/2, we can get much finer + * resolution with logithalf by representing them using p - 1/2. + * + * E.g., we can write -.00001 for phalf, and .49999 for p, but the + * difference 1/2 - .00001 gives 1.0000000000010001e-5 in binary64 + * arithmetic. So test logit(.49999) which should give the same + * answer as logithalf(-1.0000000000010001e-5), namely + * -4.000000000537333e-5, and also test logithalf(-.00001) which + * gives -4.000000000533334e-5 instead -- but don't expect + * logit(.49999) to give -4.000000000533334e-5 even though it looks + * like 1/2 - .00001. + * + * A naive implementation of logit will just use log(p/(1 - p)) and + * give the answer -4.000000000551673e-05 for .49999, which is + * wrong in a lot of digits, which happens because log is + * ill-conditioned near 1 and thus amplifies whatever relative + * error we made in computing p/(1 - p). + */ + if ((0 < p && p < 1) || tor_isinf(x)) { + if (phalf >= p - 0.5 && phalf <= p - 0.5) + CHECK_RELERR(x, logit(p)); + if (p >= 0.5 + phalf && p <= 0.5 + phalf) + CHECK_RELERR(x, logithalf(phalf)); + } + + CHECK_RELERR(-phalf, logistichalf(-x)); + if (fabs(phalf) < 0.5 || tor_isinf(x)) + CHECK_RELERR(-x, logithalf(-phalf)); + if (p < 1 || tor_isinf(x)) { + CHECK_RELERR(1 - p, logistic(-x)); + if (p > .75 || tor_isinf(x)) + CHECK_RELERR(-x, logit(1 - p)); + } else { + CHECK_LE(logistic(-x), 1e-300); + } + } + + for (i = 0; i <= 100; i++) { + double p0 = (double)i/100; + + CHECK_RELERR(logit(p0/(1 + M_E)), sample_logistic(0, 0, p0)); + CHECK_RELERR(-logit(p0/(1 + M_E)), sample_logistic(1, 0, p0)); + CHECK_RELERR(logithalf(p0*(0.5 - 1/(1 + M_E))), + sample_logistic(0, 1, p0)); + CHECK_RELERR(-logithalf(p0*(0.5 - 1/(1 + M_E))), + sample_logistic(1, 1, p0)); + } + + if (!ok) + printf("fail logit/logistic / logistic cdf/sf\n"); + + tt_assert(ok); + + done: + ; +} + +/** + * Test the cdf, sf, icdf, and isf of the LogLogistic distribution. + */ +static void +test_log_logistic(void *arg) +{ + (void) arg; + + static const struct { + /* x is a point in the support of the LogLogistic distribution */ + double x; + /* 'p' is the probability that a random variable X for a given LogLogistic + * probability ditribution will take value less-or-equal to x */ + double p; + /* 'np' is the probability that a random variable X for a given LogLogistic + * probability distribution will take value greater-or-equal to x. */ + double np; + } cases[] = { + { 0, 0, 1 }, + { 1e-300, 1e-300, 1 }, + { 1e-17, 1e-17, 1 }, + { 1e-15, 1e-15, .999999999999999 }, + { .1, .09090909090909091, .90909090909090909 }, + { .25, .2, .8 }, + { .5, .33333333333333333, .66666666666666667 }, + { .75, .42857142857142855, .5714285714285714 }, + { .9999, .49997499874993756, .5000250012500626 }, + { .99999999, .49999999749999996, .5000000025 }, + { .999999999999999, .49999999999999994, .5000000000000002 }, + { 1, .5, .5 }, + }; + double relerr_bound = 3e-15; + size_t i; + bool ok = true; + + for (i = 0; i < arraycount(cases); i++) { + double x = cases[i].x; + double p = cases[i].p; + double np = cases[i].np; + + CHECK_RELERR(p, cdf_log_logistic(x, 1, 1)); + CHECK_RELERR(p, cdf_log_logistic(x/2, .5, 1)); + CHECK_RELERR(p, cdf_log_logistic(x*2, 2, 1)); + CHECK_RELERR(p, cdf_log_logistic(sqrt(x), 1, 2)); + CHECK_RELERR(p, cdf_log_logistic(sqrt(x)/2, .5, 2)); + CHECK_RELERR(p, cdf_log_logistic(sqrt(x)*2, 2, 2)); + if (2*sqrt(DBL_MIN) < x) { + CHECK_RELERR(p, cdf_log_logistic(x*x, 1, .5)); + CHECK_RELERR(p, cdf_log_logistic(x*x/2, .5, .5)); + CHECK_RELERR(p, cdf_log_logistic(x*x*2, 2, .5)); + } + + CHECK_RELERR(np, sf_log_logistic(x, 1, 1)); + CHECK_RELERR(np, sf_log_logistic(x/2, .5, 1)); + CHECK_RELERR(np, sf_log_logistic(x*2, 2, 1)); + CHECK_RELERR(np, sf_log_logistic(sqrt(x), 1, 2)); + CHECK_RELERR(np, sf_log_logistic(sqrt(x)/2, .5, 2)); + CHECK_RELERR(np, sf_log_logistic(sqrt(x)*2, 2, 2)); + if (2*sqrt(DBL_MIN) < x) { + CHECK_RELERR(np, sf_log_logistic(x*x, 1, .5)); + CHECK_RELERR(np, sf_log_logistic(x*x/2, .5, .5)); + CHECK_RELERR(np, sf_log_logistic(x*x*2, 2, .5)); + } + + CHECK_RELERR(np, cdf_log_logistic(1/x, 1, 1)); + CHECK_RELERR(np, cdf_log_logistic(1/(2*x), .5, 1)); + CHECK_RELERR(np, cdf_log_logistic(2/x, 2, 1)); + CHECK_RELERR(np, cdf_log_logistic(1/sqrt(x), 1, 2)); + CHECK_RELERR(np, cdf_log_logistic(1/(2*sqrt(x)), .5, 2)); + CHECK_RELERR(np, cdf_log_logistic(2/sqrt(x), 2, 2)); + if (2*sqrt(DBL_MIN) < x && x < 1/(2*sqrt(DBL_MIN))) { + CHECK_RELERR(np, cdf_log_logistic(1/(x*x), 1, .5)); + CHECK_RELERR(np, cdf_log_logistic(1/(2*x*x), .5, .5)); + CHECK_RELERR(np, cdf_log_logistic(2/(x*x), 2, .5)); + } + + CHECK_RELERR(p, sf_log_logistic(1/x, 1, 1)); + CHECK_RELERR(p, sf_log_logistic(1/(2*x), .5, 1)); + CHECK_RELERR(p, sf_log_logistic(2/x, 2, 1)); + CHECK_RELERR(p, sf_log_logistic(1/sqrt(x), 1, 2)); + CHECK_RELERR(p, sf_log_logistic(1/(2*sqrt(x)), .5, 2)); + CHECK_RELERR(p, sf_log_logistic(2/sqrt(x), 2, 2)); + if (2*sqrt(DBL_MIN) < x && x < 1/(2*sqrt(DBL_MIN))) { + CHECK_RELERR(p, sf_log_logistic(1/(x*x), 1, .5)); + CHECK_RELERR(p, sf_log_logistic(1/(2*x*x), .5, .5)); + CHECK_RELERR(p, sf_log_logistic(2/(x*x), 2, .5)); + } + + CHECK_RELERR(x, icdf_log_logistic(p, 1, 1)); + CHECK_RELERR(x/2, icdf_log_logistic(p, .5, 1)); + CHECK_RELERR(x*2, icdf_log_logistic(p, 2, 1)); + CHECK_RELERR(x, icdf_log_logistic(p, 1, 1)); + CHECK_RELERR(sqrt(x)/2, icdf_log_logistic(p, .5, 2)); + CHECK_RELERR(sqrt(x)*2, icdf_log_logistic(p, 2, 2)); + CHECK_RELERR(sqrt(x), icdf_log_logistic(p, 1, 2)); + CHECK_RELERR(x*x/2, icdf_log_logistic(p, .5, .5)); + CHECK_RELERR(x*x*2, icdf_log_logistic(p, 2, .5)); + + if (np < .9) { + CHECK_RELERR(x, isf_log_logistic(np, 1, 1)); + CHECK_RELERR(x/2, isf_log_logistic(np, .5, 1)); + CHECK_RELERR(x*2, isf_log_logistic(np, 2, 1)); + CHECK_RELERR(sqrt(x), isf_log_logistic(np, 1, 2)); + CHECK_RELERR(sqrt(x)/2, isf_log_logistic(np, .5, 2)); + CHECK_RELERR(sqrt(x)*2, isf_log_logistic(np, 2, 2)); + CHECK_RELERR(x*x, isf_log_logistic(np, 1, .5)); + CHECK_RELERR(x*x/2, isf_log_logistic(np, .5, .5)); + CHECK_RELERR(x*x*2, isf_log_logistic(np, 2, .5)); + + CHECK_RELERR(1/x, icdf_log_logistic(np, 1, 1)); + CHECK_RELERR(1/(2*x), icdf_log_logistic(np, .5, 1)); + CHECK_RELERR(2/x, icdf_log_logistic(np, 2, 1)); + CHECK_RELERR(1/sqrt(x), icdf_log_logistic(np, 1, 2)); + CHECK_RELERR(1/(2*sqrt(x)), + icdf_log_logistic(np, .5, 2)); + CHECK_RELERR(2/sqrt(x), icdf_log_logistic(np, 2, 2)); + CHECK_RELERR(1/(x*x), icdf_log_logistic(np, 1, .5)); + CHECK_RELERR(1/(2*x*x), icdf_log_logistic(np, .5, .5)); + CHECK_RELERR(2/(x*x), icdf_log_logistic(np, 2, .5)); + } + + CHECK_RELERR(1/x, isf_log_logistic(p, 1, 1)); + CHECK_RELERR(1/(2*x), isf_log_logistic(p, .5, 1)); + CHECK_RELERR(2/x, isf_log_logistic(p, 2, 1)); + CHECK_RELERR(1/sqrt(x), isf_log_logistic(p, 1, 2)); + CHECK_RELERR(1/(2*sqrt(x)), isf_log_logistic(p, .5, 2)); + CHECK_RELERR(2/sqrt(x), isf_log_logistic(p, 2, 2)); + CHECK_RELERR(1/(x*x), isf_log_logistic(p, 1, .5)); + CHECK_RELERR(1/(2*x*x), isf_log_logistic(p, .5, .5)); + CHECK_RELERR(2/(x*x), isf_log_logistic(p, 2, .5)); + } + + for (i = 0; i <= 100; i++) { + double p0 = (double)i/100; + + CHECK_RELERR(0.5*p0/(1 - 0.5*p0), sample_log_logistic(0, p0)); + CHECK_RELERR((1 - 0.5*p0)/(0.5*p0), + sample_log_logistic(1, p0)); + } + + if (!ok) + printf("fail log logistic cdf/sf\n"); + + tt_assert(ok); + + done: + ; +} + +/** + * Test the cdf, sf, icdf, isf of the Weibull distribution. + */ +static void +test_weibull(void *arg) +{ + (void) arg; + + static const struct { + /* x is a point in the support of the Weibull distribution */ + double x; + /* 'p' is the probability that a random variable X for a given Weibull + * probability ditribution will take value less-or-equal to x */ + double p; + /* 'np' is the probability that a random variable X for a given Weibull + * probability distribution will take value greater-or-equal to x. */ + double np; + } cases[] = { + { 0, 0, 1 }, + { 1e-300, 1e-300, 1 }, + { 1e-17, 1e-17, 1 }, + { .1, .09516258196404043, .9048374180359595 }, + { .5, .3934693402873666, .6065306597126334 }, + { .6931471805599453, .5, .5 }, + { 1, .6321205588285577, .36787944117144233 }, + { 10, .9999546000702375, 4.5399929762484854e-5 }, + { 36, .9999999999999998, 2.319522830243569e-16 }, + { 37, .9999999999999999, 8.533047625744066e-17 }, + { 38, 1, 3.1391327920480296e-17 }, + { 100, 1, 3.720075976020836e-44 }, + { 708, 1, 3.307553003638408e-308 }, + { 710, 1, 4.47628622567513e-309 }, + { 1000, 1, 0 }, + { HUGE_VAL, 1, 0 }, + }; + double relerr_bound = 3e-15; + size_t i; + bool ok = true; + + for (i = 0; i < arraycount(cases); i++) { + double x = cases[i].x; + double p = cases[i].p; + double np = cases[i].np; + + CHECK_RELERR(p, cdf_weibull(x, 1, 1)); + CHECK_RELERR(p, cdf_weibull(x/2, .5, 1)); + CHECK_RELERR(p, cdf_weibull(x*2, 2, 1)); + /* For 0 < x < sqrt(DBL_MIN), x^2 loses lots of bits. */ + if (x <= 0 || + sqrt(DBL_MIN) <= x) { + CHECK_RELERR(p, cdf_weibull(x*x, 1, .5)); + CHECK_RELERR(p, cdf_weibull(x*x/2, .5, .5)); + CHECK_RELERR(p, cdf_weibull(x*x*2, 2, .5)); + } + CHECK_RELERR(p, cdf_weibull(sqrt(x), 1, 2)); + CHECK_RELERR(p, cdf_weibull(sqrt(x)/2, .5, 2)); + CHECK_RELERR(p, cdf_weibull(sqrt(x)*2, 2, 2)); + CHECK_RELERR(np, sf_weibull(x, 1, 1)); + CHECK_RELERR(np, sf_weibull(x/2, .5, 1)); + CHECK_RELERR(np, sf_weibull(x*2, 2, 1)); + CHECK_RELERR(np, sf_weibull(x*x, 1, .5)); + CHECK_RELERR(np, sf_weibull(x*x/2, .5, .5)); + CHECK_RELERR(np, sf_weibull(x*x*2, 2, .5)); + if (x >= 10) { + /* + * exp amplifies the error of sqrt(x)^2 + * proportionally to exp(x); for large inputs + * this is significant. + */ + double t = -expm1(-x*(2*DBL_EPSILON + DBL_EPSILON)); + relerr_bound = t + DBL_EPSILON + t*DBL_EPSILON; + if (relerr_bound < 3e-15) + /* + * The tests are written only to 16 + * decimal places anyway even if your + * `double' is, say, i387 binary80, for + * whatever reason. + */ + relerr_bound = 3e-15; + CHECK_RELERR(np, sf_weibull(sqrt(x), 1, 2)); + CHECK_RELERR(np, sf_weibull(sqrt(x)/2, .5, 2)); + CHECK_RELERR(np, sf_weibull(sqrt(x)*2, 2, 2)); + } + + if (p <= 0.75) { + /* + * For p near 1, not enough precision near 1 to + * recover x. + */ + CHECK_RELERR(x, icdf_weibull(p, 1, 1)); + CHECK_RELERR(x/2, icdf_weibull(p, .5, 1)); + CHECK_RELERR(x*2, icdf_weibull(p, 2, 1)); + } + if (p >= 0.25 && !tor_isinf(x) && np > 0) { + /* + * For p near 0, not enough precision in np + * near 1 to recover x. For 0, isf gives inf, + * even if p is precise enough for the icdf to + * work. + */ + CHECK_RELERR(x, isf_weibull(np, 1, 1)); + CHECK_RELERR(x/2, isf_weibull(np, .5, 1)); + CHECK_RELERR(x*2, isf_weibull(np, 2, 1)); + } + } + + for (i = 0; i <= 100; i++) { + double p0 = (double)i/100; + + CHECK_RELERR(3*sqrt(-log(p0/2)), sample_weibull(0, p0, 3, 2)); + CHECK_RELERR(3*sqrt(-log1p(-p0/2)), + sample_weibull(1, p0, 3, 2)); + } + + if (!ok) + printf("fail Weibull cdf/sf\n"); + + tt_assert(ok); + + done: + ; +} + +/** + * Test the cdf, sf, icdf, and isf of the generalized Pareto + * distribution. + */ +static void +test_genpareto(void *arg) +{ + (void) arg; + + struct { + /* xi is the 'xi' parameter of the generalized Pareto distribution, and the + * rest are the same as in the above tests */ + double xi, x, p, np; + } cases[] = { + { 0, 0, 0, 1 }, + { 1e-300, .004, 3.992010656008528e-3, .9960079893439915 }, + { 1e-300, .1, .09516258196404043, .9048374180359595 }, + { 1e-300, 1, .6321205588285577, .36787944117144233 }, + { 1e-300, 10, .9999546000702375, 4.5399929762484854e-5 }, + { 1e-200, 1e-16, 9.999999999999999e-17, .9999999999999999 }, + { 1e-16, 1e-200, 9.999999999999998e-201, 1 }, + { 1e-16, 1e-16, 1e-16, 1 }, + { 1e-16, .004, 3.992010656008528e-3, .9960079893439915 }, + { 1e-16, .1, .09516258196404043, .9048374180359595 }, + { 1e-16, 1, .6321205588285577, .36787944117144233 }, + { 1e-16, 10, .9999546000702375, 4.539992976248509e-5 }, + { 1e-10, 1e-6, 9.999995000001667e-7, .9999990000005 }, + { 1e-8, 1e-8, 9.999999950000001e-9, .9999999900000001 }, + { 1, 1e-300, 1e-300, 1 }, + { 1, 1e-16, 1e-16, .9999999999999999 }, + { 1, .1, .09090909090909091, .9090909090909091 }, + { 1, 1, .5, .5 }, + { 1, 10, .9090909090909091, .0909090909090909 }, + { 1, 100, .9900990099009901, .0099009900990099 }, + { 1, 1000, .999000999000999, 9.990009990009992e-4 }, + { 10, 1e-300, 1e-300, 1 }, + { 10, 1e-16, 9.999999999999995e-17, .9999999999999999 }, + { 10, .1, .06696700846319258, .9330329915368074 }, + { 10, 1, .21320655780322778, .7867934421967723 }, + { 10, 10, .3696701667040189, .6303298332959811 }, + { 10, 100, .49886285755007337, .5011371424499267 }, + { 10, 1000, .6018968102992647, .3981031897007353 }, + }; + double xi_array[] = { -1.5, -1, -1e-30, 0, 1e-30, 1, 1.5 }; + size_t i, j; + double relerr_bound = 3e-15; + bool ok = true; + + for (i = 0; i < arraycount(cases); i++) { + double xi = cases[i].xi; + double x = cases[i].x; + double p = cases[i].p; + double np = cases[i].np; + + CHECK_RELERR(p, cdf_genpareto(x, 0, 1, xi)); + CHECK_RELERR(p, cdf_genpareto(x*2, 0, 2, xi)); + CHECK_RELERR(p, cdf_genpareto(x/2, 0, .5, xi)); + CHECK_RELERR(np, sf_genpareto(x, 0, 1, xi)); + CHECK_RELERR(np, sf_genpareto(x*2, 0, 2, xi)); + CHECK_RELERR(np, sf_genpareto(x/2, 0, .5, xi)); + + if (p < .5) { + CHECK_RELERR(x, icdf_genpareto(p, 0, 1, xi)); + CHECK_RELERR(x*2, icdf_genpareto(p, 0, 2, xi)); + CHECK_RELERR(x/2, icdf_genpareto(p, 0, .5, xi)); + } + if (np < .5) { + CHECK_RELERR(x, isf_genpareto(np, 0, 1, xi)); + CHECK_RELERR(x*2, isf_genpareto(np, 0, 2, xi)); + CHECK_RELERR(x/2, isf_genpareto(np, 0, .5, xi)); + } + } + + for (i = 0; i < arraycount(xi_array); i++) { + for (j = 0; j <= 100; j++) { + double p0 = (j == 0 ? 2*DBL_MIN : (double)j/100); + + /* This is actually a check against 0, but we do <= so that the compiler + does not raise a -Wfloat-equal */ + if (fabs(xi_array[i]) <= 0) { + /* + * When xi == 0, the generalized Pareto + * distribution reduces to an + * exponential distribution. + */ + CHECK_RELERR(-log(p0/2), + sample_genpareto(0, p0, 0)); + CHECK_RELERR(-log1p(-p0/2), + sample_genpareto(1, p0, 0)); + } else { + CHECK_RELERR(expm1(-xi_array[i]*log(p0/2))/xi_array[i], + sample_genpareto(0, p0, xi_array[i])); + CHECK_RELERR((j == 0 ? DBL_MIN : + expm1(-xi_array[i]*log1p(-p0/2))/xi_array[i]), + sample_genpareto(1, p0, xi_array[i])); + } + + CHECK_RELERR(isf_genpareto(p0/2, 0, 1, xi_array[i]), + sample_genpareto(0, p0, xi_array[i])); + CHECK_RELERR(icdf_genpareto(p0/2, 0, 1, xi_array[i]), + sample_genpareto(1, p0, xi_array[i])); + } + } + + tt_assert(ok); + + done: + ; +} + +/** + * Test the deterministic sampler for uniform distribution on [a, b]. + * + * This currently only tests whether the outcome lies within [a, b]. + */ +static void +test_uniform_interval(void *arg) +{ + (void) arg; + struct { + /* Sample from a uniform distribution with parameters 'a' and 'b', using + * 't' as the sampling index. */ + double t, a, b; + } cases[] = { + { 0, 0, 0 }, + { 0, 0, 1 }, + { 0, 1.0000000000000007, 3.999999999999995 }, + { 0, 4000, 4000 }, + { 0.42475836677491291, 4000, 4000 }, + { 0, -DBL_MAX, DBL_MAX }, + { 0.25, -DBL_MAX, DBL_MAX }, + { 0.5, -DBL_MAX, DBL_MAX }, + }; + size_t i = 0; + bool ok = true; + + for (i = 0; i < arraycount(cases); i++) { + double t = cases[i].t; + double a = cases[i].a; + double b = cases[i].b; + + CHECK_LE(a, sample_uniform_interval(t, a, b)); + CHECK_LE(sample_uniform_interval(t, a, b), b); + + CHECK_LE(a, sample_uniform_interval(1 - t, a, b)); + CHECK_LE(sample_uniform_interval(1 - t, a, b), b); + + CHECK_LE(sample_uniform_interval(t, -b, -a), -a); + CHECK_LE(-b, sample_uniform_interval(t, -b, -a)); + + CHECK_LE(sample_uniform_interval(1 - t, -b, -a), -a); + CHECK_LE(-b, sample_uniform_interval(1 - t, -b, -a)); + } + + tt_assert(ok); + + done: + ; +} + +/********************** Stochastic tests ****************************/ + +/* + * Psi test, sometimes also called G-test. The psi test statistic, + * suitably scaled, has chi^2 distribution, but the psi test tends to + * have better statistical power in practice to detect deviations than + * the chi^2 test does. (The chi^2 test statistic is the first term of + * the Taylor expansion of the psi test statistic.) The psi test is + * generic, for any CDF; particular distributions might have higher- + * power tests to distinguish them from predictable deviations or bugs. + * + * We choose the psi critical value so that a single psi test has + * probability below alpha = 1% of spuriously failing even if all the + * code is correct. But the false positive rate for a suite of n tests + * is higher: 1 - Binom(0; n, alpha) = 1 - (1 - alpha)^n. For n = 10, + * this is about 10%, and for n = 100 it is well over 50%. + * + * We can drive it down by running each test twice, and accepting it if + * it passes at least once; in that case, it is as if we used Binom(2; + * 2, alpha) = alpha^2 as the false positive rate for each test, and + * for n = 10 tests, it would be 0.1%, and for n = 100 tests, still + * only 1%. + * + * The critical value for a chi^2 distribution with 100 degrees of + * freedom and false positive rate alpha = 1% was taken from: + * + * NIST/SEMATECH e-Handbook of Statistical Methods, Section + * 1.3.6.7.4 `Critical Values of the Chi-Square Distribution', + * , + * retrieved 2018-10-28. + */ + +static const size_t NSAMPLES = 100000; +/* Number of chances we give to the test to succeed. */ +static const unsigned NTRIALS = 2; +/* Number of times we want the test to pass per NTRIALS. */ +static const unsigned NPASSES_MIN = 1; + +#define PSI_DF 100 /* degrees of freedom */ +static const double PSI_CRITICAL = 135.807; /* critical value, alpha = .01 */ + +/** + * Perform a psi test on an array of sample counts, C, adding up to N + * samples, and an array of log expected probabilities, logP, + * representing the null hypothesis for the distribution of samples + * counted. Return false if the psi test rejects the null hypothesis, + * true if otherwise. + */ +static bool +psi_test(const size_t C[PSI_DF], const double logP[PSI_DF], size_t N) +{ + double psi = 0; + double c = 0; /* Kahan compensation */ + double t, u; + size_t i; + + for (i = 0; i < PSI_DF; i++) { + /* + * c*log(c/(n*p)) = (1/n) * f*log(f/p) where f = c/n is + * the frequency, and f*log(f/p) ---> 0 as f ---> 0, so + * this is a reasonable choice. Further, any mass that + * _fails_ to turn up in this bin will inflate another + * bin instead, so we don't really lose anything by + * ignoring empty bins even if they have high + * probability. + */ + if (C[i] == 0) + continue; + t = C[i]*(log((double)C[i]/N) - logP[i]) - c; + u = psi + t; + c = (u - psi) - t; + psi = u; + } + psi *= 2; + + return psi <= PSI_CRITICAL; +} + +static bool +test_stochastic_geometric_impl(double p) +{ + double logP[PSI_DF] = {0}; + unsigned ntry = NTRIALS, npass = 0; + unsigned i; + size_t j; + + /* Compute logP[i] = Geom(i + 1; p). */ + for (i = 0; i < PSI_DF - 1; i++) + logP[i] = logpmf_geometric(i + 1, p); + + /* Compute logP[n-1] = log (1 - (P[0] + P[1] + ... + P[n-2])). */ + logP[PSI_DF - 1] = log1mexp(logsumexp(logP, PSI_DF - 1)); + + while (ntry --> 0) { + size_t C[PSI_DF] = {0}; + + for (j = 0; j < NSAMPLES; j++) { + double n_tmp = ceil(geometric_sample(p)); + unsigned n = (unsigned) n_tmp; + + if (n > PSI_DF) + n = PSI_DF; + C[n - 1]++; + } + + if (psi_test(C, logP, NSAMPLES)) { + if (++npass >= NPASSES_MIN) + break; + } + } + + if (npass >= NPASSES_MIN) { + /* printf("pass %s sampler\n", "geometric"); */ + return true; + } else { + printf("fail %s sampler\n", "geometric"); + return false; + } +} + +/** + * Divide the support of dist into histogram bins in logP. Start + * at the 1st percentile and ending at the 99th percentile. Pick the bin + * boundaries using linear interpolation so that they are uniformly spaced. + * + * In each bin logP[i] we insert the expected log-probability that a sampled + * value will fall into that bin. We will use this as the null hypothesis of + * the psi test. + * + * Set logP[i] = log(CDF(x_i) - CDF(x_{i-1})), where x_-1 = -inf, x_n = + * +inf, and x_i = i*(hi - lo)/(n - 2). + */ +static void +bin_cdfs(const struct dist *dist, double lo, double hi, double *logP, size_t n) +{ +#define CDF(x) dist->ops->cdf(dist, x) +#define SF(x) dist->ops->sf(dist, x) + const double w = (hi - lo)/(n - 2); + double halfway = dist->ops->icdf(dist, 0.5); + double x_0, x_1; + size_t i; + size_t n2 = ceil_to_size_t((halfway - lo)/w); + + tor_assert(lo <= halfway); + tor_assert(halfway <= hi); + tor_assert(n2 <= n); + + x_1 = lo; + logP[0] = log(CDF(x_1) - 0); /* 0 = CDF(-inf) */ + for (i = 1; i < n2; i++) { + x_0 = x_1; + /* do the linear interpolation */ + x_1 = (i <= n/2 ? lo + i*w : hi - (n - 2 - i)*w); + /* set the expected log-probability */ + logP[i] = log(CDF(x_1) - CDF(x_0)); + } + x_0 = hi; + logP[n - 1] = log(SF(x_0) - 0); /* 0 = SF(+inf) = 1 - CDF(+inf) */ + + /* In this loop we are filling out the high part of the array. We are using + * SF because in these cases the CDF is near 1 where precision is lower. So + * instead we are using SF near 0 where the precision is higher. We have + * SF(t) = 1 - CDF(t). */ + for (i = 1; i < n - n2; i++) { + x_1 = x_0; + /* do the linear interpolation */ + x_0 = (i <= n/2 ? hi - i*w : lo + (n - 2 - i)*w); + /* set the expected log-probability */ + logP[n - i - 1] = log(SF(x_0) - SF(x_1)); + } +#undef SF +#undef CDF +} + +/** + * Draw NSAMPLES samples from dist, counting the number of samples x in + * the ith bin C[i] if x_{i-1} <= x < x_i, where x_-1 = -inf, x_n = + * +inf, and x_i = i*(hi - lo)/(n - 2). + */ +static void +bin_samples(const struct dist *dist, double lo, double hi, size_t *C, size_t n) +{ + const double w = (hi - lo)/(n - 2); + size_t i; + + for (i = 0; i < NSAMPLES; i++) { + double x = dist->ops->sample(dist); + size_t bin; + + if (x < lo) + bin = 0; + else if (x < hi) + bin = 1 + floor_to_size_t((x - lo)/w); + else + bin = n - 1; + tor_assert(bin < n); + C[bin]++; + } +} + +/** + * Carry out a Psi test on dist. + * + * Sample NSAMPLES from dist, putting them in bins from -inf to lo to + * hi to +inf, and apply up to two psi tests. True if at least one psi + * test passes; false if not. False positive rate should be bounded by + * 0.01^2 = 0.0001. + */ +static bool +test_psi_dist_sample(const struct dist *dist) +{ + double logP[PSI_DF] = {0}; + unsigned ntry = NTRIALS, npass = 0; + double lo = dist->ops->icdf(dist, 1/(double)(PSI_DF + 2)); + double hi = dist->ops->isf(dist, 1/(double)(PSI_DF + 2)); + + /* Create the null hypothesis in logP */ + bin_cdfs(dist, lo, hi, logP, PSI_DF); + + /* Now run the test */ + while (ntry --> 0) { + size_t C[PSI_DF] = {0}; + bin_samples(dist, lo, hi, C, PSI_DF); + if (psi_test(C, logP, NSAMPLES)) { + if (++npass >= NPASSES_MIN) + break; + } + } + + /* Did we fail or succeed? */ + if (npass >= NPASSES_MIN) { + /* printf("pass %s sampler\n", dist->ops->name);*/ + return true; + } else { + printf("fail %s sampler\n", dist->ops->name); + return false; + } +} + +/* This is the seed of the deterministic randomness */ +static uint32_t deterministic_rand_counter; + +/** Initialize the seed of the deterministic randomness. */ +static void +init_deterministic_rand(void) +{ + deterministic_rand_counter = crypto_rand_uint32(); +} + +/** Produce deterministic randomness for the stochastic tests using the global + * deterministic_rand_counter seed + * + * This function produces deterministic data over multiple calls iff it's + * called in the same call order with the same 'n' parameter (which is the + * case for the psi test). If not, outputs will deviate. */ +static void +crypto_rand_deterministic(char *out, size_t n) +{ + /* Use a XOF to squeeze bytes out of that silly counter */ + crypto_xof_t *xof = crypto_xof_new(); + tor_assert(xof); + crypto_xof_add_bytes(xof, (uint8_t*)&deterministic_rand_counter, + sizeof(deterministic_rand_counter)); + crypto_xof_squeeze_bytes(xof, (uint8_t*)out, n); + crypto_xof_free(xof); + + /* Increase counter for next run */ + deterministic_rand_counter++; +} + +static void +test_stochastic_uniform(void *arg) +{ + (void) arg; + + const struct uniform uniform01 = { + .base = DIST_BASE(&uniform_ops), + .a = 0, + .b = 1, + }; + const struct uniform uniform_pos = { + .base = DIST_BASE(&uniform_ops), + .a = 1.23, + .b = 4.56, + }; + const struct uniform uniform_neg = { + .base = DIST_BASE(&uniform_ops), + .a = -10, + .b = -1, + }; + const struct uniform uniform_cross = { + .base = DIST_BASE(&uniform_ops), + .a = -1.23, + .b = 4.56, + }; + const struct uniform uniform_subnormal = { + .base = DIST_BASE(&uniform_ops), + .a = 4e-324, + .b = 4e-310, + }; + const struct uniform uniform_subnormal_cross = { + .base = DIST_BASE(&uniform_ops), + .a = -4e-324, + .b = 4e-310, + }; + bool ok = true; + + init_deterministic_rand(); + MOCK(crypto_rand, crypto_rand_deterministic); + + ok &= test_psi_dist_sample(&uniform01.base); + ok &= test_psi_dist_sample(&uniform_pos.base); + ok &= test_psi_dist_sample(&uniform_neg.base); + ok &= test_psi_dist_sample(&uniform_cross.base); + ok &= test_psi_dist_sample(&uniform_subnormal.base); + ok &= test_psi_dist_sample(&uniform_subnormal_cross.base); + + tt_assert(ok); + + done: + ; +} + +static bool +test_stochastic_logistic_impl(double mu, double sigma) +{ + const struct logistic dist = { + .base = DIST_BASE(&logistic_ops), + .mu = mu, + .sigma = sigma, + }; + + /* XXX Consider some fancier logistic test. */ + return test_psi_dist_sample(&dist.base); +} + +static bool +test_stochastic_log_logistic_impl(double alpha, double beta) +{ + const struct log_logistic dist = { + .base = DIST_BASE(&log_logistic_ops), + .alpha = alpha, + .beta = beta, + }; + + /* XXX Consider some fancier log logistic test. */ + return test_psi_dist_sample(&dist.base); +} + +static bool +test_stochastic_weibull_impl(double lambda, double k) +{ + const struct weibull dist = { + .base = DIST_BASE(&weibull_ops), + .lambda = lambda, + .k = k, + }; + +/* + * XXX Consider applying a Tiku-Singh test: + * + * M.L. Tiku and M. Singh, `Testing the two-parameter + * Weibull distribution', Communications in Statistics -- + * Theory and Methods A10(9), 1981, 907--918. + *https://www.tandfonline.com/doi/pdf/10.1080/03610928108828082?needAccess=true + */ + return test_psi_dist_sample(&dist.base); +} + +static bool +test_stochastic_genpareto_impl(double mu, double sigma, double xi) +{ + const struct genpareto dist = { + .base = DIST_BASE(&genpareto_ops), + .mu = mu, + .sigma = sigma, + .xi = xi, + }; + + /* XXX Consider some fancier GPD test. */ + return test_psi_dist_sample(&dist.base); +} + +static void +test_stochastic_genpareto(void *arg) +{ + bool ok = 0; + bool tests_failed = true; + (void) arg; + + init_deterministic_rand(); + MOCK(crypto_rand, crypto_rand_deterministic); + + ok = test_stochastic_genpareto_impl(0, 1, -0.25); + tt_assert(ok); + ok = test_stochastic_genpareto_impl(0, 1, -1e-30); + tt_assert(ok); + ok = test_stochastic_genpareto_impl(0, 1, 0); + tt_assert(ok); + ok = test_stochastic_genpareto_impl(0, 1, 1e-30); + tt_assert(ok); + ok = test_stochastic_genpareto_impl(0, 1, 0.25); + tt_assert(ok); + ok = test_stochastic_genpareto_impl(-1, 1, -0.25); + tt_assert(ok); + ok = test_stochastic_genpareto_impl(1, 2, 0.25); + tt_assert(ok); + + tests_failed = false; + + done: + if (tests_failed) { + printf("seed: %"PRIu32, deterministic_rand_counter); + } + UNMOCK(crypto_rand); +} + +static void +test_stochastic_geometric(void *arg) +{ + bool ok = 0; + bool tests_failed = true; + + (void) arg; + + init_deterministic_rand(); + MOCK(crypto_rand, crypto_rand_deterministic); + + ok = test_stochastic_geometric_impl(0.1); + tt_assert(ok); + ok = test_stochastic_geometric_impl(0.5); + tt_assert(ok); + ok = test_stochastic_geometric_impl(0.9); + tt_assert(ok); + ok = test_stochastic_geometric_impl(1); + tt_assert(ok); + + tests_failed = false; + + done: + if (tests_failed) { + printf("seed: %"PRIu32, deterministic_rand_counter); + } + UNMOCK(crypto_rand); +} + +static void +test_stochastic_logistic(void *arg) +{ + bool ok = 0; + bool tests_failed = true; + (void) arg; + + init_deterministic_rand(); + MOCK(crypto_rand, crypto_rand_deterministic); + + ok = test_stochastic_logistic_impl(0, 1); + tt_assert(ok); + ok = test_stochastic_logistic_impl(0, 1e-16); + tt_assert(ok); + ok = test_stochastic_logistic_impl(1, 10); + tt_assert(ok); + ok = test_stochastic_logistic_impl(-10, 100); + tt_assert(ok); + + tests_failed = false; + + done: + if (tests_failed) { + printf("seed: %"PRIu32, deterministic_rand_counter); + } + UNMOCK(crypto_rand); +} + +static void +test_stochastic_log_logistic(void *arg) +{ + bool ok = 0; + bool tests_failed = true; + (void) arg; + + init_deterministic_rand(); + MOCK(crypto_rand, crypto_rand_deterministic); + + ok = test_stochastic_log_logistic_impl(1, 1); + tt_assert(ok); + ok = test_stochastic_log_logistic_impl(1, 10); + tt_assert(ok); + ok = test_stochastic_log_logistic_impl(M_E, 1e-1); + tt_assert(ok); + ok = test_stochastic_log_logistic_impl(exp(-10), 1e-2); + tt_assert(ok); + + tests_failed = false; + + done: + if (tests_failed) { + printf("seed: %"PRIu32, deterministic_rand_counter); + } + UNMOCK(crypto_rand); +} + +static void +test_stochastic_weibull(void *arg) +{ + bool ok = 0; + bool tests_failed = true; + (void) arg; + + init_deterministic_rand(); + MOCK(crypto_rand, crypto_rand_deterministic); + + ok = test_stochastic_weibull_impl(1, 0.5); + tt_assert(ok); + ok = test_stochastic_weibull_impl(1, 1); + tt_assert(ok); + ok = test_stochastic_weibull_impl(1, 1.5); + tt_assert(ok); + ok = test_stochastic_weibull_impl(1, 2); + tt_assert(ok); + ok = test_stochastic_weibull_impl(10, 1); + tt_assert(ok); + + tests_failed = false; + + done: + if (tests_failed) { + printf("seed: %"PRIu32, deterministic_rand_counter); + } + UNMOCK(crypto_rand); +} + +struct testcase_t prob_distr_tests[] = { + { "logit_logistics", test_logit_logistic, TT_FORK, NULL, NULL }, + { "log_logistic", test_log_logistic, TT_FORK, NULL, NULL }, + { "weibull", test_weibull, TT_FORK, NULL, NULL }, + { "genpareto", test_genpareto, TT_FORK, NULL, NULL }, + { "uniform_interval", test_uniform_interval, TT_FORK, NULL, NULL }, + END_OF_TESTCASES +}; + +struct testcase_t slow_stochastic_prob_distr_tests[] = { + { "stochastic_genpareto", test_stochastic_genpareto, TT_FORK, NULL, NULL }, + { "stochastic_geometric", test_stochastic_geometric, TT_FORK, NULL, NULL }, + { "stochastic_uniform", test_stochastic_uniform, TT_FORK, NULL, NULL }, + { "stochastic_logistic", test_stochastic_logistic, TT_FORK, NULL, NULL }, + { "stochastic_log_logistic", test_stochastic_log_logistic, TT_FORK, NULL, + NULL }, + { "stochastic_weibull", test_stochastic_weibull, TT_FORK, NULL, NULL }, + END_OF_TESTCASES +}; diff --git a/src/test/test_slow.c b/src/test/test_slow.c index 97c2912af6..39a203c726 100644 --- a/src/test/test_slow.c +++ b/src/test/test_slow.c @@ -21,6 +21,7 @@ struct testgroup_t testgroups[] = { { "slow/crypto/", slow_crypto_tests }, { "slow/process/", slow_process_tests }, + { "slow/prob_distr/", slow_stochastic_prob_distr_tests }, END_OF_GROUPS }; From dd04917851c23f16a11c0d2b0c62ea76c93c4c7d Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Tue, 27 Nov 2018 01:54:10 +0200 Subject: [PATCH 0368/2557] Use the new probability distribution code in WTF-PAD. Co-authored-by: Mike Perry Co-authored-by: Taylor R Campbell --- src/core/or/circuitpadding.c | 103 +++++++++++++++++++++-------------- 1 file changed, 62 insertions(+), 41 deletions(-) diff --git a/src/core/or/circuitpadding.c b/src/core/or/circuitpadding.c index a9d927619d..5210265ff2 100644 --- a/src/core/or/circuitpadding.c +++ b/src/core/or/circuitpadding.c @@ -1,8 +1,11 @@ /* Copyright (c) 2017 The Tor Project, Inc. */ /* See LICENSE for licensing information */ +#define CIRCUITPADDING_PRIVATE + #include #include "lib/math/fp.h" +#include "lib/math/prob_distr.h" #include "core/or/or.h" #include "core/or/circuitpadding.h" #include "core/or/circuitlist.h" @@ -374,7 +377,7 @@ circpad_distribution_sample_iat_delay(const circpad_state_t *state, * of tokens in each bin, and then a time value is chosen uniformly from * that bin's [start,end) time range. */ -static circpad_delay_t +STATIC circpad_delay_t circpad_machine_sample_delay(circpad_machineinfo_t *mi) { const circpad_state_t *state = circpad_machine_current_state(mi); @@ -487,57 +490,75 @@ circpad_machine_sample_delay(circpad_machineinfo_t *mi) static double circpad_distribution_sample(circpad_distribution_t dist) { - double p = 0; + log_fn(LOG_DEBUG,LD_CIRC, "Sampling delay with distribution %d", + dist.type); switch (dist.type) { case CIRCPAD_DIST_NONE: - return 0; - case CIRCPAD_DIST_UNIFORM: - p = crypto_rand_double(); - // param2 is upper bound, param1 is lower - /* The subtraction is exact as long as param2 and param1 are less than - * 2**53. The multiplication is accurate as long as (param2 - param1) - * is less than 2**52. (And when they are large, the low bits aren't - * important.) The result covers the full range of outputs, as long as - * p has a resolution of 1/2**32 or greater. */ - p *= (dist.param2 - dist.param1); - p += dist.param1; - return p; - case CIRCPAD_DIST_LOGISTIC: - p = crypto_rand_double(); - /* https://en.wikipedia.org/wiki/Logistic_distribution#Quantile_function - * param1 is Mu, param2 is s. */ - if (p <= 0.0) // Avoid log(0) + { + /* We should not get in here like this */ + tor_assert_nonfatal_unreached(); return 0; - return dist.param1 + dist.param2*tor_mathlog(p/(1.0-p)); + } + case CIRCPAD_DIST_UNIFORM: + { + // param2 is upper bound, param1 is lower + const struct uniform my_uniform = { + .base = DIST_BASE(&uniform_ops), + .a = dist.param1, + .b = dist.param2, + }; + return uniform_sample(&my_uniform.base); + } + case CIRCPAD_DIST_LOGISTIC: + { + /* param1 is Mu, param2 is sigma. */ + const struct logistic my_logistic = { + .base = DIST_BASE(&uniform_ops), + .mu = dist.param1, + .sigma = dist.param2, + }; + return logistic_sample(&my_logistic.base); + } case CIRCPAD_DIST_LOG_LOGISTIC: - p = crypto_rand_double(); - /* https://en.wikipedia.org/wiki/Log-logistic_distribution#Quantiles - * param1 is Alpha, param2 is Beta */ - return dist.param1 * pow(p/(1.0-p), 1.0/dist.param2); + { + /* param1 is Alpha, param2 is 1.0/Beta */ + const struct log_logistic my_log_logistic = { + .base = DIST_BASE(&log_logistic_ops), + .alpha = dist.param1, + .beta = dist.param2, + }; + return log_logistic_sample(&my_log_logistic.base); + } case CIRCPAD_DIST_GEOMETRIC: { /* param1 is 'p' (success probability) */ return geometric_sample(dist.param1); } case CIRCPAD_DIST_WEIBULL: - p = crypto_rand_double(); - /* https://en.wikipedia.org/wiki/Weibull_distribution \ - * #Cumulative_distribution_function - * param1 is k, param2 is Lambda */ - return dist.param2*pow(-tor_mathlog(1.0-p), 1.0/dist.param1); + { + /* param1 is k, param2 is Lambda */ + const struct weibull my_weibull = { + .base = DIST_BASE(&weibull_ops), + .k = dist.param1, + .lambda = dist.param2, + }; + return weibull_sample(&my_weibull.base); + } case CIRCPAD_DIST_PARETO: - p = 1.0-crypto_rand_double(); // Pareto quantile needs (0,1] - - /* https://en.wikipedia.org/wiki/Generalized_Pareto_distribution \ - * #Generating_generalized_Pareto_random_variables - * param1 is Sigma, param2 is Xi - * Since it's piecewise, we must define it for 0 (or close to 0) */ - if (fabs(dist.param2) <= 1e-22) - return -dist.param1*tor_mathlog(p); - else - return dist.param1*(pow(p, -dist.param2) - 1.0)/dist.param2; + { + /* param1 is sigma, param2 is xi, no more params for mu so we use 0 */ + const struct genpareto my_genpareto = { + .base = DIST_BASE(&weibull_ops), + .mu = 0, + .sigma = dist.param1, + .xi = dist.param2, + }; + return genpareto_sample(&my_genpareto.base); + } } + + tor_assert_nonfatal_unreached(); return 0; } @@ -1086,8 +1107,8 @@ circpad_machine_reached_padding_limit(circpad_machineinfo_t *mi) * Returns 1 if we decide to transition states (due to infinity bin), * 0 otherwise. */ -circpad_decision_t -circpad_machine_schedule_padding(circpad_machineinfo_t *mi) +MOCK_IMPL(circpad_decision_t, +circpad_machine_schedule_padding,(circpad_machineinfo_t *mi)) { circpad_delay_t in_usec = 0; struct timeval timeout; From 926fc93be5b6afab1a604ecd7c79aa6e8ae8a676 Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Fri, 7 Dec 2018 16:35:23 +0200 Subject: [PATCH 0369/2557] Concentrate all TOR_USEC_PER_SEC definitions in a single header file. Co-authored-by: Mike Perry --- src/core/or/circuitpadding.c | 5 +--- src/feature/hibernate/hibernate.c | 3 +-- src/lib/defs/include.am | 1 + src/lib/defs/time.h | 23 ++++++++++++++++++ src/lib/time/.may_include | 1 + src/lib/time/tvdiff.c | 3 +-- src/test/test_circuitpadding.c | 39 ++++++++++++++----------------- src/test/test_util.c | 2 +- 8 files changed, 47 insertions(+), 30 deletions(-) create mode 100644 src/lib/defs/time.h diff --git a/src/core/or/circuitpadding.c b/src/core/or/circuitpadding.c index 5210265ff2..9f8713f624 100644 --- a/src/core/or/circuitpadding.c +++ b/src/core/or/circuitpadding.c @@ -17,6 +17,7 @@ #include "core/or/channel.h" #include "lib/time/compat_time.h" +#include "lib/defs/time.h" #include "lib/crypt_ops/crypto_rand.h" #include "core/or/crypt_path_st.h" @@ -31,10 +32,6 @@ #include "app/config/config.h" -/* XXX: This is a dup of the constant in ./src/lib/time/tvdiff.c. - * Should/Do we have a header for time constants like this? */ -#define TOR_USEC_PER_SEC (1000000) - static inline circpad_purpose_mask_t circpad_circ_purpose_to_mask(uint8_t circ_purpose); static inline circpad_circuit_state_t circpad_circuit_state( diff --git a/src/feature/hibernate/hibernate.c b/src/feature/hibernate/hibernate.c index feeb3d92ef..f10a45f4ae 100644 --- a/src/feature/hibernate/hibernate.c +++ b/src/feature/hibernate/hibernate.c @@ -37,6 +37,7 @@ hibernating, phase 2: #include "core/or/connection_or.h" #include "feature/control/control.h" #include "lib/crypt_ops/crypto_rand.h" +#include "lib/defs/time.h" #include "feature/hibernate/hibernate.h" #include "core/mainloop/mainloop.h" #include "feature/relay/router.h" @@ -832,8 +833,6 @@ hibernate_soft_limit_reached(void) return get_accounting_bytes() >= soft_limit; } -#define TOR_USEC_PER_SEC (1000000) - /** Called when we get a SIGINT, or when bandwidth soft limit is * reached. Puts us into "loose hibernation": we don't accept new * connections, but we continue handling old ones. */ diff --git a/src/lib/defs/include.am b/src/lib/defs/include.am index 48ee7f29fc..6a7f9114ea 100644 --- a/src/lib/defs/include.am +++ b/src/lib/defs/include.am @@ -2,4 +2,5 @@ noinst_HEADERS += \ src/lib/defs/dh_sizes.h \ src/lib/defs/digest_sizes.h \ + src/lib/defs/time.h \ src/lib/defs/x25519_sizes.h diff --git a/src/lib/defs/time.h b/src/lib/defs/time.h new file mode 100644 index 0000000000..762b23feab --- /dev/null +++ b/src/lib/defs/time.h @@ -0,0 +1,23 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef TOR_TIME_DEFS_H +#define TOR_TIME_DEFS_H + +/** + * \file time.h + * + * \brief Definitions for timing-related constants. + **/ + +/** How many microseconds per second */ +#define TOR_USEC_PER_SEC (1000000) +/** How many nanoseconds per microsecond */ +#define TOR_NSEC_PER_USEC (1000) +/* How many nanoseconds per millisecond */ +#define TOR_NSEC_PER_MSEC (1000*1000) + +#endif diff --git a/src/lib/time/.may_include b/src/lib/time/.may_include index 40a18805ac..ae01431b60 100644 --- a/src/lib/time/.may_include +++ b/src/lib/time/.may_include @@ -7,6 +7,7 @@ lib/log/*.h lib/subsys/*.h lib/time/*.h lib/wallclock/*.h +lib/defs/time.h # For load_windows_system_lib. lib/fs/winlib.h \ No newline at end of file diff --git a/src/lib/time/tvdiff.c b/src/lib/time/tvdiff.c index bc8a1166e7..9dfb63c26f 100644 --- a/src/lib/time/tvdiff.c +++ b/src/lib/time/tvdiff.c @@ -11,6 +11,7 @@ #include "lib/time/tvdiff.h" #include "lib/cc/compat_compiler.h" +#include "lib/defs/time.h" #include "lib/log/log.h" #ifdef _WIN32 @@ -20,8 +21,6 @@ #include #endif -#define TOR_USEC_PER_SEC 1000000 - /** Return the difference between start->tv_sec and end->tv_sec. * Returns INT64_MAX on overflow and underflow. */ diff --git a/src/test/test_circuitpadding.c b/src/test/test_circuitpadding.c index 78f93f7b24..5693c98e41 100644 --- a/src/test/test_circuitpadding.c +++ b/src/test/test_circuitpadding.c @@ -12,6 +12,7 @@ #include #include "lib/evloop/compat_libevent.h" #include "lib/time/compat_time.h" +#include "lib/defs/time.h" #include "core/or/relay.h" #include "core/or/circuitlist.h" #include "core/or/circuitbuild.h" @@ -32,10 +33,6 @@ extern smartlist_t *connection_array; -#define USEC_PER_SEC (1000000) -#define NSEC_PER_USEC (1000) -#define NSEC_PER_MSEC (1000*1000) - circid_t get_unique_circ_id_by_chan(channel_t *chan); void helper_create_basic_machine(void); static void helper_create_conditional_machines(void); @@ -69,7 +66,7 @@ static circpad_machine_t circ_client_machine; static void timers_advance_and_run(int64_t msec_update) { - curr_mocked_time += msec_update*NSEC_PER_MSEC; + curr_mocked_time += msec_update*TOR_NSEC_PER_MSEC; monotime_coarse_set_mock_time_nsec(curr_mocked_time); monotime_set_mock_time_nsec(curr_mocked_time); timers_run_pending(); @@ -304,9 +301,9 @@ test_circuitpadding_rtt(void *arg) monotime_init(); monotime_enable_test_mocking(); - monotime_set_mock_time_nsec(1*NSEC_PER_USEC); - monotime_coarse_set_mock_time_nsec(1*NSEC_PER_USEC); - curr_mocked_time = 1*NSEC_PER_USEC; + monotime_set_mock_time_nsec(1*TOR_NSEC_PER_USEC); + monotime_coarse_set_mock_time_nsec(1*TOR_NSEC_PER_USEC); + curr_mocked_time = 1*TOR_NSEC_PER_USEC; timers_initialize(); circpad_machines_init(); @@ -994,9 +991,9 @@ test_circuitpadding_tokens(void *arg) monotime_init(); monotime_enable_test_mocking(); - monotime_set_mock_time_nsec(1*NSEC_PER_USEC); - monotime_coarse_set_mock_time_nsec(1*NSEC_PER_USEC); - curr_mocked_time = 1*NSEC_PER_USEC; + monotime_set_mock_time_nsec(1*TOR_NSEC_PER_USEC); + monotime_coarse_set_mock_time_nsec(1*TOR_NSEC_PER_USEC); + curr_mocked_time = 1*TOR_NSEC_PER_USEC; timers_initialize(); @@ -1441,9 +1438,9 @@ test_circuitpadding_negotiation(void *arg) monotime_init(); monotime_enable_test_mocking(); - monotime_set_mock_time_nsec(1*NSEC_PER_USEC); - monotime_coarse_set_mock_time_nsec(1*NSEC_PER_USEC); - curr_mocked_time = 1*NSEC_PER_USEC; + monotime_set_mock_time_nsec(1*TOR_NSEC_PER_USEC); + monotime_coarse_set_mock_time_nsec(1*TOR_NSEC_PER_USEC); + curr_mocked_time = 1*TOR_NSEC_PER_USEC; timers_initialize(); circpad_machines_init(); @@ -1562,7 +1559,7 @@ simulate_single_hop_extend(circuit_t *client, circuit_t *mid_relay, circpad_cell_event_nonpadding_received((circuit_t*)mid_relay); // Advance time a tiny bit so we can calculate an RTT - curr_mocked_time += 10 * NSEC_PER_MSEC; + curr_mocked_time += 10 * TOR_NSEC_PER_MSEC; monotime_coarse_set_mock_time_nsec(curr_mocked_time); monotime_set_mock_time_nsec(curr_mocked_time); @@ -1715,9 +1712,9 @@ test_circuitpadding_conditions(void *arg) monotime_init(); monotime_enable_test_mocking(); - monotime_set_mock_time_nsec(1*NSEC_PER_USEC); - monotime_coarse_set_mock_time_nsec(1*NSEC_PER_USEC); - curr_mocked_time = 1*NSEC_PER_USEC; + monotime_set_mock_time_nsec(1*TOR_NSEC_PER_USEC); + monotime_coarse_set_mock_time_nsec(1*TOR_NSEC_PER_USEC); + curr_mocked_time = 1*TOR_NSEC_PER_USEC; timers_initialize(); helper_create_conditional_machines(); @@ -1835,9 +1832,9 @@ test_circuitpadding_circuitsetup_machine(void *arg) monotime_init(); monotime_enable_test_mocking(); - monotime_set_mock_time_nsec(1*NSEC_PER_USEC); - monotime_coarse_set_mock_time_nsec(1*NSEC_PER_USEC); - curr_mocked_time = 1*NSEC_PER_USEC; + monotime_set_mock_time_nsec(1*TOR_NSEC_PER_USEC); + monotime_coarse_set_mock_time_nsec(1*TOR_NSEC_PER_USEC); + curr_mocked_time = 1*TOR_NSEC_PER_USEC; timers_initialize(); circpad_machines_init(); diff --git a/src/test/test_util.c b/src/test/test_util.c index b983cbb0bf..bf64cff7ef 100644 --- a/src/test/test_util.c +++ b/src/test/test_util.c @@ -19,6 +19,7 @@ #include "feature/client/transports.h" #include "lib/crypt_ops/crypto_format.h" #include "lib/crypt_ops/crypto_rand.h" +#include "lib/defs/time.h" #include "test/test.h" #include "lib/memarea/memarea.h" #include "lib/process/waitpid.h" @@ -404,7 +405,6 @@ test_util_time(void *arg) /* Assume tv_usec is an unsigned integer until proven otherwise */ #define TV_USEC_MAX UINT_MAX -#define TOR_USEC_PER_SEC 1000000 /* Overflows in the result type */ From 56a45eb4092e9c543b39e981dce798c64bbc9e5d Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Thu, 27 Dec 2018 10:45:16 +0200 Subject: [PATCH 0370/2557] Disable current padding machines. Co-authored-by: Mike Perry --- src/core/or/circuitpadding.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/core/or/circuitpadding.c b/src/core/or/circuitpadding.c index 9f8713f624..ee26558bc8 100644 --- a/src/core/or/circuitpadding.c +++ b/src/core/or/circuitpadding.c @@ -2018,6 +2018,8 @@ circpad_setup_machine_on_circ(circuit_t *on_circ, on_circ->padding_machine[machine->machine_index] = machine; } +/* These padding machines are only used for tests pending #28634. */ +#ifdef TOR_UNIT_TESTS static void circpad_circ_client_machine_init(void) { @@ -2163,6 +2165,7 @@ circpad_circ_responder_machine_init(void) circ_responder_machine->machine_num = smartlist_len(relay_padding_machines); smartlist_add(relay_padding_machines, circ_responder_machine); } +#endif /** * Initialize all of our padding machines. @@ -2180,9 +2183,10 @@ circpad_machines_init(void) relay_padding_machines = smartlist_new(); // TODO: Parse machines from consensus and torrc - +#ifdef TOR_UNIT_TESTS circpad_circ_client_machine_init(); circpad_circ_responder_machine_init(); +#endif } /** From 0658c729cf3c6be27ff774d9d219402a6cf6cf53 Mon Sep 17 00:00:00 2001 From: Mike Perry Date: Fri, 24 Aug 2018 21:37:43 +0000 Subject: [PATCH 0371/2557] Add TODO file for padding work. Note to self/others: don't merge this. Co-authored-by: George Kadianakis --- PADDING_TODO.txt | 58 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 PADDING_TODO.txt diff --git a/PADDING_TODO.txt b/PADDING_TODO.txt new file mode 100644 index 0000000000..a2e97f7ca3 --- /dev/null +++ b/PADDING_TODO.txt @@ -0,0 +1,58 @@ +TODO sketch for this branch, in rough priority order: + +- Clean up/fix XXX's and FIXMEs + - Test event entry points into circuitpad? + - Most of our events come from completely untested code :/ + +- Compat-breaking changes to be decided/done ASAP + - Option to keep circuits open if machine present + - Specify an ordered preference list of padding machines + - Specify exit policy for machine conditions? + - short_policy_t looks good, except for its flexible array member :/ + - Can we make our own struct with a small, fixed number of policy + entries? Say 3-4? Or is that a bad idea to lose this flexibility? + - Check conditions based on attached streams on the circuit + - Accept should mean "only apply if matched" + - Reject should mean "don't apply if matched" + - If a policy is specified, Reject *:* is implicit default (so reject + policies need an Accept entry). + - With no policy, Accept *:* is implicit default. + + +- Misc fixes: + - Remove circuitsetup machine (but place it in unittests -- they depend on it) + - Circuit RTT measurement will break on var_cell/EXTEND2 cells + - Are there any heuristics we can use here? + - If RELAY_EARLY is only for the first cell of an EXTEND2 series, + we can use that. But the proposal currently says MAY, but not MUST + for this behavior. + +======== 0.3.6 ======== + +- Come up with some good histograms for eg circuit setup fingerprinting, + website fingerprinting, and vanguards usage. + +- Vanguards compatibility for MiddleNodes (via changes to vanguards addon) + +- circpad_machine_validate() function to sanity-check histograms loaded from + consensus/torrc (can also be used to help guide a GA). + - Check bin construction + - no type overflow (start_usec + range_sec, etc) + - no conflicting state transitions (or overlap with cancel events) + - no use of both histograms and iat_dist + - at least two histogram bins + - min_hop vs target_hop + +- Support torrc load+serialization of state machines + - ?? + +- Support consensus load+serialization of state machines + - ?? + +- Prop #265 load balancing + +- Rephist timer stats + - Is this a privacy risk? The adversary could create lots of circuits + to find a layer2 vanguard.. Otherwise they will be spread across middles. + + From 968235ce6f2beed984d0a4290cb30a45fb04ad5d Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 2 Jan 2019 15:45:48 -0500 Subject: [PATCH 0372/2557] Explicitly ignore check_result() result in test_voting_flags_minimal Otherwise, coverity complains at is. --- src/test/test_voting_flags.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/test/test_voting_flags.c b/src/test/test_voting_flags.c index c4c4c7354e..740b96b4c0 100644 --- a/src/test/test_voting_flags.c +++ b/src/test/test_voting_flags.c @@ -104,7 +104,7 @@ static void test_voting_flags_minimal(void *arg) { flag_vote_test_cfg_t *cfg = arg; - check_result(cfg); + (void) check_result(cfg); } static void @@ -189,4 +189,3 @@ struct testcase_t voting_flags_tests[] = { T(staledesc, TT_FORK), END_OF_TESTCASES }; - From dbf1725a135405cfa3e4aea9d66e0d62fed1f360 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Thu, 3 Jan 2019 10:36:38 +0200 Subject: [PATCH 0373/2557] Completely remove 'GETINFO status/version/num-{concurring,versioning}' --- changes/ticket28757 | 5 +++++ src/feature/control/control.c | 9 --------- 2 files changed, 5 insertions(+), 9 deletions(-) create mode 100644 changes/ticket28757 diff --git a/changes/ticket28757 b/changes/ticket28757 new file mode 100644 index 0000000000..62c6d099ff --- /dev/null +++ b/changes/ticket28757 @@ -0,0 +1,5 @@ + o Removed features: + - Stop responding to 'GETINFO status/version/num-concurring' and + 'GETINFO status/version/num-versioning' control port commands, as those + were deprecated back in 0.2.0.30. Also stop listing them in output of + 'GETINFO info/names'. Resolves ticket 28757. diff --git a/src/feature/control/control.c b/src/feature/control/control.c index da62c94981..f10d5eb84a 100644 --- a/src/feature/control/control.c +++ b/src/feature/control/control.c @@ -3069,11 +3069,6 @@ getinfo_helper_events(control_connection_t *control_conn, case VS_UNKNOWN: *answer = tor_strdup("unknown"); break; default: tor_fragile_assert(); } - } else if (!strcmp(question, "status/version/num-versioning") || - !strcmp(question, "status/version/num-concurring")) { - tor_asprintf(answer, "%d", get_n_authorities(V3_DIRINFO)); - log_warn(LD_GENERAL, "%s is deprecated; it no longer gives useful " - "information", question); } } else if (!strcmp(question, "status/clients-seen")) { char *bridge_stats = geoip_get_bridge_stats_controller(time(NULL)); @@ -3366,10 +3361,6 @@ static const getinfo_item_t getinfo_items[] = { "A fresh relay/ei descriptor pair for Tor's current state. Not stored."), DOC("status/version/recommended", "List of currently recommended versions."), DOC("status/version/current", "Status of the current version."), - DOC("status/version/num-versioning", "Number of versioning authorities."), - DOC("status/version/num-concurring", - "Number of versioning authorities agreeing on the status of the " - "current version"), ITEM("address", misc, "IP address of this Tor host, if we can guess it."), ITEM("traffic/read", misc,"Bytes read since the process was started."), ITEM("traffic/written", misc, From 4e4f93d364e33bb46e271f3b960a2cf5f38f402e Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 3 Jan 2019 09:50:54 -0500 Subject: [PATCH 0374/2557] Add a #ifdef HAVE_UNISTD_H check to buffers.c Reported on tor-dev by Gisle Vanem. Bug not in any released Tor (The suggested patch used _MSC_VER, but that's not how we do stuff with autoconf. With autoconf, you detect the feature you want, rather than trying to list all the systems that do or do not have it.) --- src/lib/net/buffers_net.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/lib/net/buffers_net.c b/src/lib/net/buffers_net.c index 576dc56a52..a4c0aca3b5 100644 --- a/src/lib/net/buffers_net.c +++ b/src/lib/net/buffers_net.c @@ -21,7 +21,10 @@ #endif #include + +#ifdef HAVE_UNISTD_H #include +#endif #ifdef PARANOIA /** Helper: If PARANOIA is defined, assert that the buffer in local variable From af85a0f28fcea7d5fbb5cad2b807d75911230894 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 4 Jan 2019 15:04:00 -0500 Subject: [PATCH 0375/2557] Remove from master those changes files that will appear in 0.3.5.7 or earlier. --- changes/bug28568 | 4 ---- changes/bug28612 | 4 ---- changes/bug28974 | 3 --- changes/ticket28838 | 8 -------- changes/ticket28851 | 4 ---- changes/ticket28879 | 5 ----- changes/ticket28881 | 4 ---- changes/ticket28883 | 4 ---- changes/ticket28912 | 6 ------ changes/ticket28924 | 4 ---- changes/ticket28973 | 6 ------ 11 files changed, 52 deletions(-) delete mode 100644 changes/bug28568 delete mode 100644 changes/bug28612 delete mode 100644 changes/bug28974 delete mode 100644 changes/ticket28838 delete mode 100644 changes/ticket28851 delete mode 100644 changes/ticket28879 delete mode 100644 changes/ticket28881 delete mode 100644 changes/ticket28883 delete mode 100644 changes/ticket28912 delete mode 100644 changes/ticket28924 delete mode 100644 changes/ticket28973 diff --git a/changes/bug28568 b/changes/bug28568 deleted file mode 100644 index 919ec08903..0000000000 --- a/changes/bug28568 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (testing): - - Stop running stem's unit tests as part of "make test-stem". But continue - to run stem's unit and online tests during "make test-stem-full". - Fixes bug 28568; bugfix on 0.2.6.3-alpha. diff --git a/changes/bug28612 b/changes/bug28612 deleted file mode 100644 index 559f254234..0000000000 --- a/changes/bug28612 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (windows services): - - Make Tor start correctly as an NT service again: previously it - was broken by refactoring. Fixes bug 28612; bugfix on 0.3.5.3-alpha. - diff --git a/changes/bug28974 b/changes/bug28974 deleted file mode 100644 index 2d74f5674f..0000000000 --- a/changes/bug28974 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor bugfixes (compilation): - - Fix compilation for Android by adding a missing header to - freespace.c. Fixes bug 28974; bugfix on 0.3.5.1-alpha. diff --git a/changes/ticket28838 b/changes/ticket28838 deleted file mode 100644 index 6c290bf82b..0000000000 --- a/changes/ticket28838 +++ /dev/null @@ -1,8 +0,0 @@ - o Minor features (performance): - - Remove about 96% of the work from the function that we run at - startup to test our curve25519_basepoint implementation. Since - this function has yet to find an actual failure, we'll only - run it for 8 iterations instead of 200. Based on our profile - information, this change should save around 8% of our startup - time on typical desktops, and may have a similar effect on - other platforms. Closes ticket 28838. diff --git a/changes/ticket28851 b/changes/ticket28851 deleted file mode 100644 index bab0318662..0000000000 --- a/changes/ticket28851 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor features (performance): - - Stop re-validating our hardcoded Diffie-Hellman parameters on every - startup. Doing this wasted time and cycles, especially on low-powered - devices. Closes ticket 28851. diff --git a/changes/ticket28879 b/changes/ticket28879 deleted file mode 100644 index 126420f6ca..0000000000 --- a/changes/ticket28879 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor bugfixes (correctness): - - Fix an unreached code-path where we checked the value of "hostname" - inside send_resolved_hostnam_cell(). Previously, we used it before - checking it; now we check it first. Fixes bug 28879; bugfix on - 0.1.2.7-alpha. diff --git a/changes/ticket28881 b/changes/ticket28881 deleted file mode 100644 index 1b015a6c37..0000000000 --- a/changes/ticket28881 +++ /dev/null @@ -1,4 +0,0 @@ - o Code simplification and refactoring: - - When parsing a port configuration, make it more - obvious to static analyzer tools that we will always initialize the - address. Closes ticket 28881. diff --git a/changes/ticket28883 b/changes/ticket28883 deleted file mode 100644 index 1d8b6cb416..0000000000 --- a/changes/ticket28883 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (testing): - - Make sure that test_rebind.py actually obeys its timeout, even - when it receives a large number of log messages. Fixes bug 28883; - bugfix on 0.3.5.4-alpha. diff --git a/changes/ticket28912 b/changes/ticket28912 deleted file mode 100644 index 4119b778bc..0000000000 --- a/changes/ticket28912 +++ /dev/null @@ -1,6 +0,0 @@ - o Major bugfixes (relay, directory): - - A connection serving directory information wouldn't get reactivated after - the first chunk of data was sent (usually 32KB). Tor now always activate - the main loop event that goes through these connections as long as at - least one connection is still active. Fixes bug 28912; bugfix on - 0.3.4.1-alpha. Patch by "cypherpunks3". diff --git a/changes/ticket28924 b/changes/ticket28924 deleted file mode 100644 index 055a6cf285..0000000000 --- a/changes/ticket28924 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor features (compilation): - - When possible, place our warning flags in a separate file, to avoid - flooding verbose build logs to an unacceptable amount. Closes ticket - 28924. diff --git a/changes/ticket28973 b/changes/ticket28973 deleted file mode 100644 index b1d208ee51..0000000000 --- a/changes/ticket28973 +++ /dev/null @@ -1,6 +0,0 @@ - o Minor features (OpenSSL bug workaround): - - Work around a bug in OpenSSL 1.1.1a, which prevented the TLS 1.3 - key export function from handling long labels. When this bug - is detected, Tor will disable TLS 1.3. We recommend upgrading to - a version of OpenSSL without this bug when it becomes available. - Closes ticket 28973. From 7e753117146266fa39841b4cc9fa63febfe9e92a Mon Sep 17 00:00:00 2001 From: Taylor Yu Date: Fri, 4 Jan 2019 17:23:52 -0600 Subject: [PATCH 0376/2557] Fix typo in bootstrap message The message for the "ap_conn_proxy" bootstrap status tag was missing some text. Fixes bug 28929. Bug not in any released Tor. --- src/feature/control/control_bootstrap.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/feature/control/control_bootstrap.c b/src/feature/control/control_bootstrap.c index 49e190dc51..d7a0438513 100644 --- a/src/feature/control/control_bootstrap.c +++ b/src/feature/control/control_bootstrap.c @@ -73,7 +73,7 @@ static const struct { { BOOTSTRAP_STATUS_AP_CONN_DONE_PT, "ap_conn_done_pt", "Connected to pluggable transport to build circuits" }, { BOOTSTRAP_STATUS_AP_CONN_PROXY, "ap_conn_proxy", - "Connecting to proxy " }, + "Connecting to proxy to build circuits" }, { BOOTSTRAP_STATUS_AP_CONN_DONE_PROXY, "ap_conn_done_proxy", "Connected to proxy to build circuits" }, { BOOTSTRAP_STATUS_AP_CONN, "ap_conn", From 41e3c760a400b9df0ec7d303369b6ce8a3c3993c Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 4 Jan 2019 18:39:22 -0500 Subject: [PATCH 0377/2557] Fix a warning in test_process.c on 32-bit platforms with clang. Bug not in any released Tor. --- src/test/test_process.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/test/test_process.c b/src/test/test_process.c index 27c9935e0d..d85cb831b4 100644 --- a/src/test/test_process.c +++ b/src/test/test_process.c @@ -161,7 +161,7 @@ test_default_values(void *arg) process)); /* Default PID is 0. */ - tt_int_op(0, OP_EQ, process_get_pid(process)); + tt_u64_op(0, OP_EQ, process_get_pid(process)); /* Our arguments should be empty. */ tt_int_op(0, OP_EQ, @@ -531,7 +531,7 @@ test_exit_simple(void *arg) process_set_exit_callback(process, process_exit_callback); /* Our default is 0. */ - tt_int_op(0, OP_EQ, process_data->exit_code); + tt_u64_op(0, OP_EQ, process_data->exit_code); /* Fake that we are a running process. */ process_set_status(process, PROCESS_STATUS_RUNNING); @@ -542,7 +542,7 @@ test_exit_simple(void *arg) /* Check if our state changed and if our callback fired. */ tt_int_op(process_get_status(process), OP_EQ, PROCESS_STATUS_NOT_RUNNING); - tt_int_op(1337, OP_EQ, process_data->exit_code); + tt_u64_op(1337, OP_EQ, process_data->exit_code); done: process_set_data(process, process_data); From 671c34d9b496ecdde67527cc10b657ff6bd35c9e Mon Sep 17 00:00:00 2001 From: teor Date: Tue, 8 Jan 2019 14:45:17 +1000 Subject: [PATCH 0378/2557] lib/net: improve the docs for tor_{ersatz_,}socketpair() Add some details about IP family support, and point to tor_socketpair() from tor_ersatz_socketpair(). Closes ticket 29015. --- src/lib/net/socket.c | 4 +++- src/lib/net/socketpair.c | 7 ++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/lib/net/socket.c b/src/lib/net/socket.c index 8940e00591..385c987e68 100644 --- a/src/lib/net/socket.c +++ b/src/lib/net/socket.c @@ -458,7 +458,9 @@ get_n_open_sockets(void) * localhost is inaccessible (for example, if the networking * stack is down). And even if it succeeds, the socket pair will not * be able to read while localhost is down later (the socket pair may - * even close, depending on OS-specific timeouts). + * even close, depending on OS-specific timeouts). The socket pair + * should work on IPv4-only, IPv6-only, and dual-stack systems, as long + * as they have the standard localhost addresses. * * Returns 0 on success and -errno on failure; do not rely on the value * of errno or WSAGetLastError(). diff --git a/src/lib/net/socketpair.c b/src/lib/net/socketpair.c index 380338f15c..b23f08cab2 100644 --- a/src/lib/net/socketpair.c +++ b/src/lib/net/socketpair.c @@ -105,7 +105,12 @@ sockaddr_eq(struct sockaddr *sa1, struct sockaddr *sa2) /** * Helper used to implement socketpair on systems that lack it, by * making a direct connection to localhost. - */ + * + * See tor_socketpair() for details. + * + * The direct connection defaults to IPv4, but falls back to IPv6 if + * IPv4 is not supported. + **/ int tor_ersatz_socketpair(int family, int type, int protocol, tor_socket_t fd[2]) { From a891d81c5f62cdda37b4371ebe2782e23e4f6db1 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 9 Jan 2019 08:48:11 -0500 Subject: [PATCH 0379/2557] Changes file for 28856 --- changes/ticket28856 | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 changes/ticket28856 diff --git a/changes/ticket28856 b/changes/ticket28856 new file mode 100644 index 0000000000..b136470494 --- /dev/null +++ b/changes/ticket28856 @@ -0,0 +1,3 @@ + o Minor features (performance): + - Speed up directory parsing a little by avoiding use of the + non-inlined strcmp_len() function. Closes ticket 28856. From 44db455cc894b4a0fda80a28ed5ea55bf4d41231 Mon Sep 17 00:00:00 2001 From: teor Date: Thu, 29 Nov 2018 01:03:03 +1000 Subject: [PATCH 0380/2557] Dir: allow directory mirrors to serve future consensuses When Tor's clock is behind the clocks on the authorities, allow Tor to serve future consensuses. Fixes bug 28654; bugfix on 0.3.0.1-alpha. --- changes/bug28654 | 3 ++ src/feature/dircache/dircache.c | 71 +++++++++++++++++++++++---------- 2 files changed, 53 insertions(+), 21 deletions(-) create mode 100644 changes/bug28654 diff --git a/changes/bug28654 b/changes/bug28654 new file mode 100644 index 0000000000..4ca843309f --- /dev/null +++ b/changes/bug28654 @@ -0,0 +1,3 @@ + o Minor bugfixes (directory mirror): + - When Tor's clock is behind the clocks on the authorities, allow Tor to + serve future consensuses. Fixes bug 28654; bugfix on 0.3.0.1-alpha. diff --git a/src/feature/dircache/dircache.c b/src/feature/dircache/dircache.c index 872a88018f..ce90f49108 100644 --- a/src/feature/dircache/dircache.c +++ b/src/feature/dircache/dircache.c @@ -495,28 +495,47 @@ handle_get_frontpage(dir_connection_t *conn, const get_handler_args_t *args) } /** Warn that the cached consensus consensus of type - * flavor is too old and will not be served to clients. Rate-limit the - * warning to avoid logging an entry on every request. + * flavor too new or too old, based on is_too_new, + * and will not be served to clients. Rate-limit the warning to avoid logging + * an entry on every request. */ static void -warn_consensus_is_too_old(const struct consensus_cache_entry_t *consensus, - const char *flavor, time_t now) +warn_consensus_is_not_reasonably_live( + const struct consensus_cache_entry_t *consensus, + const char *flavor, time_t now, bool is_too_new) { -#define TOO_OLD_WARNING_INTERVAL (60*60) - static ratelim_t warned = RATELIM_INIT(TOO_OLD_WARNING_INTERVAL); +#define NOT_REASONABLY_LIVE_WARNING_INTERVAL (60*60) + static ratelim_t warned[2] = { RATELIM_INIT( + NOT_REASONABLY_LIVE_WARNING_INTERVAL), + RATELIM_INIT( + NOT_REASONABLY_LIVE_WARNING_INTERVAL) }; char timestamp[ISO_TIME_LEN+1]; - time_t valid_until; - char *dupes; + /* valid_after if is_too_new, valid_until if !is_too_new */ + time_t valid_time = 0; + char *dupes = NULL; - if (consensus_cache_entry_get_valid_until(consensus, &valid_until)) - return; + if (is_too_new) { + if (consensus_cache_entry_get_valid_after(consensus, &valid_time)) + return; + dupes = rate_limit_log(&warned[1], now); + } else { + if (consensus_cache_entry_get_valid_until(consensus, &valid_time)) + return; + dupes = rate_limit_log(&warned[0], now); + } - if ((dupes = rate_limit_log(&warned, now))) { - format_local_iso_time(timestamp, valid_until); - log_warn(LD_DIRSERV, "Our %s%sconsensus is too old, so we will not " - "serve it to clients. It was valid until %s local time and we " - "continued to serve it for up to 24 hours after it expired.%s", - flavor ? flavor : "", flavor ? " " : "", timestamp, dupes); + if (dupes) { + format_local_iso_time(timestamp, valid_time); + log_warn(LD_DIRSERV, "Our %s%sconsensus is too %s, so we will not " + "serve it to clients. It was valid %s %s local time and we " + "continued to serve it for up to 24 hours %s.%s", + flavor ? flavor : "", + flavor ? " " : "", + is_too_new ? "new" : "old", + is_too_new ? "after" : "until", + timestamp, + is_too_new ? "before it was valid" : "after it expired", + dupes); tor_free(dupes); } } @@ -859,7 +878,6 @@ handle_get_current_consensus(dir_connection_t *conn, if (req.diff_only && !cached_consensus) { write_short_http_response(conn, 404, "No such diff available"); - // XXXX warn_consensus_is_too_old(v, req.flavor, now); geoip_note_ns_response(GEOIP_REJECT_NOT_FOUND); goto done; } @@ -870,19 +888,30 @@ handle_get_current_consensus(dir_connection_t *conn, &compression_used); } - time_t fresh_until, valid_until; - int have_fresh_until = 0, have_valid_until = 0; + time_t valid_after, fresh_until, valid_until; + int have_valid_after = 0, have_fresh_until = 0, have_valid_until = 0; if (cached_consensus) { + have_valid_after = + !consensus_cache_entry_get_valid_after(cached_consensus, &valid_after); have_fresh_until = !consensus_cache_entry_get_fresh_until(cached_consensus, &fresh_until); have_valid_until = !consensus_cache_entry_get_valid_until(cached_consensus, &valid_until); } - if (cached_consensus && have_valid_until && + if (cached_consensus && have_valid_after && + !networkstatus_valid_after_is_reasonably_live(valid_after, now)) { + write_short_http_response(conn, 404, "Consensus is too new"); + warn_consensus_is_not_reasonably_live(cached_consensus, req.flavor, now, + 1); + geoip_note_ns_response(GEOIP_REJECT_NOT_FOUND); + goto done; + } else if ( + cached_consensus && have_valid_until && !networkstatus_valid_until_is_reasonably_live(valid_until, now)) { write_short_http_response(conn, 404, "Consensus is too old"); - warn_consensus_is_too_old(cached_consensus, req.flavor, now); + warn_consensus_is_not_reasonably_live(cached_consensus, req.flavor, now, + 0); geoip_note_ns_response(GEOIP_REJECT_NOT_FOUND); goto done; } From cdbd5c0af85775522845a54468492770c54b4cb4 Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Thu, 10 Jan 2019 12:28:17 +0200 Subject: [PATCH 0381/2557] Add top-level file documentation for circuitpadding.c --- src/core/or/circuitpadding.c | 38 ++++++++++++++++++++++++++++++++++++ src/core/or/circuitpadding.h | 3 ++- 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/src/core/or/circuitpadding.c b/src/core/or/circuitpadding.c index ee26558bc8..4c0736fff4 100644 --- a/src/core/or/circuitpadding.c +++ b/src/core/or/circuitpadding.c @@ -1,6 +1,44 @@ /* Copyright (c) 2017 The Tor Project, Inc. */ /* See LICENSE for licensing information */ +/** + * \file circuitpadding.c + * \brief Circuit-level padding implementation + * + * \details + * + * This file implements Tor proposal 254 "Padding Negotiation" which is heavily + * inspired by the paper "Toward an Efficient Website Fingerprinting Defense" + * by M. Juarez, M. Imani, M. Perry, C. Diaz, M. Wright. + * + * In particular the code in this file describes mechanisms for clients to + * negotiate various types of circuit-level padding from relays. + * + * Each padding type is described by a state machine (circpad_machine_t), which + * is also referred as a "padding machine" in this file. Currently, these + * state machines are hardcoded in the source code (e.g. see + * circpad_circ_client_machine_init()), but in the future we will be able to + * serialize them in the torrc or the consensus. + * + * As specified by prop#254, clients can negotiate padding with relays by using + * PADDING_NEGOTIATE cells. After successful padding negotiation, padding + * machines are assigned to the circuit in their mutable form as a + * circpad_machineinfo_t. + * + * Each state of a padding state machine can be either: + * - A histogram that specifies inter-arrival padding delays. + * - Or a parametrized probability distribution that specifies inter-arrival + * delays (see circpad_distribution_type_t). + * + * Padding machines start from the START state and finish with the END + * state. They can transition between states using the events in + * circpad_event_t. + * + * When a padding machine reaches the END state, it gets wiped from the circuit + * so that other padding machines can take over if needed (see + * circpad_machine_transitioned_to_end()). + **/ + #define CIRCUITPADDING_PRIVATE #include diff --git a/src/core/or/circuitpadding.h b/src/core/or/circuitpadding.h index 4680c6be43..6559f916d2 100644 --- a/src/core/or/circuitpadding.h +++ b/src/core/or/circuitpadding.h @@ -6,10 +6,11 @@ * \file circuitpadding.h * \brief Header file for circuitpadding.c. **/ + #ifndef TOR_CIRCUITPADDING_H #define TOR_CIRCUITPADDING_H -#include "circpad_negotiation.h" +#include "src/trunnel/circpad_negotiation.h" #include "lib/evloop/timers.h" typedef struct circuit_t circuit_t; From 5738a0ab6c7904ed9dd53ce5d045b2dedf69b4f6 Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Thu, 10 Jan 2019 12:33:43 +0200 Subject: [PATCH 0382/2557] Rename circpad_machine_t -> circpad_machine_spec_t --- src/core/or/circuit_st.h | 4 +- src/core/or/circuitpadding.c | 70 +++++++++++++++++----------------- src/core/or/circuitpadding.h | 14 +++---- src/test/test_circuitpadding.c | 17 +++++---- 4 files changed, 53 insertions(+), 52 deletions(-) diff --git a/src/core/or/circuit_st.h b/src/core/or/circuit_st.h index 0d9ad3cdd5..74f045d723 100644 --- a/src/core/or/circuit_st.h +++ b/src/core/or/circuit_st.h @@ -12,7 +12,7 @@ #include "core/or/cell_queue_st.h" struct hs_token_t; -typedef struct circpad_machine_t circpad_machine_t; +typedef struct circpad_machine_spec_t circpad_machine_spec_t; typedef struct circpad_machineinfo_t circpad_machineinfo_t; /** Number of padding state machines on a circuit. */ @@ -189,7 +189,7 @@ struct circuit_t { * * Each element of this array corresponds to a different padding machine, * and we can have up to CIRCPAD_MAX_MACHINES such machines. */ - const circpad_machine_t *padding_machine[CIRCPAD_MAX_MACHINES]; + const circpad_machine_spec_t *padding_machine[CIRCPAD_MAX_MACHINES]; /** Adaptive Padding machine info for above machines. This is the * per-circuit mutable information, such as the current state and diff --git a/src/core/or/circuitpadding.c b/src/core/or/circuitpadding.c index 4c0736fff4..c5a80d6c5d 100644 --- a/src/core/or/circuitpadding.c +++ b/src/core/or/circuitpadding.c @@ -14,9 +14,9 @@ * In particular the code in this file describes mechanisms for clients to * negotiate various types of circuit-level padding from relays. * - * Each padding type is described by a state machine (circpad_machine_t), which - * is also referred as a "padding machine" in this file. Currently, these - * state machines are hardcoded in the source code (e.g. see + * Each padding type is described by a state machine (circpad_machine_spec_t), + * which is also referred as a "padding machine" in this file. Currently, + * these state machines are hardcoded in the source code (e.g. see * circpad_circ_client_machine_init()), but in the future we will be able to * serialize them in the torrc or the consensus. * @@ -36,7 +36,7 @@ * * When a padding machine reaches the END state, it gets wiped from the circuit * so that other padding machines can take over if needed (see - * circpad_machine_transitioned_to_end()). + * circpad_machine_spec_transitioned_to_end()). **/ #define CIRCUITPADDING_PRIVATE @@ -75,7 +75,7 @@ static inline circpad_purpose_mask_t circpad_circ_purpose_to_mask(uint8_t static inline circpad_circuit_state_t circpad_circuit_state( origin_circuit_t *circ); static void circpad_setup_machine_on_circ(circuit_t *on_circ, - const circpad_machine_t *machine); + const circpad_machine_spec_t *machine); static double circpad_distribution_sample(circpad_distribution_t dist); /** Cached consensus params */ @@ -86,12 +86,12 @@ static uint16_t circpad_global_allowed_cells; static uint64_t circpad_global_padding_sent; static uint64_t circpad_global_nonpadding_sent; -/** This is the list of circpad_machine_t's parsed from consensus and torrc - * that have origin_side == 1 (ie: are for client side) */ +/** This is the list of circpad_machine_spec_t's parsed from consensus and + * torrc that have origin_side == 1 (ie: are for client side) */ STATIC smartlist_t *origin_padding_machines = NULL; -/** This is the list of circpad_machine_t's parsed from consensus and torrc - * that have origin_side == 0 (ie: are for relay side) */ +/** This is the list of circpad_machine_spec_t's parsed from consensus and + * torrc that have origin_side == 0 (ie: are for relay side) */ STATIC smartlist_t *relay_padding_machines = NULL; /** Loop over the current padding state machines using loop_var as the @@ -197,7 +197,7 @@ circpad_circuit_machineinfo_new(circuit_t *on_circ, int machine_index) STATIC const circpad_state_t * circpad_machine_current_state(const circpad_machineinfo_t *mi) { - const circpad_machine_t *machine = CIRCPAD_GET_MACHINE(mi); + const circpad_machine_spec_t *machine = CIRCPAD_GET_MACHINE(mi); if (mi->current_state == CIRCPAD_STATE_END) { return NULL; @@ -1097,7 +1097,7 @@ circpad_new_consensus_params(const networkstatus_t *ns) STATIC bool circpad_machine_reached_padding_limit(circpad_machineinfo_t *mi) { - const circpad_machine_t *machine = CIRCPAD_GET_MACHINE(mi); + const circpad_machine_spec_t *machine = CIRCPAD_GET_MACHINE(mi); /* If machine_padding_pct is non-zero, and we've sent more * than the allowed count of padding cells, then check our @@ -1233,9 +1233,9 @@ circpad_machine_schedule_padding,(circpad_machineinfo_t *mi)) * not access it. */ static void -circpad_machine_transitioned_to_end(circpad_machineinfo_t *mi) +circpad_machine_spec_transitioned_to_end(circpad_machineinfo_t *mi) { - const circpad_machine_t *machine = CIRCPAD_GET_MACHINE(mi); + const circpad_machine_spec_t *machine = CIRCPAD_GET_MACHINE(mi); /* * We allow machines to shut down and delete themselves as opposed @@ -1283,7 +1283,7 @@ circpad_machine_transitioned_to_end(circpad_machineinfo_t *mi) * Returns 1 if we transition states, 0 otherwise. */ MOCK_IMPL(circpad_decision_t, -circpad_machine_transition,(circpad_machineinfo_t *mi, +circpad_machine_spec_transition,(circpad_machineinfo_t *mi, circpad_event_t event)) { const circpad_state_t *state = @@ -1331,7 +1331,7 @@ circpad_machine_transition,(circpad_machineinfo_t *mi, /* If we transition to the end state, check to see * if this machine wants to be shut down at end */ if (s == CIRCPAD_STATE_END) { - circpad_machine_transitioned_to_end(mi); + circpad_machine_spec_transitioned_to_end(mi); /* We transitioned but we don't pad in end. Also, mi * may be freed. Returning STATE_CHANGED prevents us * from accessing it in any callers of this function. */ @@ -1485,7 +1485,7 @@ circpad_cell_event_nonpadding_sent(circuit_t *on_circ) if (!circpad_machine_remove_token(on_circ->padding_info[i])) { /* If removing a token did not cause a transition, check if * non-padding sent event should */ - circpad_machine_transition(on_circ->padding_info[i], + circpad_machine_spec_transition(on_circ->padding_info[i], CIRCPAD_EVENT_NONPADDING_SENT); } } FOR_EACH_ACTIVE_CIRCUIT_MACHINE_END; @@ -1506,7 +1506,7 @@ circpad_cell_event_nonpadding_received(circuit_t *on_circ) /* First, update any RTT estimate */ circpad_estimate_circ_rtt_on_received(on_circ, on_circ->padding_info[i]); - circpad_machine_transition(on_circ->padding_info[i], + circpad_machine_spec_transition(on_circ->padding_info[i], CIRCPAD_EVENT_NONPADDING_RECV); } FOR_EACH_ACTIVE_CIRCUIT_MACHINE_END; } @@ -1523,7 +1523,7 @@ void circpad_cell_event_padding_sent(circuit_t *on_circ) { FOR_EACH_ACTIVE_CIRCUIT_MACHINE_BEGIN(i, on_circ) { - circpad_machine_transition(on_circ->padding_info[i], + circpad_machine_spec_transition(on_circ->padding_info[i], CIRCPAD_EVENT_PADDING_SENT); } FOR_EACH_ACTIVE_CIRCUIT_MACHINE_END; } @@ -1541,7 +1541,7 @@ circpad_cell_event_padding_received(circuit_t *on_circ) { /* identical to padding sent */ FOR_EACH_ACTIVE_CIRCUIT_MACHINE_BEGIN(i, on_circ) { - circpad_machine_transition(on_circ->padding_info[i], + circpad_machine_spec_transition(on_circ->padding_info[i], CIRCPAD_EVENT_PADDING_RECV); } FOR_EACH_ACTIVE_CIRCUIT_MACHINE_END; } @@ -1558,7 +1558,7 @@ circpad_cell_event_padding_received(circuit_t *on_circ) circpad_decision_t circpad_internal_event_infinity(circpad_machineinfo_t *mi) { - return circpad_machine_transition(mi, CIRCPAD_EVENT_INFINITY); + return circpad_machine_spec_transition(mi, CIRCPAD_EVENT_INFINITY); } /** @@ -1572,7 +1572,7 @@ circpad_internal_event_infinity(circpad_machineinfo_t *mi) circpad_decision_t circpad_internal_event_bins_empty(circpad_machineinfo_t *mi) { - if (circpad_machine_transition(mi, CIRCPAD_EVENT_BINS_EMPTY) + if (circpad_machine_spec_transition(mi, CIRCPAD_EVENT_BINS_EMPTY) == CIRCPAD_STATE_CHANGED) { return CIRCPAD_STATE_CHANGED; } else { @@ -1591,7 +1591,7 @@ circpad_internal_event_bins_empty(circpad_machineinfo_t *mi) circpad_decision_t circpad_internal_event_state_length_up(circpad_machineinfo_t *mi) { - return circpad_machine_transition(mi, CIRCPAD_EVENT_LENGTH_COUNT); + return circpad_machine_spec_transition(mi, CIRCPAD_EVENT_LENGTH_COUNT); } /** @@ -1599,7 +1599,7 @@ circpad_internal_event_state_length_up(circpad_machineinfo_t *mi) */ static inline bool circpad_machine_conditions_met(origin_circuit_t *circ, - const circpad_machine_t *machine) + const circpad_machine_spec_t *machine) { if (!(circpad_circ_purpose_to_mask(TO_CIRCUIT(circ)->purpose) & machine->conditions.purpose_mask)) @@ -1740,7 +1740,7 @@ circpad_add_matching_machines(origin_circuit_t *on_circ) * machines in reverse order, so that more recently added * machines take priority over older ones. */ SMARTLIST_FOREACH_REVERSE_BEGIN(origin_padding_machines, - circpad_machine_t *, + circpad_machine_spec_t *, machine) { /* Machine definitions have a specific target machine index. * This is so event ordering is deterministic with respect @@ -2012,7 +2012,7 @@ circpad_deliver_sent_relay_cell_events(circuit_t *circ, * Initialize the states array for a circpad machine. */ void -circpad_machine_states_init(circpad_machine_t *machine, +circpad_machine_states_init(circpad_machine_spec_t *machine, circpad_statenum_t num_states) { if (BUG(num_states > CIRCPAD_MAX_MACHINE_STATES)) { @@ -2033,7 +2033,7 @@ circpad_machine_states_init(circpad_machine_t *machine, static void circpad_setup_machine_on_circ(circuit_t *on_circ, - const circpad_machine_t *machine) + const circpad_machine_spec_t *machine) { if (CIRCUIT_IS_ORIGIN(on_circ) && !machine->is_origin_side) { log_fn(LOG_WARN, LD_BUG, @@ -2061,8 +2061,8 @@ circpad_setup_machine_on_circ(circuit_t *on_circ, static void circpad_circ_client_machine_init(void) { - circpad_machine_t *circ_client_machine - = tor_malloc_zero(sizeof(circpad_machine_t)); + circpad_machine_spec_t *circ_client_machine + = tor_malloc_zero(sizeof(circpad_machine_spec_t)); // XXX: Better conditions for merge.. Or disable this machine in // merge? @@ -2115,8 +2115,8 @@ circpad_circ_client_machine_init(void) static void circpad_circ_responder_machine_init(void) { - circpad_machine_t *circ_responder_machine - = tor_malloc_zero(sizeof(circpad_machine_t)); + circpad_machine_spec_t *circ_responder_machine + = tor_malloc_zero(sizeof(circpad_machine_spec_t)); /* Shut down the machine after we've sent enough packets */ circ_responder_machine->should_negotiate_end = 1; @@ -2235,14 +2235,14 @@ circpad_machines_free(void) { if (origin_padding_machines) { SMARTLIST_FOREACH(origin_padding_machines, - circpad_machine_t *, + circpad_machine_spec_t *, m, tor_free(m->states); tor_free(m)); smartlist_free(origin_padding_machines); } if (relay_padding_machines) { SMARTLIST_FOREACH(relay_padding_machines, - circpad_machine_t *, + circpad_machine_spec_t *, m, tor_free(m->states); tor_free(m)); smartlist_free(relay_padding_machines); } @@ -2417,7 +2417,7 @@ circpad_handle_padding_negotiate(circuit_t *circ, cell_t *cell) goto err; } else if (negotiate->command == CIRCPAD_COMMAND_START) { SMARTLIST_FOREACH_BEGIN(relay_padding_machines, - const circpad_machine_t *, m) { + const circpad_machine_spec_t *, m) { if (m->machine_num == negotiate->machine_type) { circpad_setup_machine_on_circ(circ, m); goto done; @@ -2520,7 +2520,7 @@ circpad_state_serialize(const circpad_state_t *state, } char * -circpad_machine_to_string(const circpad_machine_t *machine) +circpad_machine_spec_to_string(const circpad_machine_spec_t *machine) { smartlist_t *chunks = smartlist_new(); char *out; @@ -2538,7 +2538,7 @@ circpad_machine_to_string(const circpad_machine_t *machine) } // XXX: Writeme -const circpad_machine_t * +const circpad_machine_spec_t * circpad_string_to_machine(const char *str) { (void)str; diff --git a/src/core/or/circuitpadding.h b/src/core/or/circuitpadding.h index 6559f916d2..a43be58abb 100644 --- a/src/core/or/circuitpadding.h +++ b/src/core/or/circuitpadding.h @@ -389,7 +389,7 @@ typedef struct circpad_state_t { /** * End is a pseudo-state that causes the machine to go completely * idle, and optionally get torn down (depending on the - * value of circpad_machine_t.should_negotiate_end) + * value of circpad_machine_spec_t.should_negotiate_end) * * End MUST NOT occupy a slot in the machine state array. */ @@ -529,7 +529,7 @@ typedef struct circpad_machineinfo_t { typedef uint8_t circpad_machine_num_t; /** Global state machine structure from the consensus */ -typedef struct circpad_machine_t { +typedef struct circpad_machine_spec_t { /** Global machine number */ circpad_machine_num_t machine_num; @@ -569,7 +569,7 @@ typedef struct circpad_machine_t { * Number of states this machine has (ie: length of the states array). * XXX: This field is not needed other than for safety. */ circpad_statenum_t num_states; -} circpad_machine_t; +} circpad_machine_spec_t; void circpad_new_consensus_params(const networkstatus_t *ns); @@ -608,7 +608,7 @@ void circpad_machine_event_circ_has_no_relay_early(origin_circuit_t *circ); void circpad_machines_init(void); void circpad_machines_free(void); -void circpad_machine_states_init(circpad_machine_t *machine, +void circpad_machine_states_init(circpad_machine_spec_t *machine, circpad_statenum_t num_states); void circpad_circuit_free_all_machineinfos(circuit_t *circ); @@ -617,8 +617,8 @@ bool circpad_padding_is_from_expected_hop(circuit_t *circ, crypt_path_t *from_hop); /** Serializaton functions for writing to/from torrc and consensus */ -char *circpad_machine_to_string(const circpad_machine_t *machine); -const circpad_machine_t *circpad_string_to_machine(const char *str); +char *circpad_machine_spec_to_string(const circpad_machine_spec_t *machine); +const circpad_machine_spec_t *circpad_string_to_machine(const char *str); /* Padding negotiation between client and middle */ signed_error_t circpad_handle_padding_negotiate(circuit_t *circ, cell_t *cell); @@ -637,7 +637,7 @@ MOCK_DECL(circpad_decision_t, circpad_machine_schedule_padding,(circpad_machineinfo_t *)); MOCK_DECL(circpad_decision_t, -circpad_machine_transition, (circpad_machineinfo_t *mi, +circpad_machine_spec_transition, (circpad_machineinfo_t *mi, circpad_event_t event)); circpad_decision_t circpad_send_padding_cell_for_callback( diff --git a/src/test/test_circuitpadding.c b/src/test/test_circuitpadding.c index 5693c98e41..2ab8e2445e 100644 --- a/src/test/test_circuitpadding.c +++ b/src/test/test_circuitpadding.c @@ -61,7 +61,7 @@ static node_t padding_node; static node_t non_padding_node; static channel_t dummy_channel; -static circpad_machine_t circ_client_machine; +static circpad_machine_spec_t circ_client_machine; static void timers_advance_and_run(int64_t msec_update) @@ -1514,7 +1514,7 @@ test_circuitpadding_negotiation(void *arg) client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL; SMARTLIST_FOREACH(relay_padding_machines, - circpad_machine_t *, + circpad_machine_spec_t *, m, tor_free(m->states); tor_free(m)); smartlist_free(relay_padding_machines); relay_padding_machines = smartlist_new(); @@ -1594,10 +1594,11 @@ simulate_single_hop_extend(circuit_t *client, circuit_t *mid_relay, circpad_machine_event_circ_added_hop(TO_ORIGIN_CIRCUIT(client)); } -static circpad_machine_t * +static circpad_machine_spec_t * helper_create_conditional_machine(void) { - circpad_machine_t *ret = tor_malloc_zero(sizeof(circpad_machine_t)); + circpad_machine_spec_t *ret = + tor_malloc_zero(sizeof(circpad_machine_spec_t)); /* Start, burst */ circpad_machine_states_init(ret, 2); @@ -1630,7 +1631,7 @@ helper_create_conditional_machine(void) static void helper_create_conditional_machines(void) { - circpad_machine_t *add = helper_create_conditional_machine(); + circpad_machine_spec_t *add = helper_create_conditional_machine(); origin_padding_machines = smartlist_new(); relay_padding_machines = smartlist_new(); @@ -2158,7 +2159,7 @@ test_circuitpadding_sample_distribution(void *arg) } static circpad_decision_t -circpad_machine_transition_mock(circpad_machineinfo_t *mi, +circpad_machine_spec_transition_mock(circpad_machineinfo_t *mi, circpad_event_t event) { (void) mi; @@ -2178,7 +2179,7 @@ test_circuitpadding_machine_rate_limiting(void *arg) /* Ignore machine transitions for the purposes of this function, we only * really care about padding counts */ - MOCK(circpad_machine_transition, circpad_machine_transition_mock); + MOCK(circpad_machine_spec_transition, circpad_machine_spec_transition_mock); MOCK(circpad_send_command_to_hop, circpad_send_command_to_hop_mock); /* Setup machine and circuits */ @@ -2246,7 +2247,7 @@ test_circuitpadding_global_rate_limiting(void *arg) /* Ignore machine transitions for the purposes of this function, we only * really care about padding counts */ - MOCK(circpad_machine_transition, circpad_machine_transition_mock); + MOCK(circpad_machine_spec_transition, circpad_machine_spec_transition_mock); MOCK(circuitmux_attach_circuit, circuitmux_attach_circuit_mock); MOCK(circuit_package_relay_cell, circuit_package_relay_cell_mock); From f4938179c50cc385b7599e5a03388792e46cde83 Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Thu, 10 Jan 2019 12:38:00 +0200 Subject: [PATCH 0383/2557] Rename circpad_machineinfo_t -> circpad_machine_state_t --- src/core/or/circuit_st.h | 4 +-- src/core/or/circuitpadding.c | 61 +++++++++++++++++----------------- src/core/or/circuitpadding.h | 46 +++++++++++++------------ src/test/test_circuitpadding.c | 22 ++++++------ 4 files changed, 68 insertions(+), 65 deletions(-) diff --git a/src/core/or/circuit_st.h b/src/core/or/circuit_st.h index 74f045d723..bfbd336c3a 100644 --- a/src/core/or/circuit_st.h +++ b/src/core/or/circuit_st.h @@ -13,7 +13,7 @@ struct hs_token_t; typedef struct circpad_machine_spec_t circpad_machine_spec_t; -typedef struct circpad_machineinfo_t circpad_machineinfo_t; +typedef struct circpad_machine_state_t circpad_machine_state_t; /** Number of padding state machines on a circuit. */ #define CIRCPAD_MAX_MACHINES (2) @@ -200,7 +200,7 @@ struct circuit_t { * * Each element of this array corresponds to a different padding machine, * and we can have up to CIRCPAD_MAX_MACHINES such machines. */ - circpad_machineinfo_t *padding_info[CIRCPAD_MAX_MACHINES]; + circpad_machine_state_t *padding_info[CIRCPAD_MAX_MACHINES]; }; #endif diff --git a/src/core/or/circuitpadding.c b/src/core/or/circuitpadding.c index c5a80d6c5d..6f10d3fb6a 100644 --- a/src/core/or/circuitpadding.c +++ b/src/core/or/circuitpadding.c @@ -23,7 +23,7 @@ * As specified by prop#254, clients can negotiate padding with relays by using * PADDING_NEGOTIATE cells. After successful padding negotiation, padding * machines are assigned to the circuit in their mutable form as a - * circpad_machineinfo_t. + * circpad_machine_state_t. * * Each state of a padding state machine can be either: * - A histogram that specifies inter-arrival padding delays. @@ -177,10 +177,11 @@ circpad_circuit_free_all_machineinfos(circuit_t *circ) /** * Allocate a new mutable machineinfo structure. */ -STATIC circpad_machineinfo_t * +STATIC circpad_machine_state_t * circpad_circuit_machineinfo_new(circuit_t *on_circ, int machine_index) { - circpad_machineinfo_t *mi = tor_malloc_zero(sizeof(circpad_machineinfo_t)); + circpad_machine_state_t *mi = + tor_malloc_zero(sizeof(circpad_machine_state_t)); mi->machine_index = machine_index; mi->on_circ = on_circ; @@ -195,7 +196,7 @@ circpad_circuit_machineinfo_new(circuit_t *on_circ, int machine_index) * invalid state. */ STATIC const circpad_state_t * -circpad_machine_current_state(const circpad_machineinfo_t *mi) +circpad_machine_current_state(const circpad_machine_state_t *mi) { const circpad_machine_spec_t *machine = CIRCPAD_GET_MACHINE(mi); @@ -227,7 +228,7 @@ circpad_machine_current_state(const circpad_machineinfo_t *mi) * It has a usec value of CIRCPAD_DELAY_INFINITE (UINT32_MAX). */ STATIC circpad_delay_t -circpad_histogram_bin_to_usec(const circpad_machineinfo_t *mi, +circpad_histogram_bin_to_usec(const circpad_machine_state_t *mi, circpad_hist_index_t bin) { const circpad_state_t *state = circpad_machine_current_state(mi); @@ -264,7 +265,7 @@ circpad_histogram_bin_to_usec(const circpad_machineinfo_t *mi, /** Return the midpoint of the histogram bin bin_index. */ static circpad_delay_t -circpad_get_histogram_bin_midpoint(const circpad_machineinfo_t *mi, +circpad_get_histogram_bin_midpoint(const circpad_machine_state_t *mi, int bin_index) { circpad_delay_t left_bound = circpad_histogram_bin_to_usec(mi, bin_index); @@ -285,7 +286,7 @@ circpad_get_histogram_bin_midpoint(const circpad_machineinfo_t *mi, * has range [start_usec+range_usec, CIRCPAD_DELAY_INFINITE]. */ STATIC circpad_hist_index_t -circpad_histogram_usec_to_bin(const circpad_machineinfo_t *mi, +circpad_histogram_usec_to_bin(const circpad_machine_state_t *mi, circpad_delay_t usec) { const circpad_state_t *state = circpad_machine_current_state(mi); @@ -333,7 +334,7 @@ circpad_histogram_usec_to_bin(const circpad_machineinfo_t *mi, * Called after a state transition, or if the bins are empty. */ STATIC void -circpad_machine_setup_tokens(circpad_machineinfo_t *mi) +circpad_machine_setup_tokens(circpad_machine_state_t *mi) { const circpad_state_t *state = circpad_machine_current_state(mi); @@ -365,7 +366,7 @@ circpad_machine_setup_tokens(circpad_machineinfo_t *mi) * Choose a length for this state (in cells), if specified. */ static void -circpad_choose_state_length(circpad_machineinfo_t *mi) +circpad_choose_state_length(circpad_machine_state_t *mi) { const circpad_state_t *state = circpad_machine_current_state(mi); double length; @@ -413,7 +414,7 @@ circpad_distribution_sample_iat_delay(const circpad_state_t *state, * that bin's [start,end) time range. */ STATIC circpad_delay_t -circpad_machine_sample_delay(circpad_machineinfo_t *mi) +circpad_machine_sample_delay(circpad_machine_state_t *mi) { const circpad_state_t *state = circpad_machine_current_state(mi); const circpad_hist_token_t *histogram = NULL; @@ -602,7 +603,7 @@ circpad_distribution_sample(circpad_distribution_t dist) * greater than the target, and that has tokens remaining. */ static circpad_hist_index_t -circpad_machine_first_higher_index(const circpad_machineinfo_t *mi, +circpad_machine_first_higher_index(const circpad_machine_state_t *mi, circpad_delay_t target_bin_usec) { circpad_hist_index_t bin = circpad_histogram_usec_to_bin(mi, @@ -624,7 +625,7 @@ circpad_machine_first_higher_index(const circpad_machineinfo_t *mi, * target_bin_usec, and that still has tokens remaining. */ static circpad_hist_index_t -circpad_machine_first_lower_index(const circpad_machineinfo_t *mi, +circpad_machine_first_lower_index(const circpad_machine_state_t *mi, circpad_delay_t target_bin_usec) { circpad_hist_index_t bin = circpad_histogram_usec_to_bin(mi, @@ -645,7 +646,7 @@ circpad_machine_first_lower_index(const circpad_machineinfo_t *mi, * greater than the target. */ STATIC void -circpad_machine_remove_higher_token(circpad_machineinfo_t *mi, +circpad_machine_remove_higher_token(circpad_machine_state_t *mi, circpad_delay_t target_bin_usec) { /* We need to remove the token from the first bin @@ -666,7 +667,7 @@ circpad_machine_remove_higher_token(circpad_machineinfo_t *mi, * lower than the target. */ STATIC void -circpad_machine_remove_lower_token(circpad_machineinfo_t *mi, +circpad_machine_remove_lower_token(circpad_machine_state_t *mi, circpad_delay_t target_bin_usec) { circpad_hist_index_t bin = circpad_machine_first_lower_index(mi, @@ -695,7 +696,7 @@ circpad_machine_remove_lower_token(circpad_machineinfo_t *mi, * If it is false, use bin index distance only. */ STATIC void -circpad_machine_remove_closest_token(circpad_machineinfo_t *mi, +circpad_machine_remove_closest_token(circpad_machine_state_t *mi, circpad_delay_t target_bin_usec, bool use_usec) { @@ -777,7 +778,7 @@ circpad_machine_remove_closest_token(circpad_machineinfo_t *mi, * If it is empty, do nothing. */ static void -circpad_machine_remove_exact(circpad_machineinfo_t *mi, +circpad_machine_remove_exact(circpad_machine_state_t *mi, circpad_delay_t target_bin_usec) { circpad_hist_index_t bin = circpad_histogram_usec_to_bin(mi, @@ -794,7 +795,7 @@ circpad_machine_remove_exact(circpad_machineinfo_t *mi, * otherwise returns 0. */ static circpad_decision_t -check_machine_token_supply(circpad_machineinfo_t *mi) +check_machine_token_supply(circpad_machine_state_t *mi) { uint32_t histogram_total_tokens = 0; @@ -834,7 +835,7 @@ check_machine_token_supply(circpad_machineinfo_t *mi) * Returns 1 if we transition states, 0 otherwise. */ STATIC circpad_decision_t -circpad_machine_remove_token(circpad_machineinfo_t *mi) +circpad_machine_remove_token(circpad_machine_state_t *mi) { const circpad_state_t *state = NULL; circpad_time_t current_time; @@ -961,7 +962,7 @@ circpad_send_command_to_hop,(origin_circuit_t *circ, uint8_t hopnum, * CIRCPAD_STATE_CHANGED. Otherwise return CIRCPAD_STATE_UNCHANGED. */ circpad_decision_t -circpad_send_padding_cell_for_callback(circpad_machineinfo_t *mi) +circpad_send_padding_cell_for_callback(circpad_machine_state_t *mi) { circuit_t *circ = mi->on_circ; int machine_idx = mi->machine_index; @@ -1040,7 +1041,7 @@ circpad_send_padding_cell_for_callback(circpad_machineinfo_t *mi) /** * Tor-timer compatible callback that tells us to send a padding cell. * - * Timers are associated with circpad_machineinfo_t's. When the machineinfo + * Timers are associated with circpad_machine_state_t's. When the machineinfo * is freed on a circuit, the timers are cancelled. Since the lifetime * of machineinfo is always longer than the timers, handles are not * needed. @@ -1049,7 +1050,7 @@ static void circpad_send_padding_callback(tor_timer_t *timer, void *args, const struct monotime_t *time) { - circpad_machineinfo_t *mi = ((circpad_machineinfo_t*)args); + circpad_machine_state_t *mi = ((circpad_machine_state_t*)args); (void)timer; (void)time; if (mi && mi->on_circ) { @@ -1095,7 +1096,7 @@ circpad_new_consensus_params(const networkstatus_t *ns) * Returns 1 if limits are set and we've hit them. Otherwise returns 0. */ STATIC bool -circpad_machine_reached_padding_limit(circpad_machineinfo_t *mi) +circpad_machine_reached_padding_limit(circpad_machine_state_t *mi) { const circpad_machine_spec_t *machine = CIRCPAD_GET_MACHINE(mi); @@ -1143,7 +1144,7 @@ circpad_machine_reached_padding_limit(circpad_machineinfo_t *mi) * 0 otherwise. */ MOCK_IMPL(circpad_decision_t, -circpad_machine_schedule_padding,(circpad_machineinfo_t *mi)) +circpad_machine_schedule_padding,(circpad_machine_state_t *mi)) { circpad_delay_t in_usec = 0; struct timeval timeout; @@ -1233,7 +1234,7 @@ circpad_machine_schedule_padding,(circpad_machineinfo_t *mi)) * not access it. */ static void -circpad_machine_spec_transitioned_to_end(circpad_machineinfo_t *mi) +circpad_machine_spec_transitioned_to_end(circpad_machine_state_t *mi) { const circpad_machine_spec_t *machine = CIRCPAD_GET_MACHINE(mi); @@ -1283,7 +1284,7 @@ circpad_machine_spec_transitioned_to_end(circpad_machineinfo_t *mi) * Returns 1 if we transition states, 0 otherwise. */ MOCK_IMPL(circpad_decision_t, -circpad_machine_spec_transition,(circpad_machineinfo_t *mi, +circpad_machine_spec_transition,(circpad_machine_state_t *mi, circpad_event_t event)) { const circpad_state_t *state = @@ -1364,7 +1365,7 @@ circpad_machine_spec_transition,(circpad_machineinfo_t *mi, */ static void circpad_estimate_circ_rtt_on_received(circuit_t *circ, - circpad_machineinfo_t *mi) + circpad_machine_state_t *mi) { /* Origin circuits don't estimate RTT. They could do it easily enough, * but they have no reason to use it in any delay calculations. */ @@ -1411,7 +1412,7 @@ circpad_estimate_circ_rtt_on_received(circuit_t *circ, */ static void circpad_estimate_circ_rtt_on_send(circuit_t *circ, - circpad_machineinfo_t *mi) + circpad_machine_state_t *mi) { /* Origin circuits don't estimate RTT. They could do it easily enough, * but they have no reason to use it in any delay calculations. */ @@ -1556,7 +1557,7 @@ circpad_cell_event_padding_received(circuit_t *on_circ) * Return 1 if we decide to transition, 0 otherwise. */ circpad_decision_t -circpad_internal_event_infinity(circpad_machineinfo_t *mi) +circpad_internal_event_infinity(circpad_machine_state_t *mi) { return circpad_machine_spec_transition(mi, CIRCPAD_EVENT_INFINITY); } @@ -1570,7 +1571,7 @@ circpad_internal_event_infinity(circpad_machineinfo_t *mi) * Return 1 if we decide to transition, 0 otherwise. */ circpad_decision_t -circpad_internal_event_bins_empty(circpad_machineinfo_t *mi) +circpad_internal_event_bins_empty(circpad_machine_state_t *mi) { if (circpad_machine_spec_transition(mi, CIRCPAD_EVENT_BINS_EMPTY) == CIRCPAD_STATE_CHANGED) { @@ -1589,7 +1590,7 @@ circpad_internal_event_bins_empty(circpad_machineinfo_t *mi) * Return 1 if we decide to transition, 0 otherwise. */ circpad_decision_t -circpad_internal_event_state_length_up(circpad_machineinfo_t *mi) +circpad_internal_event_state_length_up(circpad_machine_state_t *mi) { return circpad_machine_spec_transition(mi, CIRCPAD_EVENT_LENGTH_COUNT); } diff --git a/src/core/or/circuitpadding.h b/src/core/or/circuitpadding.h index a43be58abb..f3d176ebf7 100644 --- a/src/core/or/circuitpadding.h +++ b/src/core/or/circuitpadding.h @@ -88,7 +88,7 @@ typedef uint32_t circpad_delay_t; /** * Macro to clarify when we're checking the infinity bin. * - * Works with either circpad_state_t or circpad_machineinfo_t + * Works with either circpad_state_t or circpad_machine_state_t */ #define CIRCPAD_INFINITY_BIN(mi) ((mi)->histogram_len-1) @@ -237,10 +237,10 @@ typedef uint16_t circpad_statenum_t; #define CIRCPAD_MAX_HISTOGRAM_LEN (sizeof(circpad_delay_t)*8 + 1) /** - * A state of a padding machine. The information here are immutable and + * A state of a padding state machine. The information here are immutable and * represent the initial form of the state; it does not get updated as things * happen. The mutable information that gets updated in runtime are carried in - * a circpad_machineinfo_t. + * a circpad_machine_state_t. * * This struct describes the histograms and parameters of a single * state in the adaptive padding machine. Instances of this struct @@ -430,7 +430,7 @@ typedef struct circpad_state_t { * * XXX: Play with layout to minimize space on x64 Linux (most common relay). */ -typedef struct circpad_machineinfo_t { +typedef struct circpad_machine_state_t { /** The callback pointer for the padding callbacks. * * These timers stick around the machineinfo until the machineinfo's circuit @@ -514,7 +514,7 @@ typedef struct circpad_machineinfo_t { * CIRCPAD_MAX_MACHINES define). */ unsigned machine_index : 1; -} circpad_machineinfo_t; +} circpad_machine_state_t; /** Helper macro to get an actual state machine from a machineinfo */ #define CIRCPAD_GET_MACHINE(machineinfo) \ @@ -591,10 +591,12 @@ void circpad_cell_event_padding_sent(circuit_t *on_circ); void circpad_cell_event_padding_received(circuit_t *on_circ); /** Internal events are events the machines send to themselves */ -circpad_decision_t circpad_internal_event_infinity(circpad_machineinfo_t *mi); -circpad_decision_t circpad_internal_event_bins_empty(circpad_machineinfo_t *); +circpad_decision_t +circpad_internal_event_infinity(circpad_machine_state_t *mi); +circpad_decision_t +circpad_internal_event_bins_empty(circpad_machine_state_t *); circpad_decision_t circpad_internal_event_state_length_up( - circpad_machineinfo_t *); + circpad_machine_state_t *); /** Machine creation events are events that cause us to set up or * tear down padding state machines. */ @@ -634,47 +636,47 @@ bool circpad_padding_negotiated(circuit_t *circ, uint8_t response); MOCK_DECL(circpad_decision_t, -circpad_machine_schedule_padding,(circpad_machineinfo_t *)); +circpad_machine_schedule_padding,(circpad_machine_state_t *)); MOCK_DECL(circpad_decision_t, -circpad_machine_spec_transition, (circpad_machineinfo_t *mi, +circpad_machine_spec_transition, (circpad_machine_state_t *mi, circpad_event_t event)); circpad_decision_t circpad_send_padding_cell_for_callback( - circpad_machineinfo_t *mi); + circpad_machine_state_t *mi); #ifdef CIRCUITPADDING_PRIVATE STATIC circpad_delay_t -circpad_machine_sample_delay(circpad_machineinfo_t *mi); +circpad_machine_sample_delay(circpad_machine_state_t *mi); STATIC bool -circpad_machine_reached_padding_limit(circpad_machineinfo_t *mi); +circpad_machine_reached_padding_limit(circpad_machine_state_t *mi); STATIC -circpad_decision_t circpad_machine_remove_token(circpad_machineinfo_t *mi); +circpad_decision_t circpad_machine_remove_token(circpad_machine_state_t *mi); STATIC circpad_delay_t -circpad_histogram_bin_to_usec(const circpad_machineinfo_t *mi, +circpad_histogram_bin_to_usec(const circpad_machine_state_t *mi, circpad_hist_index_t bin); STATIC const circpad_state_t * -circpad_machine_current_state(const circpad_machineinfo_t *mi); +circpad_machine_current_state(const circpad_machine_state_t *mi); STATIC circpad_hist_index_t circpad_histogram_usec_to_bin( - const circpad_machineinfo_t *mi, + const circpad_machine_state_t *mi, circpad_delay_t us); -STATIC circpad_machineinfo_t *circpad_circuit_machineinfo_new( +STATIC circpad_machine_state_t *circpad_circuit_machineinfo_new( circuit_t *on_circ, int machine_index); -STATIC void circpad_machine_remove_higher_token(circpad_machineinfo_t *mi, +STATIC void circpad_machine_remove_higher_token(circpad_machine_state_t *mi, circpad_delay_t target_bin_us); -STATIC void circpad_machine_remove_lower_token(circpad_machineinfo_t *mi, +STATIC void circpad_machine_remove_lower_token(circpad_machine_state_t *mi, circpad_delay_t target_bin_us); -STATIC void circpad_machine_remove_closest_token(circpad_machineinfo_t *mi, +STATIC void circpad_machine_remove_closest_token(circpad_machine_state_t *mi, circpad_delay_t target_bin_us, bool use_usec); -STATIC void circpad_machine_setup_tokens(circpad_machineinfo_t *mi); +STATIC void circpad_machine_setup_tokens(circpad_machine_state_t *mi); MOCK_DECL(STATIC signed_error_t, circpad_send_command_to_hop,(origin_circuit_t *circ, uint8_t hopnum, diff --git a/src/test/test_circuitpadding.c b/src/test/test_circuitpadding.c index 2ab8e2445e..f4d003969e 100644 --- a/src/test/test_circuitpadding.c +++ b/src/test/test_circuitpadding.c @@ -479,7 +479,7 @@ helper_create_machine_with_big_histogram(circpad_removal_t removal_strategy) } static circpad_decision_t -circpad_machine_schedule_padding_mock(circpad_machineinfo_t *mi) +circpad_machine_schedule_padding_mock(circpad_machine_state_t *mi) { (void)mi; return 0; @@ -495,7 +495,7 @@ mock_monotime_absolute_usec(void) static void test_circuitpadding_token_removal_higher(void *arg) { - circpad_machineinfo_t *mi; + circpad_machine_state_t *mi; (void)arg; /* Mock it up */ @@ -592,7 +592,7 @@ test_circuitpadding_token_removal_higher(void *arg) static void test_circuitpadding_token_removal_lower(void *arg) { - circpad_machineinfo_t *mi; + circpad_machine_state_t *mi; (void)arg; /* Mock it up */ @@ -689,7 +689,7 @@ test_circuitpadding_token_removal_lower(void *arg) static void test_circuitpadding_closest_token_removal(void *arg) { - circpad_machineinfo_t *mi; + circpad_machine_state_t *mi; (void)arg; /* Mock it up */ @@ -794,7 +794,7 @@ test_circuitpadding_closest_token_removal(void *arg) static void test_circuitpadding_closest_token_removal_usec(void *arg) { - circpad_machineinfo_t *mi; + circpad_machine_state_t *mi; (void)arg; /* Mock it up */ @@ -902,7 +902,7 @@ test_circuitpadding_closest_token_removal_usec(void *arg) static void test_circuitpadding_token_removal_exact(void *arg) { - circpad_machineinfo_t *mi; + circpad_machine_state_t *mi; (void)arg; /* Mock it up */ @@ -963,7 +963,7 @@ void test_circuitpadding_tokens(void *arg) { const circpad_state_t *state; - circpad_machineinfo_t *mi; + circpad_machine_state_t *mi; (void)arg; /** Test plan: @@ -2114,7 +2114,7 @@ helper_circpad_circ_distribution_machine_setup(int min, int max) static void test_circuitpadding_sample_distribution(void *arg) { - circpad_machineinfo_t *mi; + circpad_machine_state_t *mi; int n_samples; int n_states; @@ -2159,7 +2159,7 @@ test_circuitpadding_sample_distribution(void *arg) } static circpad_decision_t -circpad_machine_spec_transition_mock(circpad_machineinfo_t *mi, +circpad_machine_spec_transition_mock(circpad_machine_state_t *mi, circpad_event_t event) { (void) mi; @@ -2174,7 +2174,7 @@ test_circuitpadding_machine_rate_limiting(void *arg) { (void) arg; bool retval; - circpad_machineinfo_t *mi; + circpad_machine_state_t *mi; int i; /* Ignore machine transitions for the purposes of this function, we only @@ -2242,7 +2242,7 @@ test_circuitpadding_global_rate_limiting(void *arg) { (void) arg; bool retval; - circpad_machineinfo_t *mi; + circpad_machine_state_t *mi; int i; /* Ignore machine transitions for the purposes of this function, we only From e0e0338dc42ed786979759d56e0b65f129a5df8c Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Thu, 10 Jan 2019 12:54:55 +0200 Subject: [PATCH 0384/2557] Rename crypto_rand_uint32() -> crypto_rand_u32() See https://github.com/torproject/tor/pull/624#discussion_r246453777 --- src/lib/crypt_ops/crypto_rand.c | 2 +- src/lib/crypt_ops/crypto_rand.h | 2 +- src/lib/math/prob_distr.c | 16 ++++++++-------- src/test/test_prob_distr.c | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/lib/crypt_ops/crypto_rand.c b/src/lib/crypt_ops/crypto_rand.c index 7a2c417e5a..d148dfb3a8 100644 --- a/src/lib/crypt_ops/crypto_rand.c +++ b/src/lib/crypt_ops/crypto_rand.c @@ -532,7 +532,7 @@ crypto_rand_unmocked(char *to, size_t n) * Draw an unsigned 32-bit integer uniformly at random. */ uint32_t -crypto_rand_uint32(void) +crypto_rand_u32(void) { uint32_t rand; crypto_rand((void*)&rand, sizeof(rand)); diff --git a/src/lib/crypt_ops/crypto_rand.h b/src/lib/crypt_ops/crypto_rand.h index 61fd82c806..874fcd4d08 100644 --- a/src/lib/crypt_ops/crypto_rand.h +++ b/src/lib/crypt_ops/crypto_rand.h @@ -27,7 +27,7 @@ int crypto_rand_int(unsigned int max); int crypto_rand_int_range(unsigned int min, unsigned int max); uint64_t crypto_rand_uint64_range(uint64_t min, uint64_t max); time_t crypto_rand_time_range(time_t min, time_t max); -uint32_t crypto_rand_uint32(void); +uint32_t crypto_rand_u32(void); uint64_t crypto_rand_uint64(uint64_t max); double crypto_rand_double(void); struct tor_weak_rng_t; diff --git a/src/lib/math/prob_distr.c b/src/lib/math/prob_distr.c index 832d3b4d96..f5e5218aa7 100644 --- a/src/lib/math/prob_distr.c +++ b/src/lib/math/prob_distr.c @@ -458,7 +458,7 @@ random_uniform_01(void) * system is broken. */ z = 0; - while ((x = crypto_rand_uint32()) == 0) { + while ((x = crypto_rand_u32()) == 0) { if (z >= 1088) /* Your bit sampler is broken. Go home. */ return 0; @@ -472,8 +472,8 @@ random_uniform_01(void) * occur only with measure zero in the uniform distribution on * [0, 1]. */ - hi = crypto_rand_uint32() | UINT32_C(0x80000000); - lo = crypto_rand_uint32() | UINT32_C(0x00000001); + hi = crypto_rand_u32() | UINT32_C(0x80000000); + lo = crypto_rand_u32() | UINT32_C(0x00000001); /* Round to nearest scaled significand in [2^63, 2^64]. */ s = hi*(double)4294967296 + lo; @@ -1402,7 +1402,7 @@ logistic_sample(const struct dist *dist) { const struct logistic *L = const_container_of(dist, struct logistic, base); - uint32_t s = crypto_rand_uint32(); + uint32_t s = crypto_rand_u32(); double t = random_uniform_01(); double p0 = random_uniform_01(); @@ -1460,7 +1460,7 @@ log_logistic_sample(const struct dist *dist) { const struct log_logistic *LL = const_container_of(dist, struct log_logistic, base); - uint32_t s = crypto_rand_uint32(); + uint32_t s = crypto_rand_u32(); double p0 = random_uniform_01(); return sample_log_logistic_scaleshape(s, p0, LL->alpha, LL->beta); @@ -1517,7 +1517,7 @@ weibull_sample(const struct dist *dist) { const struct weibull *W = const_container_of(dist, struct weibull, base); - uint32_t s = crypto_rand_uint32(); + uint32_t s = crypto_rand_u32(); double p0 = random_uniform_01(); return sample_weibull(s, p0, W->lambda, W->k); @@ -1574,7 +1574,7 @@ genpareto_sample(const struct dist *dist) { const struct genpareto *GP = const_container_of(dist, struct genpareto, base); - uint32_t s = crypto_rand_uint32(); + uint32_t s = crypto_rand_u32(); double p0 = random_uniform_01(); return sample_genpareto_locscale(s, p0, GP->mu, GP->sigma, GP->xi); @@ -1621,7 +1621,7 @@ genpareto_isf(const struct dist *dist, double p) double geometric_sample(double p) { - uint32_t s = crypto_rand_uint32(); + uint32_t s = crypto_rand_u32(); double p0 = random_uniform_01(); return sample_geometric(s, p0, p); } diff --git a/src/test/test_prob_distr.c b/src/test/test_prob_distr.c index bf0f9e059d..75e7e360a6 100644 --- a/src/test/test_prob_distr.c +++ b/src/test/test_prob_distr.c @@ -1107,7 +1107,7 @@ static uint32_t deterministic_rand_counter; static void init_deterministic_rand(void) { - deterministic_rand_counter = crypto_rand_uint32(); + deterministic_rand_counter = crypto_rand_u32(); } /** Produce deterministic randomness for the stochastic tests using the global From 4db9c3d63e93912a82eabeb8a9fb851d5196c3c8 Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Thu, 10 Jan 2019 13:03:41 +0200 Subject: [PATCH 0385/2557] Unittest for tor_isinf(). --- src/test/test_util.c | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/test/test_util.c b/src/test/test_util.c index bf64cff7ef..e8233d9d30 100644 --- a/src/test/test_util.c +++ b/src/test/test_util.c @@ -70,6 +70,28 @@ #define INFINITY_DBL ((double)INFINITY) #define NAN_DBL ((double)NAN) +/** Test the tor_isinf() wrapper */ +static void +test_tor_isinf(void *arg) +{ + (void) arg; + + tt_assert(tor_isinf(INFINITY_DBL)); + + tt_assert(!tor_isinf(NAN_DBL)); + tt_assert(!tor_isinf(DBL_EPSILON)); + tt_assert(!tor_isinf(DBL_MAX)); + tt_assert(!tor_isinf(DBL_MIN)); + + tt_assert(!tor_isinf(0.0)); + tt_assert(!tor_isinf(0.1)); + tt_assert(!tor_isinf(3)); + tt_assert(!tor_isinf(3.14)); + + done: + ; +} + /* XXXX this is a minimal wrapper to make the unit tests compile with the * changed tor_timegm interface. */ static time_t @@ -6191,6 +6213,7 @@ struct testcase_t util_tests[] = { UTIL_TEST(mathlog, 0), UTIL_TEST(fraction, 0), UTIL_TEST(weak_random, 0), + { "tor_isinf", test_tor_isinf, TT_FORK, NULL, NULL }, { "socket_ipv4", test_util_socket, TT_FORK, &passthrough_setup, (void*)"4" }, { "socket_ipv6", test_util_socket, TT_FORK, From 2b29bccb8553c4b80eccd3438b4b1463687b1d4c Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Thu, 10 Jan 2019 13:05:05 +0200 Subject: [PATCH 0386/2557] Misc trivial improvements around circpadding code. --- doc/tor.1.txt | 5 +++++ src/core/or/circuitpadding.c | 1 + src/core/or/relay.c | 1 + 3 files changed, 7 insertions(+) diff --git a/doc/tor.1.txt b/doc/tor.1.txt index 455356163c..6dbd4af377 100644 --- a/doc/tor.1.txt +++ b/doc/tor.1.txt @@ -1025,6 +1025,11 @@ The following options are useful only for clients (that is, if to use for "middle" hops in your normal circuits. Normal circuits include all circuits except for direct connections to directory servers. Middle hops are all hops other than exit and entry. + ++ + This is an **experimental** feature that is meant to be used by researchers + and developers to test new features in the Tor network safely. Using it + without care will strongly influence your anonymity. This feature might get + removed in the future. + The HSLayer2Node and HSLayer3Node options override this option for onion service circuits, if they are set. The vanguards addon will read this diff --git a/src/core/or/circuitpadding.c b/src/core/or/circuitpadding.c index 6f10d3fb6a..6a39a7b371 100644 --- a/src/core/or/circuitpadding.c +++ b/src/core/or/circuitpadding.c @@ -304,6 +304,7 @@ circpad_histogram_usec_to_bin(const circpad_machine_state_t *mi, else start_usec = state->start_usec; + /* The first bin (#0) has zero width and starts (and ends) at start_usec. */ if (usec <= start_usec) return 0; diff --git a/src/core/or/relay.c b/src/core/or/relay.c index 9c0f3bbbe3..00c2111955 100644 --- a/src/core/or/relay.c +++ b/src/core/or/relay.c @@ -527,6 +527,7 @@ relay_command_to_string(uint8_t command) case RELAY_COMMAND_EXTEND2: return "EXTEND2"; case RELAY_COMMAND_EXTENDED2: return "EXTENDED2"; case RELAY_COMMAND_PADDING_NEGOTIATE: return "PADDING_NEGOTIATE"; + case RELAY_COMMAND_PADDING_NEGOTIATED: return "PADDING_NEGOTIATED"; default: tor_snprintf(buf, sizeof(buf), "Unrecognized relay command %u", (unsigned)command); From ca544246020cddfee3f7fd46899dcf3a9382eb62 Mon Sep 17 00:00:00 2001 From: Taylor R Campbell Date: Thu, 10 Jan 2019 17:10:39 +0000 Subject: [PATCH 0387/2557] Fix type redefinition errors. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In file included from src/core/or/connection_edge.c:70:0: ./src/core/or/circuitpadding.h:16:26: error: redefinition of typedef ‘circuit_t’ ./src/core/or/or.h:930:26: note: previous declaration of ‘circuit_t’ was here ./src/core/or/circuitpadding.h:17:33: error: redefinition of typedef ‘origin_circuit_t’ ./src/core/or/or.h:931:33: note: previous declaration of ‘origin_circuit_t’ was here ./src/core/or/circuitpadding.h:18:23: error: redefinition of typedef ‘cell_t’ ./src/core/or/or.h:628:23: note: previous declaration of ‘cell_t’ was here typedef doesn't work for forward declarations, but plain struct without a typedef wrapper does (and unlike the _t type aliases makes it clearer for everyone whether you're talking about the struct or the pointer). --- src/core/or/circuitpadding.h | 53 +++++++++++++++++++----------------- 1 file changed, 28 insertions(+), 25 deletions(-) diff --git a/src/core/or/circuitpadding.h b/src/core/or/circuitpadding.h index f3d176ebf7..628f27ec11 100644 --- a/src/core/or/circuitpadding.h +++ b/src/core/or/circuitpadding.h @@ -13,9 +13,9 @@ #include "src/trunnel/circpad_negotiation.h" #include "lib/evloop/timers.h" -typedef struct circuit_t circuit_t; -typedef struct origin_circuit_t origin_circuit_t; -typedef struct cell_t cell_t; +struct circuit_t; +struct origin_circuit_t; +struct cell_t; /** * Signed error return with the specific property that negative @@ -440,7 +440,7 @@ typedef struct circpad_machine_state_t { tor_timer_t *padding_timer; /** The circuit for this machine */ - circuit_t *on_circ; + struct circuit_t *on_circ; /** A mutable copy of the histogram for the current state. * NULL if remove_tokens is false for that state */ @@ -576,19 +576,19 @@ void circpad_new_consensus_params(const networkstatus_t *ns); /** * The following are event call-in points that are of interest to * the state machines. They are called during cell processing. */ -void circpad_deliver_unrecognized_cell_events(circuit_t *circ, +void circpad_deliver_unrecognized_cell_events(struct circuit_t *circ, cell_direction_t dir); -void circpad_deliver_sent_relay_cell_events(circuit_t *circ, +void circpad_deliver_sent_relay_cell_events(struct circuit_t *circ, uint8_t relay_command); -void circpad_deliver_recognized_relay_cell_events(circuit_t *circ, +void circpad_deliver_recognized_relay_cell_events(struct circuit_t *circ, uint8_t relay_command, crypt_path_t *layer_hint); /** Cell events are delivered by the above delivery functions */ -void circpad_cell_event_nonpadding_sent(circuit_t *on_circ); -void circpad_cell_event_nonpadding_received(circuit_t *on_circ); -void circpad_cell_event_padding_sent(circuit_t *on_circ); -void circpad_cell_event_padding_received(circuit_t *on_circ); +void circpad_cell_event_nonpadding_sent(struct circuit_t *on_circ); +void circpad_cell_event_nonpadding_received(struct circuit_t *on_circ); +void circpad_cell_event_padding_sent(struct circuit_t *on_circ); +void circpad_cell_event_padding_received(struct circuit_t *on_circ); /** Internal events are events the machines send to themselves */ circpad_decision_t @@ -600,12 +600,13 @@ circpad_decision_t circpad_internal_event_state_length_up( /** Machine creation events are events that cause us to set up or * tear down padding state machines. */ -void circpad_machine_event_circ_added_hop(origin_circuit_t *on_circ); -void circpad_machine_event_circ_built(origin_circuit_t *circ); -void circpad_machine_event_circ_purpose_changed(origin_circuit_t *circ); -void circpad_machine_event_circ_has_streams(origin_circuit_t *circ); -void circpad_machine_event_circ_has_no_streams(origin_circuit_t *circ); -void circpad_machine_event_circ_has_no_relay_early(origin_circuit_t *circ); +void circpad_machine_event_circ_added_hop(struct origin_circuit_t *on_circ); +void circpad_machine_event_circ_built(struct origin_circuit_t *circ); +void circpad_machine_event_circ_purpose_changed(struct origin_circuit_t *circ); +void circpad_machine_event_circ_has_streams(struct origin_circuit_t *circ); +void circpad_machine_event_circ_has_no_streams(struct origin_circuit_t *circ); +void +circpad_machine_event_circ_has_no_relay_early(struct origin_circuit_t *circ); void circpad_machines_init(void); void circpad_machines_free(void); @@ -613,9 +614,9 @@ void circpad_machines_free(void); void circpad_machine_states_init(circpad_machine_spec_t *machine, circpad_statenum_t num_states); -void circpad_circuit_free_all_machineinfos(circuit_t *circ); +void circpad_circuit_free_all_machineinfos(struct circuit_t *circ); -bool circpad_padding_is_from_expected_hop(circuit_t *circ, +bool circpad_padding_is_from_expected_hop(struct circuit_t *circ, crypt_path_t *from_hop); /** Serializaton functions for writing to/from torrc and consensus */ @@ -623,14 +624,16 @@ char *circpad_machine_spec_to_string(const circpad_machine_spec_t *machine); const circpad_machine_spec_t *circpad_string_to_machine(const char *str); /* Padding negotiation between client and middle */ -signed_error_t circpad_handle_padding_negotiate(circuit_t *circ, cell_t *cell); -signed_error_t circpad_handle_padding_negotiated(circuit_t *circ, cell_t *cell, +signed_error_t circpad_handle_padding_negotiate(struct circuit_t *circ, + struct cell_t *cell); +signed_error_t circpad_handle_padding_negotiated(struct circuit_t *circ, + struct cell_t *cell, crypt_path_t *layer_hint); -signed_error_t circpad_negotiate_padding(origin_circuit_t *circ, +signed_error_t circpad_negotiate_padding(struct origin_circuit_t *circ, circpad_machine_num_t machine, uint8_t target_hopnum, uint8_t command); -bool circpad_padding_negotiated(circuit_t *circ, +bool circpad_padding_negotiated(struct circuit_t *circ, circpad_machine_num_t machine, uint8_t command, uint8_t response); @@ -667,7 +670,7 @@ STATIC circpad_hist_index_t circpad_histogram_usec_to_bin( circpad_delay_t us); STATIC circpad_machine_state_t *circpad_circuit_machineinfo_new( - circuit_t *on_circ, + struct circuit_t *on_circ, int machine_index); STATIC void circpad_machine_remove_higher_token(circpad_machine_state_t *mi, circpad_delay_t target_bin_us); @@ -679,7 +682,7 @@ STATIC void circpad_machine_remove_closest_token(circpad_machine_state_t *mi, STATIC void circpad_machine_setup_tokens(circpad_machine_state_t *mi); MOCK_DECL(STATIC signed_error_t, -circpad_send_command_to_hop,(origin_circuit_t *circ, uint8_t hopnum, +circpad_send_command_to_hop,(struct origin_circuit_t *circ, uint8_t hopnum, uint8_t relay_command, const uint8_t *payload, ssize_t payload_len)); From 948856c03ef417cc9dad9ef85b7bb5c164edb742 Mon Sep 17 00:00:00 2001 From: Taylor R Campbell Date: Thu, 10 Jan 2019 17:12:32 +0000 Subject: [PATCH 0388/2557] Fix more type redefinition errors. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In file included from ./src/core/or/or_circuit_st.h:12:0, from src/core/or/circuitlist.c:112: ./src/core/or/circuit_st.h:15:39: error: redefinition of typedef ‘circpad_machine_spec_t’ ./src/core/or/circuitpadding.h:572:3: note: previous declaration of ‘circpad_machine_spec_t’ was here ./src/core/or/circuit_st.h:16:40: error: redefinition of typedef ‘circpad_machine_state_t’ ./src/core/or/circuitpadding.h:517:3: note: previous declaration of ‘circpad_machine_state_t’ was here --- src/core/or/circuit_st.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/core/or/circuit_st.h b/src/core/or/circuit_st.h index bfbd336c3a..47639940d4 100644 --- a/src/core/or/circuit_st.h +++ b/src/core/or/circuit_st.h @@ -12,8 +12,8 @@ #include "core/or/cell_queue_st.h" struct hs_token_t; -typedef struct circpad_machine_spec_t circpad_machine_spec_t; -typedef struct circpad_machine_state_t circpad_machine_state_t; +struct circpad_machine_spec_t; +struct circpad_machine_state_t; /** Number of padding state machines on a circuit. */ #define CIRCPAD_MAX_MACHINES (2) @@ -189,7 +189,7 @@ struct circuit_t { * * Each element of this array corresponds to a different padding machine, * and we can have up to CIRCPAD_MAX_MACHINES such machines. */ - const circpad_machine_spec_t *padding_machine[CIRCPAD_MAX_MACHINES]; + const struct circpad_machine_spec_t *padding_machine[CIRCPAD_MAX_MACHINES]; /** Adaptive Padding machine info for above machines. This is the * per-circuit mutable information, such as the current state and @@ -200,7 +200,7 @@ struct circuit_t { * * Each element of this array corresponds to a different padding machine, * and we can have up to CIRCPAD_MAX_MACHINES such machines. */ - circpad_machine_state_t *padding_info[CIRCPAD_MAX_MACHINES]; + struct circpad_machine_state_t *padding_info[CIRCPAD_MAX_MACHINES]; }; #endif From 531df9590d006434b31cc81871b73c31ca9f896b Mon Sep 17 00:00:00 2001 From: Taylor R Campbell Date: Thu, 10 Jan 2019 17:12:56 +0000 Subject: [PATCH 0389/2557] Move ceil call back into the geometric sampler. Test exactly what the geometric sampler returns, because that's what the downstream callers of it are going to use. While here, also assert that the geometric sampler returns a positive integer. (Our geometric distribution is the one suported on {1, 2, 3, ...} that returns the number of trials before the first success, not the one supported on {0, 1, 2, ...} that returns the number of failures before the first success.) --- src/lib/math/prob_distr.c | 2 +- src/test/test_prob_distr.c | 12 +++++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/lib/math/prob_distr.c b/src/lib/math/prob_distr.c index f5e5218aa7..e170d000fe 100644 --- a/src/lib/math/prob_distr.c +++ b/src/lib/math/prob_distr.c @@ -1308,7 +1308,7 @@ sample_geometric(uint32_t s, double p0, double p) if (p >= 1) return 1; - return (-x/log1p(-p)); + return ceil(-x/log1p(-p)); } /*******************************************************************/ diff --git a/src/test/test_prob_distr.c b/src/test/test_prob_distr.c index 75e7e360a6..ec4e943e9a 100644 --- a/src/test/test_prob_distr.c +++ b/src/test/test_prob_distr.c @@ -958,7 +958,17 @@ test_stochastic_geometric_impl(double p) size_t C[PSI_DF] = {0}; for (j = 0; j < NSAMPLES; j++) { - double n_tmp = ceil(geometric_sample(p)); + double n_tmp = geometric_sample(p); + + /* Must be an integer. (XXX -Wfloat-equal) */ + tor_assert(ceil(n_tmp) <= n_tmp && ceil(n_tmp) >= n_tmp); + + /* Must be a positive integer. */ + tor_assert(n_tmp >= 1); + + /* Probability of getting a value in the billions is negligible. */ + tor_assert(n_tmp <= (double)UINT_MAX); + unsigned n = (unsigned) n_tmp; if (n > PSI_DF) From 0f8253bddbaae4e73fe2ff9ecf1c342e3f66b798 Mon Sep 17 00:00:00 2001 From: Taylor R Campbell Date: Thu, 10 Jan 2019 17:40:17 +0000 Subject: [PATCH 0390/2557] Use the distribution abstraction as an abstraction. --- src/core/or/circuitpadding.c | 16 ++- src/lib/math/prob_distr.c | 229 ++++++++++++++++++++++++----------- src/lib/math/prob_distr.h | 46 +++---- src/test/test_prob_distr.c | 22 ++-- 4 files changed, 196 insertions(+), 117 deletions(-) diff --git a/src/core/or/circuitpadding.c b/src/core/or/circuitpadding.c index 6a39a7b371..a5d5d24551 100644 --- a/src/core/or/circuitpadding.c +++ b/src/core/or/circuitpadding.c @@ -545,7 +545,7 @@ circpad_distribution_sample(circpad_distribution_t dist) .a = dist.param1, .b = dist.param2, }; - return uniform_sample(&my_uniform.base); + return dist_sample(&my_uniform.base); } case CIRCPAD_DIST_LOGISTIC: { @@ -555,7 +555,7 @@ circpad_distribution_sample(circpad_distribution_t dist) .mu = dist.param1, .sigma = dist.param2, }; - return logistic_sample(&my_logistic.base); + return dist_sample(&my_logistic.base); } case CIRCPAD_DIST_LOG_LOGISTIC: { @@ -565,12 +565,16 @@ circpad_distribution_sample(circpad_distribution_t dist) .alpha = dist.param1, .beta = dist.param2, }; - return log_logistic_sample(&my_log_logistic.base); + return dist_sample(&my_log_logistic.base); } case CIRCPAD_DIST_GEOMETRIC: { /* param1 is 'p' (success probability) */ - return geometric_sample(dist.param1); + const struct geometric my_geometric = { + .base = DIST_BASE(&geometric_ops), + .p = dist.param1, + }; + return dist_sample(&my_geometric.base); } case CIRCPAD_DIST_WEIBULL: { @@ -580,7 +584,7 @@ circpad_distribution_sample(circpad_distribution_t dist) .k = dist.param1, .lambda = dist.param2, }; - return weibull_sample(&my_weibull.base); + return dist_sample(&my_weibull.base); } case CIRCPAD_DIST_PARETO: { @@ -591,7 +595,7 @@ circpad_distribution_sample(circpad_distribution_t dist) .sigma = dist.param1, .xi = dist.param2, }; - return genpareto_sample(&my_genpareto.base); + return dist_sample(&my_genpareto.base); } } diff --git a/src/lib/math/prob_distr.c b/src/lib/math/prob_distr.c index e170d000fe..4263ba2074 100644 --- a/src/lib/math/prob_distr.c +++ b/src/lib/math/prob_distr.c @@ -1319,17 +1319,45 @@ sample_geometric(uint32_t s, double p0, double p) * (sample/cdf/sf/icdf/isf) as part of its dist_ops structure. */ -/** Functions for uniform distribution */ -const struct dist_ops uniform_ops = { - .name = "uniform", - .sample = uniform_sample, - .cdf = uniform_cdf, - .sf = uniform_sf, - .icdf = uniform_icdf, - .isf = uniform_isf, -}; +const char * +dist_name(const struct dist *dist) +{ + return dist->ops->name; +} double +dist_sample(const struct dist *dist) +{ + return dist->ops->sample(dist); +} + +double +dist_cdf(const struct dist *dist, double x) +{ + return dist->ops->cdf(dist, x); +} + +double +dist_sf(const struct dist *dist, double x) +{ + return dist->ops->sf(dist, x); +} + +double +dist_icdf(const struct dist *dist, double p) +{ + return dist->ops->icdf(dist, p); +} + +double +dist_isf(const struct dist *dist, double p) +{ + return dist->ops->isf(dist, p); +} + +/** Functions for uniform distribution */ + +static double uniform_sample(const struct dist *dist) { const struct uniform *U = const_container_of(dist, struct uniform, @@ -1339,7 +1367,7 @@ uniform_sample(const struct dist *dist) return sample_uniform_interval(p0, U->a, U->b); } -double +static double uniform_cdf(const struct dist *dist, double x) { const struct uniform *U = const_container_of(dist, struct uniform, @@ -1353,7 +1381,7 @@ uniform_cdf(const struct dist *dist, double x) return 1; } -double +static double uniform_sf(const struct dist *dist, double x) { const struct uniform *U = const_container_of(dist, struct uniform, @@ -1367,7 +1395,7 @@ uniform_sf(const struct dist *dist, double x) return 1; } -double +static double uniform_icdf(const struct dist *dist, double p) { const struct uniform *U = const_container_of(dist, struct uniform, @@ -1377,7 +1405,7 @@ uniform_icdf(const struct dist *dist, double p) return (p < 0.5 ? (U->a + w*p) : (U->b - w*(1 - p))); } -double +static double uniform_isf(const struct dist *dist, double p) { const struct uniform *U = const_container_of(dist, struct uniform, @@ -1387,17 +1415,18 @@ uniform_isf(const struct dist *dist, double p) return (p < 0.5 ? (U->b - w*p) : (U->a + w*(1 - p))); } -/** Functions for logistic distribution: */ -const struct dist_ops logistic_ops = { - .name = "logistic", - .sample = logistic_sample, - .cdf = logistic_cdf, - .sf = logistic_sf, - .icdf = logistic_icdf, - .isf = logistic_isf, +const struct dist_ops uniform_ops = { + .name = "uniform", + .sample = uniform_sample, + .cdf = uniform_cdf, + .sf = uniform_sf, + .icdf = uniform_icdf, + .isf = uniform_isf, }; -double +/** Functions for logistic distribution: */ + +static double logistic_sample(const struct dist *dist) { const struct logistic *L = const_container_of(dist, struct logistic, @@ -1409,7 +1438,7 @@ logistic_sample(const struct dist *dist) return sample_logistic_locscale(s, t, p0, L->mu, L->sigma); } -double +static double logistic_cdf(const struct dist *dist, double x) { const struct logistic *L = const_container_of(dist, struct logistic, @@ -1418,7 +1447,7 @@ logistic_cdf(const struct dist *dist, double x) return cdf_logistic(x, L->mu, L->sigma); } -double +static double logistic_sf(const struct dist *dist, double x) { const struct logistic *L = const_container_of(dist, struct logistic, @@ -1427,7 +1456,7 @@ logistic_sf(const struct dist *dist, double x) return sf_logistic(x, L->mu, L->sigma); } -double +static double logistic_icdf(const struct dist *dist, double p) { const struct logistic *L = const_container_of(dist, struct logistic, @@ -1436,7 +1465,7 @@ logistic_icdf(const struct dist *dist, double p) return icdf_logistic(p, L->mu, L->sigma); } -double +static double logistic_isf(const struct dist *dist, double p) { const struct logistic *L = const_container_of(dist, struct logistic, @@ -1445,17 +1474,18 @@ logistic_isf(const struct dist *dist, double p) return isf_logistic(p, L->mu, L->sigma); } -/** Functions for log-logistic distribution: */ -const struct dist_ops log_logistic_ops = { - .name = "log logistic", - .sample = log_logistic_sample, - .cdf = log_logistic_cdf, - .sf = log_logistic_sf, - .icdf = log_logistic_icdf, - .isf = log_logistic_isf, +const struct dist_ops logistic_ops = { + .name = "logistic", + .sample = logistic_sample, + .cdf = logistic_cdf, + .sf = logistic_sf, + .icdf = logistic_icdf, + .isf = logistic_isf, }; -double +/** Functions for log-logistic distribution: */ + +static double log_logistic_sample(const struct dist *dist) { const struct log_logistic *LL = const_container_of(dist, struct @@ -1466,7 +1496,7 @@ log_logistic_sample(const struct dist *dist) return sample_log_logistic_scaleshape(s, p0, LL->alpha, LL->beta); } -double +static double log_logistic_cdf(const struct dist *dist, double x) { const struct log_logistic *LL = const_container_of(dist, @@ -1475,7 +1505,7 @@ log_logistic_cdf(const struct dist *dist, double x) return cdf_log_logistic(x, LL->alpha, LL->beta); } -double +static double log_logistic_sf(const struct dist *dist, double x) { const struct log_logistic *LL = const_container_of(dist, @@ -1484,7 +1514,7 @@ log_logistic_sf(const struct dist *dist, double x) return sf_log_logistic(x, LL->alpha, LL->beta); } -double +static double log_logistic_icdf(const struct dist *dist, double p) { const struct log_logistic *LL = const_container_of(dist, @@ -1493,7 +1523,7 @@ log_logistic_icdf(const struct dist *dist, double p) return icdf_log_logistic(p, LL->alpha, LL->beta); } -double +static double log_logistic_isf(const struct dist *dist, double p) { const struct log_logistic *LL = const_container_of(dist, @@ -1502,17 +1532,18 @@ log_logistic_isf(const struct dist *dist, double p) return isf_log_logistic(p, LL->alpha, LL->beta); } -/** Functions for Weibull distribution */ -const struct dist_ops weibull_ops = { - .name = "Weibull", - .sample = weibull_sample, - .cdf = weibull_cdf, - .sf = weibull_sf, - .icdf = weibull_icdf, - .isf = weibull_isf, +const struct dist_ops log_logistic_ops = { + .name = "log logistic", + .sample = log_logistic_sample, + .cdf = log_logistic_cdf, + .sf = log_logistic_sf, + .icdf = log_logistic_icdf, + .isf = log_logistic_isf, }; -double +/** Functions for Weibull distribution */ + +static double weibull_sample(const struct dist *dist) { const struct weibull *W = const_container_of(dist, struct weibull, @@ -1523,7 +1554,7 @@ weibull_sample(const struct dist *dist) return sample_weibull(s, p0, W->lambda, W->k); } -double +static double weibull_cdf(const struct dist *dist, double x) { const struct weibull *W = const_container_of(dist, struct weibull, @@ -1532,7 +1563,7 @@ weibull_cdf(const struct dist *dist, double x) return cdf_weibull(x, W->lambda, W->k); } -double +static double weibull_sf(const struct dist *dist, double x) { const struct weibull *W = const_container_of(dist, struct weibull, @@ -1541,7 +1572,7 @@ weibull_sf(const struct dist *dist, double x) return sf_weibull(x, W->lambda, W->k); } -double +static double weibull_icdf(const struct dist *dist, double p) { const struct weibull *W = const_container_of(dist, struct weibull, @@ -1550,7 +1581,7 @@ weibull_icdf(const struct dist *dist, double p) return icdf_weibull(p, W->lambda, W->k); } -double +static double weibull_isf(const struct dist *dist, double p) { const struct weibull *W = const_container_of(dist, struct weibull, @@ -1559,17 +1590,18 @@ weibull_isf(const struct dist *dist, double p) return isf_weibull(p, W->lambda, W->k); } -/** Functions for generalized Pareto distributions */ -const struct dist_ops genpareto_ops = { - .name = "generalized Pareto", - .sample = genpareto_sample, - .cdf = genpareto_cdf, - .sf = genpareto_sf, - .icdf = genpareto_icdf, - .isf = genpareto_isf, +const struct dist_ops weibull_ops = { + .name = "Weibull", + .sample = weibull_sample, + .cdf = weibull_cdf, + .sf = weibull_sf, + .icdf = weibull_icdf, + .isf = weibull_isf, }; -double +/** Functions for generalized Pareto distributions */ + +static double genpareto_sample(const struct dist *dist) { const struct genpareto *GP = const_container_of(dist, struct genpareto, @@ -1580,7 +1612,7 @@ genpareto_sample(const struct dist *dist) return sample_genpareto_locscale(s, p0, GP->mu, GP->sigma, GP->xi); } -double +static double genpareto_cdf(const struct dist *dist, double x) { const struct genpareto *GP = const_container_of(dist, struct genpareto, @@ -1589,7 +1621,7 @@ genpareto_cdf(const struct dist *dist, double x) return cdf_genpareto(x, GP->mu, GP->sigma, GP->xi); } -double +static double genpareto_sf(const struct dist *dist, double x) { const struct genpareto *GP = const_container_of(dist, struct genpareto, @@ -1598,7 +1630,7 @@ genpareto_sf(const struct dist *dist, double x) return sf_genpareto(x, GP->mu, GP->sigma, GP->xi); } -double +static double genpareto_icdf(const struct dist *dist, double p) { const struct genpareto *GP = const_container_of(dist, struct genpareto, @@ -1607,7 +1639,7 @@ genpareto_icdf(const struct dist *dist, double p) return icdf_genpareto(p, GP->mu, GP->sigma, GP->xi); } -double +static double genpareto_isf(const struct dist *dist, double p) { const struct genpareto *GP = const_container_of(dist, struct genpareto, @@ -1616,13 +1648,70 @@ genpareto_isf(const struct dist *dist, double p) return isf_genpareto(p, GP->mu, GP->sigma, GP->xi); } -/* Deterministically sample from the geometric distribution with - * per-trial success probability p. */ -double -geometric_sample(double p) +const struct dist_ops genpareto_ops = { + .name = "generalized Pareto", + .sample = genpareto_sample, + .cdf = genpareto_cdf, + .sf = genpareto_sf, + .icdf = genpareto_icdf, + .isf = genpareto_isf, +}; + +/** Functions for geometric distribution on number of trials before success */ + +static double +geometric_sample(const struct dist *dist) { + const struct geometric *G = const_container_of(dist, struct geometric, base); uint32_t s = crypto_rand_u32(); double p0 = random_uniform_01(); - return sample_geometric(s, p0, p); + + return sample_geometric(s, p0, G->p); } +static double +geometric_cdf(const struct dist *dist, double x) +{ + const struct geometric *G = const_container_of(dist, struct geometric, base); + + if (x < 1) + return 0; + /* 1 - (1 - p)^floor(x) = 1 - e^{floor(x) log(1 - p)} */ + return -expm1(floor(x)*log1p(-G->p)); +} + +static double +geometric_sf(const struct dist *dist, double x) +{ + const struct geometric *G = const_container_of(dist, struct geometric, base); + + if (x < 1) + return 0; + /* (1 - p)^floor(x) = e^{ceil(x) log(1 - p)} */ + return exp(floor(x)*log1p(-G->p)); +} + +static double +geometric_icdf(const struct dist *dist, double p) +{ + const struct geometric *G = const_container_of(dist, struct geometric, base); + + return log1p(-p)/log1p(-G->p); +} + +static double +geometric_isf(const struct dist *dist, double p) +{ + const struct geometric *G = const_container_of(dist, struct geometric, base); + + return log(p)/log1p(-G->p); +} + +const struct dist_ops geometric_ops = { + .name = "geometric (1-based)", + .sample = geometric_sample, + .cdf = geometric_cdf, + .sf = geometric_sf, + .icdf = geometric_icdf, + .isf = geometric_isf, +}; diff --git a/src/lib/math/prob_distr.h b/src/lib/math/prob_distr.h index c2fd6c74b3..981fc2017d 100644 --- a/src/lib/math/prob_distr.h +++ b/src/lib/math/prob_distr.h @@ -21,6 +21,13 @@ struct dist { #define DIST_BASE(OPS) { .ops = (OPS) } +const char *dist_name(const struct dist *); +double dist_sample(const struct dist *); +double dist_cdf(const struct dist *, double x); +double dist_sf(const struct dist *, double x); +double dist_icdf(const struct dist *, double p); +double dist_isf(const struct dist *, double p); + struct dist_ops { const char *name; double (*sample)(const struct dist *); @@ -30,9 +37,14 @@ struct dist_ops { double (*isf)(const struct dist *, double p); }; -/* Geometric distribution */ +/* Geometric distribution on positive number of trials before first success */ -double geometric_sample(double p); +struct geometric { + struct dist base; + double p; /* success probability */ +}; + +extern const struct dist_ops geometric_ops; /* Pareto distribution */ @@ -43,12 +55,6 @@ struct genpareto { double xi; }; -double genpareto_sample(const struct dist *dist); -double genpareto_cdf(const struct dist *dist, double x); -double genpareto_sf(const struct dist *dist, double x); -double genpareto_icdf(const struct dist *dist, double p); -double genpareto_isf(const struct dist *dist, double p); - extern const struct dist_ops genpareto_ops; /* Weibull distribution */ @@ -59,12 +65,6 @@ struct weibull { double k; }; -double weibull_sample(const struct dist *dist); -double weibull_cdf(const struct dist *dist, double x); -double weibull_sf(const struct dist *dist, double x); -double weibull_icdf(const struct dist *dist, double p); -double weibull_isf(const struct dist *dist, double p); - extern const struct dist_ops weibull_ops; /* Log-logistic distribution */ @@ -75,12 +75,6 @@ struct log_logistic { double beta; }; -double log_logistic_sample(const struct dist *dist); -double log_logistic_cdf(const struct dist *dist, double x); -double log_logistic_sf(const struct dist *dist, double x); -double log_logistic_icdf(const struct dist *dist, double p); -double log_logistic_isf(const struct dist *dist, double p); - extern const struct dist_ops log_logistic_ops; /* Logistic distribution */ @@ -91,12 +85,6 @@ struct logistic { double sigma; }; -double logistic_sample(const struct dist *dist); -double logistic_cdf(const struct dist *dist, double x); -double logistic_sf(const struct dist *dist, double x); -double logistic_icdf(const struct dist *dist, double p); -double logistic_isf(const struct dist *dist, double p); - extern const struct dist_ops logistic_ops; /* Uniform distribution */ @@ -107,12 +95,6 @@ struct uniform { double b; }; -double uniform_sample(const struct dist *dist); -double uniform_cdf(const struct dist *dist, double x); -double uniform_sf(const struct dist *dist, double x); -double uniform_icdf(const struct dist *dist, double p); -double uniform_isf(const struct dist *dist, double p); - extern const struct dist_ops uniform_ops; /** Only by unittests */ diff --git a/src/test/test_prob_distr.c b/src/test/test_prob_distr.c index ec4e943e9a..fe3969518c 100644 --- a/src/test/test_prob_distr.c +++ b/src/test/test_prob_distr.c @@ -942,6 +942,10 @@ psi_test(const size_t C[PSI_DF], const double logP[PSI_DF], size_t N) static bool test_stochastic_geometric_impl(double p) { + const struct geometric geometric = { + .base = DIST_BASE(&geometric_ops), + .p = p, + }; double logP[PSI_DF] = {0}; unsigned ntry = NTRIALS, npass = 0; unsigned i; @@ -958,7 +962,7 @@ test_stochastic_geometric_impl(double p) size_t C[PSI_DF] = {0}; for (j = 0; j < NSAMPLES; j++) { - double n_tmp = geometric_sample(p); + double n_tmp = dist_sample(&geometric.base); /* Must be an integer. (XXX -Wfloat-equal) */ tor_assert(ceil(n_tmp) <= n_tmp && ceil(n_tmp) >= n_tmp); @@ -1006,10 +1010,10 @@ test_stochastic_geometric_impl(double p) static void bin_cdfs(const struct dist *dist, double lo, double hi, double *logP, size_t n) { -#define CDF(x) dist->ops->cdf(dist, x) -#define SF(x) dist->ops->sf(dist, x) +#define CDF(x) dist_cdf(dist, x) +#define SF(x) dist_sf(dist, x) const double w = (hi - lo)/(n - 2); - double halfway = dist->ops->icdf(dist, 0.5); + double halfway = dist_icdf(dist, 0.5); double x_0, x_1; size_t i; size_t n2 = ceil_to_size_t((halfway - lo)/w); @@ -1057,7 +1061,7 @@ bin_samples(const struct dist *dist, double lo, double hi, size_t *C, size_t n) size_t i; for (i = 0; i < NSAMPLES; i++) { - double x = dist->ops->sample(dist); + double x = dist_sample(dist); size_t bin; if (x < lo) @@ -1084,8 +1088,8 @@ test_psi_dist_sample(const struct dist *dist) { double logP[PSI_DF] = {0}; unsigned ntry = NTRIALS, npass = 0; - double lo = dist->ops->icdf(dist, 1/(double)(PSI_DF + 2)); - double hi = dist->ops->isf(dist, 1/(double)(PSI_DF + 2)); + double lo = dist_icdf(dist, 1/(double)(PSI_DF + 2)); + double hi = dist_isf(dist, 1/(double)(PSI_DF + 2)); /* Create the null hypothesis in logP */ bin_cdfs(dist, lo, hi, logP, PSI_DF); @@ -1102,10 +1106,10 @@ test_psi_dist_sample(const struct dist *dist) /* Did we fail or succeed? */ if (npass >= NPASSES_MIN) { - /* printf("pass %s sampler\n", dist->ops->name);*/ + /* printf("pass %s sampler\n", dist_name(dist));*/ return true; } else { - printf("fail %s sampler\n", dist->ops->name); + printf("fail %s sampler\n", dist_name(dist)); return false; } } From d82a8a7f9d268728b2447b2dbbaa346140784f9b Mon Sep 17 00:00:00 2001 From: Taylor R Campbell Date: Thu, 10 Jan 2019 18:08:20 +0000 Subject: [PATCH 0391/2557] Add some more type checking. NOTE: This commit breaks the build, because there was a mistake in an earlier change of exactly the sort that this is meant to detect! I'm leaving it broken for illustration. --- src/core/or/circuitpadding.c | 12 ++++++------ src/lib/math/prob_distr.h | 20 ++++++++++++++++++++ src/test/test_prob_distr.c | 22 +++++++++++----------- 3 files changed, 37 insertions(+), 17 deletions(-) diff --git a/src/core/or/circuitpadding.c b/src/core/or/circuitpadding.c index a5d5d24551..4080614338 100644 --- a/src/core/or/circuitpadding.c +++ b/src/core/or/circuitpadding.c @@ -541,7 +541,7 @@ circpad_distribution_sample(circpad_distribution_t dist) { // param2 is upper bound, param1 is lower const struct uniform my_uniform = { - .base = DIST_BASE(&uniform_ops), + .base = UNIFORM(my_uniform), .a = dist.param1, .b = dist.param2, }; @@ -551,7 +551,7 @@ circpad_distribution_sample(circpad_distribution_t dist) { /* param1 is Mu, param2 is sigma. */ const struct logistic my_logistic = { - .base = DIST_BASE(&uniform_ops), + .base = LOGISTIC(my_uniform), .mu = dist.param1, .sigma = dist.param2, }; @@ -561,7 +561,7 @@ circpad_distribution_sample(circpad_distribution_t dist) { /* param1 is Alpha, param2 is 1.0/Beta */ const struct log_logistic my_log_logistic = { - .base = DIST_BASE(&log_logistic_ops), + .base = LOG_LOGISTIC(my_log_logistic), .alpha = dist.param1, .beta = dist.param2, }; @@ -571,7 +571,7 @@ circpad_distribution_sample(circpad_distribution_t dist) { /* param1 is 'p' (success probability) */ const struct geometric my_geometric = { - .base = DIST_BASE(&geometric_ops), + .base = GEOMETRIC(my_geometric), .p = dist.param1, }; return dist_sample(&my_geometric.base); @@ -580,7 +580,7 @@ circpad_distribution_sample(circpad_distribution_t dist) { /* param1 is k, param2 is Lambda */ const struct weibull my_weibull = { - .base = DIST_BASE(&weibull_ops), + .base = WEIBULL(my_weibull), .k = dist.param1, .lambda = dist.param2, }; @@ -590,7 +590,7 @@ circpad_distribution_sample(circpad_distribution_t dist) { /* param1 is sigma, param2 is xi, no more params for mu so we use 0 */ const struct genpareto my_genpareto = { - .base = DIST_BASE(&weibull_ops), + .base = GENPARETO(my_weibull), .mu = 0, .sigma = dist.param1, .xi = dist.param2, diff --git a/src/lib/math/prob_distr.h b/src/lib/math/prob_distr.h index 981fc2017d..66acb796fd 100644 --- a/src/lib/math/prob_distr.h +++ b/src/lib/math/prob_distr.h @@ -20,6 +20,8 @@ struct dist { }; #define DIST_BASE(OPS) { .ops = (OPS) } +#define DIST_BASE_TYPED(OPS, OBJ, TYPE) \ + DIST_BASE((OPS) + 0*sizeof(&(OBJ) - (const TYPE *)&(OBJ))) const char *dist_name(const struct dist *); double dist_sample(const struct dist *); @@ -46,6 +48,9 @@ struct geometric { extern const struct dist_ops geometric_ops; +#define GEOMETRIC(OBJ) \ + DIST_BASE_TYPED(&geometric_ops, OBJ, struct geometric) + /* Pareto distribution */ struct genpareto { @@ -57,6 +62,9 @@ struct genpareto { extern const struct dist_ops genpareto_ops; +#define GENPARETO(OBJ) \ + DIST_BASE_TYPED(&genpareto_ops, OBJ, struct genpareto) + /* Weibull distribution */ struct weibull { @@ -67,6 +75,9 @@ struct weibull { extern const struct dist_ops weibull_ops; +#define WEIBULL(OBJ) \ + DIST_BASE_TYPED(&weibull_ops, OBJ, struct weibull) + /* Log-logistic distribution */ struct log_logistic { @@ -77,6 +88,9 @@ struct log_logistic { extern const struct dist_ops log_logistic_ops; +#define LOG_LOGISTIC(OBJ) \ + DIST_BASE_TYPED(&log_logistic_ops, OBJ, struct log_logistic) + /* Logistic distribution */ struct logistic { @@ -87,6 +101,9 @@ struct logistic { extern const struct dist_ops logistic_ops; +#define LOGISTIC(OBJ) \ + DIST_BASE_TYPED(&logistic_ops, OBJ, struct logistic) + /* Uniform distribution */ struct uniform { @@ -97,6 +114,9 @@ struct uniform { extern const struct dist_ops uniform_ops; +#define UNIFORM(OBJ) \ + DIST_BASE_TYPED(&uniform_ops, OBJ, struct uniform) + /** Only by unittests */ #ifdef PROB_DISTR_PRIVATE diff --git a/src/test/test_prob_distr.c b/src/test/test_prob_distr.c index fe3969518c..ff23f01033 100644 --- a/src/test/test_prob_distr.c +++ b/src/test/test_prob_distr.c @@ -943,7 +943,7 @@ static bool test_stochastic_geometric_impl(double p) { const struct geometric geometric = { - .base = DIST_BASE(&geometric_ops), + .base = GEOMETRIC(geometric), .p = p, }; double logP[PSI_DF] = {0}; @@ -1151,32 +1151,32 @@ test_stochastic_uniform(void *arg) (void) arg; const struct uniform uniform01 = { - .base = DIST_BASE(&uniform_ops), + .base = UNIFORM(uniform01), .a = 0, .b = 1, }; const struct uniform uniform_pos = { - .base = DIST_BASE(&uniform_ops), + .base = UNIFORM(uniform_pos), .a = 1.23, .b = 4.56, }; const struct uniform uniform_neg = { - .base = DIST_BASE(&uniform_ops), + .base = UNIFORM(uniform_neg), .a = -10, .b = -1, }; const struct uniform uniform_cross = { - .base = DIST_BASE(&uniform_ops), + .base = UNIFORM(uniform_cross), .a = -1.23, .b = 4.56, }; const struct uniform uniform_subnormal = { - .base = DIST_BASE(&uniform_ops), + .base = UNIFORM(uniform_subnormal), .a = 4e-324, .b = 4e-310, }; const struct uniform uniform_subnormal_cross = { - .base = DIST_BASE(&uniform_ops), + .base = UNIFORM(uniform_subnormal_cross), .a = -4e-324, .b = 4e-310, }; @@ -1202,7 +1202,7 @@ static bool test_stochastic_logistic_impl(double mu, double sigma) { const struct logistic dist = { - .base = DIST_BASE(&logistic_ops), + .base = LOGISTIC(dist), .mu = mu, .sigma = sigma, }; @@ -1215,7 +1215,7 @@ static bool test_stochastic_log_logistic_impl(double alpha, double beta) { const struct log_logistic dist = { - .base = DIST_BASE(&log_logistic_ops), + .base = LOG_LOGISTIC(dist), .alpha = alpha, .beta = beta, }; @@ -1228,7 +1228,7 @@ static bool test_stochastic_weibull_impl(double lambda, double k) { const struct weibull dist = { - .base = DIST_BASE(&weibull_ops), + .base = WEIBULL(dist), .lambda = lambda, .k = k, }; @@ -1248,7 +1248,7 @@ static bool test_stochastic_genpareto_impl(double mu, double sigma, double xi) { const struct genpareto dist = { - .base = DIST_BASE(&genpareto_ops), + .base = GENPARETO(dist), .mu = mu, .sigma = sigma, .xi = xi, From 9728d3f8ac395d7157d30f9b73117b58d704432c Mon Sep 17 00:00:00 2001 From: Taylor R Campbell Date: Thu, 10 Jan 2019 18:11:36 +0000 Subject: [PATCH 0392/2557] Fix wrong bases. --- src/core/or/circuitpadding.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/or/circuitpadding.c b/src/core/or/circuitpadding.c index 4080614338..36fcbb5030 100644 --- a/src/core/or/circuitpadding.c +++ b/src/core/or/circuitpadding.c @@ -551,7 +551,7 @@ circpad_distribution_sample(circpad_distribution_t dist) { /* param1 is Mu, param2 is sigma. */ const struct logistic my_logistic = { - .base = LOGISTIC(my_uniform), + .base = LOGISTIC(my_logistic), .mu = dist.param1, .sigma = dist.param2, }; @@ -590,7 +590,7 @@ circpad_distribution_sample(circpad_distribution_t dist) { /* param1 is sigma, param2 is xi, no more params for mu so we use 0 */ const struct genpareto my_genpareto = { - .base = GENPARETO(my_weibull), + .base = GENPARETO(my_genpareto), .mu = 0, .sigma = dist.param1, .xi = dist.param2, From 4dc189a9047931599f524a7c7dc5b447f3915409 Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Fri, 11 Jan 2019 12:06:14 +0200 Subject: [PATCH 0393/2557] Clarify immutability of global padding machine specs. --- src/core/or/circuit_st.h | 4 +++- src/core/or/circuitpadding.c | 12 ++++++++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/src/core/or/circuit_st.h b/src/core/or/circuit_st.h index 47639940d4..29bcaa098f 100644 --- a/src/core/or/circuit_st.h +++ b/src/core/or/circuit_st.h @@ -185,7 +185,9 @@ struct circuit_t { /** Adaptive Padding state machines: these are immutable. The state machines * that come from the consensus are saved to a global structure, to avoid - * per-circuit allocations. This merely points to the global copy. + * per-circuit allocations. This merely points to the global copy in + * origin_padding_machines or relay_padding_machines that should never + * change or get deallocated. * * Each element of this array corresponds to a different padding machine, * and we can have up to CIRCPAD_MAX_MACHINES such machines. */ diff --git a/src/core/or/circuitpadding.c b/src/core/or/circuitpadding.c index 36fcbb5030..0dadc52139 100644 --- a/src/core/or/circuitpadding.c +++ b/src/core/or/circuitpadding.c @@ -87,11 +87,19 @@ static uint64_t circpad_global_padding_sent; static uint64_t circpad_global_nonpadding_sent; /** This is the list of circpad_machine_spec_t's parsed from consensus and - * torrc that have origin_side == 1 (ie: are for client side) */ + * torrc that have origin_side == 1 (ie: are for client side). + * + * The machines in this smartlist are considered immutable and they are used + * as-is by circuits so they should not change or get deallocated in Tor's + * runtime and as long as circuits are alive. */ STATIC smartlist_t *origin_padding_machines = NULL; /** This is the list of circpad_machine_spec_t's parsed from consensus and - * torrc that have origin_side == 0 (ie: are for relay side) */ + * torrc that have origin_side == 0 (ie: are for relay side). + * + * The machines in this smartlist are considered immutable and they are used + * as-is by circuits so they should not change or get deallocated in Tor's + * runtime and as long as circuits are alive. */ STATIC smartlist_t *relay_padding_machines = NULL; /** Loop over the current padding state machines using loop_var as the From 5dd926caa670002c6d2f15ac78e2282c00c352b6 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 11 Jan 2019 09:08:51 -0500 Subject: [PATCH 0394/2557] Include the beginnings of a FAQ about which timer to use. --- src/lib/time/compat_time.h | 96 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) diff --git a/src/lib/time/compat_time.h b/src/lib/time/compat_time.h index 44fab62de5..c5337e9998 100644 --- a/src/lib/time/compat_time.h +++ b/src/lib/time/compat_time.h @@ -15,6 +15,102 @@ * of tens of milliseconds. */ +/* Q: Should you use monotime or monotime_coarse as your source? + * + * A: Generally, you get better precision with monotime, but better + * performance with monotime_coarse. + * + * Q: Should you use monotime_t or monotime_coarse_t directly? Should you use + * usec? msec? "stamp units?" + * + * A: Using monotime_t and monotime_coarse_t directly is most time-efficient, + * since no conversion needs to happen. But they can potentially use more + * memory than you would need for a usec/msec/"stamp unit" count. + * + * Converting to usec or msec on some platforms, and working with them in + * general, creates a risk of doing a 64-bit division. 64-bit division is + * expensive on 32-bit platforms, which still do exist. + * + * The "stamp unit" type is designed to give a type that is cheap to convert + * from monotime_coarse, has resolution of about 1-2ms, and fits nicely in a + * 32-bit integer. Its downside is that it does not correspond directly + * to a natural unit of time. + * + * There is not much point in using "coarse usec" or "coarse nsec", since the + * current coarse monotime implementations give you on the order of + * milliseconds of precision. + * + * Q: So, what backends is monotime_coarse using? + * + * A: Generally speaking, it uses "whatever monotonic-ish time implemenation + * does not require a context switch." The various implementations provide + * this by having a view of the current time in a read-only memory page that + * is updated with a frequency corresponding to the kernel's tick count. + * + * On Windows, monotime_coarse uses GetCount64() [or GetTickCount() on + * obsolete systems]. MSDN claims that the resolution is "typically in the + * range of 10-16 msec", but it has said that for years. Storing + * monotime_coarse_t uses 8 bytes. + * + * On OSX/iOS, monotime_coarse uses uses mach_approximate_time() where + * available, and falls back to regular monotime. The precision is not + * documented, but the implementation is open-source: it reads from a page + * that the kernel updates. Storing monotime_coarse_t uses 8 bytes. + * + * On unixy systems, monotime_coarse uses clock_gettime() with + * CLOCK_MONOTONIC_COARSE where available, and falls back to CLOCK_MONOTONIC. + * It typically uses vdso tricks to read from a page that the kernel updates. + * Its precision fixed, but you can get it with clock_getres(): on my Linux + * desktop, it claims to be 1 msec, but it will depend on the system HZ + * setting. Storing monotime_coarse_t uses 16 bytes. + * + * [TODO: Try CLOCK_MONOTONIC_FAST on foobsd.] + * + * Q: What backends is regular monotonic time using? + * + * A: In general, regular monotime uses something that requires a system call. + * On platforms where system calls are cheap, you win! Otherwise, you lose. + * + * On Windows, monotonic time uses QuereyPerformanceCounter. Storing + * monotime_t costs 8 bytes. + * + * On OSX/Apple, monotonic time uses mach_absolute_time. Storing + * monotime_t costs 8 bytes. + * + * On unixy systems, monotonic time uses CLOCK_MONOTONIC. Storing + * monotime_t costs 16 bytes. + * + * Q: Tell me about the costs of converting to a 64-bit nsec, usec, or msec + * count. + * + * A: Windows, coarse: Cheap, since it's all multiplication. + * + * Windows, precise: Expensive on 32-bit: it needs 64-bit division. + * + * Apple, all: Expensive on 32-bit: it needs 64-bit division. + * + * Unixy, all: Fairly cheap, since the only division required is dividing + * tv_nsec 1000, and nanoseconds-per-second fits in a 32-bit value. + * + * All, "timestamp units": Cheap everywhere: it never divides. + * + * Q: This is only somewhat related, but how much precision could I hope for + * from a libevent time.? + * + * A: Actually, it's _very_ related if you're timing in order to have a + * timeout happen. + * + * On Windows, it uses select: you could in theory have a microsecond + * resolution, but it usually isn't that accurate. + * + * On OSX, iOS, and BSD, you have kqueue: You could in theory have a nanosecond + * resolution, but it usually isn't that accurate. + * + * On Linux, you have epoll: It has a millisecond resolution. Some recent + * Libevents can also use timerfd for higher resolution if + * EVENT_BASE_FLAG_PRECISE_TIMER is set: Tor doesn't set that flag. + */ + #ifndef TOR_COMPAT_TIME_H #define TOR_COMPAT_TIME_H From df1a8a657eadcc156032aa558ae0395cde878ee8 Mon Sep 17 00:00:00 2001 From: Kris Katterjohn Date: Fri, 4 Jan 2019 13:48:08 -0600 Subject: [PATCH 0395/2557] Actually close the stdout pipe on error in process_unix_exec When cleaning up after an error in process_unix_exec, the stdin pipe was being double closed instead of closing both the stdin and stdout pipes. This occurred in two places. Signed-off-by: Kris Katterjohn --- src/lib/process/process_unix.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/lib/process/process_unix.c b/src/lib/process/process_unix.c index 57ca69a768..dd4ccbf603 100644 --- a/src/lib/process/process_unix.c +++ b/src/lib/process/process_unix.c @@ -183,8 +183,8 @@ process_unix_exec(process_t *process) close(stdin_pipe[1]); /** Cleanup standard out pipe. */ - close(stdin_pipe[0]); - close(stdin_pipe[1]); + close(stdout_pipe[0]); + close(stdout_pipe[1]); return PROCESS_STATUS_ERROR; } @@ -281,8 +281,8 @@ process_unix_exec(process_t *process) close(stdin_pipe[1]); /** Cleanup standard out pipe. */ - close(stdin_pipe[0]); - close(stdin_pipe[1]); + close(stdout_pipe[0]); + close(stdout_pipe[1]); /** Cleanup standard error pipe. */ close(stderr_pipe[0]); From 1bcececdd8e2ff4d6e921270984ba7800c914f64 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Sat, 5 Jan 2019 11:01:07 +0200 Subject: [PATCH 0396/2557] Fix CID 1442277 --- changes/bug28989 | 5 +++++ src/test/test_hs_service.c | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+) create mode 100644 changes/bug28989 diff --git a/changes/bug28989 b/changes/bug28989 new file mode 100644 index 0000000000..3e3ccccaf3 --- /dev/null +++ b/changes/bug28989 @@ -0,0 +1,5 @@ + o Minor bugfixes (unit tests): + - Instead of relying on hs_free_all() to clean up all onion service + objects we created in test_build_descriptors(), deallocate + them one by one. This lets Coverity know that we are not leaking memory + here and fixes CID 1442277. Fixes bug 28989; bugfix on 0.3.5.1-alpha. diff --git a/src/test/test_hs_service.c b/src/test/test_hs_service.c index b2aafc1cd6..3357ca2c56 100644 --- a/src/test/test_hs_service.c +++ b/src/test/test_hs_service.c @@ -289,6 +289,20 @@ helper_create_service(void) return service; } +/* Helper: Deallocate a given service object, its child objects and + * remove it from onion service map. + * */ +static void +helper_destroy_service(hs_service_t *service) +{ + if (!service) + return; + + remove_service(get_hs_service_map(), service); + + hs_service_free(service); +} + /* Helper: Return a newly allocated service object with clients. */ static hs_service_t * helper_create_service_with_clients(int num_clients) @@ -1626,6 +1640,7 @@ test_build_descriptors(void *arg) { int ret; time_t now = time(NULL); + hs_service_t *last_service = NULL; (void) arg; @@ -1650,6 +1665,7 @@ test_build_descriptors(void *arg) * is disabled. */ { hs_service_t *service = helper_create_service(); + last_service = service; service_descriptor_free(service->desc_current); service->desc_current = NULL; @@ -1660,12 +1676,16 @@ test_build_descriptors(void *arg) hs_desc_superencrypted_data_t *superencrypted; superencrypted = &service->desc_current->desc->superencrypted_data; tt_int_op(smartlist_len(superencrypted->clients), OP_EQ, 16); + + helper_destroy_service(service); + last_service = NULL; } /* Generate a valid number of fake auth clients when the number of * clients is zero. */ { hs_service_t *service = helper_create_service_with_clients(0); + last_service = service; service_descriptor_free(service->desc_current); service->desc_current = NULL; @@ -1673,12 +1693,16 @@ test_build_descriptors(void *arg) hs_desc_superencrypted_data_t *superencrypted; superencrypted = &service->desc_current->desc->superencrypted_data; tt_int_op(smartlist_len(superencrypted->clients), OP_EQ, 16); + + helper_destroy_service(service); + last_service = NULL; } /* Generate a valid number of fake auth clients when the number of * clients is not a multiple of 16. */ { hs_service_t *service = helper_create_service_with_clients(20); + last_service = service; service_descriptor_free(service->desc_current); service->desc_current = NULL; @@ -1686,12 +1710,16 @@ test_build_descriptors(void *arg) hs_desc_superencrypted_data_t *superencrypted; superencrypted = &service->desc_current->desc->superencrypted_data; tt_int_op(smartlist_len(superencrypted->clients), OP_EQ, 32); + + helper_destroy_service(service); + last_service = NULL; } /* Do not generate any fake desc client when the number of clients is * a multiple of 16 but not zero. */ { hs_service_t *service = helper_create_service_with_clients(32); + last_service = service; service_descriptor_free(service->desc_current); service->desc_current = NULL; @@ -1699,9 +1727,13 @@ test_build_descriptors(void *arg) hs_desc_superencrypted_data_t *superencrypted; superencrypted = &service->desc_current->desc->superencrypted_data; tt_int_op(smartlist_len(superencrypted->clients), OP_EQ, 32); + + helper_destroy_service(service); + last_service = NULL; } done: + helper_destroy_service(last_service); hs_free_all(); } From b69d28ae830ee0fea642395f0e7c2e89cc8a68d0 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Sat, 5 Jan 2019 11:02:18 +0200 Subject: [PATCH 0397/2557] Fix a typo --- src/test/test_hs_service.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/test_hs_service.c b/src/test/test_hs_service.c index 3357ca2c56..ccb4d93feb 100644 --- a/src/test/test_hs_service.c +++ b/src/test/test_hs_service.c @@ -267,7 +267,7 @@ helper_clone_authorized_client(const hs_service_authorized_client_t *client) /* Helper: Return a newly allocated service object with the identity keypair * sets and the current descriptor. Then register it to the global map. - * Caller should us hs_free_all() to free this service or remove it from the + * Caller should use hs_free_all() to free this service or remove it from the * global map before freeing. */ static hs_service_t * helper_create_service(void) From b269ab5aaeee65a3a0b1e5e0923d9dc7898c232e Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Mon, 14 Jan 2019 15:07:19 +0200 Subject: [PATCH 0398/2557] Remove TODO file. All remaining tasks are now part of #28632. --- PADDING_TODO.txt | 58 ------------------------------------------------ 1 file changed, 58 deletions(-) delete mode 100644 PADDING_TODO.txt diff --git a/PADDING_TODO.txt b/PADDING_TODO.txt deleted file mode 100644 index a2e97f7ca3..0000000000 --- a/PADDING_TODO.txt +++ /dev/null @@ -1,58 +0,0 @@ -TODO sketch for this branch, in rough priority order: - -- Clean up/fix XXX's and FIXMEs - - Test event entry points into circuitpad? - - Most of our events come from completely untested code :/ - -- Compat-breaking changes to be decided/done ASAP - - Option to keep circuits open if machine present - - Specify an ordered preference list of padding machines - - Specify exit policy for machine conditions? - - short_policy_t looks good, except for its flexible array member :/ - - Can we make our own struct with a small, fixed number of policy - entries? Say 3-4? Or is that a bad idea to lose this flexibility? - - Check conditions based on attached streams on the circuit - - Accept should mean "only apply if matched" - - Reject should mean "don't apply if matched" - - If a policy is specified, Reject *:* is implicit default (so reject - policies need an Accept entry). - - With no policy, Accept *:* is implicit default. - - -- Misc fixes: - - Remove circuitsetup machine (but place it in unittests -- they depend on it) - - Circuit RTT measurement will break on var_cell/EXTEND2 cells - - Are there any heuristics we can use here? - - If RELAY_EARLY is only for the first cell of an EXTEND2 series, - we can use that. But the proposal currently says MAY, but not MUST - for this behavior. - -======== 0.3.6 ======== - -- Come up with some good histograms for eg circuit setup fingerprinting, - website fingerprinting, and vanguards usage. - -- Vanguards compatibility for MiddleNodes (via changes to vanguards addon) - -- circpad_machine_validate() function to sanity-check histograms loaded from - consensus/torrc (can also be used to help guide a GA). - - Check bin construction - - no type overflow (start_usec + range_sec, etc) - - no conflicting state transitions (or overlap with cancel events) - - no use of both histograms and iat_dist - - at least two histogram bins - - min_hop vs target_hop - -- Support torrc load+serialization of state machines - - ?? - -- Support consensus load+serialization of state machines - - ?? - -- Prop #265 load balancing - -- Rephist timer stats - - Is this a privacy risk? The adversary could create lots of circuits - to find a layer2 vanguard.. Otherwise they will be spread across middles. - - From 59a88b3c3a31d2fc195d199f6c7aa62a8764e6dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Mon, 14 Jan 2019 17:59:59 +0100 Subject: [PATCH 0399/2557] Rename TYPE to TRANSPORT in PT STATUS messages. See: https://bugs.torproject.org/28181 --- src/feature/client/transports.c | 12 ++++++------ src/test/test_pt.c | 12 ++++++------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/feature/client/transports.c b/src/feature/client/transports.c index 8a8bcd9f7f..a961b21895 100644 --- a/src/feature/client/transports.c +++ b/src/feature/client/transports.c @@ -1236,13 +1236,13 @@ parse_status_line(const char *line, managed_proxy_t *mp) goto done; } - /* We check if we received the TYPE parameter, which is the only *required* - * value. */ - const config_line_t *type = config_line_find(values, "TYPE"); + /* We check if we received the TRANSPORT parameter, which is the only + * *required* value. */ + const config_line_t *type = config_line_find(values, "TRANSPORT"); if (! type) { log_warn(LD_PT, "Managed proxy \"%s\" wrote a STATUS line without " - "TYPE: %s", mp->argv[0], escaped(data)); + "TRANSPORT: %s", mp->argv[0], escaped(data)); goto done; } @@ -1250,8 +1250,8 @@ parse_status_line(const char *line, managed_proxy_t *mp) config_line_prepend(&values, "PT", mp->argv[0]); status_message = kvline_encode(values, KV_QUOTED); - /* We have checked that TYPE is there, we can now emit the STATUS event via - * the control port. */ + /* We have checked that TRANSPORT is there, we can now emit the STATUS event + * via the control port. */ control_event_pt_status(status_message); done: diff --git a/src/test/test_pt.c b/src/test/test_pt.c index 60a044aeca..5edf85815c 100644 --- a/src/test/test_pt.c +++ b/src/test/test_pt.c @@ -311,9 +311,9 @@ process_read_stdout_replacement(process_t *process, buf_t *buffer) buf_add_string(buffer, "LOG SEVERITY=info MESSAGE=\"info msg\"\n"); buf_add_string(buffer, "LOG SEVERITY=debug MESSAGE=\"debug msg\"\n"); } else if (times_called <= 8) { - buf_add_string(buffer, "STATUS TYPE=a K_1=a K_2=b K_3=\"foo bar\"\n"); - buf_add_string(buffer, "STATUS TYPE=b K_1=a K_2=b K_3=\"foo bar\"\n"); - buf_add_string(buffer, "STATUS TYPE=c K_1=a K_2=b K_3=\"foo bar\"\n"); + buf_add_string(buffer, "STATUS TRANSPORT=a K_1=a K_2=b K_3=\"foo bar\"\n"); + buf_add_string(buffer, "STATUS TRANSPORT=b K_1=a K_2=b K_3=\"foo bar\"\n"); + buf_add_string(buffer, "STATUS TRANSPORT=c K_1=a K_2=b K_3=\"foo bar\"\n"); } return (int)buf_datalen(buffer); @@ -445,13 +445,13 @@ test_pt_configure_proxy(void *arg) tt_str_op(smartlist_get(controlevent_msgs, 10), OP_EQ, "650 PT_STATUS " - "PT= TYPE=a K_1=a K_2=b K_3=\"foo bar\"\r\n"); + "PT= TRANSPORT=a K_1=a K_2=b K_3=\"foo bar\"\r\n"); tt_str_op(smartlist_get(controlevent_msgs, 11), OP_EQ, "650 PT_STATUS " - "PT= TYPE=b K_1=a K_2=b K_3=\"foo bar\"\r\n"); + "PT= TRANSPORT=b K_1=a K_2=b K_3=\"foo bar\"\r\n"); tt_str_op(smartlist_get(controlevent_msgs, 12), OP_EQ, "650 PT_STATUS " - "PT= TYPE=c K_1=a K_2=b K_3=\"foo bar\"\r\n"); + "PT= TRANSPORT=c K_1=a K_2=b K_3=\"foo bar\"\r\n"); { /* check that the transport info were saved properly in the tor state */ config_line_t *transport_in_state = NULL; From 136e6e7cebbba00e7bd33cb1bc1e660e01e966fd Mon Sep 17 00:00:00 2001 From: rl1987 Date: Sat, 15 Dec 2018 10:47:30 +0200 Subject: [PATCH 0400/2557] Run shellcheck for stuff in scripts/ as part of 'make check' --- Makefile.am | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/Makefile.am b/Makefile.am index a945130213..272724340b 100644 --- a/Makefile.am +++ b/Makefile.am @@ -215,7 +215,16 @@ doxygen: test: all $(top_builddir)/src/test/test -check-local: check-spaces check-changes check-includes +shellcheck: + # Only use shellcheck if it is present + if command -v shellcheck; then \ + find $(top_srcdir)/scripts/ -name "*.sh" -exec shellcheck {} +; \ + if [ -d "$(top_srcdir)/scripts/test" ]; then \ + shellcheck $(top_srcdir)/scripts/test/cov-diff $(top_builddir)/scripts/test/coverage; \ + fi; \ + fi + +check-local: check-spaces check-changes check-includes shellcheck need-chutney-path: @if test ! -d "$$CHUTNEY_PATH"; then \ From 78574557296dc004e1eab95c4839d5cf40c96638 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Sat, 15 Dec 2018 11:08:16 +0200 Subject: [PATCH 0401/2557] Add shellcheck to Travis CI builds --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 2ea529e252..474f6867fd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -131,6 +131,7 @@ addons: - docbook-xsl - docbook-xml - xmlto + - shellcheck ## (OSX only) homebrew: packages: @@ -154,6 +155,7 @@ addons: ## Always installed, because manual brew installs are hard to get right - asciidoc - xmlto + - shellcheck ## (OSX only) Use the default OSX image ## See https://docs.travis-ci.com/user/reference/osx#os-x-version From 30a925fe96ed69284a915c8e80d0f7ef7b556786 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Sat, 15 Dec 2018 13:53:37 +0200 Subject: [PATCH 0402/2557] Add changes file --- changes/ticket28058 | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 changes/ticket28058 diff --git a/changes/ticket28058 b/changes/ticket28058 new file mode 100644 index 0000000000..00ac595864 --- /dev/null +++ b/changes/ticket28058 @@ -0,0 +1,2 @@ + o Testing: + - Run shellcheck for stuff in scripts/ directory. Closes ticket 28058. From 7fbe7a256383eebae35c8eec31fe88807c82d394 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Mon, 7 Jan 2019 11:09:33 +0200 Subject: [PATCH 0403/2557] In updateFallbackDirs.py, say 'fallback list' instead of 'whitelist' --- changes/bug24953 | 4 ++++ scripts/maint/updateFallbackDirs.py | 15 ++++++++++----- 2 files changed, 14 insertions(+), 5 deletions(-) create mode 100644 changes/bug24953 diff --git a/changes/bug24953 b/changes/bug24953 new file mode 100644 index 0000000000..d142dfd6cc --- /dev/null +++ b/changes/bug24953 @@ -0,0 +1,4 @@ + o Minor bugfixes (fallback scripts): + - In updateFallbackDirs.py, call the filter file a "fallback list" + instead of a "whitelist" in check_existing mode. + Fixes bug 24953; bugfix on 0.3.0.3-alpha. diff --git a/scripts/maint/updateFallbackDirs.py b/scripts/maint/updateFallbackDirs.py index 14372d0e83..e93db7ab3f 100755 --- a/scripts/maint/updateFallbackDirs.py +++ b/scripts/maint/updateFallbackDirs.py @@ -1594,7 +1594,7 @@ class CandidateList(dict): """ Apply the fallback whitelist_obj to this fallback list, passing exact to is_in_whitelist(). """ excluded_count = 0 - logging.debug('Applying whitelist') + logging.debug('Applying fallback list') # parse the whitelist whitelist = self.load_relaylist(whitelist_obj) filtered_fallbacks = [] @@ -1609,14 +1609,18 @@ class CandidateList(dict): else: # exclude excluded_count += 1 - log_excluded('Excluding %s: not in whitelist.', + log_excluded('Excluding %s: not in fallback list.', f._fpr) self.fallbacks = filtered_fallbacks return excluded_count @staticmethod - def summarise_filters(initial_count, excluded_count): - return '/* Whitelist excluded %d of %d candidates. */'%( + def summarise_filters(initial_count, excluded_count, check_existing): + list_type = 'Whitelist' + if check_existing: + list_type = 'Fallback list' + + return '/* %s excluded %d of %d candidates. */'%(list_type, excluded_count, initial_count) # calculate each fallback's measured bandwidth based on the median @@ -2284,7 +2288,8 @@ def list_fallbacks(whitelist, exact=False): # instead, there will be an info-level log during the eligibility check. initial_count = len(candidates.fallbacks) excluded_count = candidates.apply_filter_lists(whitelist, exact=exact) - print candidates.summarise_filters(initial_count, excluded_count) + print candidates.summarise_filters(initial_count, excluded_count, + whitelist['check_existing']) eligible_count = len(candidates.fallbacks) # calculate the measured bandwidth of each relay, From d47e7863bd859b6f7918a55e0ce0c4ee078ab96b Mon Sep 17 00:00:00 2001 From: rl1987 Date: Tue, 8 Jan 2019 13:28:43 +0200 Subject: [PATCH 0404/2557] Depending on script mode, either use 'whitelist' or 'fallback list' --- scripts/maint/updateFallbackDirs.py | 32 +++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/scripts/maint/updateFallbackDirs.py b/scripts/maint/updateFallbackDirs.py index e93db7ab3f..930a0a7275 100755 --- a/scripts/maint/updateFallbackDirs.py +++ b/scripts/maint/updateFallbackDirs.py @@ -1594,7 +1594,11 @@ class CandidateList(dict): """ Apply the fallback whitelist_obj to this fallback list, passing exact to is_in_whitelist(). """ excluded_count = 0 - logging.debug('Applying fallback list') + list_type = 'whitelist' + if whitelist_obj['check_existing']: + list_type = 'fallback list' + + logging.debug('Applying {}'.format(list_type)) # parse the whitelist whitelist = self.load_relaylist(whitelist_obj) filtered_fallbacks = [] @@ -1609,8 +1613,8 @@ class CandidateList(dict): else: # exclude excluded_count += 1 - log_excluded('Excluding %s: not in fallback list.', - f._fpr) + log_excluded('Excluding %s: not in %s.', + f._fpr, list_type) self.fallbacks = filtered_fallbacks return excluded_count @@ -2150,7 +2154,7 @@ class CandidateList(dict): fallback_count))) def summarise_fallbacks(self, eligible_count, operator_count, failed_count, - guard_count, target_count): + guard_count, target_count, check_existing): s = '' # Report: # whether we checked consensus download times @@ -2202,12 +2206,15 @@ class CandidateList(dict): s += '\n' s += '*/' if fallback_count < MIN_FALLBACK_COUNT: + list_type = 'whitelist' + if check_existing: + list_type = 'fallback list' # We must have a minimum number of fallbacks so they are always # reachable, and are in diverse locations s += '\n' s += '#error Fallback Count %d is too low. '%(fallback_count) s += 'Must be at least %d for diversity. '%(MIN_FALLBACK_COUNT) - s += 'Try adding entries to the whitelist, ' + s += 'Try adding entries to %s, '%(list_type) s += 'or setting INCLUDE_UNLISTED_ENTRIES = True.' return s @@ -2215,14 +2222,16 @@ def process_existing(): logging.basicConfig(level=logging.INFO) logging.getLogger('stem').setLevel(logging.INFO) whitelist = {'data': parse_fallback_file(FALLBACK_FILE_NAME), - 'name': FALLBACK_FILE_NAME} + 'name': FALLBACK_FILE_NAME, + 'check_existing' : True} list_fallbacks(whitelist, exact=True) def process_default(): logging.basicConfig(level=logging.WARNING) logging.getLogger('stem').setLevel(logging.WARNING) whitelist = {'data': read_from_file(WHITELIST_FILE_NAME, MAX_LIST_FILE_SIZE), - 'name': WHITELIST_FILE_NAME} + 'name': WHITELIST_FILE_NAME, + 'check_existing': False} list_fallbacks(whitelist, exact=False) ## Main Function @@ -2248,7 +2257,11 @@ def list_fallbacks(whitelist, exact=False): """ Fetches required onionoo documents and evaluates the fallback directory criteria for each of the relays, passing exact to apply_filter_lists(). """ - print "/* type=fallback */" + if whitelist['check_existing']: + print "/* type=fallback */" + else: + print "/* type=whitelist */" + print ("/* version={} */" .format(cleanse_c_multiline_comment(FALLBACK_FORMAT_VERSION))) now = datetime.datetime.utcnow() @@ -2342,7 +2355,8 @@ def list_fallbacks(whitelist, exact=False): if len(candidates.fallbacks) > 0: print candidates.summarise_fallbacks(eligible_count, operator_count, failed_count, guard_count, - target_count) + target_count, + whitelist['check_existing']) else: print '/* No Fallbacks met criteria */' From ba5889011853bc1d86892b5379a87766b0380be5 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 15 Jan 2019 13:15:22 -0500 Subject: [PATCH 0405/2557] Forward-port changelog and releasenotes entries from last Monday --- ChangeLog | 383 +++++++++++++++++++ ReleaseNotes | 1017 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 1400 insertions(+) diff --git a/ChangeLog b/ChangeLog index a1667aa30a..090498c859 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,386 @@ +Changes in version 0.3.3.11 - 2018-01-07 + Tor 0.3.3.11 backports numerous fixes from later versions of Tor. + numerous fixes, including an important fix for anyone using OpenSSL + 1.1.1. Anyone running an earlier version of Tor 0.3.3 should upgrade + to this version, or to a later series. + + As a reminder, support the Tor 0.3.3 series will end on 22 Feb 2019. + We anticipate that this will be the last release of Tor 0.3.3, unless + some major bug is before then. Some time between now and then, users + should switch to either the Tor 0.3.4 series (supported until at least + 10 June 2019), or the Tor 0.3.5 series, which will receive long-term + support until at least 1 Feb 2022. + + o Major bugfixes (OpenSSL, portability, backport from 0.3.5.5-alpha): + - Fix our usage of named groups when running as a TLS 1.3 client in + OpenSSL 1.1.1. Previously, we only initialized EC groups when + running as a relay, which caused clients to fail to negotiate TLS + 1.3 with relays. Fixes bug 28245; bugfix on 0.2.9.15 (when TLS 1.3 + support was added). + + o Major bugfixes (restart-in-process, backport from 0.3.5.1-alpha): + - Fix a use-after-free error that could be caused by passing Tor an + impossible set of options that would fail during options_act(). + Fixes bug 27708; bugfix on 0.3.3.1-alpha. + + o Minor features (continuous integration, backport from 0.3.5.1-alpha): + - Only run one online rust build in Travis, to reduce network + errors. Skip offline rust builds on Travis for Linux gcc, because + they're redundant. Implements ticket 27252. + - Skip gcc on OSX in Travis CI, because it's rarely used. Skip a + duplicate hardening-off build in Travis on Tor 0.2.9. Skip gcc on + Linux with default settings, because all the non-default builds + use gcc on Linux. Implements ticket 27252. + + o Minor features (continuous integration, backport from 0.3.5.3-alpha): + - Use the Travis Homebrew addon to install packages on macOS during + Travis CI. The package list is the same, but the Homebrew addon + does not do a `brew update` by default. Implements ticket 27738. + + o Minor features (fallback directory list, backport from 0.3.5.6-rc): + - Replace the 150 fallbacks originally introduced in Tor + 0.3.3.1-alpha in January 2018 (of which ~115 were still + functional), with a list of 157 fallbacks (92 new, 65 existing, 85 + removed) generated in December 2018. Closes ticket 24803. + + o Minor features (geoip): + - Update geoip and geoip6 to the January 3 2019 Maxmind GeoLite2 + Country database. Closes ticket 29012. + + o Minor features (OpenSSL bug workaround, backport from 0.3.5.7): + - Work around a bug in OpenSSL 1.1.1a, which prevented the TLS 1.3 + key export function from handling long labels. When this bug is + detected, Tor will disable TLS 1.3. We recommend upgrading to a + version of OpenSSL without this bug when it becomes available. + Closes ticket 28973. + + o Minor bugfixes (relay statistics, backport from 0.3.5.7): + - Update relay descriptor on bandwidth changes only when the uptime + is smaller than 24h, in order to reduce the efficiency of guard + discovery attacks. Fixes bug 24104; bugfix on 0.1.1.6-alpha. + + o Minor bugfixes (C correctness, backport from 0.3.5.4-alpha): + - Avoid undefined behavior in an end-of-string check when parsing + the BEGIN line in a directory object. Fixes bug 28202; bugfix + on 0.2.0.3-alpha. + + o Minor bugfixes (code safety, backport from 0.3.5.3-alpha): + - Rewrite our assertion macros so that they no longer suppress the + compiler's -Wparentheses warnings. Fixes bug 27709; bugfix + + o Minor bugfixes (compilation, backport from 0.3.5.5-alpha): + - Initialize a variable unconditionally in aes_new_cipher(), since + some compilers cannot tell that we always initialize it before + use. Fixes bug 28413; bugfix on 0.2.9.3-alpha. + + o Minor bugfixes (directory authority, backport from 0.3.5.4-alpha): + - Log additional info when we get a relay that shares an ed25519 ID + with a different relay, instead making a BUG() warning. Fixes bug + 27800; bugfix on 0.3.2.1-alpha. + + o Minor bugfixes (directory permissions, backport form 0.3.5.3-alpha): + - When a user requests a group-readable DataDirectory, give it to + them. Previously, when the DataDirectory and the CacheDirectory + were the same, the default setting (0) for + CacheDirectoryGroupReadable would override the setting for + DataDirectoryGroupReadable. Fixes bug 26913; bugfix + on 0.3.3.1-alpha. + + o Minor bugfixes (onion service v3, backport from 0.3.5.1-alpha): + - When the onion service directory can't be created or has the wrong + permissions, do not log a stack trace. Fixes bug 27335; bugfix + on 0.3.2.1-alpha. + + o Minor bugfixes (onion service v3, backport from 0.3.5.2-alpha): + - Close all SOCKS request (for the same .onion) if the newly fetched + descriptor is unusable. Before that, we would close only the first + one leaving the other hanging and let to time out by themselves. + Fixes bug 27410; bugfix on 0.3.2.1-alpha. + + o Minor bugfixes (onion service v3, backport from 0.3.5.3-alpha): + - Don't warn so loudly when Tor is unable to decode an onion + descriptor. This can now happen as a normal use case if a client + gets a descriptor with client authorization but the client is not + authorized. Fixes bug 27550; bugfix on 0.3.5.1-alpha. + + o Minor bugfixes (onion service v3, backport from 0.3.5.6-rc): + - When deleting an ephemeral onion service (DEL_ONION), do not close + any rendezvous circuits in order to let the existing client + connections finish by themselves or closed by the application. The + HS v2 is doing that already so now we have the same behavior for + all versions. Fixes bug 28619; bugfix on 0.3.3.1-alpha. + + o Minor bugfixes (HTTP tunnel): + - Fix a bug warning when closing an HTTP tunnel connection due to + an HTTP request we couldn't handle. Fixes bug 26470; bugfix on + 0.3.2.1-alpha. + + o Minor bugfixes (memory leaks, backport from 0.3.5.5-alpha): + - Fix a harmless memory leak in libtorrunner.a. Fixes bug 28419; + bugfix on 0.3.3.1-alpha. Patch from Martin Kepplinger. + + o Minor bugfixes (netflow padding, backport from 0.3.5.1-alpha): + - Ensure circuitmux queues are empty before scheduling or sending + padding. Fixes bug 25505; bugfix on 0.3.1.1-alpha. + + o Minor bugfixes (protover, backport from 0.3.5.3-alpha): + - Reject protocol names containing bytes other than alphanumeric + characters and hyphens ([A-Za-z0-9-]). Fixes bug 27316; bugfix + on 0.2.9.4-alpha. + + o Minor bugfixes (rust, backport from 0.3.5.1-alpha): + - Compute protover votes correctly in the rust version of the + protover code. Previously, the protover rewrite in 24031 allowed + repeated votes from the same voter for the same protocol version + to be counted multiple times in protover_compute_vote(). Fixes bug + 27649; bugfix on 0.3.3.5-rc. + - Reject protover names that contain invalid characters. Fixes bug + 27687; bugfix on 0.3.3.1-alpha. + + o Minor bugfixes (rust, backport from 0.3.5.2-alpha): + - protover_all_supported() would attempt to allocate up to 16GB on + some inputs, leading to a potential memory DoS. Fixes bug 27206; + bugfix on 0.3.3.5-rc. + + o Minor bugfixes (rust, backport from 0.3.5.4-alpha): + - Fix a potential null dereference in protover_all_supported(). Add + a test for it. Fixes bug 27804; bugfix on 0.3.3.1-alpha. + - Return a string that can be safely freed by C code, not one + created by the rust allocator, in protover_all_supported(). Fixes + bug 27740; bugfix on 0.3.3.1-alpha. + - Fix an API mismatch in the rust implementation of + protover_compute_vote(). This bug could have caused crashes on any + directory authorities running Tor with Rust (which we do not yet + recommend). Fixes bug 27741; bugfix on 0.3.3.6. + + o Minor bugfixes (testing, backport from 0.3.5.1-alpha): + - If a unit test running in a subprocess exits abnormally or with a + nonzero status code, treat the test as having failed, even if the + test reported success. Without this fix, memory leaks don't cause + the tests to fail, even with LeakSanitizer. Fixes bug 27658; + bugfix on 0.2.2.4-alpha. + + o Minor bugfixes (testing, backport from 0.3.5.4-alpha): + - Treat backtrace test failures as expected on BSD-derived systems + (NetBSD, OpenBSD, and macOS/Darwin) until we solve bug 17808. + (FreeBSD failures have been treated as expected since 18204 in + 0.2.8.) Fixes bug 27948; bugfix on 0.2.5.2-alpha. + + o Minor bugfixes (unit tests, guard selection, backport from 0.3.5.6-rc): + - Stop leaking memory in an entry guard unit test. Fixes bug 28554; + bugfix on 0.3.0.1-alpha. + + +Changes in version 0.3.4.10 - 2018-01-07 + Tor 0.3.4.9 is the second stable release in its series; it backports + numerous fixes, including an important fix for relays, and for anyone + using OpenSSL 1.1.1. Anyone running an earlier version of Tor 0.3.4 + should upgrade. + + As a reminder, the Tor 0.3.4 series will be supported until 10 June + 2019. Some time between now and then, users should switch to the Tor + 0.3.5 series, which will receive long-term support until at least 1 + Feb 2022. + + o Major bugfixes (OpenSSL, portability, backport from 0.3.5.5-alpha): + - Fix our usage of named groups when running as a TLS 1.3 client in + OpenSSL 1.1.1. Previously, we only initialized EC groups when + running as a relay, which caused clients to fail to negotiate TLS + 1.3 with relays. Fixes bug 28245; bugfix on 0.2.9.15 (when TLS 1.3 + support was added). + + o Major bugfixes (relay, directory, backport from 0.3.5.7): + - Always reactivate linked connections in the main loop so long as + any linked connection has been active. Previously, connections + serving directory information wouldn't get reactivated after the + first chunk of data was sent (usually 32KB), which would prevent + clients from bootstrapping. Fixes bug 28912; bugfix on + 0.3.4.1-alpha. Patch by "cypherpunks3". + + o Minor features (continuous integration, Windows, backport from 0.3.5.6-rc): + - Always show the configure and test logs, and upload them as build + artifacts, when building for Windows using Appveyor CI. + Implements 28459. + + o Minor features (controller, backport from 0.3.5.1-alpha): + - For purposes of CIRC_BW-based dropped cell detection, track half- + closed stream ids, and allow their ENDs, SENDMEs, DATA and path + bias check cells to arrive without counting it as dropped until + either the END arrives, or the windows are empty. Closes + ticket 25573. + + o Minor features (fallback directory list, backport from 0.3.5.6-rc): + - Replace the 150 fallbacks originally introduced in Tor + 0.3.3.1-alpha in January 2018 (of which ~115 were still + functional), with a list of 157 fallbacks (92 new, 65 existing, 85 + removed) generated in December 2018. Closes ticket 24803. + + o Minor features (geoip): + - Update geoip and geoip6 to the November 6 2018 Maxmind GeoLite2 + Country database. Closes ticket 28395. + + o Minor features (OpenSSL bug workaround, backport from 0.3.5.7): + - Work around a bug in OpenSSL 1.1.1a, which prevented the TLS 1.3 + key export function from handling long labels. When this bug is + detected, Tor will disable TLS 1.3. We recommend upgrading to a + version of OpenSSL without this bug when it becomes available. + Closes ticket 28973. + + o Minor bugfixes (compilation, backport from 0.3.5.5-alpha): + - Initialize a variable unconditionally in aes_new_cipher(), since + some compilers cannot tell that we always initialize it before + use. Fixes bug 28413; bugfix on 0.2.9.3-alpha. + + o Minor bugfixes (connection, relay, backport from 0.3.5.5-alpha): + - Avoid a logging a BUG() stacktrace when closing connection held + open because the write side is rate limited but not the read side. + Now, the connection read side is simply shut down until Tor is + able to flush the connection and close it. Fixes bug 27750; bugfix + on 0.3.4.1-alpha. + + o Minor bugfixes (continuous integration, Windows, backport from 0.3.5.5-alpha): + - Manually configure the zstd compiler options, when building using + mingw on Appveyor Windows CI. The MSYS2 mingw zstd package does + not come with a pkg-config file. Fixes bug 28454; bugfix + on 0.3.4.1-alpha. + - Stop using an external OpenSSL install, and stop installing MSYS2 + packages, when building using mingw on Appveyor Windows CI. Fixes + bug 28399; bugfix on 0.3.4.1-alpha. + + o Minor bugfixes (continuous integration, Windows, backport from 0.3.5.6-rc): + - Explicitly specify the path to the OpenSSL library and do not + download OpenSSL from Pacman, but instead use the library that is + already provided by AppVeyor. Fixes bug 28574; bugfix on master. + + o Minor bugfixes (directory permissions, backport form 0.3.5.3-alpha): + - When a user requests a group-readable DataDirectory, give it to + them. Previously, when the DataDirectory and the CacheDirectory + were the same, the default setting (0) for + CacheDirectoryGroupReadable would override the setting for + DataDirectoryGroupReadable. Fixes bug 26913; bugfix + on 0.3.3.1-alpha. + + o Minor bugfixes (memory leaks, backport from 0.3.5.5-alpha): + - Fix a harmless memory leak in libtorrunner.a. Fixes bug 28419; + bugfix on 0.3.3.1-alpha. Patch from Martin Kepplinger. + + o Minor bugfixes (onion service v3, backport from 0.3.5.3-alpha): + - Don't warn so loudly when Tor is unable to decode an onion + descriptor. This can now happen as a normal use case if a client + gets a descriptor with client authorization but the client is not + authorized. Fixes bug 27550; bugfix on 0.3.5.1-alpha. + + o Minor bugfixes (onion service v3, backport from 0.3.5.6-rc): + - When deleting an ephemeral onion service (DEL_ONION), do not close + any rendezvous circuits in order to let the existing client + connections finish by themselves or closed by the application. The + HS v2 is doing that already so now we have the same behavior for + all versions. Fixes bug 28619; bugfix on 0.3.3.1-alpha. + + o Minor bugfixes (relay statistics, backport from 0.3.5.7): + - Update relay descriptor on bandwidth changes only when the uptime + is smaller than 24h, in order to reduce the efficiency of guard + discovery attacks. Fixes bug 24104; bugfix on 0.1.1.6-alpha. + + o Minor bugfixes (unit tests, guard selection, backport from 0.3.5.6-rc): + - Stop leaking memory in an entry guard unit test. Fixes bug 28554; + bugfix on 0.3.0.1-alpha. + + +Changes in version 0.3.5.7 - 2019-01-07 + Tor 0.3.5.7 is the first stable release in its series; it includes + compilation and portability fixes, and a fix for a severe problem + affecting directory caches. + + The Tor 0.3.5 series includes several new features and performance + improvements, including client authorization for v3 onion services, + cleanups to bootstrap reporting, support for improved bandwidth- + measurement tools, experimental support for NSS in place of OpenSSL, + and much more. It also begins a full reorganization of Tor's code + layout, for improved modularity and maintainability in the future. + Finally, there is the usual set of performance improvements and + bugfixes that we try to do in every release series. + + There are a couple of changes in the 0.3.5 that may affect + compatibility. First, the default version for newly created onion + services is now v3. Use the HiddenServiceVersion option if you want to + override this. Second, some log messages related to bootstrapping have + changed; if you use stem, you may need to update to the latest version + so it will recognize them. + + We have designated 0.3.5 as a "long-term support" (LTS) series: we + will continue to patch major bugs in typical configurations of 0.3.5 + until at least 1 Feb 2022. (We do not plan to provide long-term + support for embedding, Rust support, NSS support, running a directory + authority, or unsupported platforms. For these, you will need to stick + with the latest stable release.) + + Below are the changes since 0.3.5.6-rc. For a complete list of changes + since 0.3.4.9, see the ReleaseNotes file. + + o Major bugfixes (relay, directory): + - Always reactivate linked connections in the main loop so long as + any linked connection has been active. Previously, connections + serving directory information wouldn't get reactivated after the + first chunk of data was sent (usually 32KB), which would prevent + clients from bootstrapping. Fixes bug 28912; bugfix on + 0.3.4.1-alpha. Patch by "cypherpunks3". + + o Minor features (compilation): + - When possible, place our warning flags in a separate file, to + avoid flooding verbose build logs. Closes ticket 28924. + + o Minor features (geoip): + - Update geoip and geoip6 to the January 3 2019 Maxmind GeoLite2 + Country database. Closes ticket 29012. + + o Minor features (OpenSSL bug workaround): + - Work around a bug in OpenSSL 1.1.1a, which prevented the TLS 1.3 + key export function from handling long labels. When this bug is + detected, Tor will disable TLS 1.3. We recommend upgrading to a + version of OpenSSL without this bug when it becomes available. + Closes ticket 28973. + + o Minor features (performance): + - Remove about 96% of the work from the function that we run at + startup to test our curve25519_basepoint implementation. Since + this function has yet to find an actual failure, we now only run + it for 8 iterations instead of 200. Based on our profile + information, this change should save around 8% of our startup time + on typical desktops, and may have a similar effect on other + platforms. Closes ticket 28838. + - Stop re-validating our hardcoded Diffie-Hellman parameters on + every startup. Doing this wasted time and cycles, especially on + low-powered devices. Closes ticket 28851. + + o Minor bugfixes (compilation): + - Fix compilation for Android by adding a missing header to + freespace.c. Fixes bug 28974; bugfix on 0.3.5.1-alpha. + + o Minor bugfixes (correctness): + - Fix an unreached code path where we checked the value of + "hostname" inside send_resolved_hostname_cell(). Previously, we + used it before checking it; now we check it first. Fixes bug + 28879; bugfix on 0.1.2.7-alpha. + + o Minor bugfixes (testing): + - Make sure that test_rebind.py actually obeys its timeout, even + when it receives a large number of log messages. Fixes bug 28883; + bugfix on 0.3.5.4-alpha. + - Stop running stem's unit tests as part of "make test-stem", but + continue to run stem's unit and online tests during "make test- + stem-full". Fixes bug 28568; bugfix on 0.2.6.3-alpha. + + o Minor bugfixes (windows services): + - Make Tor start correctly as an NT service again: previously it was + broken by refactoring. Fixes bug 28612; bugfix on 0.3.5.3-alpha. + + o Code simplification and refactoring: + - When parsing a port configuration, make it more obvious to static + analyzer tools that we always initialize the address. Closes + ticket 28881. + + Changes in version 0.3.5.6-rc - 2018-12-18 Tor 0.3.5.6-rc fixes numerous small bugs in earlier versions of Tor. It is the first release candidate in the 0.3.5.x series; if no further diff --git a/ReleaseNotes b/ReleaseNotes index e0a25a74b7..7c8ab01661 100644 --- a/ReleaseNotes +++ b/ReleaseNotes @@ -2,6 +2,1023 @@ This document summarizes new features and bugfixes in each stable release of Tor. If you want to see more detailed descriptions of the changes in each development snapshot, see the ChangeLog file. +Changes in version 0.3.3.11 - 2018-01-07 + Tor 0.3.3.11 backports numerous fixes from later versions of Tor. + numerous fixes, including an important fix for anyone using OpenSSL + 1.1.1. Anyone running an earlier version of Tor 0.3.3 should upgrade + to this version, or to a later series. + + As a reminder, support the Tor 0.3.3 series will end on 22 Feb 2019. + We anticipate that this will be the last release of Tor 0.3.3, unless + some major bug is before then. Some time between now and then, users + should switch to either the Tor 0.3.4 series (supported until at least + 10 June 2019), or the Tor 0.3.5 series, which will receive long-term + support until at least 1 Feb 2022. + + o Major bugfixes (OpenSSL, portability, backport from 0.3.5.5-alpha): + - Fix our usage of named groups when running as a TLS 1.3 client in + OpenSSL 1.1.1. Previously, we only initialized EC groups when + running as a relay, which caused clients to fail to negotiate TLS + 1.3 with relays. Fixes bug 28245; bugfix on 0.2.9.15 (when TLS 1.3 + support was added). + + o Major bugfixes (restart-in-process, backport from 0.3.5.1-alpha): + - Fix a use-after-free error that could be caused by passing Tor an + impossible set of options that would fail during options_act(). + Fixes bug 27708; bugfix on 0.3.3.1-alpha. + + o Minor features (continuous integration, backport from 0.3.5.1-alpha): + - Only run one online rust build in Travis, to reduce network + errors. Skip offline rust builds on Travis for Linux gcc, because + they're redundant. Implements ticket 27252. + - Skip gcc on OSX in Travis CI, because it's rarely used. Skip a + duplicate hardening-off build in Travis on Tor 0.2.9. Skip gcc on + Linux with default settings, because all the non-default builds + use gcc on Linux. Implements ticket 27252. + + o Minor features (continuous integration, backport from 0.3.5.3-alpha): + - Use the Travis Homebrew addon to install packages on macOS during + Travis CI. The package list is the same, but the Homebrew addon + does not do a `brew update` by default. Implements ticket 27738. + + o Minor features (fallback directory list, backport from 0.3.5.6-rc): + - Replace the 150 fallbacks originally introduced in Tor + 0.3.3.1-alpha in January 2018 (of which ~115 were still + functional), with a list of 157 fallbacks (92 new, 65 existing, 85 + removed) generated in December 2018. Closes ticket 24803. + + o Minor features (geoip): + - Update geoip and geoip6 to the January 3 2019 Maxmind GeoLite2 + Country database. Closes ticket 29012. + + o Minor features (OpenSSL bug workaround, backport from 0.3.5.7): + - Work around a bug in OpenSSL 1.1.1a, which prevented the TLS 1.3 + key export function from handling long labels. When this bug is + detected, Tor will disable TLS 1.3. We recommend upgrading to a + version of OpenSSL without this bug when it becomes available. + Closes ticket 28973. + + o Minor bugfixes (relay statistics, backport from 0.3.5.7): + - Update relay descriptor on bandwidth changes only when the uptime + is smaller than 24h, in order to reduce the efficiency of guard + discovery attacks. Fixes bug 24104; bugfix on 0.1.1.6-alpha. + + o Minor bugfixes (C correctness, backport from 0.3.5.4-alpha): + - Avoid undefined behavior in an end-of-string check when parsing + the BEGIN line in a directory object. Fixes bug 28202; bugfix + on 0.2.0.3-alpha. + + o Minor bugfixes (code safety, backport from 0.3.5.3-alpha): + - Rewrite our assertion macros so that they no longer suppress the + compiler's -Wparentheses warnings. Fixes bug 27709; bugfix + + o Minor bugfixes (compilation, backport from 0.3.5.5-alpha): + - Initialize a variable unconditionally in aes_new_cipher(), since + some compilers cannot tell that we always initialize it before + use. Fixes bug 28413; bugfix on 0.2.9.3-alpha. + + o Minor bugfixes (directory authority, backport from 0.3.5.4-alpha): + - Log additional info when we get a relay that shares an ed25519 ID + with a different relay, instead making a BUG() warning. Fixes bug + 27800; bugfix on 0.3.2.1-alpha. + + o Minor bugfixes (directory permissions, backport form 0.3.5.3-alpha): + - When a user requests a group-readable DataDirectory, give it to + them. Previously, when the DataDirectory and the CacheDirectory + were the same, the default setting (0) for + CacheDirectoryGroupReadable would override the setting for + DataDirectoryGroupReadable. Fixes bug 26913; bugfix + on 0.3.3.1-alpha. + + o Minor bugfixes (onion service v3, backport from 0.3.5.1-alpha): + - When the onion service directory can't be created or has the wrong + permissions, do not log a stack trace. Fixes bug 27335; bugfix + on 0.3.2.1-alpha. + + o Minor bugfixes (onion service v3, backport from 0.3.5.2-alpha): + - Close all SOCKS request (for the same .onion) if the newly fetched + descriptor is unusable. Before that, we would close only the first + one leaving the other hanging and let to time out by themselves. + Fixes bug 27410; bugfix on 0.3.2.1-alpha. + + o Minor bugfixes (onion service v3, backport from 0.3.5.3-alpha): + - Don't warn so loudly when Tor is unable to decode an onion + descriptor. This can now happen as a normal use case if a client + gets a descriptor with client authorization but the client is not + authorized. Fixes bug 27550; bugfix on 0.3.5.1-alpha. + + o Minor bugfixes (onion service v3, backport from 0.3.5.6-rc): + - When deleting an ephemeral onion service (DEL_ONION), do not close + any rendezvous circuits in order to let the existing client + connections finish by themselves or closed by the application. The + HS v2 is doing that already so now we have the same behavior for + all versions. Fixes bug 28619; bugfix on 0.3.3.1-alpha. + + o Minor bugfixes (HTTP tunnel): + - Fix a bug warning when closing an HTTP tunnel connection due to + an HTTP request we couldn't handle. Fixes bug 26470; bugfix on + 0.3.2.1-alpha. + + o Minor bugfixes (memory leaks, backport from 0.3.5.5-alpha): + - Fix a harmless memory leak in libtorrunner.a. Fixes bug 28419; + bugfix on 0.3.3.1-alpha. Patch from Martin Kepplinger. + + o Minor bugfixes (netflow padding, backport from 0.3.5.1-alpha): + - Ensure circuitmux queues are empty before scheduling or sending + padding. Fixes bug 25505; bugfix on 0.3.1.1-alpha. + + o Minor bugfixes (protover, backport from 0.3.5.3-alpha): + - Reject protocol names containing bytes other than alphanumeric + characters and hyphens ([A-Za-z0-9-]). Fixes bug 27316; bugfix + on 0.2.9.4-alpha. + + o Minor bugfixes (rust, backport from 0.3.5.1-alpha): + - Compute protover votes correctly in the rust version of the + protover code. Previously, the protover rewrite in 24031 allowed + repeated votes from the same voter for the same protocol version + to be counted multiple times in protover_compute_vote(). Fixes bug + 27649; bugfix on 0.3.3.5-rc. + - Reject protover names that contain invalid characters. Fixes bug + 27687; bugfix on 0.3.3.1-alpha. + + o Minor bugfixes (rust, backport from 0.3.5.2-alpha): + - protover_all_supported() would attempt to allocate up to 16GB on + some inputs, leading to a potential memory DoS. Fixes bug 27206; + bugfix on 0.3.3.5-rc. + + o Minor bugfixes (rust, backport from 0.3.5.4-alpha): + - Fix a potential null dereference in protover_all_supported(). Add + a test for it. Fixes bug 27804; bugfix on 0.3.3.1-alpha. + - Return a string that can be safely freed by C code, not one + created by the rust allocator, in protover_all_supported(). Fixes + bug 27740; bugfix on 0.3.3.1-alpha. + - Fix an API mismatch in the rust implementation of + protover_compute_vote(). This bug could have caused crashes on any + directory authorities running Tor with Rust (which we do not yet + recommend). Fixes bug 27741; bugfix on 0.3.3.6. + + o Minor bugfixes (testing, backport from 0.3.5.1-alpha): + - If a unit test running in a subprocess exits abnormally or with a + nonzero status code, treat the test as having failed, even if the + test reported success. Without this fix, memory leaks don't cause + the tests to fail, even with LeakSanitizer. Fixes bug 27658; + bugfix on 0.2.2.4-alpha. + + o Minor bugfixes (testing, backport from 0.3.5.4-alpha): + - Treat backtrace test failures as expected on BSD-derived systems + (NetBSD, OpenBSD, and macOS/Darwin) until we solve bug 17808. + (FreeBSD failures have been treated as expected since 18204 in + 0.2.8.) Fixes bug 27948; bugfix on 0.2.5.2-alpha. + + o Minor bugfixes (unit tests, guard selection, backport from 0.3.5.6-rc): + - Stop leaking memory in an entry guard unit test. Fixes bug 28554; + bugfix on 0.3.0.1-alpha. + + +Changes in version 0.3.4.10 - 2018-01-07 + Tor 0.3.4.9 is the second stable release in its series; it backports + numerous fixes, including an important fix for relays, and for anyone + using OpenSSL 1.1.1. Anyone running an earlier version of Tor 0.3.4 + should upgrade. + + As a reminder, the Tor 0.3.4 series will be supported until 10 June + 2019. Some time between now and then, users should switch to the Tor + 0.3.5 series, which will receive long-term support until at least 1 + Feb 2022. + + o Major bugfixes (OpenSSL, portability, backport from 0.3.5.5-alpha): + - Fix our usage of named groups when running as a TLS 1.3 client in + OpenSSL 1.1.1. Previously, we only initialized EC groups when + running as a relay, which caused clients to fail to negotiate TLS + 1.3 with relays. Fixes bug 28245; bugfix on 0.2.9.15 (when TLS 1.3 + support was added). + + o Major bugfixes (relay, directory, backport from 0.3.5.7): + - Always reactivate linked connections in the main loop so long as + any linked connection has been active. Previously, connections + serving directory information wouldn't get reactivated after the + first chunk of data was sent (usually 32KB), which would prevent + clients from bootstrapping. Fixes bug 28912; bugfix on + 0.3.4.1-alpha. Patch by "cypherpunks3". + + o Minor features (continuous integration, Windows, backport from 0.3.5.6-rc): + - Always show the configure and test logs, and upload them as build + artifacts, when building for Windows using Appveyor CI. + Implements 28459. + + o Minor features (controller, backport from 0.3.5.1-alpha): + - For purposes of CIRC_BW-based dropped cell detection, track half- + closed stream ids, and allow their ENDs, SENDMEs, DATA and path + bias check cells to arrive without counting it as dropped until + either the END arrives, or the windows are empty. Closes + ticket 25573. + + o Minor features (fallback directory list, backport from 0.3.5.6-rc): + - Replace the 150 fallbacks originally introduced in Tor + 0.3.3.1-alpha in January 2018 (of which ~115 were still + functional), with a list of 157 fallbacks (92 new, 65 existing, 85 + removed) generated in December 2018. Closes ticket 24803. + + o Minor features (geoip): + - Update geoip and geoip6 to the November 6 2018 Maxmind GeoLite2 + Country database. Closes ticket 28395. + + o Minor features (OpenSSL bug workaround, backport from 0.3.5.7): + - Work around a bug in OpenSSL 1.1.1a, which prevented the TLS 1.3 + key export function from handling long labels. When this bug is + detected, Tor will disable TLS 1.3. We recommend upgrading to a + version of OpenSSL without this bug when it becomes available. + Closes ticket 28973. + + o Minor bugfixes (compilation, backport from 0.3.5.5-alpha): + - Initialize a variable unconditionally in aes_new_cipher(), since + some compilers cannot tell that we always initialize it before + use. Fixes bug 28413; bugfix on 0.2.9.3-alpha. + + o Minor bugfixes (connection, relay, backport from 0.3.5.5-alpha): + - Avoid a logging a BUG() stacktrace when closing connection held + open because the write side is rate limited but not the read side. + Now, the connection read side is simply shut down until Tor is + able to flush the connection and close it. Fixes bug 27750; bugfix + on 0.3.4.1-alpha. + + o Minor bugfixes (continuous integration, Windows, backport from 0.3.5.5-alpha): + - Manually configure the zstd compiler options, when building using + mingw on Appveyor Windows CI. The MSYS2 mingw zstd package does + not come with a pkg-config file. Fixes bug 28454; bugfix + on 0.3.4.1-alpha. + - Stop using an external OpenSSL install, and stop installing MSYS2 + packages, when building using mingw on Appveyor Windows CI. Fixes + bug 28399; bugfix on 0.3.4.1-alpha. + + o Minor bugfixes (continuous integration, Windows, backport from 0.3.5.6-rc): + - Explicitly specify the path to the OpenSSL library and do not + download OpenSSL from Pacman, but instead use the library that is + already provided by AppVeyor. Fixes bug 28574; bugfix on master. + + o Minor bugfixes (directory permissions, backport form 0.3.5.3-alpha): + - When a user requests a group-readable DataDirectory, give it to + them. Previously, when the DataDirectory and the CacheDirectory + were the same, the default setting (0) for + CacheDirectoryGroupReadable would override the setting for + DataDirectoryGroupReadable. Fixes bug 26913; bugfix + on 0.3.3.1-alpha. + + o Minor bugfixes (memory leaks, backport from 0.3.5.5-alpha): + - Fix a harmless memory leak in libtorrunner.a. Fixes bug 28419; + bugfix on 0.3.3.1-alpha. Patch from Martin Kepplinger. + + o Minor bugfixes (onion service v3, backport from 0.3.5.3-alpha): + - Don't warn so loudly when Tor is unable to decode an onion + descriptor. This can now happen as a normal use case if a client + gets a descriptor with client authorization but the client is not + authorized. Fixes bug 27550; bugfix on 0.3.5.1-alpha. + + o Minor bugfixes (onion service v3, backport from 0.3.5.6-rc): + - When deleting an ephemeral onion service (DEL_ONION), do not close + any rendezvous circuits in order to let the existing client + connections finish by themselves or closed by the application. The + HS v2 is doing that already so now we have the same behavior for + all versions. Fixes bug 28619; bugfix on 0.3.3.1-alpha. + + o Minor bugfixes (relay statistics, backport from 0.3.5.7): + - Update relay descriptor on bandwidth changes only when the uptime + is smaller than 24h, in order to reduce the efficiency of guard + discovery attacks. Fixes bug 24104; bugfix on 0.1.1.6-alpha. + + o Minor bugfixes (unit tests, guard selection, backport from 0.3.5.6-rc): + - Stop leaking memory in an entry guard unit test. Fixes bug 28554; + bugfix on 0.3.0.1-alpha. + + +Changes in version 0.3.5.7 - 2019-01-07 + Tor 0.3.5.7 is the first stable release in its series; it includes + compilation and portability fixes, and a fix for a severe problem + affecting directory caches. + + The Tor 0.3.5 series includes several new features and performance + improvements, including client authorization for v3 onion services, + cleanups to bootstrap reporting, support for improved bandwidth- + measurement tools, experimental support for NSS in place of OpenSSL, + and much more. It also begins a full reorganization of Tor's code + layout, for improved modularity and maintainability in the future. + Finally, there is the usual set of performance improvements and + bugfixes that we try to do in every release series. + + There are a couple of changes in the 0.3.5 that may affect + compatibility. First, the default version for newly created onion + services is now v3. Use the HiddenServiceVersion option if you want to + override this. Second, some log messages related to bootstrapping have + changed; if you use stem, you may need to update to the latest version + so it will recognize them. + + We have designated 0.3.5 as a "long-term support" (LTS) series: we + will continue to patch major bugs in typical configurations of 0.3.5 + until at least 1 Feb 2022. (We do not plan to provide long-term + support for embedding, Rust support, NSS support, running a directory + authority, or unsupported platforms. For these, you will need to stick + with the latest stable release.) + + Below are the changes since 0.3.4.9. For a complete list of changes + since 0.3.5.6-rc, see the ChangeLog file. + + o Major features (bootstrap): + - Don't report directory progress until after a connection to a + relay or bridge has succeeded. Previously, we'd report 80% + progress based on cached directory information when we couldn't + even connect to the network. Closes ticket 27169. + + o Major features (new code layout): + - Nearly all of Tor's source code has been moved around into more + logical places. The "common" directory is now divided into a set + of libraries in "lib", and files in the "or" directory have been + split into "core" (logic absolutely needed for onion routing), + "feature" (independent modules in Tor), and "app" (to configure + and invoke the rest of Tor). See doc/HACKING/CodeStructure.md for + more information. Closes ticket 26481. + + This refactoring is not complete: although the libraries have been + refactored to be acyclic, the main body of Tor is still too + interconnected. We will attempt to improve this in the future. + + o Major features (onion services v3): + - Implement onion service client authorization at the descriptor + level: only authorized clients can decrypt a service's descriptor + to find out how to contact it. A new torrc option was added to + control this client side: ClientOnionAuthDir . On the + service side, if the "authorized_clients/" directory exists in the + onion service directory path, client configurations are read from + the files within. See the manpage for more details. Closes ticket + 27547. Patch done by Suphanat Chunhapanya (haxxpop). + - Improve revision counter generation in next-gen onion services. + Onion services can now scale by hosting multiple instances on + different hosts without synchronization between them, which was + previously impossible because descriptors would get rejected by + HSDirs. Addresses ticket 25552. + - Version 3 onion services can now use the per-service + HiddenServiceExportCircuitID option to differentiate client + circuits. It communicates with the service by using the HAProxy + protocol to assign virtual IP addresses to inbound client + circuits. Closes ticket 4700. Patch by Mahrud Sayrafi. + + o Major features (onion services, UI change): + - For a newly created onion service, the default version is now 3. + Tor still supports existing version 2 services, but the operator + now needs to set "HiddenServiceVersion 2" in order to create a new + version 2 service. For existing services, Tor now learns the + version by reading the key file. Closes ticket 27215. + + o Major features (portability, cryptography, experimental, TLS): + - Tor now has the option to compile with the NSS library instead of + OpenSSL. This feature is experimental, and we expect that bugs may + remain. It is mainly intended for environments where Tor's + performance is not CPU-bound, and where NSS is already known to be + installed. To try it out, configure Tor with the --enable-nss + flag. Closes tickets 26631, 26815, and 26816. + + If you are experimenting with this option and using an old cached + consensus, Tor may fail to start. To solve this, delete your + "cached-consensus" and "cached-microdesc-consensus" files, + (if present), and restart Tor. + + o Major features (relay, UI change): + - Relays no longer run as exits by default. If the "ExitRelay" + option is auto (or unset), and no exit policy is specified with + ExitPolicy or ReducedExitPolicy, we now treat ExitRelay as 0. + Previously in this case, we allowed exit traffic and logged a + warning message. Closes ticket 21530. Patch by Neel Chauhan. + - Tor now validates that the ContactInfo config option is valid UTF- + 8 when parsing torrc. Closes ticket 27428. + + o Major bugfixes (compilation): + - Fix compilation on ARM (and other less-used CPUs) when compiling + with OpenSSL before 1.1. Fixes bug 27781; bugfix on 0.3.4.1-alpha. + + o Major bugfixes (compilation, rust): + - Rust tests can now build and run successfully with the + --enable-fragile-hardening option enabled. Doing this currently + requires the rust beta channel; it will be possible with stable + rust once Rust version 1.31 is released. Patch from Alex Crichton. + Fixes bugs 27272, 27273, and 27274. Bugfix on 0.3.1.1-alpha. + + o Major bugfixes (directory authority): + - Actually check that the address we get from DirAuthority + configuration line is valid IPv4. Explicitly disallow DirAuthority + address to be a DNS hostname. Fixes bug 26488; bugfix + on 0.1.2.10-rc. + + o Major bugfixes (embedding, main loop): + - When DisableNetwork becomes set, actually disable periodic events + that are already enabled. (Previously, we would refrain from + enabling new ones, but we would leave the old ones turned on.) + Fixes bug 28348; bugfix on 0.3.4.1-alpha. + + o Major bugfixes (main loop, bootstrap): + - Make sure Tor bootstraps and works properly if only the + ControlPort is set. Prior to this fix, Tor would only bootstrap + when a client port was set (Socks, Trans, NATD, DNS or HTTPTunnel + port). Fixes bug 27849; bugfix on 0.3.4.1-alpha. + + o Major bugfixes (onion service v3): + - On an intro point for a version 3 onion service, stop closing + introduction circuits on a NACK. This lets the client decide + whether to reuse the circuit or discard it. Previously, we closed + intro circuits when sending NACKs. Fixes bug 27841; bugfix on + 0.3.2.1-alpha. Patch by Neel Chaunan. + + o Major bugfixes (OpenSSL, portability): + - Fix our usage of named groups when running as a TLS 1.3 client in + OpenSSL 1.1.1. Previously, we only initialized EC groups when + running as a relay, which caused clients to fail to negotiate TLS + 1.3 with relays. Fixes bug 28245; bugfix on 0.2.9.15 (when TLS 1.3 + support was added). + + o Major bugfixes (relay bandwidth statistics): + - When we close relayed circuits, report the data in the circuit + queues as being written in our relay bandwidth stats. This + mitigates guard discovery and other attacks that close circuits + for the explicit purpose of noticing this discrepancy in + statistics. Fixes bug 23512; bugfix on 0.0.8pre3. + + o Major bugfixes (relay): + - When our write bandwidth limit is exhausted, stop writing on the + connection. Previously, we had a typo in the code that would make + us stop reading instead, leading to relay connections being stuck + indefinitely and consuming kernel RAM. Fixes bug 28089; bugfix + on 0.3.4.1-alpha. + - Always reactivate linked connections in the main loop so long as + any linked connection has been active. Previously, connections + serving directory information wouldn't get reactivated after the + first chunk of data was sent (usually 32KB), which would prevent + clients from bootstrapping. Fixes bug 28912; bugfix on + 0.3.4.1-alpha. Patch by "cypherpunks3". + + o Major bugfixes (restart-in-process): + - Fix a use-after-free error that could be caused by passing Tor an + impossible set of options that would fail during options_act(). + Fixes bug 27708; bugfix on 0.3.3.1-alpha. + + o Minor features (admin tools): + - Add a new --key-expiration option to print the expiration date of + the signing cert in an ed25519_signing_cert file. Resolves + issue 19506. + + o Minor features (build): + - If you pass the "--enable-pic" option to configure, Tor will try + to tell the compiler to build position-independent code suitable + to link into a dynamic library. (The default remains -fPIE, for + code suitable for a relocatable executable.) Closes ticket 23846. + + o Minor features (code correctness, testing): + - Tor's build process now includes a "check-includes" make target to + verify that no module of Tor relies on any headers from a higher- + level module. We hope to use this feature over time to help + refactor our codebase. Closes ticket 26447. + + o Minor features (code layout): + - We have a new "lowest-level" error-handling API for use by code + invoked from within the logging module. With this interface, the + logging code is no longer at risk of calling into itself if a + failure occurs while it is trying to log something. Closes + ticket 26427. + + o Minor features (compilation): + - When possible, place our warning flags in a separate file, to + avoid flooding verbose build logs. Closes ticket 28924. + - Tor's configure script now supports a --with-malloc= option to + select your malloc implementation. Supported options are + "tcmalloc", "jemalloc", "openbsd" (deprecated), and "system" (the + default). Addresses part of ticket 20424. Based on a patch from + Alex Xu. + + o Minor features (config): + - The "auto" keyword in torrc is now case-insensitive. Closes + ticket 26663. + + o Minor features (continuous integration): + - Add a Travis CI build for --enable-nss on Linux gcc. Closes + ticket 27751. + - Add new CI job to Travis configuration to run stem-based + integration tests. Closes ticket 27913. + - Use the Travis Homebrew addon to install packages on macOS during + Travis CI. The package list is the same, but the Homebrew addon + does not do a `brew update` by default. Implements ticket 27738. + - Report what program produced the mysterious core file that we + occasionally see on Travis CI during make distcheck. Closes + ticket 28024. + - Don't do a distcheck with --disable-module-dirauth in Travis. + Implements ticket 27252. + - Install libcap-dev and libseccomp2-dev so these optional + dependencies get tested on Travis CI. Closes ticket 26560. + - Only run one online rust build in Travis, to reduce network + errors. Skip offline rust builds on Travis for Linux gcc, because + they're redundant. Implements ticket 27252. + - Skip gcc on OSX in Travis CI, because it's rarely used. Skip a + duplicate hardening-off build in Travis on Tor 0.2.9. Skip gcc on + Linux with default settings, because all the non-default builds + use gcc on Linux. Implements ticket 27252. + + o Minor features (continuous integration, Windows): + - Always show the configure and test logs, and upload them as build + artifacts, when building for Windows using Appveyor CI. + Implements 28459. + - Build tor on Windows Server 2012 R2 and Windows Server 2016 using + Appveyor's CI. Closes ticket 28318. + + o Minor features (controller): + - Emit CIRC_BW events as soon as we detect that we processed an + invalid or otherwise dropped cell on a circuit. This allows + vanguards and other controllers to react more quickly to dropped + cells. Closes ticket 27678. + - For purposes of CIRC_BW-based dropped cell detection, track half- + closed stream ids, and allow their ENDs, SENDMEs, DATA and path + bias check cells to arrive without counting it as dropped until + either the END arrives, or the windows are empty. Closes + ticket 25573. + - Implement a 'GETINFO md/all' controller command to enable getting + all known microdescriptors. Closes ticket 8323. + - The GETINFO command now support an "uptime" argument, to return + Tor's uptime in seconds. Closes ticket 25132. + + o Minor features (denial-of-service avoidance): + - Make our OOM handler aware of the DNS cache so that it doesn't + fill up the memory. This check is important for our DoS mitigation + subsystem. Closes ticket 18642. Patch by Neel Chauhan. + + o Minor features (development): + - Tor's makefile now supports running the "clippy" Rust style tool + on our Rust code. Closes ticket 22156. + + o Minor features (directory authority): + - There is no longer an artificial upper limit on the length of + bandwidth lines. Closes ticket 26223. + - When a bandwidth file is used to obtain the bandwidth measurements, + include this bandwidth file headers in the votes. Closes + ticket 3723. + - Improved support for networks with only a single authority or a + single fallback directory. Patch from Gabriel Somlo. Closes + ticket 25928. + + o Minor features (embedding API): + - The Tor controller API now supports a function to launch Tor with + a preconstructed owning controller FD, so that embedding + applications don't need to manage controller ports and + authentication. Closes ticket 24204. + - The Tor controller API now has a function that returns the name + and version of the backend implementing the API. Closes + ticket 26947. + + o Minor features (fallback directory list): + - Replace the 150 fallbacks originally introduced in Tor + 0.3.3.1-alpha in January 2018 (of which ~115 were still + functional), with a list of 157 fallbacks (92 new, 65 existing, 85 + removed) generated in December 2018. Closes ticket 24803. + + o Minor features (geoip): + - Update geoip and geoip6 to the January 3 2019 Maxmind GeoLite2 + Country database. Closes ticket 29012. + + o Minor features (memory management): + - Get Libevent to use the same memory allocator as Tor, by calling + event_set_mem_functions() during initialization. Resolves + ticket 8415. + + o Minor features (memory usage): + - When not using them, store legacy TAP public onion keys in DER- + encoded format, rather than as expanded public keys. This should + save several megabytes on typical clients. Closes ticket 27246. + + o Minor features (OpenSSL bug workaround): + - Work around a bug in OpenSSL 1.1.1a, which prevented the TLS 1.3 + key export function from handling long labels. When this bug is + detected, Tor will disable TLS 1.3. We recommend upgrading to a + version of OpenSSL without this bug when it becomes available. + Closes ticket 28973. + + o Minor features (OpenSSL): + - When possible, use RFC5869 HKDF implementation from OpenSSL rather + than our own. Resolves ticket 19979. + + o Minor features (performance): + - Remove about 96% of the work from the function that we run at + startup to test our curve25519_basepoint implementation. Since + this function has yet to find an actual failure, we now only run + it for 8 iterations instead of 200. Based on our profile + information, this change should save around 8% of our startup time + on typical desktops, and may have a similar effect on other + platforms. Closes ticket 28838. + - Stop re-validating our hardcoded Diffie-Hellman parameters on + every startup. Doing this wasted time and cycles, especially on + low-powered devices. Closes ticket 28851. + + o Minor features (Rust, code quality): + - Improve rust code quality in the rust protover implementation by + making it more idiomatic. Includes changing an internal API to + take &str instead of &String. Closes ticket 26492. + + o Minor features (testing): + - Add scripts/test/chutney-git-bisect.sh, for bisecting using + chutney. Implements ticket 27211. + + o Minor features (tor-resolve): + - The tor-resolve utility can now be used with IPv6 SOCKS proxies. + Side-effect of the refactoring for ticket 26526. + + o Minor features (UI): + - Log each included configuration file or directory as we read it, + to provide more visibility about where Tor is reading from. Patch + from Unto Sten; closes ticket 27186. + - Lower log level of "Scheduler type KIST has been enabled" to INFO. + Closes ticket 26703. + + o Minor bugfixes (32-bit OSX and iOS, timing): + - Fix an integer overflow bug in our optimized 32-bit millisecond- + difference algorithm for 32-bit Apple platforms. Previously, it + would overflow when calculating the difference between two times + more than 47 days apart. Fixes part of bug 27139; bugfix + on 0.3.4.1-alpha. + - Improve the precision of our 32-bit millisecond difference + algorithm for 32-bit Apple platforms. Fixes part of bug 27139; + bugfix on 0.3.4.1-alpha. + - Relax the tolerance on the mainloop/update_time_jumps test when + running on 32-bit Apple platforms. Fixes part of bug 27139; bugfix + on 0.3.4.1-alpha. + + o Minor bugfixes (bootstrap): + - Try harder to get descriptors in non-exit test networks, by using + the mid weight for the third hop when there are no exits. Fixes + bug 27237; bugfix on 0.2.6.2-alpha. + + o Minor bugfixes (C correctness): + - Avoid casting smartlist index to int implicitly, as it may trigger + a warning (-Wshorten-64-to-32). Fixes bug 26282; bugfix on + 0.2.3.13-alpha, 0.2.7.1-alpha and 0.2.1.1-alpha. + - Use time_t for all values in + predicted_ports_prediction_time_remaining(). Rework the code that + computes difference between durations/timestamps. Fixes bug 27165; + bugfix on 0.3.1.1-alpha. + + o Minor bugfixes (client, memory usage): + - When not running as a directory cache, there is no need to store + the text of the current consensus networkstatus in RAM. + Previously, however, clients would store it anyway, at a cost of + over 5 MB. Now, they do not. Fixes bug 27247; bugfix + on 0.3.0.1-alpha. + + o Minor bugfixes (client, ReachableAddresses): + - Instead of adding a "reject *:*" line to ReachableAddresses when + loading the configuration, add one to the policy after parsing it + in parse_reachable_addresses(). This prevents extra "reject *.*" + lines from accumulating on reloads. Fixes bug 20874; bugfix on + 0.1.1.5-alpha. Patch by Neel Chauhan. + + o Minor bugfixes (code quality): + - Rename sandbox_getaddrinfo() and other functions to no longer + misleadingly suggest that they are sandbox-only. Fixes bug 26525; + bugfix on 0.2.7.1-alpha. + + o Minor bugfixes (code safety): + - Rewrite our assertion macros so that they no longer suppress the + compiler's -Wparentheses warnings. Fixes bug 27709; bugfix + on 0.0.6. + + o Minor bugfixes (compilation): + - Initialize a variable unconditionally in aes_new_cipher(), since + some compilers cannot tell that we always initialize it before + use. Fixes bug 28413; bugfix on 0.2.9.3-alpha. + + o Minor bugfixes (configuration): + - Refuse to start with relative file paths and RunAsDaemon set + (regression from the fix for bug 22731). Fixes bug 28298; bugfix + on 0.3.3.1-alpha. + + o Minor bugfixes (configuration, Onion Services): + - In rend_service_parse_port_config(), disallow any input to remain + after address-port pair was parsed. This will catch address and + port being whitespace-separated by mistake of the user. Fixes bug + 27044; bugfix on 0.2.9.10. + + o Minor bugfixes (connection, relay): + - Avoid a logging a BUG() stacktrace when closing connection held + open because the write side is rate limited but not the read side. + Now, the connection read side is simply shut down until Tor is + able to flush the connection and close it. Fixes bug 27750; bugfix + on 0.3.4.1-alpha. + + o Minor bugfixes (continuous integration, Windows): + - Stop reinstalling identical packages in our Windows CI. Fixes bug + 27464; bugfix on 0.3.4.1-alpha. + - Install only the necessary mingw packages during our appveyor + builds. This change makes the build a little faster, and prevents + a conflict with a preinstalled mingw openssl that appveyor now + ships. Fixes bugs 27765 and 27943; bugfix on 0.3.4.2-alpha. + - Explicitly specify the path to the OpenSSL library and do not + download OpenSSL from Pacman, but instead use the library that is + already provided by AppVeyor. Fixes bug 28574; bugfix on master. + - Manually configure the zstd compiler options, when building using + mingw on Appveyor Windows CI. The MSYS2 mingw zstd package does + not come with a pkg-config file. Fixes bug 28454; bugfix + on 0.3.4.1-alpha. + - Stop using an external OpenSSL install, and stop installing MSYS2 + packages, when building using mingw on Appveyor Windows CI. Fixes + bug 28399; bugfix on 0.3.4.1-alpha. + + o Minor bugfixes (controller): + - Consider all routerinfo errors other than "not a server" to be + transient for the purpose of "GETINFO exit-policy/*" controller + request. Print stacktrace in the unlikely case of failing to + recompute routerinfo digest. Fixes bug 27034; bugfix + on 0.3.4.1-alpha. + + o Minor bugfixes (correctness): + - Fix an unreached code path where we checked the value of + "hostname" inside send_resolved_hostname_cell(). Previously, we + used it before checking it; now we check it first. Fixes bug + 28879; bugfix on 0.1.2.7-alpha. + + o Minor bugfixes (directory connection shutdown): + - Avoid a double-close when shutting down a stalled directory + connection. Fixes bug 26896; bugfix on 0.3.4.1-alpha. + + o Minor bugfixes (directory permissions): + - When a user requests a group-readable DataDirectory, give it to + them. Previously, when the DataDirectory and the CacheDirectory + were the same, the default setting (0) for + CacheDirectoryGroupReadable would override the setting for + DataDirectoryGroupReadable. Fixes bug 26913; bugfix + on 0.3.3.1-alpha. + + o Minor bugfixes (HTTP tunnel): + - Fix a bug warning when closing an HTTP tunnel connection due to an + HTTP request we couldn't handle. Fixes bug 26470; bugfix + on 0.3.2.1-alpha. + + o Minor bugfixes (ipv6): + - In addrs_in_same_network_family(), we choose the subnet size based + on the IP version (IPv4 or IPv6). Previously, we chose a fixed + subnet size of /16 for both IPv4 and IPv6 addresses. Fixes bug + 15518; bugfix on 0.2.3.1-alpha. Patch by Neel Chauhan. + + o Minor bugfixes (Linux seccomp2 sandbox): + - Permit the "shutdown()" system call, which is apparently used by + OpenSSL under some circumstances. Fixes bug 28183; bugfix + on 0.2.5.1-alpha. + + o Minor bugfixes (logging): + - Stop talking about the Named flag in log messages. Clients have + ignored the Named flag since 0.3.2. Fixes bug 28441; bugfix + on 0.3.2.1-alpha. + - As a precaution, do an early return from log_addr_has_changed() if + Tor is running as client. Also, log a stack trace for debugging as + this function should only be called when Tor runs as server. Fixes + bug 26892; bugfix on 0.1.1.9-alpha. + - Refrain from mentioning bug 21018 in the logs, as it is already + fixed. Fixes bug 25477; bugfix on 0.2.9.8. + + o Minor bugfixes (logging, documentation): + - When SafeLogging is enabled, scrub IP address in + channel_tls_process_netinfo_cell(). Also, add a note to manpage + that scrubbing is not guaranteed on loglevels below Notice. Fixes + bug 26882; bugfix on 0.2.4.10-alpha. + + o Minor bugfixes (memory leaks): + - Fix a harmless memory leak in libtorrunner.a. Fixes bug 28419; + bugfix on 0.3.3.1-alpha. Patch from Martin Kepplinger. + - Fix a small memory leak when calling Tor with --dump-config. Fixes + bug 27893; bugfix on 0.3.2.1-alpha. + + o Minor bugfixes (netflow padding): + - Ensure circuitmux queues are empty before scheduling or sending + padding. Fixes bug 25505; bugfix on 0.3.1.1-alpha. + + o Minor bugfixes (onion service v2): + - Log at level "info", not "warning", in the case that we do not + have a consensus when a .onion request comes in. This can happen + normally while bootstrapping. Fixes bug 27040; bugfix + on 0.2.8.2-alpha. + + o Minor bugfixes (onion service v3): + - When deleting an ephemeral onion service (DEL_ONION), do not close + any rendezvous circuits in order to let the existing client + connections finish by themselves or closed by the application. The + HS v2 is doing that already so now we have the same behavior for + all versions. Fixes bug 28619; bugfix on 0.3.3.1-alpha. + - Build the service descriptor's signing key certificate before + uploading, so we always have a fresh one: leaving no chances for + it to expire service side. Fixes bug 27838; bugfix + on 0.3.2.1-alpha. + - Stop dumping a stack trace when trying to connect to an intro + point without having a descriptor for it. Fixes bug 27774; bugfix + on 0.3.2.1-alpha. + - When selecting a v3 rendezvous point, don't only look at the + protover, but also check whether the curve25519 onion key is + present. This way we avoid picking a relay that supports the v3 + rendezvous but for which we don't have the microdescriptor. Fixes + bug 27797; bugfix on 0.3.2.1-alpha. + - Close all SOCKS request (for the same .onion) if the newly fetched + descriptor is unusable. Before that, we would close only the first + one leaving the other hanging and let to time out by themselves. + Fixes bug 27410; bugfix on 0.3.2.1-alpha. + - When the onion service directory can't be created or has the wrong + permissions, do not log a stack trace. Fixes bug 27335; bugfix + on 0.3.2.1-alpha. + - When replacing a descriptor in the client cache, make sure to + close all client introduction circuits for the old descriptor, so + we don't end up with unusable leftover circuits. Fixes bug 27471; + bugfix on 0.3.2.1-alpha. + + o Minor bugfixes (OS compatibility): + - Properly handle configuration changes that move a listener to/from + wildcard IP address. If the first attempt to bind a socket fails, + close the old listener and try binding the socket again. Fixes bug + 17873; bugfix on 0.0.8pre-1. + + o Minor bugfixes (performance):: + - Rework node_is_a_configured_bridge() to no longer call + node_get_all_orports(), which was performing too many memory + allocations. Fixes bug 27224; bugfix on 0.2.3.9. + + o Minor bugfixes (protover): + - Reject protocol names containing bytes other than alphanumeric + characters and hyphens ([A-Za-z0-9-]). Fixes bug 27316; bugfix + on 0.2.9.4-alpha. + + o Minor bugfixes (protover, rust): + - Reject extra commas in version strings. Fixes bug 27197; bugfix + on 0.3.3.3-alpha. + - protover_all_supported() would attempt to allocate up to 16GB on + some inputs, leading to a potential memory DoS. Fixes bug 27206; + bugfix on 0.3.3.5-rc. + - Compute protover votes correctly in the rust version of the + protover code. Previously, the protover rewrite in 24031 allowed + repeated votes from the same voter for the same protocol version + to be counted multiple times in protover_compute_vote(). Fixes bug + 27649; bugfix on 0.3.3.5-rc. + - Reject protover names that contain invalid characters. Fixes bug + 27687; bugfix on 0.3.3.1-alpha. + + o Minor bugfixes (relay shutdown, systemd): + - Notify systemd of ShutdownWaitLength so it can be set to longer + than systemd's TimeoutStopSec. In Tor's systemd service file, set + TimeoutSec to 60 seconds to allow Tor some time to shut down. + Fixes bug 28113; bugfix on 0.2.6.2-alpha. + + o Minor bugfixes (relay statistics): + - Update relay descriptor on bandwidth changes only when the uptime + is smaller than 24h, in order to reduce the efficiency of guard + discovery attacks. Fixes bug 24104; bugfix on 0.1.1.6-alpha. + + o Minor bugfixes (relay): + - Consider the fact that we'll be making direct connections to our + entry and guard nodes when computing the fraction of nodes that + have their descriptors. Also, if we are using bridges and there is + at least one bridge with a full descriptor, treat the fraction of + guards available as 100%. Fixes bug 25886; bugfix on 0.2.4.10-alpha. + Patch by Neel Chauhan. + - Update the message logged on relays when DirCache is disabled. + Since 0.3.3.5-rc, authorities require DirCache (V2Dir) for the + Guard flag. Fixes bug 24312; bugfix on 0.3.3.5-rc. + + o Minor bugfixes (testing): + - Stop running stem's unit tests as part of "make test-stem", but + continue to run stem's unit and online tests during "make test- + stem-full". Fixes bug 28568; bugfix on 0.2.6.3-alpha. + - Stop leaking memory in an entry guard unit test. Fixes bug 28554; + bugfix on 0.3.0.1-alpha. + - Make the hs_service tests use the same time source when creating + the introduction point and when testing it. Now tests work better + on very slow systems like ARM or Travis. Fixes bug 27810; bugfix + on 0.3.2.1-alpha. + - Revise the "conditionvar_timeout" test so that it succeeds even on + heavily loaded systems where the test threads are not scheduled + within 200 msec. Fixes bug 27073; bugfix on 0.2.6.3-alpha. + - Fix two unit tests to work when HOME environment variable is not + set. Fixes bug 27096; bugfix on 0.2.8.1-alpha. + - If a unit test running in a subprocess exits abnormally or with a + nonzero status code, treat the test as having failed, even if the + test reported success. Without this fix, memory leaks don't cause + the tests to fail, even with LeakSanitizer. Fixes bug 27658; + bugfix on 0.2.2.4-alpha. + - When logging a version mismatch in our openssl_version tests, + report the actual offending version strings. Fixes bug 26152; + bugfix on 0.2.9.1-alpha. + - Fix forking tests on Windows when there is a space somewhere in + the path. Fixes bug 26437; bugfix on 0.2.2.4-alpha. + + o Minor bugfixes (Windows): + - Correctly identify Windows 8.1, Windows 10, and Windows Server + 2008 and later from their NT versions. Fixes bug 28096; bugfix on + 0.2.2.34; reported by Keifer Bly. + - On recent Windows versions, the GetVersionEx() function may report + an earlier Windows version than the running OS. To avoid user + confusion, add "[or later]" to Tor's version string on affected + versions of Windows. Fixes bug 28096; bugfix on 0.2.2.34; reported + by Keifer Bly. + - Remove Windows versions that were never supported by the + GetVersionEx() function. Stop duplicating the latest Windows + version in get_uname(). Fixes bug 28096; bugfix on 0.2.2.34; + reported by Keifer Bly. + + o Code simplification and refactoring: + - When parsing a port configuration, make it more obvious to static + analyzer tools that we always initialize the address. Closes + ticket 28881. + - Divide more large Tor source files -- especially ones that span + multiple areas of functionality -- into smaller parts, including + onion.c and main.c. Closes ticket 26747. + - Divide the "routerparse.c" module into separate modules for each + group of parsed objects. Closes ticket 27924. + - Move protover_rust.c to the same place protover.c was moved to. + Closes ticket 27814. + - Split directory.c into separate pieces for client, server, and + common functionality. Closes ticket 26744. + - Split the non-statistics-related parts from the rephist.c and + geoip.c modules. Closes ticket 27892. + - Split the router.c file into relay-only and shared components, to + help with future modularization. Closes ticket 27864. + - Divide the routerlist.c and dirserv.c modules into smaller parts. + Closes ticket 27799. + - 'updateFallbackDirs.py' now ignores the blacklist file, as it's not + longer needed. Closes ticket 26502. + - Include paths to header files within Tor are now qualified by + directory within the top-level src directory. + - Many structures have been removed from the centralized "or.h" + header, and moved into their own headers. This will allow us to + reduce the number of places in the code that rely on each + structure's contents and layout. Closes ticket 26383. + - Remove ATTR_NONNULL macro from codebase. Resolves ticket 26527. + - Remove GetAdaptersAddresses_fn_t. The code that used it was + removed as part of the 26481 refactor. Closes ticket 27467. + - Rework Tor SOCKS server code to use Trunnel and benefit from + autogenerated functions for parsing and generating SOCKS wire + format. New implementation is cleaner, more maintainable and + should be less prone to heartbleed-style vulnerabilities. + Implements a significant fraction of ticket 3569. + - Split sampled_guards_update_from_consensus() and + select_entry_guard_for_circuit() into subfunctions. In + entry_guards_update_primary() unite three smartlist enumerations + into one and move smartlist comparison code out of the function. + Closes ticket 21349. + - Tor now assumes that you have standards-conformant stdint.h and + inttypes.h headers when compiling. Closes ticket 26626. + - Unify our bloom filter logic. Previously we had two copies of this + code: one for routerlist filtering, and one for address set + calculations. Closes ticket 26510. + - Use the simpler strcmpstart() helper in + rend_parse_v2_service_descriptor instead of strncmp(). Closes + ticket 27630. + - Utility functions that can perform a DNS lookup are now wholly + separated from those that can't, in separate headers and C + modules. Closes ticket 26526. + + o Documentation: + - In the tor-resolve(1) manpage, fix the reference to socks- + extensions.txt by adding a web URL. Resolves ticket 27853. + - Mention that we require Python to be 2.7 or newer for some + integration tests that we ship with Tor. Resolves ticket 27677. + - Copy paragraph and URL to Tor's code of conduct document from + CONTRIBUTING to new CODE_OF_CONDUCT file. Resolves ticket 26638. + - Remove old instructions from INSTALL document. Closes ticket 26588. + - Warn users that they should not include MyFamily line(s) in their + torrc when running Tor bridge. Closes ticket 26908. + + o Removed features: + - Tor no longer supports building with the dmalloc library. For + debugging memory issues, we suggest using gperftools or msan + instead. Closes ticket 26426. + - Tor no longer attempts to run on Windows environments without the + GetAdaptersAddresses() function. This function has existed since + Windows XP, which is itself already older than we support. + - Remove Tor2web functionality for version 2 onion services. The + Tor2webMode and Tor2webRendezvousPoints options are now obsolete. + (This feature was never shipped in vanilla Tor and it was only + possible to use this feature by building the support at compile + time. Tor2webMode is not implemented for version 3 onion services.) + Closes ticket 26367. + + o Testing: + - Increase logging and tag all log entries with timestamps in + test_rebind.py. Provides diagnostics for issue 28229. + + o Code simplification and refactoring (shared random, dirauth): + - Change many tor_assert() to use BUG() instead. The idea is to not + crash a dirauth but rather scream loudly with a stacktrace and let + it continue run. The shared random subsystem is very resilient and + if anything wrong happens with it, at worst a non coherent value + will be put in the vote and discarded by the other authorities. + Closes ticket 19566. + + o Documentation (onion services): + - Improve HSv3 client authorization by making some options more + explicit and detailed. Closes ticket 28026. Patch by Mike Tigas. + - Document in the man page that changing ClientOnionAuthDir value or + adding a new file in the directory will not work at runtime upon + sending a HUP if Sandbox 1. Closes ticket 28128. + - Note in the man page that the only real way to fully revoke an + onion service v3 client authorization is by restarting the tor + process. Closes ticket 28275. + + Changes in version 0.3.4.9 - 2018-11-02 Tor 0.3.4.9 is the second stable release in its series; it backports numerous fixes, including a fix for a bandwidth management bug that From 6440fdb8de64f7a42924e2ff1ee2d960c18e6888 Mon Sep 17 00:00:00 2001 From: Mike Perry Date: Tue, 15 Jan 2019 22:50:52 +0000 Subject: [PATCH 0406/2557] Changes file for Ticket 28142. --- changes/ticket28142 | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 changes/ticket28142 diff --git a/changes/ticket28142 b/changes/ticket28142 new file mode 100644 index 0000000000..b74b2bd47e --- /dev/null +++ b/changes/ticket28142 @@ -0,0 +1,10 @@ + o Major features (circuit padding): + - Implement preliminary support for the circuit padding portion of + Proposal 254. The implementation supports Adaptive Padding (aka + WTF-PAD) state machines for use between experimental clients and + relays. Support is also provided for APE-style state machines that + use probability distributions instead of histograms to specify + inter-packet delay. At the moment, Tor does not provide any padding + state machines that are used in normal operation -- this feature + exists solely for experimentation in this release. Closes + ticket 28142. From 31e784915eaf90bf5794ff4f96989b9ff509ec8e Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 15 Jan 2019 18:26:08 -0500 Subject: [PATCH 0407/2557] Use [ -n "$x" ] in place of [ ! -z "$x" ] in chutney-git-bisect Closes 29099. --- scripts/test/chutney-git-bisect.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/test/chutney-git-bisect.sh b/scripts/test/chutney-git-bisect.sh index dc1319a27a..dcf8ab1102 100755 --- a/scripts/test/chutney-git-bisect.sh +++ b/scripts/test/chutney-git-bisect.sh @@ -15,21 +15,21 @@ # Skips the test if fails (default no skip). CHUTNEY_TRIES=3 -if [ ! -z "$1" ]; then +if [ -n "$1" ]; then CHUTNEY_TRIES="$1" fi -if [ ! -z "$2" ]; then +if [ -n "$2" ]; then cd "$2" || exit fi CHUTNEY_TEST_CMD="make test-network-all" -if [ ! -z "$3" ]; then +if [ -n "$3" ]; then CHUTNEY_TEST_CMD="$CHUTNEY_PATH/tools/test-network.sh --flavour $3" fi CHUTNEY_SKIP_ON_FAIL_CMD="true" -if [ ! -z "$4" ]; then +if [ -n "$4" ]; then CHUTNEY_SKIP_ON_FAIL_CMD="$CHUTNEY_PATH/tools/test-network.sh --flavour $4" fi From 2dd23086f13ecfc9843a9612e208a3592ac46141 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 15 Jan 2019 19:18:00 -0500 Subject: [PATCH 0408/2557] Remove fallback scripts and whitelist They have been extracted to a new fallback-scripts.git repository. Closes ticket 27914. --- changes/ticket27914 | 4 + scripts/maint/fallback.whitelist | 1064 --------- scripts/maint/generateFallbackDirLine.py | 38 - scripts/maint/lookupFallbackDirContact.py | 28 - scripts/maint/updateFallbackDirs.py | 2383 --------------------- 5 files changed, 4 insertions(+), 3513 deletions(-) create mode 100644 changes/ticket27914 delete mode 100644 scripts/maint/fallback.whitelist delete mode 100755 scripts/maint/generateFallbackDirLine.py delete mode 100755 scripts/maint/lookupFallbackDirContact.py delete mode 100755 scripts/maint/updateFallbackDirs.py diff --git a/changes/ticket27914 b/changes/ticket27914 new file mode 100644 index 0000000000..433e9657af --- /dev/null +++ b/changes/ticket27914 @@ -0,0 +1,4 @@ + o Removed features: + - The scripts used to generate and maintain the list of fallback + directories have been extracted into a new "fallback-scripts" + repository. Closes ticket 27914. diff --git a/scripts/maint/fallback.whitelist b/scripts/maint/fallback.whitelist deleted file mode 100644 index 60d3e7bb85..0000000000 --- a/scripts/maint/fallback.whitelist +++ /dev/null @@ -1,1064 +0,0 @@ -# updateFallbackDirs.py directory mirror whitelist -# -# At least one of these keys must match for a directory mirror to be included -# in the fallback list: -# id -# ipv4 -# ipv6 -# The ports and nickname are ignored. Missing or extra ipv6 addresses -# are ignored. -# -# The latest relay details from Onionoo are included in the generated list. -# -# To check the hard-coded fallback list (for testing), use: -# $ updateFallbackDirs.py check_existing -# -# If a relay operator wants their relay to be a FallbackDir, -# enter the following information here: -# : orport= id= ( ipv6=[]: )? -# or use: -# scripts/maint/generateFallbackDirLine.py fingerprint ... - -# https://lists.torproject.org/pipermail/tor-relays/2015-December/008362.html -# https://trac.torproject.org/projects/tor/ticket/22321#comment:22 -78.47.18.110:443 orport=80 id=F8D27B163B9247B232A2EEE68DD8B698695C28DE ipv6=[2a01:4f8:120:4023::110]:80 # fluxe3 -131.188.40.188:1443 orport=80 id=EBE718E1A49EE229071702964F8DB1F318075FF8 ipv6=[2001:638:a000:4140::ffff:188]:80 # fluxe4 - -# https://lists.torproject.org/pipermail/tor-relays/2015-December/008366.html -5.39.88.19:9030 orport=9001 id=7CB8C31432A796731EA7B6BF4025548DFEB25E0C ipv6=[2001:41d0:8:9a13::1]:9050 - -# https://lists.torproject.org/pipermail/tor-relays/2015-December/008370.html -# https://lists.torproject.org/pipermail/tor-relays/2016-January/008517.html -# https://lists.torproject.org/pipermail/tor-relays/2016-January/008555.html -212.47.237.95:9030 orport=9001 id=3F5D8A879C58961BB45A3D26AC41B543B40236D6 -212.47.237.95:9130 orport=9101 id=6FB38EB22E57EF7ED5EF00238F6A48E553735D88 - -# https://lists.torproject.org/pipermail/tor-relays/2015-December/008372.html -# IPv6 tunnel available on request (is this a good idea?) -108.53.208.157:80 orport=443 id=4F0DB7E687FC7C0AE55C8F243DA8B0EB27FBF1F2 - -# https://lists.torproject.org/pipermail/tor-relays/2015-December/008373.html -167.114.35.28:9030 orport=9001 id=E65D300F11E1DB12C534B0146BDAB6972F1A8A48 - -# https://lists.torproject.org/pipermail/tor-relays/2015-December/008378.html -144.76.14.145:110 orport=143 id=14419131033443AE6E21DA82B0D307F7CAE42BDB ipv6=[2a01:4f8:190:9490::dead]:443 - -# https://lists.torproject.org/pipermail/tor-relays/2015-December/008379.html -# Email sent directly to teor, verified using relay contact info -91.121.84.137:4951 orport=4051 id=6DE61A6F72C1E5418A66BFED80DFB63E4C77668F - -# https://lists.torproject.org/pipermail/tor-relays/2015-December/008381.html -# Sent additional emails to teor with updated relays -81.7.11.96:9030 orport=9001 id=8FA37B93397015B2BC5A525C908485260BE9F422 # Doedel22 -# 9F5068310818ED7C70B0BC4087AB55CB12CB4377 not found in current consensus -178.254.19.101:80 orport=443 id=F9246DEF2B653807236DA134F2AEAB103D58ABFE # Freebird31 -178.254.19.101:9030 orport=9001 id=0C475BA4D3AA3C289B716F95954CAD616E50C4E5 # Freebird32 -81.7.14.253:9001 orport=443 id=1AE039EE0B11DB79E4B4B29CBA9F752864A0259E # Ichotolot60 -81.7.11.186:1080 orport=443 id=B86137AE9681701901C6720E55C16805B46BD8E3 # BeastieJoy60 -85.25.213.211:465 orport=80 id=CE47F0356D86CF0A1A2008D97623216D560FB0A8 # BeastieJoy61 -85.25.159.65:995 orport=80 id=52BFADA8BEAA01BA46C8F767F83C18E2FE50C1B9 # BeastieJoy63 -81.7.3.67:993 orport=443 id=A2E6BB5C391CD46B38C55B4329C35304540771F1 # BeastieJoy62 -81.7.14.31:9001 orport=443 id=7600680249A22080ECC6173FBBF64D6FCF330A61 # Ichotolot62 - -# https://lists.torproject.org/pipermail/tor-relays/2015-December/008382.html -51.255.33.237:9091 orport=9001 id=A360C21FA87FFA2046D92C17086A6B47E5C68109 - -# https://lists.torproject.org/pipermail/tor-relays/2015-December/008383.html -81.7.14.246:80 orport=443 id=CE75BF0972ADD52AF8807602374E495C815DB304 ipv6=[2a02:180:a:51::dead]:443 - -# https://lists.torproject.org/pipermail/tor-relays/2015-December/008384.html -# Sent additional email to teor with fingerprint change -149.202.98.161:80 orport=443 id=FC64CD763F8C1A319BFBBF62551684F4E1E42332 ipv6=[2001:41d0:8:4528::161]:443 -193.111.136.162:80 orport=443 id=C79552275DFCD486B942510EF663ED36ACA1A84B ipv6=[2001:4ba0:cafe:10d0::1]:443 - -# https://lists.torproject.org/pipermail/tor-relays/2015-December/008416.html -185.100.84.212:80 orport=443 id=330CD3DB6AD266DC70CDB512B036957D03D9BC59 ipv6=[2a06:1700:0:7::1]:443 - -# https://lists.torproject.org/pipermail/tor-relays/2015-December/008417.html -178.16.208.56:80 orport=443 id=2CDCFED0142B28B002E89D305CBA2E26063FADE2 ipv6=[2a00:1c20:4089:1234:cd49:b58a:9ebe:67ec]:443 -178.16.208.57:80 orport=443 id=92CFD9565B24646CAC2D172D3DB503D69E777B8A ipv6=[2a00:1c20:4089:1234:7825:2c5d:1ecd:c66f]:443 - -# https://lists.torproject.org/pipermail/tor-relays/2016-January/008513.html -178.62.173.203:9030 orport=9001 id=DD85503F2D1F52EF9EAD621E942298F46CD2FC10 ipv6=[2a03:b0c0:0:1010::a4:b001]:9001 - -# https://lists.torproject.org/pipermail/tor-relays/2016-January/008534.html -5.9.110.236:9030 orport=9001 id=0756B7CD4DFC8182BE23143FAC0642F515182CEB ipv6=[2a01:4f8:162:51e2::2]:9001 - -# https://lists.torproject.org/pipermail/tor-relays/2016-January/008542.html -178.62.199.226:80 orport=443 id=CBEFF7BA4A4062045133C053F2D70524D8BBE5BE ipv6=[2a03:b0c0:2:d0::b7:5001]:443 - -# Email sent directly to teor, verified using relay contact info -94.23.204.175:9030 orport=9001 id=5665A3904C89E22E971305EE8C1997BCA4123C69 - -# Email sent directly to teor, verified using relay contact info -171.25.193.77:80 orport=443 id=A10C4F666D27364036B562823E5830BC448E046A ipv6=[2001:67c:289c:3::77]:443 -171.25.193.78:80 orport=443 id=A478E421F83194C114F41E94F95999672AED51FE ipv6=[2001:67c:289c:3::78]:443 -171.25.193.20:80 orport=443 id=DD8BD7307017407FCC36F8D04A688F74A0774C02 ipv6=[2001:67c:289c::20]:443 -# same machine as DD8BD7307017407FCC36F8D04A688F74A0774C02 -171.25.193.25:80 orport=443 id=185663B7C12777F052B2C2D23D7A239D8DA88A0F ipv6=[2001:67c:289c::25]:443 - -# Email sent directly to teor, verified using relay contact info -212.47.229.2:9030 orport=9001 id=20462CBA5DA4C2D963567D17D0B7249718114A68 ipv6=[2001:bc8:4400:2100::f03]:9001 -93.115.97.242:9030 orport=9001 id=B5212DB685A2A0FCFBAE425738E478D12361710D -46.28.109.231:9030 orport=9001 id=F70B7C5CD72D74C7F9F2DC84FA9D20D51BA13610 ipv6=[2a02:2b88:2:1::4205:1]:9001 - -# Email sent directly to teor, verified using relay contact info -85.235.250.88:80 orport=443 id=72B2B12A3F60408BDBC98C6DF53988D3A0B3F0EE # TykRelay01 -185.96.88.29:80 orport=443 id=86C281AD135058238D7A337D546C902BE8505DDE # TykRelay051 -# This fallback opted-in in previous releases, then changed its details, -# and so we blacklisted it. Now we want to whitelist changes. -# Assume details update is permanent -185.96.180.29:80 orport=443 id=F93D8F37E35C390BCAD9F9069E13085B745EC216 # TykRelay06 - -# Email sent directly to teor, verified using relay contact info -185.11.180.67:80 orport=9001 id=794D8EA8343A4E820320265D05D4FA83AB6D1778 - -# Email sent directly to teor, verified using relay contact info -178.16.208.62:80 orport=443 id=5CF8AFA5E4B0BB88942A44A3F3AAE08C3BDFD60B ipv6=[2a00:1c20:4089:1234:a6a4:2926:d0af:dfee]:443 -46.165.221.166:80 orport=443 id=EE5F897C752D46BCFF531641B853FC6BC78DD4A7 -178.16.208.60:80 orport=443 id=B44FBE5366AD98B46D829754FA4AC599BAE41A6A ipv6=[2a00:1c20:4089:1234:67bc:79f3:61c0:6e49]:443 -178.16.208.55:80 orport=443 id=C4AEA05CF380BAD2230F193E083B8869B4A29937 ipv6=[2a00:1c20:4089:1234:7b2c:11c5:5221:903e]:443 -178.16.208.61:80 orport=443 id=3B52392E2256C35CDCF7801FF898FC88CE6D431A ipv6=[2a00:1c20:4089:1234:2712:a3d0:666b:88a6]:443 -81.89.96.88:80 orport=443 id=55ED4BB49F6D3F36D8D9499BE43500E017A5EF82 ipv6=[2a02:180:1:1:14c5:b0b7:2d7d:5f3a]:443 -209.222.8.196:80 orport=443 id=C86D2F3DEFE287A0EEB28D4887AF14E35C172733 ipv6=[2001:19f0:1620:41c1:426c:5adf:2ed5:4e88]:443 -81.89.96.89:80 orport=443 id=28651F419F5A1CF74511BB500C58112192DD4943 ipv6=[2a02:180:1:1:2ced:24e:32ea:a03b]:443 -46.165.221.166:9030 orport=9001 id=8C7106C880FE8AA1319DD71B59623FCB8914C9F1 -178.16.208.62:80 orport=443 id=5CF8AFA5E4B0BB88942A44A3F3AAE08C3BDFD60B ipv6=[2a00:1c20:4089:1234:a6a4:2926:d0af:dfee]:443" -46.165.221.166:80 orport=443 id=EE5F897C752D46BCFF531641B853FC6BC78DD4A7 -178.16.208.60:80 orport=443 id=B44FBE5366AD98B46D829754FA4AC599BAE41A6A ipv6=[2a00:1c20:4089:1234:67bc:79f3:61c0:6e49]:443 -178.16.208.55:80 orport=443 id=C4AEA05CF380BAD2230F193E083B8869B4A29937 ipv6=[2a00:1c20:4089:1234:7b2c:11c5:5221:903e]:443 -178.16.208.61:80 orport=443 id=3B52392E2256C35CDCF7801FF898FC88CE6D431A ipv6=[2a00:1c20:4089:1234:2712:a3d0:666b:88a6]:443 -81.89.96.88:80 orport=443 id=55ED4BB49F6D3F36D8D9499BE43500E017A5EF82 ipv6=[2a02:180:1:1:14c5:b0b7:2d7d:5f3a]:443 -209.222.8.196:80 orport=443 id=C86D2F3DEFE287A0EEB28D4887AF14E35C172733 ipv6=[2001:19f0:1620:41c1:426c:5adf:2ed5:4e88]:443 -81.89.96.89:80 orport=443 id=28651F419F5A1CF74511BB500C58112192DD4943 ipv6=[2a02:180:1:1:2ced:24e:32ea:a03b]:443 -46.165.221.166:9030 orport=9001 id=8C7106C880FE8AA1319DD71B59623FCB8914C9F1 -178.16.208.56:80 orport=443 id=2CDCFED0142B28B002E89D305CBA2E26063FADE2 ipv6=[2a00:1c20:4089:1234:cd49:b58a:9ebe:67ec]:443 -178.16.208.58:80 orport=443 id=A4C98CEA3F34E05299417E9F885A642C88EF6029 ipv6=[2a00:1c20:4089:1234:cdae:1b3e:cc38:3d45]:443 -178.16.208.57:80 orport=443 id=92CFD9565B24646CAC2D172D3DB503D69E777B8A ipv6=[2a00:1c20:4089:1234:7825:2c5d:1ecd:c66f]:443 -178.16.208.59:80 orport=443 id=136F9299A5009A4E0E96494E723BDB556FB0A26B ipv6=[2a00:1c20:4089:1234:bff6:e1bb:1ce3:8dc6]:443 - -# Email sent directly to teor, verified using relay contact info -5.39.76.158:80 orport=443 id=C41F60F8B00E7FEF5CCC5BC6BB514CA1B8AAB651 - -# Email sent directly to teor, verified using relay contact info -109.163.234.2:80 orport=443 id=14F92FF956105932E9DEC5B82A7778A0B1BD9A52 -109.163.234.4:80 orport=443 id=4888770464F0E900EFEF1BA181EA873D13F7713C -109.163.234.5:80 orport=443 id=5EB8D862E70981B8690DEDEF546789E26AB2BD24 -109.163.234.7:80 orport=443 id=23038A7F2845EBA2234ECD6651BD4A7762F51B18 -109.163.234.8:80 orport=443 id=0818DAE0E2DDF795AEDEAC60B15E71901084F281 -109.163.234.9:80 orport=443 id=ABF7FBF389C9A747938B639B20E80620B460B2A9 -62.102.148.67:80 orport=443 id=4A0C3E177AF684581EF780981AEAF51A98A6B5CF -# Assume details update is permanent -77.247.181.166:80 orport=443 id=77131D7E2EC1CA9B8D737502256DA9103599CE51 # CriticalMass -77.247.181.164:80 orport=443 id=204DFD2A2C6A0DC1FA0EACB495218E0B661704FD # HaveHeart -77.247.181.162:80 orport=443 id=7BFB908A3AA5B491DA4CA72CCBEE0E1F2A939B55 # sofia - -# https://twitter.com/biotimylated/status/718994247500718080 -212.47.252.149:9030 orport=9001 id=2CAC39BAA996791CEFAADC9D4754D65AF5EB77C0 - -# Email sent directly to teor, verified using relay contact info -46.165.230.5:80 orport=443 id=A0F06C2FADF88D3A39AA3072B406F09D7095AC9E - -# Email sent directly to teor, verified using relay contact info -94.242.246.24:23 orport=8080 id=EC116BCB80565A408CE67F8EC3FE3B0B02C3A065 ipv6=[2a01:608:ffff:ff07::1:24]:9004 -94.242.246.23:443 orport=9001 id=F65E0196C94DFFF48AFBF2F5F9E3E19AAE583FD0 ipv6=[2a01:608:ffff:ff07::1:23]:9003 -85.248.227.164:444 orport=9002 id=B84F248233FEA90CAD439F292556A3139F6E1B82 ipv6=[2a00:1298:8011:212::164]:9004 -85.248.227.163:443 orport=9001 id=C793AB88565DDD3C9E4C6F15CCB9D8C7EF964CE9 ipv6=[2a00:1298:8011:212::163]:9003 - -# Email sent directly to teor, verified using relay contact info -148.251.190.229:9030 orport=9010 id=BF0FB582E37F738CD33C3651125F2772705BB8E8 ipv6=[2a01:4f8:211:c68::2]:9010 - -# Email sent directly to teor, verified using relay contact info -5.79.68.161:81 orport=443 id=9030DCF419F6E2FBF84F63CBACBA0097B06F557E ipv6=[2001:1af8:4700:a012:1::1]:443 -5.79.68.161:9030 orport=9001 id=B7EC0C02D7D9F1E31B0C251A6B058880778A0CD1 ipv6=[2001:1af8:4700:a012:1::1]:9001 - -# Email sent directly to teor, verified using relay contact info -62.210.92.11:9030 orport=9001 id=0266B0660F3F20A7D1F3D8335931C95EF50F6C6B ipv6=[2001:bc8:338c::1]:9001 -62.210.92.11:9130 orport=9101 id=387B065A38E4DAA16D9D41C2964ECBC4B31D30FF ipv6=[2001:bc8:338c::1]:9101 - -# Email sent directly to teor, verified using relay contact info -188.165.194.195:9030 orport=9001 id=49E7AD01BB96F6FE3AB8C3B15BD2470B150354DF - -# Message sent directly to teor, verified using relay contact info -95.215.44.110:80 orport=443 id=D56AA4A1AA71961F5279FB70A6DCF7AD7B993EB5 -95.215.44.122:80 orport=443 id=998D8FE06B867AA3F8D257A7D28FFF16964D53E2 -95.215.44.111:80 orport=443 id=A7C7FD510B20BC8BE8F2A1D911364E1A23FBD09F - -# Email sent directly to teor, verified using relay contact info -86.59.119.88:80 orport=443 id=ACD889D86E02EDDAB1AFD81F598C0936238DC6D0 -86.59.119.83:80 orport=443 id=FC9AC8EA0160D88BCCFDE066940D7DD9FA45495B - -# Email sent directly to teor, verified using relay contact info -193.11.164.243:9030 orport=9001 id=FFA72BD683BC2FCF988356E6BEC1E490F313FB07 ipv6=[2001:6b0:7:125::243]:9001 -109.105.109.162:52860 orport=60784 id=32EE911D968BE3E016ECA572BB1ED0A9EE43FC2F ipv6=[2001:948:7:2::163]:5001 - -# Email sent directly to teor, verified using relay contact info -146.0.32.144:9030 orport=9001 id=35E8B344F661F4F2E68B17648F35798B44672D7E - -# Email sent directly to teor, verified using relay contact info -46.252.26.2:45212 orport=49991 id=E589316576A399C511A9781A73DA4545640B479D - -# Email sent directly to teor, verified using relay contact info -89.187.142.208:80 orport=443 id=64186650FFE4469EBBE52B644AE543864D32F43C - -# Email sent directly to teor -# Assume details update is permanent -212.51.134.123:9030 orport=9001 id=50586E25BE067FD1F739998550EDDCB1A14CA5B2 # Jans - -# Email sent directly to teor, verified using relay contact info -46.101.143.173:80 orport=443 id=F960DF50F0FD4075AC9B505C1D4FFC8384C490FB - -# Email sent directly to teor, verified using relay contact info -193.171.202.146:9030 orport=9001 id=01A9258A46E97FF8B2CAC7910577862C14F2C524 - -# Email sent directly to teor, verified using relay contact info -# Assume details update is permanent -197.231.221.211:9030 orport=443 id=BC630CBBB518BE7E9F4E09712AB0269E9DC7D626 # IPredator - -# Email sent directly to teor, verified using relay contact info -185.61.138.18:8080 orport=4443 id=2541759BEC04D37811C2209A88E863320271EC9C - -# Email sent directly to teor, verified using relay contact info -193.11.114.45:9031 orport=9002 id=80AAF8D5956A43C197104CEF2550CD42D165C6FB -193.11.114.43:9030 orport=9001 id=12AD30E5D25AA67F519780E2111E611A455FDC89 ipv6=[2001:6b0:30:1000::99]:9050 -193.11.114.46:9032 orport=9003 id=B83DC1558F0D34353BB992EF93AFEAFDB226A73E - -# Email sent directly to teor, verified using relay contact info -138.201.250.33:9012 orport=9011 id=2BA2C8E96B2590E1072AECE2BDB5C48921BF8510 - -# Email sent directly to teor, verified using relay contact info -37.221.162.226:9030 orport=9001 id=D64366987CB39F61AD21DBCF8142FA0577B92811 - -# Email sent directly to teor, verified using relay contact info -91.219.237.244:80 orport=443 id=92ECC9E0E2AF81BB954719B189AC362E254AD4A5 - -# Email sent directly to teor, verified using relay contact info -185.21.100.50:9030 orport=9001 id=58ED9C9C35E433EE58764D62892B4FFD518A3CD0 ipv6=[2a00:1158:2:cd00:0:74:6f:72]:443 - -# Email sent directly to teor, verified using relay contact info -193.35.52.53:9030 orport=9001 id=DAA39FC00B196B353C2A271459C305C429AF09E4 - -# Email sent directly to teor, verified using relay contact info -134.119.3.164:9030 orport=9001 id=D1B8AAA98C65F3DF7D8BB3AF881CAEB84A33D8EE - -# Email sent directly to teor, verified using relay contact info -173.212.254.192:31336 orport=31337 id=99E246DB480B313A3012BC3363093CC26CD209C7 - -# Email sent directly to teor, verified using relay contact info -178.62.22.36:80 orport=443 id=A0766C0D3A667A3232C7D569DE94A28F9922FCB1 ipv6=[2a03:b0c0:1:d0::174:1]:9050 -188.166.23.127:80 orport=443 id=8672E8A01B4D3FA4C0BBE21C740D4506302EA487 ipv6=[2a03:b0c0:2:d0::27b:7001]:9050 -198.199.64.217:80 orport=443 id=B1D81825CFD7209BD1B4520B040EF5653C204A23 ipv6=[2604:a880:400:d0::1a9:b001]:9050 -159.203.32.149:80 orport=443 id=55C7554AFCEC1062DCBAC93E67B2E03C6F330EFC ipv6=[2604:a880:cad:d0::105:f001]:9050 - -# Email sent directly to teor, verified using relay contact info -5.196.31.80:9030 orport=9900 id=DFB2EB472643FAFCD5E73D2E37D51DB67203A695 ipv6=[2001:41d0:52:400::a65]:9900 - -# Email sent directly to teor, verified using relay contact info -188.138.112.60:1433 orport=1521 id=C414F28FD2BEC1553024299B31D4E726BEB8E788 - -# Email sent directly to teor, verified using relay contact info -213.61.66.118:9031 orport=9001 id=30648BC64CEDB3020F4A405E4AB2A6347FB8FA22 -213.61.66.117:9032 orport=9002 id=6E44A52E3D1FF7683FE5C399C3FB5E912DE1C6B4 -213.61.66.115:9034 orport=9004 id=480CCC94CEA04D2DEABC0D7373868E245D4C2AE2 -213.61.66.116:9033 orport=9003 id=A9DEB920B42B4EC1DE6249034039B06D61F38690 - -# Email sent directly to teor, verified using relay contact info -136.243.187.165:9030 orport=443 id=1AC65257D7BFDE7341046625470809693A8ED83E - -# Email sent directly to teor, verified using relay contact info -212.47.230.49:9030 orport=9001 id=3D6D0771E54056AEFC28BB1DE816951F11826E97 - -# Email sent directly to teor, verified using relay contact info -192.99.55.69:80 orport=443 id=0682DE15222A4A4A0D67DBA72A8132161992C023 -192.99.59.140:80 orport=443 id=3C9148DA49F20654730FAC83FFF693A4D49D0244 -51.254.215.13:80 orport=443 id=73C30C8ABDD6D9346C822966DE73B9F82CB6178A -51.254.215.129:80 orport=443 id=7B4491D05144B20AE8519AE784B94F0525A8BB79 -192.99.59.139:80 orport=443 id=82EC878ADA7C205146B9F5193A7310867FAA0D7B -51.254.215.124:80 orport=443 id=98999EBE89B5FA9AA0C58421F0B46C3D0AF51CBA -51.254.214.208:80 orport=443 id=C3F0D1417848EAFC41277A73DEB4A9F2AEC23DDF -192.99.59.141:80 orport=443 id=F45426551795B9DA78BEDB05CD5F2EACED8132E4 -192.99.59.14:80 orport=443 id=161A1B29A37EBF096D2F8A9B1E176D6487FE42AE - -# Email sent directly to teor, verified using relay contact info -151.80.42.103:9030 orport=9001 id=9007C1D8E4F03D506A4A011B907A9E8D04E3C605 ipv6=[2001:41d0:e:f67::114]:9001 - -# Email sent directly to teor, verified using relay contact info -176.31.159.231:80 orport=443 id=D5DBCC0B4F029F80C7B8D33F20CF7D97F0423BB1 -176.31.159.230:80 orport=443 id=631748AFB41104D77ADBB7E5CD4F8E8AE876E683 -195.154.79.128:80 orport=443 id=C697612CA5AED06B8D829FCC6065B9287212CB2F -195.154.9.161:80 orport=443 id=B6295A9960F89BD0C743EEBC5670450EA6A34685 -46.148.18.74:8080 orport=443 id=6CACF0B5F03C779672F3C5C295F37C8D234CA3F7 - -# Email sent directly to teor, verified using relay contact info -37.187.102.108:80 orport=443 id=F4263275CF54A6836EE7BD527B1328836A6F06E1 ipv6=[2001:41d0:a:266c::1]:443 # EvilMoe -212.47.241.21:80 orport=443 id=892F941915F6A0C6E0958E52E0A9685C190CF45C # EvilMoe - -# Email sent directly to teor, verified using relay contact info -212.129.38.254:9030 orport=9001 id=FDF845FC159C0020E2BDDA120C30C5C5038F74B4 - -# Email sent directly to teor -37.157.195.87:8030 orport=443 id=12FD624EE73CEF37137C90D38B2406A66F68FAA2 # thanatosCZ -5.189.169.190:8030 orport=8080 id=8D79F73DCD91FC4F5017422FAC70074D6DB8DD81 # thanatosDE - -# Email sent directly to teor, verified using relay contact info -37.187.7.74:80 orport=443 id=AEA43CB1E47BE5F8051711B2BF01683DB1568E05 ipv6=[2001:41d0:a:74a::1]:443 - -# Email sent directly to teor, verified using relay contact info -185.66.250.141:9030 orport=9001 id=B1726B94885CE3AC3910CA8B60622B97B98E2529 - -# Email sent directly to teor, verified using relay contact info -# Email sent directly to Phoul -185.104.120.7:9030 orport=443 id=445F1C853966624FB3CF1E12442570DC553CC2EC ipv6=[2a06:3000::120:7]:443 -185.104.120.2:9030 orport=21 id=518FF8708698E1DA09C823C36D35DF89A2CAD956 ipv6=[2a06:3000::120:2]:443 -185.104.120.4:9030 orport=9001 id=F92B3CB9BBE0CB22409843FB1AE4DBCD5EFAC835 ipv6=[2a06:3000::120:4]:443 -185.104.120.3:9030 orport=21 id=707C1B61AC72227B34487B56D04BAA3BA1179CE8 ipv6=[2a06:3000::120:3]:443 -185.104.120.5:80 orport=443 id=3EBDF84DE3B16F0EBF7D51450F07913A02EFDA6C ipv6=[2a06:3000::120:5]:443 -185.104.120.60:80 orport=443 id=D05C9C7068EB5A45F670D5E38A14907EE6223141 ipv6=[2a06:3000::120:60]:443 - - -# Email sent directly to teor, verified using relay contact info -37.187.102.186:9030 orport=9001 id=489D94333DF66D57FFE34D9D59CC2D97E2CB0053 ipv6=[2001:41d0:a:26ba::1]:9001 - -# Email sent directly to teor, verified using relay contact info -198.96.155.3:8080 orport=5001 id=BCEDF6C193AA687AE471B8A22EBF6BC57C2D285E - -# Email sent directly to teor, verified using relay contact info -212.83.154.33:8888 orport=443 id=3C79699D4FBC37DE1A212D5033B56DAE079AC0EF -212.83.154.33:8080 orport=8443 id=322C6E3A973BC10FC36DE3037AD27BC89F14723B - -# Email sent directly to teor, verified using relay contact info -51.255.41.65:9030 orport=9001 id=9231DF741915AA1630031A93026D88726877E93A - -# Email sent directly to teor, verified using relay contact info -78.142.142.246:80 orport=443 id=5A5E03355C1908EBF424CAF1F3ED70782C0D2F74 - -# Email sent directly to teor, verified using relay contact info -195.154.97.91:80 orport=443 id=BD33C50D50DCA2A46AAED54CA319A1EFEBF5D714 - -# Email sent directly to teor, verified using relay contact info -62.210.129.246:80 orport=443 id=79E169B25E4C7CE99584F6ED06F379478F23E2B8 - -# Email sent directly to teor, verified using relay contact info -5.196.74.215:9030 orport=9001 id=5818055DFBAF0FA7F67E8125FD63E3E7F88E28F6 - -# Email sent directly to teor, verified using relay contact info -212.47.233.86:9030 orport=9001 id=B4CAFD9CBFB34EC5DAAC146920DC7DFAFE91EA20 - -# Email sent directly to teor, verified using relay contact info -85.214.206.219:9030 orport=9001 id=98F8D5F359949E41DE8DF3DBB1975A86E96A84A0 - -# Email sent directly to teor, verified using relay contact info -46.166.170.4:80 orport=443 id=19F42DB047B72C7507F939F5AEA5CD1FA4656205 -46.166.170.5:80 orport=443 id=DA705AD4591E7B4708FA2CAC3D53E81962F3E6F6 - -# Email sent directly to teor, verified using relay contact info -5.189.157.56:80 orport=443 id=77F6D6A6B6EAFB8F5DADDC07A918BBF378ED6725 - -# Email sent directly to teor, verified using relay contact info -46.28.110.244:80 orport=443 id=9F7D6E6420183C2B76D3CE99624EBC98A21A967E -185.13.39.197:80 orport=443 id=001524DD403D729F08F7E5D77813EF12756CFA8D -95.130.12.119:80 orport=443 id=587E0A9552E4274B251F29B5B2673D38442EE4BF - -# Email sent directly to teor, verified using relay contact info -212.129.62.232:80 orport=443 id=B143D439B72D239A419F8DCE07B8A8EB1B486FA7 - -# Email sent directly to teor, verified using relay contact info -91.219.237.229:80 orport=443 id=1ECD73B936CB6E6B3CD647CC204F108D9DF2C9F7 - -# Email sent directly to teor, verified using relay contact info -178.62.197.82:80 orport=443 id=0D3EBA17E1C78F1E9900BABDB23861D46FCAF163 - -# Email sent directly to teor, verified using relay contact info -82.223.21.74:9030 orport=9001 id=7A32C9519D80CA458FC8B034A28F5F6815649A98 ipv6=[2001:470:53e0::cafe]:9050 - -# Email sent directly to teor, verified using relay contact info -146.185.177.103:80 orport=9030 id=9EC5E097663862DF861A18C32B37C5F82284B27D - -# Email sent directly to teor, verified using relay contact info -37.187.22.87:9030 orport=9001 id=36B9E7AC1E36B62A9D6F330ABEB6012BA7F0D400 ipv6=[2001:41d0:a:1657::1]:9001 - -# Email sent directly to teor, verified using relay contact info -37.59.46.159:9030 orport=9001 id=CBD0D1BD110EC52963082D839AC6A89D0AE243E7 - -# Email sent directly to teor, verified using relay contact info -212.47.250.243:9030 orport=9001 id=5B33EDBAEA92F446768B3753549F3B813836D477 -# Confirm with operator before adding these -#163.172.133.36:9030 orport=9001 id=D8C2BD36F01FA86F4401848A0928C4CB7E5FDFF9 -#158.69.216.70:9030 orport=9001 id=0ACE25A978D4422C742D6BC6345896719BF6A7EB - -# Email sent directly to teor, verified using relay contact info -5.199.142.236:9030 orport=9001 id=F4C0EDAA0BF0F7EC138746F8FEF1CE26C7860265 - -# Email sent directly to teor, verified using relay contact info -46.8.249.10:80 orport=443 id=31670150090A7C3513CB7914B9610E786391A95D - -# Email sent directly to teor, verified using relay contact info -144.76.163.93:9030 orport=9001 id=22F08CF09764C4E8982640D77F71ED72FF26A9AC - -# Email sent directly to teor, verified using relay contact info -46.4.24.161:9030 orport=9001 id=DB4C76A3AD7E234DA0F00D6F1405D8AFDF4D8DED -46.4.24.161:9031 orport=9002 id=7460F3D12EBE861E4EE073F6233047AACFE46AB4 -46.38.51.132:9030 orport=9001 id=810DEFA7E90B6C6C383C063028EC397A71D7214A -163.172.194.53:9030 orport=9001 id=8C00FA7369A7A308F6A137600F0FA07990D9D451 ipv6=[2001:bc8:225f:142:6c69:7461:7669:73]:9001 - -# Email sent directly to teor, verified using relay contact info -176.10.107.180:9030 orport=9001 id=3D7E274A87D9A89AF064C13D1EE4CA1F184F2600 - -# Email sent directly to teor, verified using relay contact info -46.28.207.19:80 orport=443 id=5B92FA5C8A49D46D235735504C72DBB3472BA321 -46.28.207.141:80 orport=443 id=F69BED36177ED727706512BA6A97755025EEA0FB -46.28.205.170:80 orport=443 id=AF322D83A4D2048B22F7F1AF5F38AFF4D09D0B76 -95.183.48.12:80 orport=443 id=7187CED1A3871F837D0E60AC98F374AC541CB0DA - -# Email sent directly to teor, verified using relay contact info -93.180.156.84:9030 orport=9001 id=8844D87E9B038BE3270938F05AF797E1D3C74C0F - -# Email sent directly to teor, verified using relay contact info -37.187.115.157:9030 orport=9001 id=D5039E1EBFD96D9A3F9846BF99EC9F75EDDE902A - -# Email sent directly to teor, verified using relay contact info -5.34.183.205:80 orport=443 id=DDD7871C1B7FA32CB55061E08869A236E61BDDF8 - -# Email sent directly to teor, verified using relay contact info -51.254.246.203:9030 orport=9001 id=47B596B81C9E6277B98623A84B7629798A16E8D5 - -# Email sent directly to teor, verified using relay contact info -5.9.146.203:80 orport=443 id=1F45542A24A61BF9408F1C05E0DCE4E29F2CBA11 - -# Email sent directly to teor, verified using relay contact info -# Updated details from atlas based on ticket #20010 -163.172.176.167:80 orport=443 id=230A8B2A8BA861210D9B4BA97745AEC217A94207 -163.172.149.155:80 orport=443 id=0B85617241252517E8ECF2CFC7F4C1A32DCD153F -163.172.149.122:80 orport=443 id=A9406A006D6E7B5DA30F2C6D4E42A338B5E340B2 - -# Email sent directly to teor, verified using relay contact info -204.11.50.131:9030 orport=9001 id=185F2A57B0C4620582602761097D17DB81654F70 - -# Email sent directly to teor, verified using relay contact info -151.236.222.217:44607 orport=9001 id=94D58704C2589C130C9C39ED148BD8EA468DBA54 - -# Email sent directly to teor, verified using relay contact info -185.35.202.221:9030 orport=9001 id=C13B91384CDD52A871E3ECECE4EF74A7AC7DCB08 ipv6=[2a02:ed06::221]:9001 - -# Email sent directly to teor, verified using relay contact info -5.9.151.241:9030 orport=4223 id=9BF04559224F0F1C3C953D641F1744AF0192543A ipv6=[2a01:4f8:190:34f0::2]:4223 - -# Email sent directly to teor, verified using relay contact info -89.40.71.149:8081 orport=8080 id=EC639EDAA5121B47DBDF3D6B01A22E48A8CB6CC7 - -# Email sent directly to teor, verified using relay contact info -92.222.20.130:80 orport=443 id=0639612FF149AA19DF3BCEA147E5B8FED6F3C87C - -# Email sent directly to teor, verified using relay contact info -80.112.155.100:9030 orport=9001 id=53B000310984CD86AF47E5F3CD0BFF184E34B383 ipv6=[2001:470:7b02::38]:9001 - -# Email sent directly to teor, verified using relay contact info -83.212.99.68:80 orport=443 id=DDBB2A38252ADDA53E4492DDF982CA6CC6E10EC0 ipv6=[2001:648:2ffc:1225:a800:bff:fe3d:67b5]:443 - -# Email sent directly to teor, verified using relay contact info -95.130.11.147:9030 orport=443 id=6B697F3FF04C26123466A5C0E5D1F8D91925967A - -# Email sent directly to teor, verified using relay contact info -128.199.55.207:9030 orport=9001 id=BCEF908195805E03E92CCFE669C48738E556B9C5 ipv6=[2a03:b0c0:2:d0::158:3001]:9001 - -# Email sent directly to teor, verified using relay contact info -178.32.216.146:9030 orport=9001 id=17898F9A2EBC7D69DAF87C00A1BD2FABF3C9E1D2 - -# Email sent directly to teor, verified using relay contact info -212.83.40.238:9030 orport=9001 id=F409FA7902FD89270E8DE0D7977EA23BC38E5887 - -# Email sent directly to teor, verified using relay contact info -204.8.156.142:80 orport=443 id=94C4B7B8C50C86A92B6A20107539EE2678CF9A28 - -# Email sent directly to teor, verified using relay contact info -80.240.139.111:80 orport=443 id=DD3BE7382C221F31723C7B294310EF9282B9111B - -# Email sent directly to teor, verified using relay contact info -185.97.32.18:9030 orport=9001 id=04250C3835019B26AA6764E85D836088BE441088 - -# Email sent directly to teor -149.56.45.200:9030 orport=9001 id=FE296180018833AF03A8EACD5894A614623D3F76 ipv6=[2607:5300:201:3000::17d3]:9002 # PiotrTorpotkinOne - -# Email sent directly to teor, verified using relay contact info -81.2.209.10:443 orport=80 id=B6904ADD4C0D10CDA7179E051962350A69A63243 ipv6=[2001:15e8:201:1::d10a]:80 - -# Email sent directly to teor, verified using relay contact info -# IPv6 address unreliable -195.154.164.243:80 orport=443 id=AC66FFA4AB35A59EBBF5BF4C70008BF24D8A7A5C #ipv6=[2001:bc8:399f:f000::1]:993 -138.201.26.2:80 orport=443 id=6D3A3ED5671E4E3F58D4951438B10AE552A5FA0F -81.7.16.182:80 orport=443 id=51E1CF613FD6F9F11FE24743C91D6F9981807D82 ipv6=[2a02:180:1:1::517:10b6]:993 -134.119.36.135:80 orport=443 id=763C9556602BD6207771A7A3D958091D44C43228 ipv6=[2a00:1158:3::2a8]:993 -46.228.199.19:80 orport=443 id=E26AFC5F718E21AC502899B20C653AEFF688B0D2 ipv6=[2001:4ba0:cafe:4a::1]:993 -37.200.98.5:80 orport=443 id=231C2B9C8C31C295C472D031E06964834B745996 ipv6=[2a00:1158:3::11a]:993 -46.23.70.195:80 orport=443 id=C9933B3725239B6FAB5227BA33B30BE7B48BB485 -185.15.244.124:80 orport=443 id=935BABE2564F82016C19AEF63C0C40B5753BA3D2 ipv6=[2001:4ba0:cafe:e35::1]:993 -195.154.116.232:80 orport=443 id=B35C5739C8C5AB72094EB2B05738FD1F8EEF6EBD ipv6=[2001:bc8:399f:200::1]:993 -195.154.121.198:80 orport=443 id=0C77421C890D16B6D201283A2244F43DF5BC89DD ipv6=[2001:bc8:399f:100::1]:993 -37.187.20.59:80 orport=443 id=91D23D8A539B83D2FB56AA67ECD4D75CC093AC55 ipv6=[2001:41d0:a:143b::1]:993 -217.12.208.117:80 orport=443 id=E6E18151300F90C235D3809F90B31330737CEB43 ipv6=[2a00:1ca8:a7::1bb]:993 -81.7.10.251:80 orport=443 id=8073670F8F852971298F8AF2C5B23AE012645901 ipv6=[2a02:180:1:1::517:afb]:993 -46.36.39.50:80 orport=443 id=ED4B0DBA79AEF5521564FA0231455DCFDDE73BB6 ipv6=[2a02:25b0:aaaa:aaaa:8d49:b692:4852:0]:995 -91.194.90.103:80 orport=443 id=75C4495F4D80522CA6F6A3FB349F1B009563F4B7 ipv6=[2a02:c205:3000:5449::1]:993 -163.172.25.118:80 orport=22 id=0CF8F3E6590F45D50B70F2F7DA6605ECA6CD408F -188.138.88.42:80 orport=443 id=70C55A114C0EF3DC5784A4FAEE64388434A3398F -81.7.13.84:80 orport=443 id=0C1E7DD9ED0676C788933F68A9985ED853CA5812 ipv6=[2a02:180:1:1::5b8f:538c]:993 -213.246.56.95:80 orport=443 id=27E6E8E19C46751E7312420723C6162FF3356A4C ipv6=[2a00:c70:1:213:246:56:95:1]:993 -94.198.100.18:80 orport=443 id=BAACCB29197DB833F107E410E2BFAE5009EE7583 -217.12.203.46:80 orport=443 id=6A29FD8C00D573E6C1D47852345B0E5275BA3307 -212.117.180.107:80 orport=443 id=0B454C7EBA58657B91133A587C1BDAEDC6E23142 -217.12.199.190:80 orport=443 id=A37C47B03FF31CA6937D3D68366B157997FE7BCD ipv6=[2a02:27a8:0:2::486]:993 -216.230.230.247:80 orport=443 id=4C7BF55B1BFF47993DFF995A2926C89C81E4F04A -69.30.215.42:80 orport=443 id=510176C07005D47B23E6796F02C93241A29AA0E9 ipv6=[2604:4300:a:2e::2]:993 -89.46.100.162:80 orport=443 id=6B7191639E179965FD694612C9B2C8FB4267B27D -107.181.174.22:80 orport=443 id=5A551BF2E46BF26CC50A983F7435CB749C752553 ipv6=[2607:f7a0:3:4::4e]:993 - -# Email sent directly to teor, verified using relay contact info -212.238.208.48:9030 orport=9001 id=F406219CDD339026D160E53FCA0EF6857C70F109 ipv6=[2001:984:a8fb:1:ba27:ebff:feac:c109]:9001 - -# Email sent directly to teor -176.158.236.102:9030 orport=9001 id=DC163DDEF4B6F0C6BC226F9F6656A5A30C5C5686 # Underworld - -# Email sent directly to teor, verified using relay contact info -91.229.20.27:9030 orport=9001 id=9A0D54D3A6D2E0767596BF1515E6162A75B3293F - -# Email sent directly to teor, verified using relay contact info -80.127.137.19:80 orport=443 id=6EF897645B79B6CB35E853B32506375014DE3621 ipv6=[2001:981:47c1:1::6]:443 - -# Email sent directly to teor -163.172.138.22:80 orport=443 id=16102E458460349EE45C0901DAA6C30094A9BBEA ipv6=[2001:bc8:4400:2100::1:3]:443 # mkultra - -# Email sent directly to teor, verified using relay contact info -97.74.237.196:9030 orport=9001 id=2F0F32AB1E5B943CA7D062C03F18960C86E70D94 - -# Email sent directly to teor, verified using relay contact info -192.187.124.98:9030 orport=9001 id=FD1871854BFC06D7B02F10742073069F0528B5CC - -# Email sent directly to teor, verified using relay contact info -178.62.98.160:9030 orport=9001 id=8B92044763E880996A988831B15B2B0E5AD1544A - -# Email sent directly to teor, verified using relay contact info -163.172.217.50:9030 orport=9001 id=02ECD99ECD596013A8134D46531560816ECC4BE6 - -# Email sent directly to teor, verified using relay contact info -185.100.86.100:80 orport=443 id=0E8C0C8315B66DB5F703804B3889A1DD66C67CE0 -185.100.84.82:80 orport=443 id=7D05A38E39FC5D29AFE6BE487B9B4DC9E635D09E - -# Email sent directly to teor, verified using relay contact info -78.24.75.53:9030 orport=9001 id=DEB73705B2929AE9BE87091607388939332EF123 - -# Email sent directly to teor, verified using relay contact info -46.101.237.246:9030 orport=9001 id=75F1992FD3F403E9C082A5815EB5D12934CDF46C ipv6=[2a03:b0c0:3:d0::208:5001]:9050 -178.62.86.96:9030 orport=9001 id=439D0447772CB107B886F7782DBC201FA26B92D1 ipv6=[2a03:b0c0:1:d0::3cf:7001]:9050 - -# Email sent directly to teor, verified using relay contact info -# Very low bandwidth, stale consensues, excluded to cut down on warnings -#91.233.106.121:80 orport=443 id=896364B7996F5DFBA0E15D1A2E06D0B98B555DD6 - -# Email sent directly to teor, verified using relay contact info -167.114.113.48:9030 orport=403 id=2EC0C66EA700C44670444280AABAB1EC78B722A0 - -# Email sent directly to teor, verified using relay contact info -# Assume details update is permanent -213.141.138.174:9030 orport=9001 id=BD552C165E2ED2887D3F1CCE9CFF155DDA2D86E6 # Schakalium - -# Email sent directly to teor, verified using relay contact info -95.128.43.164:80 orport=443 id=616081EC829593AF4232550DE6FFAA1D75B37A90 ipv6=[2a02:ec0:209:10::4]:443 - -# Email sent directly to teor, verified using relay contact info -166.82.21.200:9030 orport=9029 id=D5C33F3E203728EDF8361EA868B2939CCC43FAFB - -# Email sent directly to teor, verified using relay contact info -91.121.54.8:9030 orport=9001 id=CBEE0F3303C8C50462A12107CA2AE061831931BC - -# Email sent directly to teor, verified using relay contact info -178.217.184.32:8080 orport=443 id=8B7F47AE1A5D954A3E58ACDE0865D09DBA5B738D - -# Email sent directly to teor, verified using relay contact info -85.10.201.47:9030 orport=9001 id=D8B7A3A6542AA54D0946B9DC0257C53B6C376679 ipv6=[2a01:4f8:a0:43eb::beef]:9001 - -# Email sent directly to teor, verified using relay contact info -120.29.217.46:80 orport=443 id=5E853C94AB1F655E9C908924370A0A6707508C62 - -# Email sent directly to teor, verified using relay contact info -37.153.1.10:9030 orport=9001 id=9772EFB535397C942C3AB8804FB35CFFAD012438 - -# Email sent directly to teor, verified using relay contact info -92.222.4.102:9030 orport=9001 id=1A6B8B8272632D8AD38442027F822A367128405C - -# Email sent directly to teor, verified using relay contact info -31.31.78.49:80 orport=443 id=46791D156C9B6C255C2665D4D8393EC7DBAA7798 - -# Email sent directly to teor -192.160.102.169:80 orport=9001 id=C0192FF43E777250084175F4E59AC1BA2290CE38 ipv6=[2620:132:300c:c01d::9]:9002 # manipogo -192.160.102.166:80 orport=9001 id=547DA56F6B88B6C596B3E3086803CDA4F0EF8F21 ipv6=[2620:132:300c:c01d::6]:9002 # chaucer -192.160.102.170:80 orport=9001 id=557ACEC850F54EEE65839F83CACE2B0825BE811E ipv6=[2620:132:300c:c01d::a]:9002 # ogopogo -192.160.102.164:80 orport=9001 id=823AA81E277F366505545522CEDC2F529CE4DC3F ipv6=[2620:132:300c:c01d::4]:9002 # snowfall -192.160.102.165:80 orport=9001 id=C90CA3B7FE01A146B8268D56977DC4A2C024B9EA ipv6=[2620:132:300c:c01d::5]:9002 # cowcat -192.160.102.168:80 orport=9001 id=F6A358DD367B3282D6EF5824C9D45E1A19C7E815 ipv6=[2620:132:300c:c01d::8]:9002 # prawksi - -# Email sent directly to teor, verified using relay contact info -136.243.214.137:80 orport=443 id=B291D30517D23299AD7CEE3E60DFE60D0E3A4664 - -# Email sent directly to teor, verified using relay contact info -192.87.28.28:9030 orport=9001 id=ED2338CAC2711B3E331392E1ED2831219B794024 -192.87.28.82:9030 orport=9001 id=844AE9CAD04325E955E2BE1521563B79FE7094B7 - -# Email sent directly to teor, verified using relay contact info -192.87.28.28:9030 orport=9001 id=ED2338CAC2711B3E331392E1ED2831219B794024 -# same machine as ED2338CAC2711B3E331392E1ED2831219B794024 -192.87.28.82:9030 orport=9001 id=844AE9CAD04325E955E2BE1521563B79FE7094B7 - -# https://twitter.com/kosjoli/status/719507270904758272 -85.10.202.87:9030 orport=9001 id=971AFB23C168DCD8EDA17473C1C452B359DE3A5A -176.9.5.116:9030 orport=9001 id=A1EB8D8F1EE28DB98BBB1EAA3B4BEDD303BAB911 -46.4.111.124:9030 orport=9001 id=D9065F9E57899B3D272AA212317AF61A9B14D204 - -# Email sent directly to teor, verified using relay contact info -185.100.85.61:80 orport=443 id=025B66CEBC070FCB0519D206CF0CF4965C20C96E - -# Email sent directly to teor, verified using relay contact info -108.166.168.158:80 orport=443 id=CDAB3AE06A8C9C6BF817B3B0F1877A4B91465699 - -# Email sent directly to teor, verified using relay contact info -91.219.236.222:80 orport=443 id=20704E7DD51501DC303FA51B738D7B7E61397CF6 - -# Email sent directly to teor, verified using relay contact info -185.14.185.240:9030 orport=443 id=D62FB817B0288085FAC38A6DC8B36DCD85B70260 -192.34.63.137:9030 orport=443 id=ABCB4965F1FEE193602B50A365425105C889D3F8 -128.199.197.16:9030 orport=443 id=DEE5298B3BA18CDE651421CD2DCB34A4A69F224D - -# Email sent directly to teor, verified using relay contact info -185.13.38.75:9030 orport=9001 id=D2A1703758A0FBBA026988B92C2F88BAB59F9361 - -# Email sent directly to teor, verified using relay contact info -128.204.39.106:9030 orport=9001 id=6F0F3C09AF9580F7606B34A7678238B3AF7A57B7 - -# Email sent directly to teor, verified using relay contact info -198.50.191.95:80 orport=443 id=39F096961ED2576975C866D450373A9913AFDC92 - -# Email sent directly to teor, verified using relay contact info -167.114.66.61:9696 orport=443 id=DE6CD5F09DF26076F26321B0BDFBE78ACD935C65 ipv6=[2607:5300:100::78d]:443 - -# Email sent directly to teor, verified using relay contact info -66.111.2.20:9030 orport=9001 id=9A68B85A02318F4E7E87F2828039FBD5D75B0142 -66.111.2.16:9030 orport=9001 id=3F092986E9B87D3FDA09B71FA3A602378285C77A - -# Email sent directly to teor, verified using relay contact info -92.222.38.67:80 orport=443 id=DED6892FF89DBD737BA689698A171B2392EB3E82 - -# Email sent directly to teor, verified using relay contact info -212.47.228.115:9030 orport=443 id=BCA017ACDA48330D02BB70716639ED565493E36E - -# Email sent directly to teor, verified using relay contact info -185.100.84.175:80 orport=443 id=39B59AF4FE54FAD8C5085FA9C15FDF23087250DB - -# Email sent directly to teor, verified using relay contact info -166.70.207.2:9030 orport=9001 id=E3DB2E354B883B59E8DC56B3E7A353DDFD457812 - -# Emails sent directly to teor, verified using relay contact info -69.162.139.9:9030 orport=9001 id=4791FC0692EAB60DF2BCCAFF940B95B74E7654F6 ipv6=[2607:f128:40:1212::45a2:8b09]:9001 - -# Email sent directly to teor, verified using relay contact info -213.239.217.18:1338 orport=1337 id=C37BC191AC389179674578C3E6944E925FE186C2 ipv6=[2a01:4f8:a0:746a:101:1:1:1]:1337 - -# Email sent directly to teor, verified using relay contact info -# Assume details update is permanent -188.40.128.246:9030 orport=9001 id=AD19490C7DBB26D3A68EFC824F67E69B0A96E601 ipv6=[2a01:4f8:221:1ac1:dead:beef:7005:9001]:9001 # sputnik -129.13.131.140:80 orport=443 id=F2DFE5FA1E4CF54F8E761A6D304B9B4EC69BDAE8 ipv6=[2a00:1398:5:f604:cafe:cafe:cafe:9001]:443 # AlleKochenKaffee - -# Email sent directly to teor, verified using relay contact info -88.198.253.13:9030 orport=9001 id=DF924196D69AAE3C00C115A9CCDF7BB62A175310 ipv6=[2a01:4f8:11a:b1f::2]:9001 - -# Email sent directly to teor, verified using relay contact info -185.100.86.128:9030 orport=9001 id=9B31F1F1C1554F9FFB3455911F82E818EF7C7883 -46.36.36.127:9030 orport=9001 id=C80DF89B21FF932DEC0D7821F679B6C79E1449C3 - -# Email sent directly to teor, verified using relay contact info -176.10.104.240:80 orport=443 id=0111BA9B604669E636FFD5B503F382A4B7AD6E80 -176.10.104.240:8080 orport=8443 id=AD86CD1A49573D52A7B6F4A35750F161AAD89C88 -176.10.104.243:8080 orport=8443 id=95DA61AEF23A6C851028C1AA88AD8593F659E60F -94.230.208.147:80 orport=443 id=9AA3FF35E7A549D2337E962333D366E102FE4D50 ipv6=[2a02:418:6017::147]:443 - -# Email sent directly to teor, verified using relay contact info -107.170.101.39:9030 orport=443 id=30973217E70AF00EBE51797FF6D9AA720A902EAA - -# Email sent directly to teor -193.70.112.165:80 orport=443 id=F10BDE279AE71515DDCCCC61DC19AC8765F8A3CC # ParkBenchInd001 - -# Email sent directly to teor -185.220.101.6:10006 orport=20006 id=C08DE49658E5B3CFC6F2A952B453C4B608C9A16A # niftyvolcanorabbit -185.220.101.13:10013 orport=20013 id=71AB4726D830FAE776D74AEF790CF04D8E0151B4 # niftycottontail -185.220.101.5:10005 orport=20005 id=1084200B44021D308EA4253F256794671B1D099A # niftyhedgehog -185.220.101.9:10009 orport=20009 id=14877C6384A9E793F422C8D1DDA447CACA4F7C4B # niftywoodmouse -185.220.101.8:10008 orport=20008 id=24E91955D969AEA1D80413C64FE106FAE7FD2EA9 # niftymouse -185.220.101.1:10001 orport=20001 id=28F4F392F8F19E3FBDE09616D9DB8143A1E2DDD3 # niftycottonmouse -185.220.101.21:10021 orport=20021 id=348B89013EDDD99E4755951D1EC284D9FED71226 # niftysquirrel -185.220.101.10:10010 orport=20010 id=4031460683AE9E0512D3620C2758D98758AC6C93 # niftyeuropeanrabbit -185.220.101.34:10034 orport=20034 id=47C42E2094EE482E7C9B586B10BABFB67557030B # niftyquokka -185.220.101.18:10018 orport=20018 id=5D5006E4992F2F97DF4F8B926C3688870EB52BD8 # niftyplagiodontia -185.220.101.28:10028 orport=20028 id=609E598FB6A00BCF7872906B602B705B64541C50 # niftychipmunk -185.220.101.20:10020 orport=20020 id=619349D82424C601CAEB94161A4CF778993DAEE7 # niftytucotuco -185.220.101.17:10017 orport=20017 id=644DECC5A1879C0FE23DE927DD7049F58BBDF349 # niftyhutia -185.220.101.0:10000 orport=20000 id=6E94866ED8CA098BACDFD36D4E8E2B459B8A734E # niftybeaver -185.220.101.30:10030 orport=20030 id=71CFDEB4D9E00CCC3E31EC4E8A29E109BBC1FB36 # niftypedetidae -185.220.101.29:10029 orport=20029 id=7DC52AE6667A30536BA2383CD102CFC24F20AD71 # niftyllipika -185.220.101.41:10041 orport=20041 id=7E281CD2C315C4F7A84BC7C8721C3BC974DDBFA3 # niftyporcupine -185.220.101.25:10025 orport=20025 id=8EE0534532EA31AA5172B1892F53B2F25C76EB02 # niftyjerboa -185.220.101.33:10033 orport=20033 id=906DCB390F2BA987AE258D745E60BAAABAD31DE8 # niftyquokka -185.220.101.26:10026 orport=20026 id=92A6085EABAADD928B6F8E871540A1A41CBC08BA # niftypedetes -185.220.101.40:10040 orport=20040 id=9A857254F379194D1CD76F4A79A20D2051BEDA3F # niftynutria -185.220.101.42:10042 orport=20042 id=9B816A5B3EB20B8E4E9B9D1FBA299BD3F40F0320 # niftypygmyjerboa -185.220.101.2:10002 orport=20002 id=B740BCECC4A9569232CDD45C0E1330BA0D030D33 # niftybunny -185.220.101.32:10032 orport=20032 id=B771AA877687F88E6F1CA5354756DF6C8A7B6B24 # niftypika -185.220.101.12:10012 orport=20012 id=BC82F2190DE2E97DE65F49B4A95572374BDC0789 # niftycapybara -185.220.101.22:10022 orport=20022 id=CA37CD46799449D83B6B98B8C22C649906307888 # niftyjackrabbit -185.220.101.4:10004 orport=20004 id=CDA2EA326E2272C57ACB26773D7252C211795B78 # niftygerbil -185.220.101.14:10014 orport=20014 id=E7EBA5D8A4E09684D11A1DF24F75362817333768 # niftyhare -185.220.101.16:10016 orport=20016 id=EC1997D51892E4607C68E800549A1E7E4694005A # niftyguineapig -185.220.101.24:10024 orport=20024 id=FDA70EC93DB01E3CB418CB6943B0C68464B18B4C # niftyrat - -# Email sent directly to teor, verified using relay contact info -198.232.165.2:9030 orport=9001 id=30C19B81981F450C402306E2E7CFB6C3F79CB6B2 - -# Emails sent directly to teor, verified using relay contact info -51.254.101.242:9002 orport=9001 id=4CC9CC9195EC38645B699A33307058624F660CCF - -# Emails sent directly to teor, verified using relay contact info -# Updated IP https://trac.torproject.org/projects/tor/ticket/24805#comment:16 -94.130.186.5:80 orport=443 id=6A7551EEE18F78A9813096E82BF84F740D32B911 - -# Email sent directly to teor, verified using relay contact info -173.255.245.116:9030 orport=9001 id=91E4015E1F82DAF0121D62267E54A1F661AB6DC7 - -# Email sent directly to teor, verified using relay contact info -62.216.5.120:9030 orport=9001 id=D032D4D617140D6B828FC7C4334860E45E414FBE - -# Email sent directly to teor, verified using relay contact info -51.254.136.195:80 orport=443 id=7BB70F8585DFC27E75D692970C0EEB0F22983A63 - -# Email sent directly to teor, verified using relay contact info -5.196.88.122:9030 orport=9001 id=0C2C599AFCB26F5CFC2C7592435924C1D63D9484 ipv6=[2001:41d0:a:fb7a::1]:9001 - -# Email sent directly to teor, verified using relay contact info -5.9.158.75:80 orport=443 id=1AF72E8906E6C49481A791A6F8F84F8DFEBBB2BA ipv6=[2a01:4f8:190:514a::2]:443 -5.9.158.75:9030 orport=9001 id=D11D11877769B9E617537B4B46BFB92B443DE33D ipv6=[2a01:4f8:190:514a::2]:9001 - -# Email sent directly to teor, verified using relay contact info -46.101.169.151:9030 orport=9001 id=D760C5B436E42F93D77EF2D969157EEA14F9B39C ipv6=[2a03:b0c0:3:d0::74f:a001]:9001 - -# Email sent directly to teor, verified using relay contact info -199.249.223.81:80 orport=443 id=F7447E99EB5CBD4D5EB913EE0E35AC642B5C1EF3 -199.249.223.79:80 orport=443 id=D33292FEDE24DD40F2385283E55C87F85C0943B6 -199.249.223.78:80 orport=443 id=EC15DB62D9101481F364DE52EB8313C838BDDC29 -199.249.223.77:80 orport=443 id=CC4A3AE960E3617F49BF9887B79186C14CBA6813 -199.249.223.76:80 orport=443 id=43209F6D50C657A56FE79AF01CA69F9EF19BD338 -199.249.223.75:80 orport=443 id=60D3667F56AEC5C69CF7E8F557DB21DDF6C36060 -199.249.223.74:80 orport=443 id=5F4CD12099AF20FAF9ADFDCEC65316A376D0201C -199.249.223.73:80 orport=443 id=5649CB2158DA94FB747415F26628BEC07FA57616 -199.249.223.72:80 orport=443 id=B028707969D8ED84E6DEA597A884F78AAD471971 -199.249.223.71:80 orport=443 id=B6320E44A230302C7BF9319E67597A9B87882241 -199.249.223.60:80 orport=443 id=B7047FBDE9C53C39011CA84E5CB2A8E3543066D0 -199.249.223.61:80 orport=443 id=40E7D6CE5085E4CDDA31D51A29D1457EB53F12AD -199.249.223.62:80 orport=443 id=0077BCBA7244DB3E6A5ED2746E86170066684887 -199.249.223.63:80 orport=443 id=1DB25DF59DAA01B5BE3D3CEB8AFED115940EBE8B -199.249.223.64:80 orport=443 id=9F2856F6D2B89AD4EF6D5723FAB167DB5A53519A -199.249.223.65:80 orport=443 id=9D21F034C3BFF4E7737D08CF775DC1745706801F -199.249.223.66:80 orport=443 id=C5A53BCC174EF8FD0DCB223E4AA929FA557DEDB2 -199.249.223.67:80 orport=443 id=155D6F57425F16C0624D77777641E4EB1B47C6F0 -199.249.223.68:80 orport=443 id=DF20497E487A979995D851A5BCEC313DF7E5BC51 -199.249.223.69:80 orport=443 id=7FA8E7E44F1392A4E40FFC3B69DB3B00091B7FD3 - -# https://lists.torproject.org/pipermail/tor-relays/2016-December/011114.html -86.105.212.130:9030 orport=443 id=9C900A7F6F5DD034CFFD192DAEC9CCAA813DB022 - -# Email sent directly to teor, verified using relay contact info -178.33.183.251:80 orport=443 id=DD823AFB415380A802DCAEB9461AE637604107FB ipv6=[2001:41d0:2:a683::251]:443 - -# Email sent directly to teor, verified using relay contact info -31.185.104.19:80 orport=443 id=9EAD5B2D3DBD96DBC80DCE423B0C345E920A758D -# same machine as 9EAD5B2D3DBD96DBC80DCE423B0C345E920A758D -31.185.104.20:80 orport=443 id=ADB2C26629643DBB9F8FE0096E7D16F9414B4F8D -31.185.104.21:80 orport=443 id=C2AAB088555850FC434E68943F551072042B85F1 -31.185.104.22:80 orport=443 id=5BA3A52760A0EABF7E7C3ED3048A77328FF0F148 - -# Email sent directly to teor, verified using relay contact info -185.34.60.114:80 orport=443 id=7F7A695DF6F2B8640A70B6ADD01105BC2EBC5135 - -# https://lists.torproject.org/pipermail/tor-relays/2017-December/013939.html -94.142.242.84:80 orport=443 id=AA0D167E03E298F9A8CD50F448B81FBD7FA80D56 ipv6=[2a02:898:24:84::1]:443 # rejozenger - -# Email sent directly to teor, verified using relay contact info -185.129.62.62:9030 orport=9001 id=ACDD9E85A05B127BA010466C13C8C47212E8A38F ipv6=[2a06:d380:0:3700::62]:9001 - -# Email sent directly to teor, verified using relay contact info -# The e84 part of the IPv6 address does not have a leading 0 in the consensus -81.30.158.213:9030 orport=9001 id=789EA6C9AE9ADDD8760903171CFA9AC5741B0C70 ipv6=[2001:4ba0:cafe:e84::1]:9001 - -# https://lists.torproject.org/pipermail/tor-relays/2016-December/011209.html -5.9.159.14:9030 orport=9001 id=0F100F60C7A63BED90216052324D29B08CFCF797 - -# Email sent directly to teor, verified using relay contact info -45.62.255.25:80 orport=443 id=3473ED788D9E63361D1572B7E82EC54338953D2A - -# Email sent directly to teor, verified using relay contact info -217.79.179.177:9030 orport=9001 id=3E53D3979DB07EFD736661C934A1DED14127B684 ipv6=[2001:4ba0:fff9:131:6c4f::90d3]:9001 - -# Email sent directly to teor, verified using relay contact info -212.47.244.38:8080 orport=443 id=E81EF60A73B3809F8964F73766B01BAA0A171E20 -163.172.157.213:8080 orport=443 id=4623A9EC53BFD83155929E56D6F7B55B5E718C24 -163.172.139.104:8080 orport=443 id=68F175CCABE727AA2D2309BCD8789499CEE36ED7 - -# Email sent directly to teor, verified using relay contact info -163.172.223.200:80 orport=443 id=998BF3ED7F70E33D1C307247B9626D9E7573C438 -195.154.122.54:80 orport=443 id=64E99CB34C595A02A3165484BD1215E7389322C6 - -# Email sent directly to teor, verified using relay contact info -# Email sent directly to Phoul -185.100.86.128:9030 orport=9001 id=9B31F1F1C1554F9FFB3455911F82E818EF7C7883 -185.100.85.101:9030 orport=9001 id=4061C553CA88021B8302F0814365070AAE617270 - -# Email sent directly to teor, verified using relay contact info -89.163.247.43:9030 orport=9001 id=BC7ACFAC04854C77167C7D66B7E471314ED8C410 ipv6=[2001:4ba0:fff7:25::5]:9001 - -# Email sent directly to teor, verified using relay contact info -95.85.8.226:80 orport=443 id=1211AC1BBB8A1AF7CBA86BCE8689AA3146B86423 - -# Email sent directly to teor, verified using relay contact info -85.214.151.72:9030 orport=9001 id=722D365140C8C52DBB3C9FF6986E3CEFFE2BA812 - -# email sent directly to teor -72.52.75.27:9030 orport=9001 id=8567AD0A6369ED08527A8A8533A5162AC00F7678 # piecoopdotnet - -# Email sent directly to teor, verified using relay contact info -5.9.146.203:80 orport=443 id=1F45542A24A61BF9408F1C05E0DCE4E29F2CBA11 -5.9.159.14:9030 orport=9001 id=0F100F60C7A63BED90216052324D29B08CFCF797 - -# Email sent directly to teor, verified using relay contact info -# Assume details update is permanent -5.9.147.226:9030 orport=9001 id=B0553175AADB0501E5A61FC61CEA3970BE130FF2 ipv6=[2a01:4f8:190:30e1::2]:9001 # zwiubel - -# https://trac.torproject.org/projects/tor/ticket/22527#comment:1 -199.184.246.250:80 orport=443 id=1F6ABD086F40B890A33C93CC4606EE68B31C9556 ipv6=[2620:124:1009:1::171]:443 - -# https://trac.torproject.org/projects/tor/ticket/24695 -163.172.53.84:143 orport=21 id=1C90D3AEADFF3BCD079810632C8B85637924A58E ipv6=[2001:bc8:24f8::]:21 # Multivac - -# Email sent directly to teor -54.36.237.163:80 orport=443 id=DB2682153AC0CCAECD2BD1E9EBE99C6815807A1E # GermanCraft2 - -# Email sent directly to teor -62.138.7.171:9030 orport=9001 id=9844B981A80B3E4B50897098E2D65167E6AEF127 # 0x3d004 -62.138.7.171:8030 orport=8001 id=9285B22F7953D7874604EEE2B470609AD81C74E9 # 0x3d005 -91.121.23.100:9030 orport=9001 id=3711E80B5B04494C971FB0459D4209AB7F2EA799 # 0x3d002 -91.121.23.100:8030 orport=8001 id=CFBBA0D858F02E40B1432A65F6D13C9BDFE7A46B # 0x3d001 -51.15.13.245:9030 orport=9001 id=CED527EAC230E7B56E5B363F839671829C3BA01B # 0x3d006 -51.15.13.245:8030 orport=8001 id=8EBB8D1CF48FE2AB95C451DA8F10DB6235F40F8A # 0x3d007 - -# Email sent directly to teor -104.192.5.248:9030 orport=9001 id=BF735F669481EE1CCC348F0731551C933D1E2278 # Freeway11 - -# Email sent directly to teor -# https://lists.torproject.org/pipermail/tor-relays/2017-December/013961.html -178.17.174.14:9030 orport=9001 id=B06F093A3D4DFAD3E923F4F28A74901BD4F74EB1 # TorExitMoldova -178.17.170.156:9030 orport=9001 id=41C59606AFE1D1AA6EC6EF6719690B856F0B6587 # TorExitMoldova2 - -# Email sent directly to teor -163.172.221.44:59030 orport=59001 id=164604F5C86FC8CC9C0288BD9C02311958427597 # altego - -# Email sent directly to teor -46.38.237.221:9030 orport=9001 id=D30E9D4D639068611D6D96861C95C2099140B805 # mine - -# https://lists.torproject.org/pipermail/tor-relays/2017-December/013911.html -# https://lists.torproject.org/pipermail/tor-relays/2017-December/013912.html -199.249.223.62:80 orport=443 id=0077BCBA7244DB3E6A5ED2746E86170066684887 # Quintex13 -199.249.224.45:80 orport=443 id=041646640AB306EA74B001966E86169B04CC88D2 # QuintexAirVPN26 -199.249.223.67:80 orport=443 id=155D6F57425F16C0624D77777641E4EB1B47C6F0 # Quintex18 -199.249.223.45:80 orport=443 id=1AE949967F82BBE7534A3D6BA77A7EBE1CED4369 # Quintex36 -199.249.223.63:80 orport=443 id=1DB25DF59DAA01B5BE3D3CEB8AFED115940EBE8B # Quintex14 -199.249.224.63:80 orport=443 id=1E5136DDC52FAE1219208F0A6BADB0BA62587EE6 # Quintex43 -199.249.224.46:80 orport=443 id=2ED4D25766973713EB8C56A290BF07E06B85BF12 # QuintexAirVPN27 -199.249.223.42:80 orport=443 id=3687FEC7E73F61AC66F7AE251E7DEE6BBD8C0252 # Quintex33 -199.249.223.49:80 orport=443 id=36D68478366CB8627866757EBCE7FB3C17FC1CB8 # Quintex40 -199.249.224.49:80 orport=443 id=3CA0D15567024D2E0B557DC0CF3E962B37999A79 # QuintexAirVPN30 -199.249.223.61:80 orport=443 id=40E7D6CE5085E4CDDA31D51A29D1457EB53F12AD # Quintex12 -199.249.223.76:80 orport=443 id=43209F6D50C657A56FE79AF01CA69F9EF19BD338 # QuintexAirVPN5 -199.249.224.41:80 orport=443 id=54A4820B46E65509BF3E2B892E66930A41759DE9 # QuintexAirVPN22 -199.249.223.73:80 orport=443 id=5649CB2158DA94FB747415F26628BEC07FA57616 # QuintexAirVPN8 -199.249.223.74:80 orport=443 id=5F4CD12099AF20FAF9ADFDCEC65316A376D0201C # QuintexAirVPN7 -199.249.223.75:80 orport=443 id=60D3667F56AEC5C69CF7E8F557DB21DDF6C36060 # QuintexAirVPN6 -199.249.223.46:80 orport=443 id=66E19E8C4773086F669A1E06A3F8C23B6C079129 # Quintex37 -199.249.224.65:80 orport=443 id=764BF8A03868F84C8F323C1A676AA254B80DC3BF # Quintex45 -199.249.223.48:80 orport=443 id=7A3DD280EA4CD4DD16EF8C67B93D9BDE184D1A81 # Quintex39 -199.249.224.68:80 orport=443 id=7E6E9A6FDDB8DC7C92F0CFCC3CBE76C29F061799 # Quintex48 -199.249.223.69:80 orport=443 id=7FA8E7E44F1392A4E40FFC3B69DB3B00091B7FD3 # Quintex20 -199.249.223.44:80 orport=443 id=8B80169BEF71450FC4069A190853523B7AEA45E1 # Quintex35 -199.249.224.60:80 orport=443 id=9314BD9503B9014261A65C221D77E57389DBCCC1 # Quintex50 -199.249.224.40:80 orport=443 id=9C1E7D92115D431385B8CAEA6A7C15FB89CE236B # QuintexAirVPN21 -199.249.223.65:80 orport=443 id=9D21F034C3BFF4E7737D08CF775DC1745706801F # Quintex16 -199.249.224.67:80 orport=443 id=9E2D7C6981269404AA1970B53891701A20424EF8 # Quintex47 -199.249.223.64:80 orport=443 id=9F2856F6D2B89AD4EF6D5723FAB167DB5A53519A # Quintex15 -199.249.224.48:80 orport=443 id=A0DB820FEC87C0405F7BF05DEE5E4ADED2BB9904 # QuintexAirVPN29 -199.249.224.64:80 orport=443 id=A4A393FEF48640961AACE92D041934B55348CEF9 # Quintex44 -199.249.223.72:80 orport=443 id=B028707969D8ED84E6DEA597A884F78AAD471971 # QuintexAirVPN9 -199.249.223.40:80 orport=443 id=B0CD9F9B5B60651ADC5919C0F1EAA87DBA1D9249 # Quintex31 -199.249.224.61:80 orport=443 id=B2197C23A4FF5D1C49EE45BA7688BA8BCCD89A0B # Quintex41 -199.249.223.71:80 orport=443 id=B6320E44A230302C7BF9319E67597A9B87882241 # QuintexAirVPN10 -199.249.223.60:80 orport=443 id=B7047FBDE9C53C39011CA84E5CB2A8E3543066D0 # Quintex11 -199.249.224.66:80 orport=443 id=C78AFFEEE320EA0F860961763E613FD2FAC855F5 # Quintex46 -199.249.224.44:80 orport=443 id=CB7C0D841FE376EF43F7845FF201B0290C0A239E # QuintexAirVPN25 -199.249.223.47:80 orport=443 id=CC14C97F1D23EE97766828FC8ED8582E21E11665 # Quintex38 -199.249.223.77:80 orport=443 id=CC4A3AE960E3617F49BF9887B79186C14CBA6813 # QuintexAirVPN4 -199.249.223.41:80 orport=443 id=D25210CE07C49F2A4F2BC7A506EB0F5EA7F5E2C2 # Quintex32 -199.249.223.79:80 orport=443 id=D33292FEDE24DD40F2385283E55C87F85C0943B6 # QuintexAirVPN2 -199.249.224.47:80 orport=443 id=D6FF2697CEA5C0C7DA84797C2E71163814FC2466 # QuintexAirVPN28 -199.249.223.68:80 orport=443 id=DF20497E487A979995D851A5BCEC313DF7E5BC51 # Quintex19 -199.249.223.43:80 orport=443 id=E480D577F58E782A5BC4FA6F49A6650E9389302F # Quintex34 -199.249.224.69:80 orport=443 id=EABC2DD0D47B5DB11F2D37EB3C60C2A4D91C10F2 # Quintex49 -199.249.223.78:80 orport=443 id=EC15DB62D9101481F364DE52EB8313C838BDDC29 # QuintexAirVPN3 -199.249.224.42:80 orport=443 id=F21DE9C7DE31601D9716781E17E24380887883D1 # QuintexAirVPN23 -199.249.223.81:80 orport=443 id=F7447E99EB5CBD4D5EB913EE0E35AC642B5C1EF3 # QuintexAirVPN1 -199.249.224.43:80 orport=443 id=FDD700C791CC6BB0AC1C2099A82CBC367AD4B764 # QuintexAirVPN24 -199.249.224.62:80 orport=443 id=FE00A3A835680E67FBBC895A724E2657BB253E97 # Quintex42 -199.249.223.66:80 orport=443 id=C5A53BCC174EF8FD0DCB223E4AA929FA557DEDB2 # Quintex17 - -# https://lists.torproject.org/pipermail/tor-relays/2017-December/013914.html -# https://lists.torproject.org/pipermail/tor-relays/2018-January/014063.html -5.196.23.64:9030 orport=9001 id=775B0FAFDE71AADC23FFC8782B7BEB1D5A92733E # Aerodynamik01 -217.182.75.181:9030 orport=9001 id=EFEACD781604EB80FBC025EDEDEA2D523AEAAA2F # Aerodynamik02 -193.70.43.76:9030 orport=9001 id=484A10BA2B8D48A5F0216674C8DD50EF27BC32F3 # Aerodynamik03 -149.56.141.138:9030 orport=9001 id=1938EBACBB1A7BFA888D9623C90061130E63BB3F # Aerodynamik04 -54.37.73.111:9030 orport=9001 id=92412EA1B9AA887D462B51D816777002F4D58907 # Aerodynamik05 -54.37.17.235:9030 orport=9001 id=360CBA08D1E24F513162047BDB54A1015E531534 # Aerodynamik06 - -# https://lists.torproject.org/pipermail/tor-relays/2017-December/013917.html -104.200.20.46:80 orport=9001 id=78E2BE744A53631B4AAB781468E94C52AB73968B # bynumlawtor - -# https://lists.torproject.org/pipermail/tor-relays/2017-December/013929.html -139.99.130.178:80 orport=443 id=867B95CACD64653FEEC4D2CEFC5C49B4620307A7 # coffswifi2 - -# https://lists.torproject.org/pipermail/tor-relays/2017-December/013946.html -172.98.193.43:80 orport=443 id=5E56738E7F97AA81DEEF59AF28494293DFBFCCDF # Backplane - -# Email sent directly to teor -62.210.254.132:80 orport=443 id=8456DFA94161CDD99E480C2A2992C366C6564410 # turingmachine - -# https://lists.torproject.org/pipermail/tor-relays/2017-December/013960.html -51.15.205.214:9030 orport=9001 id=8B6556601612F1E2AFCE2A12FFFAF8482A76DD1F ipv6=[2001:bc8:4400:2500::5:b07]:9001 # titania1 -51.15.205.214:9031 orport=9002 id=5E363D72488276160D062DDD2DFA25CFEBAF5EA9 ipv6=[2001:bc8:4400:2500::5:b07]:9002 # titania2 - -# https://lists.torproject.org/pipermail/tor-relays/2017-December/014000.html -24.117.231.229:34175 orport=45117 id=CE24412AD69444954B4015E293AE53DDDAFEA3D6 # Anosognosia - -# https://lists.torproject.org/pipermail/tor-relays/2018-January/014012.html -128.31.0.13:80 orport=443 id=A53C46F5B157DD83366D45A8E99A244934A14C46 # csailmitexit - -# Email sent directly to teor -82.247.103.117:110 orport=995 id=C9B3C1661A9577BA24C1C2C6123918921A495509 # Casper01 -109.238.2.79:110 orport=995 id=7520892E3DD133D0B0464D01A158B54B8E2A8B75 # Casper02 -51.15.179.153:110 orport=995 id=BB60F5BA113A0B8B44B7B37DE3567FE561E92F78 # Casper04 - -# Email sent directly to teor -80.127.107.179:80 orport=443 id=BC6B2E2F62ACC5EDECBABE64DA1E48F84DD98B78 ipv6=[2001:981:4a22:c::6]:443 # TVISION02 - -# https://lists.torproject.org/pipermail/tor-relays/2018-January/014020.html -37.120.174.249:80 orport=443 id=11DF0017A43AF1F08825CD5D973297F81AB00FF3 ipv6=[2a03:4000:6:724c:df98:15f9:b34d:443]:443 # gGDHjdcC6zAlM8k08lX - -# These fallbacks opted-in in previous releases, then changed their details, -# and so we blacklisted them. Now we want to whitelist changes. -# Assume details update is permanent -85.230.184.93:9030 orport=443 id=855BC2DABE24C861CD887DB9B2E950424B49FC34 # Logforme -176.31.180.157:143 orport=22 id=E781F4EC69671B3F1864AE2753E0890351506329 ipv6=[2001:41d0:8:eb9d::1]:22 # armbrust - -# https://lists.torproject.org/pipermail/tor-relays/2018-January/014024.html -82.161.212.209:9030 orport=9001 id=4E8CE6F5651E7342C1E7E5ED031E82078134FB0D ipv6=[2001:980:d7ed:1:ff:b0ff:fe00:d0b]:9001 # ymkeo - -# https://lists.torproject.org/pipermail/tor-relays/2018-January/014055.html -37.157.255.35:9030 orport=9090 id=361D33C96D0F161275EE67E2C91EE10B276E778B # cxx4freedom - -# https://lists.torproject.org/pipermail/tor-relays/2018-January/014064.html -87.118.122.120:80 orport=443 id=A2A6616723B511D8E068BB71705191763191F6B2 # otheontelth - -# https://lists.torproject.org/pipermail/tor-relays/2018-January/014069.html -185.100.86.182:9030 orport=8080 id=E51620B90DCB310138ED89EDEDD0A5C361AAE24E # NormalCitizen - -# https://lists.torproject.org/pipermail/tor-relays/2018-January/014267.html -51.15.72.211:80 orport=9001 id=D122094E396DF8BA560843E7B983B0EA649B7DF9 ipv6=[2001:bc8:4700:2300::1b:f09]:9001 # gjtorrelay - -# Email sent directly to Phoul -185.34.33.2:9265 orport=31415 id=D71B1CA1C9DC7E8CA64158E106AD770A21160FEE # lqdn - -# Email sent directly to Phoul -78.156.110.135:9091 orport=9090 id=F48FD1AED068496D51D1384BC7497C04E4985DA6 # SkynetC2 - -# Email sent directly to Phoul -5.200.21.144:80 orport=443 id=0C039F35C2E40DCB71CD8A07E97C7FD7787D42D6 # libel -64.79.152.132:80 orport=443 id=375DCBB2DBD94E5263BC0C015F0C9E756669617E # ebola - -# https://lists.torproject.org/pipermail/tor-relays/2018-June/015524.html -132.248.241.5:9030 orport=9001 id=4661DE96D3F8E923994B05218F23760C8D7935A4 - -# https://lists.torproject.org/pipermail/tor-relays/2018-June/015522.html -96.253.78.108:80 orport=442 id=924B24AFA7F075D059E8EEB284CC400B33D3D036 - -# Email sent directly to Phoul -163.172.218.10:9030 orport=9001 id=78809B6C50CB6491DB3A72C60EC39DC85BF72D1F ipv6=[2001:bc8:3f23:1100::1]:9001 -163.172.218.10:9130 orport=9101 id=B247BA9E0AEA93E6D7BF4080CFBB964034AF2B28 ipv6=[2001:bc8:3f23:1100::1]:9101 - -# Email sent directly to Phoul -158.255.212.178:8080 orport=8443 id=D941D380E5228E7B4D372AF4D484629A96DC48B9 ipv6=[2a03:f80:ed15:158:255:212:178:2]:8443 - -# Email sent directly to Phoul -45.79.108.130:9030 orport=9001 id=AEDAC7081AE14B8D241ECF0FF17A2858AB4383D0 ipv6=[2600:3c01:e000:131::8000:0]:9001 - -# Email sent directly to Phoul -51.254.147.57:80 orport=443 id=EB80A8D52F07238B576C42CEAB98ADD084EE075E -217.182.51.248:80 orport=443 id=D6BA940D3255AB40DC5EE5B0B285FA143E1F9865 - -# https://lists.torproject.org/pipermail/tor-relays/2018-June/015541.html -195.191.81.7:9030 orport=9001 id=41A3C16269C7B63DB6EB741DBDDB4E1F586B1592 ipv6=[2a00:1908:fffc:ffff:c0a6:ccff:fe62:e1a1]:9001 -51.254.96.208:9030 orport=9001 id=8101421BEFCCF4C271D5483C5AABCAAD245BBB9D ipv6=[2001:41d0:401:3100::30dc]:9001 -163.172.154.162:9030 orport=9001 id=F741E5124CB12700DA946B78C9B2DD175D6CD2A1 ipv6=[2001:bc8:4400:2100::17:419]:9001 -51.15.78.0:9030 orport=9001 id=15BE17C99FACE24470D40AF782D6A9C692AB36D6 ipv6=[2001:bc8:4700:2300::16:c0b]:9001 -54.37.139.118:9030 orport=9001 id=90A5D1355C4B5840E950EB61E673863A6AE3ACA1 ipv6=[2001:41d0:601:1100::1b8]:9001 -51.38.65.160:9030 orport=9001 id=3CB4193EF4E239FCEDC4DC43468E0B0D6B67ACC3 ipv6=[2001:41d0:801:2000::f6e]:9001 - -# Email sent directly to Phoul -54.37.138.138:8080 orport=993 id=1576BE143D8727745BB2BCDDF183291B3C3EFEFC - -# Email sent directly to Phoul -67.215.255.140:9030 orport=9001 id=23917BB3F3994BC61F0C9D7AD19B069F9E150D26 - -# Email sent directly to Phoul -195.154.105.170:9030 orport=9001 id=E947C029087FA1C3499BEF5D4372947C51223D8F - -# Email sent directly to Phoul -23.129.64.101:80 orport=443 id=2EB20285FE55927B7AECC47BB94F22534FBC3941 ipv6=[2620:18c:0:1001::101]:443 -23.129.64.102:80 orport=443 id=CA9739E2805A3CD73CF75BBCB6785C32394240E3 ipv6=[2620:18c:0:1001::102]:443 -23.129.64.103:80 orport=443 id=8ED84B53BD9556CCBB036073A1AD508EC27CBE52 ipv6=[2620:18c:0:1001::103]:443 - -# Email sent directly to Phoul -37.139.8.104:9030 orport=9001 id=7088D485934E8A403B81531F8C90BDC75FA43C98 ipv6=[2a03:b0c0:0:1010::24c:1001]:9001 - -# Email sent directly to Phoul -178.254.7.88:9030 orpport=9001 id=85A885433E50B1874F11CEC9BE98451E24660976 - -# https://lists.torproject.org/pipermail/tor-relays/2018-August/015869.html -5.45.111.149:80 orport=443 id=D405FCCF06ADEDF898DF2F29C9348DCB623031BA ipv6=[2a03:4000:6:2388:df98:15f9:b34d:443]:443 - -# https://trac.torproject.org/projects/tor/ticket/27297 -37.252.185.182:9030 orport=8080 id=113143469021882C3A4B82F084F8125B08EE471E ipv6=[2a00:63c1:a:182::2]:8080 - -# Email sent directly to Phoul -139.99.130.178:80 orport=443 id=867B95CACD64653FEEC4D2CEFC5C49B4620307A7 - -# Email sent directly to Phoul -104.131.11.214:9030 orport=8080 id=32828476F4F84E15C42B4C360A5CD8DE4C3C2BE7 - -# Email sent directly to Phoul / Teor -178.175.139.122:80 orport=443 id=490FB3FAAF8837407D94CA7E1DEF025DEF0F3516 ipv6=[2a00:1dc0:3002::3]:443 - -# Email sent directly to Phoul -192.42.116.16:80 orport=443 id=81B75D534F91BFB7C57AB67DA10BCEF622582AE8 - -# https://lists.torproject.org/pipermail/tor-relays/2018-November/016610.html -24.117.194.80:80 orport=443 id=B6C4C9A43658F686F8892CA5666717532F72979C diff --git a/scripts/maint/generateFallbackDirLine.py b/scripts/maint/generateFallbackDirLine.py deleted file mode 100755 index b856c938bf..0000000000 --- a/scripts/maint/generateFallbackDirLine.py +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env python - -# Generate a fallback directory whitelist/blacklist line for every fingerprint -# passed as an argument. -# -# Usage: -# generateFallbackDirLine.py fingerprint ... - -import sys -import urllib2 - -import stem.descriptor.remote -import stem.util.tor_tools - -if len(sys.argv) <= 1: - print('Usage: %s fingerprint ...' % sys.argv[0]) - sys.exit(1) - -for fingerprint in sys.argv[1:]: - if not stem.util.tor_tools.is_valid_fingerprint(fingerprint): - print("'%s' isn't a valid relay fingerprint" % fingerprint) - sys.exit(1) - - try: - desc = stem.descriptor.remote.get_server_descriptors(fingerprint).run()[0] - except urllib2.HTTPError as exc: - if exc.code == 404: - print('# %s not found in recent descriptors' % fingerprint) - continue - else: - raise - - if not desc.dir_port: - print("# %s needs a DirPort" % fingerprint) - else: - ipv6_addresses = [(address, port) for address, port, is_ipv6 in desc.or_addresses if is_ipv6] - ipv6_field = ' ipv6=[%s]:%s' % ipv6_addresses[0] if ipv6_addresses else '' - print('%s:%s orport=%s id=%s%s # %s' % (desc.address, desc.dir_port, desc.or_port, fingerprint, ipv6_field, desc.nickname)) diff --git a/scripts/maint/lookupFallbackDirContact.py b/scripts/maint/lookupFallbackDirContact.py deleted file mode 100755 index 14c53d1282..0000000000 --- a/scripts/maint/lookupFallbackDirContact.py +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env python - -# Lookup fallback directory contact lines for every fingerprint passed as an -# argument. -# -# Usage: -# lookupFallbackDirContact.py fingerprint ... - -import sys - -import stem.descriptor.remote as remote - -if len(sys.argv) <= 1: - print "Usage: {} fingerprint ...".format(sys.argv[0]) - sys.exit(-1) - -# we need descriptors, because the consensus does not have contact infos -descriptor_list = remote.get_server_descriptors(fingerprints=sys.argv[1:]).run() - -descriptor_list_fingerprints = [] -for d in descriptor_list: - assert d.fingerprint in sys.argv[1:] - descriptor_list_fingerprints.append(d.fingerprint) - print "{} {}".format(d.fingerprint, d.contact) - -for fingerprint in sys.argv[1:]: - if fingerprint not in descriptor_list_fingerprints: - print "{} not found in current descriptors".format(fingerprint) diff --git a/scripts/maint/updateFallbackDirs.py b/scripts/maint/updateFallbackDirs.py deleted file mode 100755 index 930a0a7275..0000000000 --- a/scripts/maint/updateFallbackDirs.py +++ /dev/null @@ -1,2383 +0,0 @@ -#!/usr/bin/env python - -# Usage: -# -# Regenerate the list: -# scripts/maint/updateFallbackDirs.py > src/app/config/fallback_dirs.inc 2> fallback_dirs.log -# -# Check the existing list: -# scripts/maint/updateFallbackDirs.py check_existing > fallback_dirs.inc.ok 2> fallback_dirs.log -# mv fallback_dirs.inc.ok src/app/config/fallback_dirs.inc -# -# This script should be run from a stable, reliable network connection, -# with no other network activity (and not over tor). -# If this is not possible, please disable: -# PERFORM_IPV4_DIRPORT_CHECKS and PERFORM_IPV6_DIRPORT_CHECKS -# -# Needs dateutil, stem, and potentially other python packages. -# Optionally uses ipaddress (python 3 builtin) or py2-ipaddress (package) -# for netblock analysis. -# -# After running this script, read the logs to make sure the fallbacks aren't -# dominated by a single netblock or port. - -# Script by weasel, April 2015 -# Portions by gsathya & karsten, 2013 -# https://trac.torproject.org/projects/tor/attachment/ticket/8374/dir_list.2.py -# Modifications by teor, 2015 - -import StringIO -import string -import re -import datetime -import gzip -import os.path -import json -import math -import sys -import urllib -import urllib2 -import hashlib -import dateutil.parser -import copy -import re - -from stem.descriptor import DocumentHandler -from stem.descriptor.remote import get_consensus, get_server_descriptors, MAX_FINGERPRINTS - -import logging -logging.root.name = '' - -HAVE_IPADDRESS = False -try: - # python 3 builtin, or install package py2-ipaddress - # there are several ipaddress implementations for python 2 - # with slightly different semantics with str typed text - # fortunately, all our IP addresses are in unicode - import ipaddress - HAVE_IPADDRESS = True -except ImportError: - # if this happens, we avoid doing netblock analysis - logging.warning('Unable to import ipaddress, please install py2-ipaddress.' + - ' A fallback list will be created, but optional netblock' + - ' analysis will not be performed.') - -## Top-Level Configuration - -# We use semantic versioning: https://semver.org -# In particular: -# * major changes include removing a mandatory field, or anything else that -# would break an appropriately tolerant parser, -# * minor changes include adding a field, -# * patch changes include changing header comments or other unstructured -# content -FALLBACK_FORMAT_VERSION = '2.0.0' -SECTION_SEPARATOR_BASE = '=====' -SECTION_SEPARATOR_COMMENT = '/* ' + SECTION_SEPARATOR_BASE + ' */' - -# Output all candidate fallbacks, or only output selected fallbacks? -OUTPUT_CANDIDATES = False - -# Perform DirPort checks over IPv4? -# Change this to False if IPv4 doesn't work for you, or if you don't want to -# download a consensus for each fallback -# Don't check ~1000 candidates when OUTPUT_CANDIDATES is True -PERFORM_IPV4_DIRPORT_CHECKS = False if OUTPUT_CANDIDATES else True - -# Perform DirPort checks over IPv6? -# If you know IPv6 works for you, set this to True -# This will exclude IPv6 relays without an IPv6 DirPort configured -# So it's best left at False until #18394 is implemented -# Don't check ~1000 candidates when OUTPUT_CANDIDATES is True -PERFORM_IPV6_DIRPORT_CHECKS = False if OUTPUT_CANDIDATES else False - -# Must relays be running now? -MUST_BE_RUNNING_NOW = (PERFORM_IPV4_DIRPORT_CHECKS - or PERFORM_IPV6_DIRPORT_CHECKS) - -# Clients have been using microdesc consensuses by default for a while now -DOWNLOAD_MICRODESC_CONSENSUS = True - -# If a relay delivers an invalid consensus, if it will become valid less than -# this many seconds in the future, or expired less than this many seconds ago, -# accept the relay as a fallback. For the consensus expiry check to be -# accurate, the machine running this script needs an accurate clock. -# -# Relays on 0.3.0 and later return a 404 when they are about to serve a -# consensus that expired more than 24 hours ago. 0.2.9 and earlier relays -# will serve consensuses that are very old. -# -# Relays on 0.3.5.6-rc? and later return a 404 when they are about to serve a -# consensus that will become valid more than 24 hours in the future. Older -# relays don't serve future consensuses. -# -# A 404 makes relays fail the download check. We use a tolerance of 24 hours, -# so that 0.2.9 relays also fail the download check if they serve a consensus -# that is not reasonably live. -# -# REASONABLY_LIVE_TIME should never be more than Tor's REASONABLY_LIVE_TIME, -# (24 hours), because clients reject consensuses that are older than that. -# Clients on 0.3.5.5-alpha? and earlier also won't select guards from -# consensuses that have expired, but can bootstrap if they already have guards -# in their state file. -REASONABLY_LIVE_TIME = 24*60*60 - -# Output fallback name, flags, bandwidth, and ContactInfo in a C comment? -OUTPUT_COMMENTS = True if OUTPUT_CANDIDATES else False - -# Output matching ContactInfo in fallbacks list? -# Useful if you're trying to contact operators -CONTACT_COUNT = True if OUTPUT_CANDIDATES else False - -# How the list should be sorted: -# fingerprint: is useful for stable diffs of fallback lists -# measured_bandwidth: is useful when pruning the list based on bandwidth -# contact: is useful for contacting operators once the list has been pruned -OUTPUT_SORT_FIELD = 'contact' if OUTPUT_CANDIDATES else 'fingerprint' - -## OnionOO Settings - -ONIONOO = 'https://onionoo.torproject.org/' -#ONIONOO = 'https://onionoo.thecthulhu.com/' - -# Don't bother going out to the Internet, just use the files available locally, -# even if they're very old -LOCAL_FILES_ONLY = False - -## Whitelist / Blacklist Filter Settings - -# The whitelist contains entries that are included if all attributes match -# (IPv4, dirport, orport, id, and optionally IPv6 and IPv6 orport) - -# What happens to entries not in whitelist? -# When True, they are included, when False, they are excluded -INCLUDE_UNLISTED_ENTRIES = True if OUTPUT_CANDIDATES else False - -WHITELIST_FILE_NAME = 'scripts/maint/fallback.whitelist' -FALLBACK_FILE_NAME = 'src/app/config/fallback_dirs.inc' - -# The number of bytes we'll read from a filter file before giving up -MAX_LIST_FILE_SIZE = 1024 * 1024 - -## Eligibility Settings - -# Require fallbacks to have the same address and port for a set amount of time -# We used to have this at 1 week, but that caused many fallback failures, which -# meant that we had to rebuild the list more often. We want fallbacks to be -# stable for 2 years, so we set it to a few months. -# -# If a relay changes address or port, that's it, it's not useful any more, -# because clients can't find it -ADDRESS_AND_PORT_STABLE_DAYS = 90 -# We ignore relays that have been down for more than this period -MAX_DOWNTIME_DAYS = 0 if MUST_BE_RUNNING_NOW else 7 -# FallbackDirs must have a time-weighted-fraction that is greater than or -# equal to: -# Mirrors that are down half the time are still useful half the time -CUTOFF_RUNNING = .50 -CUTOFF_V2DIR = .50 -# Guard flags are removed for some time after a relay restarts, so we ignore -# the guard flag. -CUTOFF_GUARD = .00 -# FallbackDirs must have a time-weighted-fraction that is less than or equal -# to: -# .00 means no bad exits -PERMITTED_BADEXIT = .00 - -# older entries' weights are adjusted with ALPHA^(age in days) -AGE_ALPHA = 0.99 - -# this factor is used to scale OnionOO entries to [0,1] -ONIONOO_SCALE_ONE = 999. - -## Fallback Count Limits - -# The target for these parameters is 20% of the guards in the network -# This is around 200 as of October 2015 -_FB_POG = 0.2 -FALLBACK_PROPORTION_OF_GUARDS = None if OUTPUT_CANDIDATES else _FB_POG - -# Limit the number of fallbacks (eliminating lowest by advertised bandwidth) -MAX_FALLBACK_COUNT = None if OUTPUT_CANDIDATES else 200 -# Emit a C #error if the number of fallbacks is less than expected -MIN_FALLBACK_COUNT = 0 if OUTPUT_CANDIDATES else MAX_FALLBACK_COUNT*0.5 - -# The maximum number of fallbacks on the same address, contact, or family -# -# With 150 fallbacks, this means each operator sees 5% of client bootstraps. -# For comparison: -# - We try to limit guard and exit operators to 5% of the network -# - The directory authorities used to see 11% of client bootstraps each -# -# We also don't want too much of the list to go down if a single operator -# has to move all their relays. -MAX_FALLBACKS_PER_IP = 1 -MAX_FALLBACKS_PER_IPV4 = MAX_FALLBACKS_PER_IP -MAX_FALLBACKS_PER_IPV6 = MAX_FALLBACKS_PER_IP -MAX_FALLBACKS_PER_CONTACT = 7 -MAX_FALLBACKS_PER_FAMILY = 7 - -## Fallback Bandwidth Requirements - -# Any fallback with the Exit flag has its bandwidth multiplied by this fraction -# to make sure we aren't further overloading exits -# (Set to 1.0, because we asked that only lightly loaded exits opt-in, -# and the extra load really isn't that much for large relays.) -EXIT_BANDWIDTH_FRACTION = 1.0 - -# If a single fallback's bandwidth is too low, it's pointless adding it -# We expect fallbacks to handle an extra 10 kilobytes per second of traffic -# Make sure they can support fifty times the expected extra load -# -# We convert this to a consensus weight before applying the filter, -# because all the bandwidth amounts are specified by the relay -MIN_BANDWIDTH = 50.0 * 10.0 * 1024.0 - -# Clients will time out after 30 seconds trying to download a consensus -# So allow fallback directories half that to deliver a consensus -# The exact download times might change based on the network connection -# running this script, but only by a few seconds -# There is also about a second of python overhead -CONSENSUS_DOWNLOAD_SPEED_MAX = 15.0 -# If the relay fails a consensus check, retry the download -# This avoids delisting a relay due to transient network conditions -CONSENSUS_DOWNLOAD_RETRY = True - -## Parsing Functions - -def parse_ts(t): - return datetime.datetime.strptime(t, "%Y-%m-%d %H:%M:%S") - -def remove_bad_chars(raw_string, bad_char_list): - # Remove each character in the bad_char_list - cleansed_string = raw_string - for c in bad_char_list: - cleansed_string = cleansed_string.replace(c, '') - return cleansed_string - -def cleanse_unprintable(raw_string): - # Remove all unprintable characters - cleansed_string = '' - for c in raw_string: - if c in string.printable: - cleansed_string += c - return cleansed_string - -def cleanse_whitespace(raw_string): - # Replace all whitespace characters with a space - cleansed_string = raw_string - for c in string.whitespace: - cleansed_string = cleansed_string.replace(c, ' ') - return cleansed_string - -def cleanse_c_multiline_comment(raw_string): - cleansed_string = raw_string - # Embedded newlines should be removed by tor/onionoo, but let's be paranoid - cleansed_string = cleanse_whitespace(cleansed_string) - # ContactInfo and Version can be arbitrary binary data - cleansed_string = cleanse_unprintable(cleansed_string) - # Prevent a malicious / unanticipated string from breaking out - # of a C-style multiline comment - # This removes '/*' and '*/' and '//' - bad_char_list = '*/' - # Prevent a malicious string from using C nulls - bad_char_list += '\0' - # Avoid confusing parsers by making sure there is only one comma per fallback - bad_char_list += ',' - # Avoid confusing parsers by making sure there is only one equals per field - bad_char_list += '=' - # Be safer by removing bad characters entirely - cleansed_string = remove_bad_chars(cleansed_string, bad_char_list) - # Some compilers may further process the content of comments - # There isn't much we can do to cover every possible case - # But comment-based directives are typically only advisory - return cleansed_string - -def cleanse_c_string(raw_string): - cleansed_string = raw_string - # Embedded newlines should be removed by tor/onionoo, but let's be paranoid - cleansed_string = cleanse_whitespace(cleansed_string) - # ContactInfo and Version can be arbitrary binary data - cleansed_string = cleanse_unprintable(cleansed_string) - # Prevent a malicious address/fingerprint string from breaking out - # of a C-style string - bad_char_list = '"' - # Prevent a malicious string from using escapes - bad_char_list += '\\' - # Prevent a malicious string from using C nulls - bad_char_list += '\0' - # Avoid confusing parsers by making sure there is only one comma per fallback - bad_char_list += ',' - # Avoid confusing parsers by making sure there is only one equals per field - bad_char_list += '=' - # Be safer by removing bad characters entirely - cleansed_string = remove_bad_chars(cleansed_string, bad_char_list) - # Some compilers may further process the content of strings - # There isn't much we can do to cover every possible case - # But this typically only results in changes to the string data - return cleansed_string - -## OnionOO Source Functions - -# a dictionary of source metadata for each onionoo query we've made -fetch_source = {} - -# register source metadata for 'what' -# assumes we only retrieve one document for each 'what' -def register_fetch_source(what, url, relays_published, version): - fetch_source[what] = {} - fetch_source[what]['url'] = url - fetch_source[what]['relays_published'] = relays_published - fetch_source[what]['version'] = version - -# list each registered source's 'what' -def fetch_source_list(): - return sorted(fetch_source.keys()) - -# given 'what', provide a multiline C comment describing the source -def describe_fetch_source(what): - desc = '/*' - desc += '\n' - desc += 'Onionoo Source: ' - desc += cleanse_c_multiline_comment(what) - desc += ' Date: ' - desc += cleanse_c_multiline_comment(fetch_source[what]['relays_published']) - desc += ' Version: ' - desc += cleanse_c_multiline_comment(fetch_source[what]['version']) - desc += '\n' - desc += 'URL: ' - desc += cleanse_c_multiline_comment(fetch_source[what]['url']) - desc += '\n' - desc += '*/' - return desc - -## File Processing Functions - -def write_to_file(str, file_name, max_len): - try: - with open(file_name, 'w') as f: - f.write(str[0:max_len]) - except EnvironmentError, error: - logging.error('Writing file %s failed: %d: %s'% - (file_name, - error.errno, - error.strerror) - ) - -def read_from_file(file_name, max_len): - try: - if os.path.isfile(file_name): - with open(file_name, 'r') as f: - return f.read(max_len) - except EnvironmentError, error: - logging.info('Loading file %s failed: %d: %s'% - (file_name, - error.errno, - error.strerror) - ) - return None - -def parse_fallback_file(file_name): - file_data = read_from_file(file_name, MAX_LIST_FILE_SIZE) - file_data = cleanse_unprintable(file_data) - file_data = remove_bad_chars(file_data, '\n"\0') - file_data = re.sub('/\*.*?\*/', '', file_data) - file_data = file_data.replace(',', '\n') - file_data = file_data.replace(' weight=10', '') - return file_data - -def load_possibly_compressed_response_json(response): - if response.info().get('Content-Encoding') == 'gzip': - buf = StringIO.StringIO( response.read() ) - f = gzip.GzipFile(fileobj=buf) - return json.load(f) - else: - return json.load(response) - -def load_json_from_file(json_file_name): - # An exception here may be resolved by deleting the .last_modified - # and .json files, and re-running the script - try: - with open(json_file_name, 'r') as f: - return json.load(f) - except EnvironmentError, error: - raise Exception('Reading not-modified json file %s failed: %d: %s'% - (json_file_name, - error.errno, - error.strerror) - ) - -## OnionOO Functions - -def datestr_to_datetime(datestr): - # Parse datetimes like: Fri, 02 Oct 2015 13:34:14 GMT - if datestr is not None: - dt = dateutil.parser.parse(datestr) - else: - # Never modified - use start of epoch - dt = datetime.datetime.utcfromtimestamp(0) - # strip any timezone out (in case they're supported in future) - dt = dt.replace(tzinfo=None) - return dt - -def onionoo_fetch(what, **kwargs): - params = kwargs - params['type'] = 'relay' - #params['limit'] = 10 - params['first_seen_days'] = '%d-'%(ADDRESS_AND_PORT_STABLE_DAYS) - params['last_seen_days'] = '-%d'%(MAX_DOWNTIME_DAYS) - params['flag'] = 'V2Dir' - url = ONIONOO + what + '?' + urllib.urlencode(params) - - # Unfortunately, the URL is too long for some OS filenames, - # but we still don't want to get files from different URLs mixed up - base_file_name = what + '-' + hashlib.sha1(url).hexdigest() - - full_url_file_name = base_file_name + '.full_url' - MAX_FULL_URL_LENGTH = 1024 - - last_modified_file_name = base_file_name + '.last_modified' - MAX_LAST_MODIFIED_LENGTH = 64 - - json_file_name = base_file_name + '.json' - - if LOCAL_FILES_ONLY: - # Read from the local file, don't write to anything - response_json = load_json_from_file(json_file_name) - else: - # store the full URL to a file for debugging - # no need to compare as long as you trust SHA-1 - write_to_file(url, full_url_file_name, MAX_FULL_URL_LENGTH) - - request = urllib2.Request(url) - request.add_header('Accept-encoding', 'gzip') - - # load the last modified date from the file, if it exists - last_mod_date = read_from_file(last_modified_file_name, - MAX_LAST_MODIFIED_LENGTH) - if last_mod_date is not None: - request.add_header('If-modified-since', last_mod_date) - - # Parse last modified date - last_mod = datestr_to_datetime(last_mod_date) - - # Not Modified and still recent enough to be useful - # Onionoo / Globe used to use 6 hours, but we can afford a day - required_freshness = datetime.datetime.utcnow() - # strip any timezone out (to match dateutil.parser) - required_freshness = required_freshness.replace(tzinfo=None) - required_freshness -= datetime.timedelta(hours=24) - - # Make the OnionOO request - response_code = 0 - try: - response = urllib2.urlopen(request) - response_code = response.getcode() - except urllib2.HTTPError, error: - response_code = error.code - if response_code == 304: # not modified - pass - else: - raise Exception("Could not get " + url + ": " - + str(error.code) + ": " + error.reason) - - if response_code == 200: # OK - last_mod = datestr_to_datetime(response.info().get('Last-Modified')) - - # Check for freshness - if last_mod < required_freshness: - if last_mod_date is not None: - # This check sometimes fails transiently, retry the script if it does - date_message = "Outdated data: last updated " + last_mod_date - else: - date_message = "No data: never downloaded " - raise Exception(date_message + " from " + url) - - # Process the data - if response_code == 200: # OK - - response_json = load_possibly_compressed_response_json(response) - - with open(json_file_name, 'w') as f: - # use the most compact json representation to save space - json.dump(response_json, f, separators=(',',':')) - - # store the last modified date in its own file - if response.info().get('Last-modified') is not None: - write_to_file(response.info().get('Last-Modified'), - last_modified_file_name, - MAX_LAST_MODIFIED_LENGTH) - - elif response_code == 304: # Not Modified - - response_json = load_json_from_file(json_file_name) - - else: # Unexpected HTTP response code not covered in the HTTPError above - raise Exception("Unexpected HTTP response code to " + url + ": " - + str(response_code)) - - register_fetch_source(what, - url, - response_json['relays_published'], - response_json['version']) - - return response_json - -def fetch(what, **kwargs): - #x = onionoo_fetch(what, **kwargs) - # don't use sort_keys, as the order of or_addresses is significant - #print json.dumps(x, indent=4, separators=(',', ': ')) - #sys.exit(0) - - return onionoo_fetch(what, **kwargs) - -## Fallback Candidate Class - -class Candidate(object): - CUTOFF_ADDRESS_AND_PORT_STABLE = (datetime.datetime.utcnow() - - datetime.timedelta(ADDRESS_AND_PORT_STABLE_DAYS)) - - def __init__(self, details): - for f in ['fingerprint', 'nickname', 'last_changed_address_or_port', - 'consensus_weight', 'or_addresses', 'dir_address']: - if not f in details: raise Exception("Document has no %s field."%(f,)) - - if not 'contact' in details: - details['contact'] = None - if not 'flags' in details or details['flags'] is None: - details['flags'] = [] - if (not 'advertised_bandwidth' in details - or details['advertised_bandwidth'] is None): - # relays without advertised bandwidth have it calculated from their - # consensus weight - details['advertised_bandwidth'] = 0 - if (not 'effective_family' in details - or details['effective_family'] is None): - details['effective_family'] = [] - if not 'platform' in details: - details['platform'] = None - details['last_changed_address_or_port'] = parse_ts( - details['last_changed_address_or_port']) - self._data = details - self._stable_sort_or_addresses() - - self._fpr = self._data['fingerprint'] - self._running = self._guard = self._v2dir = 0. - self._split_dirport() - self._compute_orport() - if self.orport is None: - raise Exception("Failed to get an orport for %s."%(self._fpr,)) - self._compute_ipv6addr() - if not self.has_ipv6(): - logging.debug("Failed to get an ipv6 address for %s."%(self._fpr,)) - self._compute_version() - self._extra_info_cache = None - - def _stable_sort_or_addresses(self): - # replace self._data['or_addresses'] with a stable ordering, - # sorting the secondary addresses in string order - # leave the received order in self._data['or_addresses_raw'] - self._data['or_addresses_raw'] = self._data['or_addresses'] - or_address_primary = self._data['or_addresses'][:1] - # subsequent entries in the or_addresses array are in an arbitrary order - # so we stabilise the addresses by sorting them in string order - or_addresses_secondaries_stable = sorted(self._data['or_addresses'][1:]) - or_addresses_stable = or_address_primary + or_addresses_secondaries_stable - self._data['or_addresses'] = or_addresses_stable - - def get_fingerprint(self): - return self._fpr - - # is_valid_ipv[46]_address by gsathya, karsten, 2013 - @staticmethod - def is_valid_ipv4_address(address): - if not isinstance(address, (str, unicode)): - return False - - # check if there are four period separated values - if address.count(".") != 3: - return False - - # checks that each value in the octet are decimal values between 0-255 - for entry in address.split("."): - if not entry.isdigit() or int(entry) < 0 or int(entry) > 255: - return False - elif entry[0] == "0" and len(entry) > 1: - return False # leading zeros, for instance in "1.2.3.001" - - return True - - @staticmethod - def is_valid_ipv6_address(address): - if not isinstance(address, (str, unicode)): - return False - - # remove brackets - address = address[1:-1] - - # addresses are made up of eight colon separated groups of four hex digits - # with leading zeros being optional - # https://en.wikipedia.org/wiki/IPv6#Address_format - - colon_count = address.count(":") - - if colon_count > 7: - return False # too many groups - elif colon_count != 7 and not "::" in address: - return False # not enough groups and none are collapsed - elif address.count("::") > 1 or ":::" in address: - return False # multiple groupings of zeros can't be collapsed - - found_ipv4_on_previous_entry = False - for entry in address.split(":"): - # If an IPv6 address has an embedded IPv4 address, - # it must be the last entry - if found_ipv4_on_previous_entry: - return False - if not re.match("^[0-9a-fA-f]{0,4}$", entry): - if not Candidate.is_valid_ipv4_address(entry): - return False - else: - found_ipv4_on_previous_entry = True - - return True - - def _split_dirport(self): - # Split the dir_address into dirip and dirport - (self.dirip, _dirport) = self._data['dir_address'].split(':', 2) - self.dirport = int(_dirport) - - def _compute_orport(self): - # Choose the first ORPort that's on the same IPv4 address as the DirPort. - # In rare circumstances, this might not be the primary ORPort address. - # However, _stable_sort_or_addresses() ensures we choose the same one - # every time, even if onionoo changes the order of the secondaries. - self._split_dirport() - self.orport = None - for i in self._data['or_addresses']: - if i != self._data['or_addresses'][0]: - logging.debug('Secondary IPv4 Address Used for %s: %s'%(self._fpr, i)) - (ipaddr, port) = i.rsplit(':', 1) - if (ipaddr == self.dirip) and Candidate.is_valid_ipv4_address(ipaddr): - self.orport = int(port) - return - - def _compute_ipv6addr(self): - # Choose the first IPv6 address that uses the same port as the ORPort - # Or, choose the first IPv6 address in the list - # _stable_sort_or_addresses() ensures we choose the same IPv6 address - # every time, even if onionoo changes the order of the secondaries. - self.ipv6addr = None - self.ipv6orport = None - # Choose the first IPv6 address that uses the same port as the ORPort - for i in self._data['or_addresses']: - (ipaddr, port) = i.rsplit(':', 1) - if (port == self.orport) and Candidate.is_valid_ipv6_address(ipaddr): - self.ipv6addr = ipaddr - self.ipv6orport = int(port) - return - # Choose the first IPv6 address in the list - for i in self._data['or_addresses']: - (ipaddr, port) = i.rsplit(':', 1) - if Candidate.is_valid_ipv6_address(ipaddr): - self.ipv6addr = ipaddr - self.ipv6orport = int(port) - return - - def _compute_version(self): - # parse the version out of the platform string - # The platform looks like: "Tor 0.2.7.6 on Linux" - self._data['version'] = None - if self._data['platform'] is None: - return - # be tolerant of weird whitespacing, use a whitespace split - tokens = self._data['platform'].split() - for token in tokens: - vnums = token.split('.') - # if it's at least a.b.c.d, with potentially an -alpha-dev, -alpha, -rc - if (len(vnums) >= 4 and vnums[0].isdigit() and vnums[1].isdigit() and - vnums[2].isdigit()): - self._data['version'] = token - return - - # From #20509 - # bug #20499 affects versions from 0.2.9.1-alpha-dev to 0.2.9.4-alpha-dev - # and version 0.3.0.0-alpha-dev - # Exhaustive lists are hard to get wrong - STALE_CONSENSUS_VERSIONS = ['0.2.9.1-alpha-dev', - '0.2.9.2-alpha', - '0.2.9.2-alpha-dev', - '0.2.9.3-alpha', - '0.2.9.3-alpha-dev', - '0.2.9.4-alpha', - '0.2.9.4-alpha-dev', - '0.3.0.0-alpha-dev' - ] - - def is_valid_version(self): - # call _compute_version before calling this - # is the version of the relay a version we want as a fallback? - # checks both recommended versions and bug #20499 / #20509 - # - # if the relay doesn't have a recommended version field, exclude the relay - if not self._data.has_key('recommended_version'): - log_excluded('%s not a candidate: no recommended_version field', - self._fpr) - return False - if not self._data['recommended_version']: - log_excluded('%s not a candidate: version not recommended', self._fpr) - return False - # if the relay doesn't have version field, exclude the relay - if not self._data.has_key('version'): - log_excluded('%s not a candidate: no version field', self._fpr) - return False - if self._data['version'] in Candidate.STALE_CONSENSUS_VERSIONS: - logging.warning('%s not a candidate: version delivers stale consensuses', - self._fpr) - return False - return True - - @staticmethod - def _extract_generic_history(history, which='unknown'): - # given a tree like this: - # { - # "1_month": { - # "count": 187, - # "factor": 0.001001001001001001, - # "first": "2015-02-27 06:00:00", - # "interval": 14400, - # "last": "2015-03-30 06:00:00", - # "values": [ - # 999, - # 999 - # ] - # }, - # "1_week": { - # "count": 169, - # "factor": 0.001001001001001001, - # "first": "2015-03-23 07:30:00", - # "interval": 3600, - # "last": "2015-03-30 07:30:00", - # "values": [ ...] - # }, - # "1_year": { - # "count": 177, - # "factor": 0.001001001001001001, - # "first": "2014-04-11 00:00:00", - # "interval": 172800, - # "last": "2015-03-29 00:00:00", - # "values": [ ...] - # }, - # "3_months": { - # "count": 185, - # "factor": 0.001001001001001001, - # "first": "2014-12-28 06:00:00", - # "interval": 43200, - # "last": "2015-03-30 06:00:00", - # "values": [ ...] - # } - # }, - # extract exactly one piece of data per time interval, - # using smaller intervals where available. - # - # returns list of (age, length, value) dictionaries. - - generic_history = [] - - periods = history.keys() - periods.sort(key = lambda x: history[x]['interval']) - now = datetime.datetime.utcnow() - newest = now - for p in periods: - h = history[p] - interval = datetime.timedelta(seconds = h['interval']) - this_ts = parse_ts(h['last']) - - if (len(h['values']) != h['count']): - logging.warning('Inconsistent value count in %s document for %s' - %(p, which)) - for v in reversed(h['values']): - if (this_ts <= newest): - agt1 = now - this_ts - agt2 = interval - agetmp1 = (agt1.microseconds + (agt1.seconds + agt1.days * 24 * 3600) - * 10**6) / 10**6 - agetmp2 = (agt2.microseconds + (agt2.seconds + agt2.days * 24 * 3600) - * 10**6) / 10**6 - generic_history.append( - { 'age': agetmp1, - 'length': agetmp2, - 'value': v - }) - newest = this_ts - this_ts -= interval - - if (this_ts + interval != parse_ts(h['first'])): - logging.warning('Inconsistent time information in %s document for %s' - %(p, which)) - - #print json.dumps(generic_history, sort_keys=True, - # indent=4, separators=(',', ': ')) - return generic_history - - @staticmethod - def _avg_generic_history(generic_history): - a = [] - for i in generic_history: - if i['age'] > (ADDRESS_AND_PORT_STABLE_DAYS * 24 * 3600): - continue - if (i['length'] is not None - and i['age'] is not None - and i['value'] is not None): - w = i['length'] * math.pow(AGE_ALPHA, i['age']/(3600*24)) - a.append( (i['value'] * w, w) ) - - sv = math.fsum(map(lambda x: x[0], a)) - sw = math.fsum(map(lambda x: x[1], a)) - - if sw == 0.0: - svw = 0.0 - else: - svw = sv/sw - return svw - - def _add_generic_history(self, history): - periods = r['read_history'].keys() - periods.sort(key = lambda x: r['read_history'][x]['interval'] ) - - print periods - - def add_running_history(self, history): - pass - - def add_uptime(self, uptime): - logging.debug('Adding uptime %s.'%(self._fpr,)) - - # flags we care about: Running, V2Dir, Guard - if not 'flags' in uptime: - logging.debug('No flags in document for %s.'%(self._fpr,)) - return - - for f in ['Running', 'Guard', 'V2Dir']: - if not f in uptime['flags']: - logging.debug('No %s in flags for %s.'%(f, self._fpr,)) - return - - running = self._extract_generic_history(uptime['flags']['Running'], - '%s-Running'%(self._fpr)) - guard = self._extract_generic_history(uptime['flags']['Guard'], - '%s-Guard'%(self._fpr)) - v2dir = self._extract_generic_history(uptime['flags']['V2Dir'], - '%s-V2Dir'%(self._fpr)) - if 'BadExit' in uptime['flags']: - badexit = self._extract_generic_history(uptime['flags']['BadExit'], - '%s-BadExit'%(self._fpr)) - - self._running = self._avg_generic_history(running) / ONIONOO_SCALE_ONE - self._guard = self._avg_generic_history(guard) / ONIONOO_SCALE_ONE - self._v2dir = self._avg_generic_history(v2dir) / ONIONOO_SCALE_ONE - self._badexit = None - if 'BadExit' in uptime['flags']: - self._badexit = self._avg_generic_history(badexit) / ONIONOO_SCALE_ONE - - def is_candidate(self): - try: - if (MUST_BE_RUNNING_NOW and not self.is_running()): - log_excluded('%s not a candidate: not running now, unable to check ' + - 'DirPort consensus download', self._fpr) - return False - if (self._data['last_changed_address_or_port'] > - self.CUTOFF_ADDRESS_AND_PORT_STABLE): - log_excluded('%s not a candidate: changed address/port recently (%s)', - self._fpr, self._data['last_changed_address_or_port']) - return False - if self._running < CUTOFF_RUNNING: - log_excluded('%s not a candidate: running avg too low (%lf)', - self._fpr, self._running) - return False - if self._v2dir < CUTOFF_V2DIR: - log_excluded('%s not a candidate: v2dir avg too low (%lf)', - self._fpr, self._v2dir) - return False - if self._badexit is not None and self._badexit > PERMITTED_BADEXIT: - log_excluded('%s not a candidate: badexit avg too high (%lf)', - self._fpr, self._badexit) - return False - # this function logs a message depending on which check fails - if not self.is_valid_version(): - return False - if self._guard < CUTOFF_GUARD: - log_excluded('%s not a candidate: guard avg too low (%lf)', - self._fpr, self._guard) - return False - if (not self._data.has_key('consensus_weight') - or self._data['consensus_weight'] < 1): - log_excluded('%s not a candidate: consensus weight invalid', self._fpr) - return False - except BaseException as e: - logging.warning("Exception %s when checking if fallback is a candidate", - str(e)) - return False - return True - - def id_matches(self, id, exact=False): - """ Does this fallback's id match id? - exact is ignored. """ - return self._fpr == id - - def ipv4_addr_matches(self, ipv4_addr, exact=False): - """ Does this fallback's IPv4 address match ipv4_addr? - exact is ignored. """ - return self.dirip == ipv4_addr - - def ipv4_dirport_matches(self, ipv4_dirport, exact=False): - """ Does this fallback's IPv4 dirport match ipv4_dirport? - If exact is False, always return True. """ - if exact: - return self.dirport == int(ipv4_dirport) - else: - return True - - def ipv4_and_dirport_matches(self, ipv4_addr, ipv4_dirport, exact=False): - """ Does this fallback's IPv4 address match ipv4_addr? - If exact is True, also check ipv4_dirport. """ - ipv4_match = self.ipv4_addr_matches(ipv4_addr, exact=exact) - if exact: - return ipv4_match and self.ipv4_dirport_matches(ipv4_dirport, - exact=exact) - else: - return ipv4_match - - def ipv4_orport_matches(self, ipv4_orport, exact=False): - """ Does this fallback's IPv4 orport match ipv4_orport? - If exact is False, always return True. """ - if exact: - return self.orport == int(ipv4_orport) - else: - return True - - def ipv4_and_orport_matches(self, ipv4_addr, ipv4_orport, exact=False): - """ Does this fallback's IPv4 address match ipv4_addr? - If exact is True, also check ipv4_orport. """ - ipv4_match = self.ipv4_addr_matches(ipv4_addr, exact=exact) - if exact: - return ipv4_match and self.ipv4_orport_matches(ipv4_orport, - exact=exact) - else: - return ipv4_match - - def ipv6_addr_matches(self, ipv6_addr, exact=False): - """ Does this fallback's IPv6 address match ipv6_addr? - Both addresses must be present to match. - exact is ignored. """ - if self.has_ipv6() and ipv6_addr is not None: - # Check that we have a bracketed IPv6 address without a port - assert(ipv6_addr.startswith('[') and ipv6_addr.endswith(']')) - return self.ipv6addr == ipv6_addr - else: - return False - - def ipv6_orport_matches(self, ipv6_orport, exact=False): - """ Does this fallback's IPv6 orport match ipv6_orport? - Both ports must be present to match. - If exact is False, always return True. """ - if exact: - return (self.has_ipv6() and ipv6_orport is not None and - self.ipv6orport == int(ipv6_orport)) - else: - return True - - def ipv6_and_orport_matches(self, ipv6_addr, ipv6_orport, exact=False): - """ Does this fallback's IPv6 address match ipv6_addr? - If exact is True, also check ipv6_orport. """ - ipv6_match = self.ipv6_addr_matches(ipv6_addr, exact=exact) - if exact: - return ipv6_match and self.ipv6_orport_matches(ipv6_orport, - exact=exact) - else: - return ipv6_match - - def entry_matches_exact(self, entry): - """ Is entry an exact match for this fallback? - A fallback is an exact match for entry if each key in entry matches: - ipv4 - dirport - orport - id - ipv6 address and port (if present in the fallback or the whitelist) - If the fallback has an ipv6 key, the whitelist line must also have - it, otherwise they don't match. - - Logs a warning-level message if the fallback would be an exact match, - but one of the id, ipv4, ipv4 orport, ipv4 dirport, or ipv6 orport - have changed. """ - if not self.id_matches(entry['id'], exact=True): - # can't log here unless we match an IP and port, because every relay's - # fingerprint is compared to every entry's fingerprint - if self.ipv4_and_orport_matches(entry['ipv4'], - entry['orport'], - exact=True): - logging.warning('%s excluded: has OR %s:%d changed fingerprint to ' + - '%s?', entry['id'], self.dirip, self.orport, - self._fpr) - if self.ipv6_and_orport_matches(entry.get('ipv6_addr'), - entry.get('ipv6_orport'), - exact=True): - logging.warning('%s excluded: has OR %s changed fingerprint to ' + - '%s?', entry['id'], entry['ipv6'], self._fpr) - return False - if not self.ipv4_addr_matches(entry['ipv4'], exact=True): - logging.warning('%s excluded: has it changed IPv4 from %s to %s?', - self._fpr, entry['ipv4'], self.dirip) - return False - if not self.ipv4_dirport_matches(entry['dirport'], exact=True): - logging.warning('%s excluded: has it changed DirPort from %s:%d to ' + - '%s:%d?', self._fpr, self.dirip, int(entry['dirport']), - self.dirip, self.dirport) - return False - if not self.ipv4_orport_matches(entry['orport'], exact=True): - logging.warning('%s excluded: has it changed ORPort from %s:%d to ' + - '%s:%d?', self._fpr, self.dirip, int(entry['orport']), - self.dirip, self.orport) - return False - if entry.has_key('ipv6') and self.has_ipv6(): - # if both entry and fallback have an ipv6 address, compare them - if not self.ipv6_and_orport_matches(entry['ipv6_addr'], - entry['ipv6_orport'], - exact=True): - logging.warning('%s excluded: has it changed IPv6 ORPort from %s ' + - 'to %s:%d?', self._fpr, entry['ipv6'], - self.ipv6addr, self.ipv6orport) - return False - # if the fallback has an IPv6 address but the whitelist entry - # doesn't, or vice versa, the whitelist entry doesn't match - elif entry.has_key('ipv6') and not self.has_ipv6(): - logging.warning('%s excluded: has it lost its former IPv6 address %s?', - self._fpr, entry['ipv6']) - return False - elif not entry.has_key('ipv6') and self.has_ipv6(): - logging.warning('%s excluded: has it gained an IPv6 address %s:%d?', - self._fpr, self.ipv6addr, self.ipv6orport) - return False - return True - - def entry_matches_fuzzy(self, entry): - """ Is entry a fuzzy match for this fallback? - A fallback is a fuzzy match for entry if at least one of these keys - in entry matches: - id - ipv4 - ipv6 (if present in both the fallback and whitelist) - The ports and nickname are ignored. Missing or extra ipv6 addresses - are ignored. - - Doesn't log any warning messages. """ - if self.id_matches(entry['id'], exact=False): - return True - if self.ipv4_addr_matches(entry['ipv4'], exact=False): - return True - if entry.has_key('ipv6') and self.has_ipv6(): - # if both entry and fallback have an ipv6 address, compare them - if self.ipv6_addr_matches(entry['ipv6_addr'], exact=False): - return True - return False - - def is_in_whitelist(self, relaylist, exact=False): - """ If exact is True (existing fallback list), check if this fallback is - an exact match for any whitelist entry, using entry_matches_exact(). - - If exact is False (new fallback whitelist), check if this fallback is - a fuzzy match for any whitelist entry, using entry_matches_fuzzy(). """ - for entry in relaylist: - if exact: - if self.entry_matches_exact(entry): - return True - else: - if self.entry_matches_fuzzy(entry): - return True - return False - - def cw_to_bw_factor(self): - # any relays with a missing or zero consensus weight are not candidates - # any relays with a missing advertised bandwidth have it set to zero - return self._data['advertised_bandwidth'] / self._data['consensus_weight'] - - # since advertised_bandwidth is reported by the relay, it can be gamed - # to avoid this, use the median consensus weight to bandwidth factor to - # estimate this relay's measured bandwidth, and make that the upper limit - def measured_bandwidth(self, median_cw_to_bw_factor): - cw_to_bw= median_cw_to_bw_factor - # Reduce exit bandwidth to make sure we're not overloading them - if self.is_exit(): - cw_to_bw *= EXIT_BANDWIDTH_FRACTION - measured_bandwidth = self._data['consensus_weight'] * cw_to_bw - if self._data['advertised_bandwidth'] != 0: - # limit advertised bandwidth (if available) to measured bandwidth - return min(measured_bandwidth, self._data['advertised_bandwidth']) - else: - return measured_bandwidth - - def set_measured_bandwidth(self, median_cw_to_bw_factor): - self._data['measured_bandwidth'] = self.measured_bandwidth( - median_cw_to_bw_factor) - - def is_exit(self): - return 'Exit' in self._data['flags'] - - def is_guard(self): - return 'Guard' in self._data['flags'] - - def is_running(self): - return 'Running' in self._data['flags'] - - # does this fallback have an IPv6 address and orport? - def has_ipv6(self): - return self.ipv6addr is not None and self.ipv6orport is not None - - # strip leading and trailing brackets from an IPv6 address - # safe to use on non-bracketed IPv6 and on IPv4 addresses - # also convert to unicode, and make None appear as '' - @staticmethod - def strip_ipv6_brackets(ip): - if ip is None: - return unicode('') - if len(ip) < 2: - return unicode(ip) - if ip[0] == '[' and ip[-1] == ']': - return unicode(ip[1:-1]) - return unicode(ip) - - # are ip_a and ip_b in the same netblock? - # mask_bits is the size of the netblock - # takes both IPv4 and IPv6 addresses - # the versions of ip_a and ip_b must be the same - # the mask must be valid for the IP version - @staticmethod - def netblocks_equal(ip_a, ip_b, mask_bits): - if ip_a is None or ip_b is None: - return False - ip_a = Candidate.strip_ipv6_brackets(ip_a) - ip_b = Candidate.strip_ipv6_brackets(ip_b) - a = ipaddress.ip_address(ip_a) - b = ipaddress.ip_address(ip_b) - if a.version != b.version: - raise Exception('Mismatching IP versions in %s and %s'%(ip_a, ip_b)) - if mask_bits > a.max_prefixlen: - logging.error('Bad IP mask %d for %s and %s'%(mask_bits, ip_a, ip_b)) - mask_bits = a.max_prefixlen - if mask_bits < 0: - logging.error('Bad IP mask %d for %s and %s'%(mask_bits, ip_a, ip_b)) - mask_bits = 0 - a_net = ipaddress.ip_network('%s/%d'%(ip_a, mask_bits), strict=False) - return b in a_net - - # is this fallback's IPv4 address (dirip) in the same netblock as other's - # IPv4 address? - # mask_bits is the size of the netblock - def ipv4_netblocks_equal(self, other, mask_bits): - return Candidate.netblocks_equal(self.dirip, other.dirip, mask_bits) - - # is this fallback's IPv6 address (ipv6addr) in the same netblock as - # other's IPv6 address? - # Returns False if either fallback has no IPv6 address - # mask_bits is the size of the netblock - def ipv6_netblocks_equal(self, other, mask_bits): - if not self.has_ipv6() or not other.has_ipv6(): - return False - return Candidate.netblocks_equal(self.ipv6addr, other.ipv6addr, mask_bits) - - # is this fallback's IPv4 DirPort the same as other's IPv4 DirPort? - def dirport_equal(self, other): - return self.dirport == other.dirport - - # is this fallback's IPv4 ORPort the same as other's IPv4 ORPort? - def ipv4_orport_equal(self, other): - return self.orport == other.orport - - # is this fallback's IPv6 ORPort the same as other's IPv6 ORPort? - # Returns False if either fallback has no IPv6 address - def ipv6_orport_equal(self, other): - if not self.has_ipv6() or not other.has_ipv6(): - return False - return self.ipv6orport == other.ipv6orport - - # does this fallback have the same DirPort, IPv4 ORPort, or - # IPv6 ORPort as other? - # Ignores IPv6 ORPort if either fallback has no IPv6 address - def port_equal(self, other): - return (self.dirport_equal(other) or self.ipv4_orport_equal(other) - or self.ipv6_orport_equal(other)) - - # return a list containing IPv4 ORPort, DirPort, and IPv6 ORPort (if present) - def port_list(self): - ports = [self.dirport, self.orport] - if self.has_ipv6() and not self.ipv6orport in ports: - ports.append(self.ipv6orport) - return ports - - # does this fallback share a port with other, regardless of whether the - # port types match? - # For example, if self's IPv4 ORPort is 80 and other's DirPort is 80, - # return True - def port_shared(self, other): - for p in self.port_list(): - if p in other.port_list(): - return True - return False - - # log how long it takes to download a consensus from dirip:dirport - # returns True if the download failed, False if it succeeded within max_time - @staticmethod - def fallback_consensus_download_speed(dirip, dirport, nickname, fingerprint, - max_time): - download_failed = False - # some directory mirrors respond to requests in ways that hang python - # sockets, which is why we log this line here - logging.info('Initiating %sconsensus download from %s (%s:%d) %s.', - 'microdesc ' if DOWNLOAD_MICRODESC_CONSENSUS else '', - nickname, dirip, dirport, fingerprint) - # there appears to be about 1 second of overhead when comparing stem's - # internal trace time and the elapsed time calculated here - TIMEOUT_SLOP = 1.0 - start = datetime.datetime.utcnow() - try: - consensus = get_consensus( - endpoints = [(dirip, dirport)], - timeout = (max_time + TIMEOUT_SLOP), - validate = True, - retries = 0, - fall_back_to_authority = False, - document_handler = DocumentHandler.BARE_DOCUMENT, - microdescriptor = DOWNLOAD_MICRODESC_CONSENSUS - ).run()[0] - end = datetime.datetime.utcnow() - time_since_expiry = (end - consensus.valid_until).total_seconds() - time_until_valid = (consensus.valid_after - end).total_seconds() - except Exception, stem_error: - end = datetime.datetime.utcnow() - log_excluded('Unable to retrieve a consensus from %s: %s', nickname, - stem_error) - status = 'error: "%s"' % (stem_error) - level = logging.WARNING - download_failed = True - elapsed = (end - start).total_seconds() - if download_failed: - # keep the error failure status, and avoid using the variables - pass - elif elapsed > max_time: - status = 'too slow' - level = logging.WARNING - download_failed = True - elif (time_since_expiry > 0): - status = 'outdated consensus, expired %ds ago'%(int(time_since_expiry)) - if time_since_expiry <= REASONABLY_LIVE_TIME: - status += ', tolerating up to %ds'%(REASONABLY_LIVE_TIME) - level = logging.INFO - else: - status += ', invalid' - level = logging.WARNING - download_failed = True - elif (time_until_valid > 0): - status = 'future consensus, valid in %ds'%(int(time_until_valid)) - if time_until_valid <= REASONABLY_LIVE_TIME: - status += ', tolerating up to %ds'%(REASONABLY_LIVE_TIME) - level = logging.INFO - else: - status += ', invalid' - level = logging.WARNING - download_failed = True - else: - status = 'ok' - level = logging.DEBUG - logging.log(level, 'Consensus download: %0.1fs %s from %s (%s:%d) %s, ' + - 'max download time %0.1fs.', elapsed, status, nickname, - dirip, dirport, fingerprint, max_time) - return download_failed - - # does this fallback download the consensus fast enough? - def check_fallback_download_consensus(self): - # include the relay if we're not doing a check, or we can't check (IPv6) - ipv4_failed = False - ipv6_failed = False - if PERFORM_IPV4_DIRPORT_CHECKS: - ipv4_failed = Candidate.fallback_consensus_download_speed(self.dirip, - self.dirport, - self._data['nickname'], - self._fpr, - CONSENSUS_DOWNLOAD_SPEED_MAX) - if self.has_ipv6() and PERFORM_IPV6_DIRPORT_CHECKS: - # Clients assume the IPv6 DirPort is the same as the IPv4 DirPort - ipv6_failed = Candidate.fallback_consensus_download_speed(self.ipv6addr, - self.dirport, - self._data['nickname'], - self._fpr, - CONSENSUS_DOWNLOAD_SPEED_MAX) - return ((not ipv4_failed) and (not ipv6_failed)) - - # if this fallback has not passed a download check, try it again, - # and record the result, available in get_fallback_download_consensus - def try_fallback_download_consensus(self): - if not self.get_fallback_download_consensus(): - self._data['download_check'] = self.check_fallback_download_consensus() - - # did this fallback pass the download check? - def get_fallback_download_consensus(self): - # if we're not performing checks, return True - if not PERFORM_IPV4_DIRPORT_CHECKS and not PERFORM_IPV6_DIRPORT_CHECKS: - return True - # if we are performing checks, but haven't done one, return False - if not self._data.has_key('download_check'): - return False - return self._data['download_check'] - - # output an optional header comment and info for this fallback - # try_fallback_download_consensus before calling this - def fallbackdir_line(self, fallbacks, prefilter_fallbacks): - s = '' - if OUTPUT_COMMENTS: - s += self.fallbackdir_comment(fallbacks, prefilter_fallbacks) - # if the download speed is ok, output a C string - # if it's not, but we OUTPUT_COMMENTS, output a commented-out C string - if self.get_fallback_download_consensus() or OUTPUT_COMMENTS: - s += self.fallbackdir_info(self.get_fallback_download_consensus()) - return s - - # output a header comment for this fallback - def fallbackdir_comment(self, fallbacks, prefilter_fallbacks): - # /* - # nickname - # flags - # adjusted bandwidth, consensus weight - # [contact] - # [identical contact counts] - # */ - # Multiline C comment - s = '/*' - s += '\n' - s += cleanse_c_multiline_comment(self._data['nickname']) - s += '\n' - s += 'Flags: ' - s += cleanse_c_multiline_comment(' '.join(sorted(self._data['flags']))) - s += '\n' - # this is an adjusted bandwidth, see calculate_measured_bandwidth() - bandwidth = self._data['measured_bandwidth'] - weight = self._data['consensus_weight'] - s += 'Bandwidth: %.1f MByte/s, Consensus Weight: %d'%( - bandwidth/(1024.0*1024.0), - weight) - s += '\n' - if self._data['contact'] is not None: - s += cleanse_c_multiline_comment(self._data['contact']) - if CONTACT_COUNT: - fallback_count = len([f for f in fallbacks - if f._data['contact'] == self._data['contact']]) - if fallback_count > 1: - s += '\n' - s += '%d identical contacts listed' % (fallback_count) - - # output the fallback info C string for this fallback - # this is the text that would go after FallbackDir in a torrc - # if this relay failed the download test and we OUTPUT_COMMENTS, - # comment-out the returned string - def fallbackdir_info(self, dl_speed_ok): - # "address:dirport orport=port id=fingerprint" - # (insert additional madatory fields here) - # "[ipv6=addr:orport]" - # (insert additional optional fields here) - # /* nickname=name */ - # /* extrainfo={0,1} */ - # (insert additional comment fields here) - # /* ===== */ - # , - # - # Do we want a C string, or a commented-out string? - c_string = dl_speed_ok - comment_string = not dl_speed_ok and OUTPUT_COMMENTS - # If we don't want either kind of string, bail - if not c_string and not comment_string: - return '' - s = '' - # Comment out the fallback directory entry if it's too slow - # See the debug output for which address and port is failing - if comment_string: - s += '/* Consensus download failed or was too slow:\n' - # Multi-Line C string with trailing comma (part of a string list) - # This makes it easier to diff the file, and remove IPv6 lines using grep - # Integers don't need escaping - s += '"%s orport=%d id=%s"'%( - cleanse_c_string(self._data['dir_address']), - self.orport, - cleanse_c_string(self._fpr)) - s += '\n' - # (insert additional madatory fields here) - if self.has_ipv6(): - s += '" ipv6=%s:%d"'%(cleanse_c_string(self.ipv6addr), self.ipv6orport) - s += '\n' - # (insert additional optional fields here) - if not comment_string: - s += '/* ' - s += 'nickname=%s'%(cleanse_c_string(self._data['nickname'])) - if not comment_string: - s += ' */' - s += '\n' - # if we know that the fallback is an extrainfo cache, flag it - # and if we don't know, assume it is not - if not comment_string: - s += '/* ' - s += 'extrainfo=%d'%(1 if self._extra_info_cache else 0) - if not comment_string: - s += ' */' - s += '\n' - # (insert additional comment fields here) - # The terminator and comma must be the last line in each fallback entry - if not comment_string: - s += '/* ' - s += SECTION_SEPARATOR_BASE - if not comment_string: - s += ' */' - s += '\n' - s += ',' - if comment_string: - s += '\n' - s += '*/' - return s - -## Fallback Candidate List Class - -class CandidateList(dict): - def __init__(self): - pass - - def _add_relay(self, details): - if not 'dir_address' in details: return - c = Candidate(details) - self[ c.get_fingerprint() ] = c - - def _add_uptime(self, uptime): - try: - fpr = uptime['fingerprint'] - except KeyError: - raise Exception("Document has no fingerprint field.") - - try: - c = self[fpr] - except KeyError: - logging.debug('Got unknown relay %s in uptime document.'%(fpr,)) - return - - c.add_uptime(uptime) - - def _add_details(self): - logging.debug('Loading details document.') - d = fetch('details', - fields=('fingerprint,nickname,contact,last_changed_address_or_port,' + - 'consensus_weight,advertised_bandwidth,or_addresses,' + - 'dir_address,recommended_version,flags,effective_family,' + - 'platform')) - logging.debug('Loading details document done.') - - if not 'relays' in d: raise Exception("No relays found in document.") - - for r in d['relays']: self._add_relay(r) - - def _add_uptimes(self): - logging.debug('Loading uptime document.') - d = fetch('uptime') - logging.debug('Loading uptime document done.') - - if not 'relays' in d: raise Exception("No relays found in document.") - for r in d['relays']: self._add_uptime(r) - - def add_relays(self): - self._add_details() - self._add_uptimes() - - def count_guards(self): - guard_count = 0 - for fpr in self.keys(): - if self[fpr].is_guard(): - guard_count += 1 - return guard_count - - # Find fallbacks that fit the uptime, stability, and flags criteria, - # and make an array of them in self.fallbacks - def compute_fallbacks(self): - self.fallbacks = map(lambda x: self[x], - filter(lambda x: self[x].is_candidate(), - self.keys())) - - # sort fallbacks by their consensus weight to advertised bandwidth factor, - # lowest to highest - # used to find the median cw_to_bw_factor() - def sort_fallbacks_by_cw_to_bw_factor(self): - self.fallbacks.sort(key=lambda f: f.cw_to_bw_factor()) - - # sort fallbacks by their measured bandwidth, highest to lowest - # calculate_measured_bandwidth before calling this - # this is useful for reviewing candidates in priority order - def sort_fallbacks_by_measured_bandwidth(self): - self.fallbacks.sort(key=lambda f: f._data['measured_bandwidth'], - reverse=True) - - # sort fallbacks by the data field data_field, lowest to highest - def sort_fallbacks_by(self, data_field): - self.fallbacks.sort(key=lambda f: f._data[data_field]) - - @staticmethod - def load_relaylist(file_obj): - """ Read each line in the file, and parse it like a FallbackDir line: - an IPv4 address and optional port: - : - which are parsed into dictionary entries: - ipv4= - dirport= - followed by a series of key=value entries: - orport= - id= - ipv6=: - each line's key/value pairs are placed in a dictonary, - (of string -> string key/value pairs), - and these dictionaries are placed in an array. - comments start with # and are ignored. """ - file_data = file_obj['data'] - file_name = file_obj['name'] - relaylist = [] - if file_data is None: - return relaylist - for line in file_data.split('\n'): - relay_entry = {} - # ignore comments - line_comment_split = line.split('#') - line = line_comment_split[0] - # cleanup whitespace - line = cleanse_whitespace(line) - line = line.strip() - if len(line) == 0: - continue - for item in line.split(' '): - item = item.strip() - if len(item) == 0: - continue - key_value_split = item.split('=') - kvl = len(key_value_split) - if kvl < 1 or kvl > 2: - print '#error Bad %s item: %s, format is key=value.'%( - file_name, item) - if kvl == 1: - # assume that entries without a key are the ipv4 address, - # perhaps with a dirport - ipv4_maybe_dirport = key_value_split[0] - ipv4_maybe_dirport_split = ipv4_maybe_dirport.split(':') - dirl = len(ipv4_maybe_dirport_split) - if dirl < 1 or dirl > 2: - print '#error Bad %s IPv4 item: %s, format is ipv4:port.'%( - file_name, item) - if dirl >= 1: - relay_entry['ipv4'] = ipv4_maybe_dirport_split[0] - if dirl == 2: - relay_entry['dirport'] = ipv4_maybe_dirport_split[1] - elif kvl == 2: - relay_entry[key_value_split[0]] = key_value_split[1] - # split ipv6 addresses and orports - if key_value_split[0] == 'ipv6': - ipv6_orport_split = key_value_split[1].rsplit(':', 1) - ipv6l = len(ipv6_orport_split) - if ipv6l != 2: - print '#error Bad %s IPv6 item: %s, format is [ipv6]:orport.'%( - file_name, item) - relay_entry['ipv6_addr'] = ipv6_orport_split[0] - relay_entry['ipv6_orport'] = ipv6_orport_split[1] - relaylist.append(relay_entry) - return relaylist - - def apply_filter_lists(self, whitelist_obj, exact=False): - """ Apply the fallback whitelist_obj to this fallback list, - passing exact to is_in_whitelist(). """ - excluded_count = 0 - list_type = 'whitelist' - if whitelist_obj['check_existing']: - list_type = 'fallback list' - - logging.debug('Applying {}'.format(list_type)) - # parse the whitelist - whitelist = self.load_relaylist(whitelist_obj) - filtered_fallbacks = [] - for f in self.fallbacks: - in_whitelist = f.is_in_whitelist(whitelist, exact=exact) - if in_whitelist: - # include - filtered_fallbacks.append(f) - elif INCLUDE_UNLISTED_ENTRIES: - # include - filtered_fallbacks.append(f) - else: - # exclude - excluded_count += 1 - log_excluded('Excluding %s: not in %s.', - f._fpr, list_type) - self.fallbacks = filtered_fallbacks - return excluded_count - - @staticmethod - def summarise_filters(initial_count, excluded_count, check_existing): - list_type = 'Whitelist' - if check_existing: - list_type = 'Fallback list' - - return '/* %s excluded %d of %d candidates. */'%(list_type, - excluded_count, initial_count) - - # calculate each fallback's measured bandwidth based on the median - # consensus weight to advertised bandwidth ratio - def calculate_measured_bandwidth(self): - self.sort_fallbacks_by_cw_to_bw_factor() - median_fallback = self.fallback_median(True) - if median_fallback is not None: - median_cw_to_bw_factor = median_fallback.cw_to_bw_factor() - else: - # this will never be used, because there are no fallbacks - median_cw_to_bw_factor = None - for f in self.fallbacks: - f.set_measured_bandwidth(median_cw_to_bw_factor) - - # remove relays with low measured bandwidth from the fallback list - # calculate_measured_bandwidth for each relay before calling this - def remove_low_bandwidth_relays(self): - if MIN_BANDWIDTH is None: - return - above_min_bw_fallbacks = [] - for f in self.fallbacks: - if f._data['measured_bandwidth'] >= MIN_BANDWIDTH: - above_min_bw_fallbacks.append(f) - else: - # the bandwidth we log here is limited by the relay's consensus weight - # as well as its adverttised bandwidth. See set_measured_bandwidth - # for details - log_excluded('%s not a candidate: bandwidth %.1fMByte/s too low, ' + - 'must be at least %.1fMByte/s', f._fpr, - f._data['measured_bandwidth']/(1024.0*1024.0), - MIN_BANDWIDTH/(1024.0*1024.0)) - self.fallbacks = above_min_bw_fallbacks - - # the minimum fallback in the list - # call one of the sort_fallbacks_* functions before calling this - def fallback_min(self): - if len(self.fallbacks) > 0: - return self.fallbacks[-1] - else: - return None - - # the median fallback in the list - # call one of the sort_fallbacks_* functions before calling this - def fallback_median(self, require_advertised_bandwidth): - # use the low-median when there are an evan number of fallbacks, - # for consistency with the bandwidth authorities - if len(self.fallbacks) > 0: - median_position = (len(self.fallbacks) - 1) / 2 - if not require_advertised_bandwidth: - return self.fallbacks[median_position] - # if we need advertised_bandwidth but this relay doesn't have it, - # move to a fallback with greater consensus weight until we find one - while not self.fallbacks[median_position]._data['advertised_bandwidth']: - median_position += 1 - if median_position >= len(self.fallbacks): - return None - return self.fallbacks[median_position] - else: - return None - - # the maximum fallback in the list - # call one of the sort_fallbacks_* functions before calling this - def fallback_max(self): - if len(self.fallbacks) > 0: - return self.fallbacks[0] - else: - return None - - # return a new bag suitable for storing attributes - @staticmethod - def attribute_new(): - return dict() - - # get the count of attribute in attribute_bag - # if attribute is None or the empty string, return 0 - @staticmethod - def attribute_count(attribute, attribute_bag): - if attribute is None or attribute == '': - return 0 - if attribute not in attribute_bag: - return 0 - return attribute_bag[attribute] - - # does attribute_bag contain more than max_count instances of attribute? - # if so, return False - # if not, return True - # if attribute is None or the empty string, or max_count is invalid, - # always return True - @staticmethod - def attribute_allow(attribute, attribute_bag, max_count=1): - if attribute is None or attribute == '' or max_count <= 0: - return True - elif CandidateList.attribute_count(attribute, attribute_bag) >= max_count: - return False - else: - return True - - # add attribute to attribute_bag, incrementing the count if it is already - # present - # if attribute is None or the empty string, or count is invalid, - # do nothing - @staticmethod - def attribute_add(attribute, attribute_bag, count=1): - if attribute is None or attribute == '' or count <= 0: - pass - attribute_bag.setdefault(attribute, 0) - attribute_bag[attribute] += count - - # make sure there are only MAX_FALLBACKS_PER_IP fallbacks per IPv4 address, - # and per IPv6 address - # there is only one IPv4 address on each fallback: the IPv4 DirPort address - # (we choose the IPv4 ORPort which is on the same IPv4 as the DirPort) - # there is at most one IPv6 address on each fallback: the IPv6 ORPort address - # we try to match the IPv4 ORPort, but will use any IPv6 address if needed - # (clients only use the IPv6 ORPort) - # if there is no IPv6 address, only the IPv4 address is checked - # return the number of candidates we excluded - def limit_fallbacks_same_ip(self): - ip_limit_fallbacks = [] - ip_list = CandidateList.attribute_new() - for f in self.fallbacks: - if (CandidateList.attribute_allow(f.dirip, ip_list, - MAX_FALLBACKS_PER_IPV4) - and CandidateList.attribute_allow(f.ipv6addr, ip_list, - MAX_FALLBACKS_PER_IPV6)): - ip_limit_fallbacks.append(f) - CandidateList.attribute_add(f.dirip, ip_list) - if f.has_ipv6(): - CandidateList.attribute_add(f.ipv6addr, ip_list) - elif not CandidateList.attribute_allow(f.dirip, ip_list, - MAX_FALLBACKS_PER_IPV4): - log_excluded('Eliminated %s: already have %d fallback(s) on IPv4 %s' - %(f._fpr, CandidateList.attribute_count(f.dirip, ip_list), - f.dirip)) - elif (f.has_ipv6() and - not CandidateList.attribute_allow(f.ipv6addr, ip_list, - MAX_FALLBACKS_PER_IPV6)): - log_excluded('Eliminated %s: already have %d fallback(s) on IPv6 %s' - %(f._fpr, CandidateList.attribute_count(f.ipv6addr, - ip_list), - f.ipv6addr)) - original_count = len(self.fallbacks) - self.fallbacks = ip_limit_fallbacks - return original_count - len(self.fallbacks) - - # make sure there are only MAX_FALLBACKS_PER_CONTACT fallbacks for each - # ContactInfo - # if there is no ContactInfo, allow the fallback - # this check can be gamed by providing no ContactInfo, or by setting the - # ContactInfo to match another fallback - # However, given the likelihood that relays with the same ContactInfo will - # go down at similar times, its usefulness outweighs the risk - def limit_fallbacks_same_contact(self): - contact_limit_fallbacks = [] - contact_list = CandidateList.attribute_new() - for f in self.fallbacks: - if CandidateList.attribute_allow(f._data['contact'], contact_list, - MAX_FALLBACKS_PER_CONTACT): - contact_limit_fallbacks.append(f) - CandidateList.attribute_add(f._data['contact'], contact_list) - else: - log_excluded( - 'Eliminated %s: already have %d fallback(s) on ContactInfo %s' - %(f._fpr, CandidateList.attribute_count(f._data['contact'], - contact_list), - f._data['contact'])) - original_count = len(self.fallbacks) - self.fallbacks = contact_limit_fallbacks - return original_count - len(self.fallbacks) - - # make sure there are only MAX_FALLBACKS_PER_FAMILY fallbacks per effective - # family - # if there is no family, allow the fallback - # we use effective family, which ensures mutual family declarations - # but the check can be gamed by not declaring a family at all - # if any indirect families exist, the result depends on the order in which - # fallbacks are sorted in the list - def limit_fallbacks_same_family(self): - family_limit_fallbacks = [] - fingerprint_list = CandidateList.attribute_new() - for f in self.fallbacks: - if CandidateList.attribute_allow(f._fpr, fingerprint_list, - MAX_FALLBACKS_PER_FAMILY): - family_limit_fallbacks.append(f) - CandidateList.attribute_add(f._fpr, fingerprint_list) - for family_fingerprint in f._data['effective_family']: - CandidateList.attribute_add(family_fingerprint, fingerprint_list) - else: - # we already have a fallback with this fallback in its effective - # family - log_excluded( - 'Eliminated %s: already have %d fallback(s) in effective family' - %(f._fpr, CandidateList.attribute_count(f._fpr, fingerprint_list))) - original_count = len(self.fallbacks) - self.fallbacks = family_limit_fallbacks - return original_count - len(self.fallbacks) - - # try once to get the descriptors for fingerprint_list using stem - # returns an empty list on exception - @staticmethod - def get_fallback_descriptors_once(fingerprint_list): - desc_list = get_server_descriptors(fingerprints=fingerprint_list).run(suppress=True) - return desc_list - - # try up to max_retries times to get the descriptors for fingerprint_list - # using stem. Stops retrying when all descriptors have been retrieved. - # returns a list containing the descriptors that were retrieved - @staticmethod - def get_fallback_descriptors(fingerprint_list, max_retries=5): - # we can't use stem's retries=, because we want to support more than 96 - # descriptors - # - # add an attempt for every MAX_FINGERPRINTS (or part thereof) in the list - max_retries += (len(fingerprint_list) + MAX_FINGERPRINTS - 1) / MAX_FINGERPRINTS - remaining_list = fingerprint_list - desc_list = [] - for _ in xrange(max_retries): - if len(remaining_list) == 0: - break - new_desc_list = CandidateList.get_fallback_descriptors_once(remaining_list[0:MAX_FINGERPRINTS]) - for d in new_desc_list: - try: - remaining_list.remove(d.fingerprint) - except ValueError: - # warn and ignore if a directory mirror returned a bad descriptor - logging.warning("Directory mirror returned unwanted descriptor %s, ignoring", - d.fingerprint) - continue - desc_list.append(d) - return desc_list - - # find the fallbacks that cache extra-info documents - # Onionoo doesn't know this, so we have to use stem - def mark_extra_info_caches(self): - fingerprint_list = [ f._fpr for f in self.fallbacks ] - logging.info("Downloading fallback descriptors to find extra-info caches") - desc_list = CandidateList.get_fallback_descriptors(fingerprint_list) - for d in desc_list: - self[d.fingerprint]._extra_info_cache = d.extra_info_cache - missing_descriptor_list = [ f._fpr for f in self.fallbacks - if f._extra_info_cache is None ] - for f in missing_descriptor_list: - logging.warning("No descriptor for {}. Assuming extrainfo=0.".format(f)) - - # try a download check on each fallback candidate in order - # stop after max_count successful downloads - # but don't remove any candidates from the array - def try_download_consensus_checks(self, max_count): - dl_ok_count = 0 - for f in self.fallbacks: - f.try_fallback_download_consensus() - if f.get_fallback_download_consensus(): - # this fallback downloaded a consensus ok - dl_ok_count += 1 - if dl_ok_count >= max_count: - # we have enough fallbacks - return - - # put max_count successful candidates in the fallbacks array: - # - perform download checks on each fallback candidate - # - retry failed candidates if CONSENSUS_DOWNLOAD_RETRY is set - # - eliminate failed candidates - # - if there are more than max_count candidates, eliminate lowest bandwidth - # - if there are fewer than max_count candidates, leave only successful - # Return the number of fallbacks that failed the consensus check - def perform_download_consensus_checks(self, max_count): - self.sort_fallbacks_by_measured_bandwidth() - self.try_download_consensus_checks(max_count) - if CONSENSUS_DOWNLOAD_RETRY: - # try unsuccessful candidates again - # we could end up with more than max_count successful candidates here - self.try_download_consensus_checks(max_count) - # now we have at least max_count successful candidates, - # or we've tried them all - original_count = len(self.fallbacks) - self.fallbacks = filter(lambda x: x.get_fallback_download_consensus(), - self.fallbacks) - # some of these failed the check, others skipped the check, - # if we already had enough successful downloads - failed_count = original_count - len(self.fallbacks) - self.fallbacks = self.fallbacks[:max_count] - return failed_count - - # return a string that describes a/b as a percentage - @staticmethod - def describe_percentage(a, b): - if b != 0: - return '%d/%d = %.0f%%'%(a, b, (a*100.0)/b) - else: - # technically, 0/0 is undefined, but 0.0% is a sensible result - return '%d/%d = %.0f%%'%(a, b, 0.0) - - # return a dictionary of lists of fallbacks by IPv4 netblock - # the dictionary is keyed by the fingerprint of an arbitrary fallback - # in each netblock - # mask_bits is the size of the netblock - def fallbacks_by_ipv4_netblock(self, mask_bits): - netblocks = {} - for f in self.fallbacks: - found_netblock = False - for b in netblocks.keys(): - # we found an existing netblock containing this fallback - if f.ipv4_netblocks_equal(self[b], mask_bits): - # add it to the list - netblocks[b].append(f) - found_netblock = True - break - # make a new netblock based on this fallback's fingerprint - if not found_netblock: - netblocks[f._fpr] = [f] - return netblocks - - # return a dictionary of lists of fallbacks by IPv6 netblock - # where mask_bits is the size of the netblock - def fallbacks_by_ipv6_netblock(self, mask_bits): - netblocks = {} - for f in self.fallbacks: - # skip fallbacks without IPv6 addresses - if not f.has_ipv6(): - continue - found_netblock = False - for b in netblocks.keys(): - # we found an existing netblock containing this fallback - if f.ipv6_netblocks_equal(self[b], mask_bits): - # add it to the list - netblocks[b].append(f) - found_netblock = True - break - # make a new netblock based on this fallback's fingerprint - if not found_netblock: - netblocks[f._fpr] = [f] - return netblocks - - # log a message about the proportion of fallbacks in each IPv4 netblock, - # where mask_bits is the size of the netblock - def describe_fallback_ipv4_netblock_mask(self, mask_bits): - fallback_count = len(self.fallbacks) - shared_netblock_fallback_count = 0 - most_frequent_netblock = None - netblocks = self.fallbacks_by_ipv4_netblock(mask_bits) - for b in netblocks.keys(): - if len(netblocks[b]) > 1: - # how many fallbacks are in a netblock with other fallbacks? - shared_netblock_fallback_count += len(netblocks[b]) - # what's the netblock with the most fallbacks? - if (most_frequent_netblock is None - or len(netblocks[b]) > len(netblocks[most_frequent_netblock])): - most_frequent_netblock = b - logging.debug('Fallback IPv4 addresses in the same /%d:'%(mask_bits)) - for f in netblocks[b]: - logging.debug('%s - %s', f.dirip, f._fpr) - if most_frequent_netblock is not None: - logging.warning('There are %s fallbacks in the IPv4 /%d containing %s'%( - CandidateList.describe_percentage( - len(netblocks[most_frequent_netblock]), - fallback_count), - mask_bits, - self[most_frequent_netblock].dirip)) - if shared_netblock_fallback_count > 0: - logging.warning(('%s of fallbacks are in an IPv4 /%d with other ' + - 'fallbacks')%(CandidateList.describe_percentage( - shared_netblock_fallback_count, - fallback_count), - mask_bits)) - - # log a message about the proportion of fallbacks in each IPv6 netblock, - # where mask_bits is the size of the netblock - def describe_fallback_ipv6_netblock_mask(self, mask_bits): - fallback_count = len(self.fallbacks_with_ipv6()) - shared_netblock_fallback_count = 0 - most_frequent_netblock = None - netblocks = self.fallbacks_by_ipv6_netblock(mask_bits) - for b in netblocks.keys(): - if len(netblocks[b]) > 1: - # how many fallbacks are in a netblock with other fallbacks? - shared_netblock_fallback_count += len(netblocks[b]) - # what's the netblock with the most fallbacks? - if (most_frequent_netblock is None - or len(netblocks[b]) > len(netblocks[most_frequent_netblock])): - most_frequent_netblock = b - logging.debug('Fallback IPv6 addresses in the same /%d:'%(mask_bits)) - for f in netblocks[b]: - logging.debug('%s - %s', f.ipv6addr, f._fpr) - if most_frequent_netblock is not None: - logging.warning('There are %s fallbacks in the IPv6 /%d containing %s'%( - CandidateList.describe_percentage( - len(netblocks[most_frequent_netblock]), - fallback_count), - mask_bits, - self[most_frequent_netblock].ipv6addr)) - if shared_netblock_fallback_count > 0: - logging.warning(('%s of fallbacks are in an IPv6 /%d with other ' + - 'fallbacks')%(CandidateList.describe_percentage( - shared_netblock_fallback_count, - fallback_count), - mask_bits)) - - # log a message about the proportion of fallbacks in each IPv4 /8, /16, - # and /24 - def describe_fallback_ipv4_netblocks(self): - # this doesn't actually tell us anything useful - #self.describe_fallback_ipv4_netblock_mask(8) - self.describe_fallback_ipv4_netblock_mask(16) - #self.describe_fallback_ipv4_netblock_mask(24) - - # log a message about the proportion of fallbacks in each IPv6 /12 (RIR), - # /23 (smaller RIR blocks), /32 (LIR), /48 (Customer), and /64 (Host) - # https://www.iana.org/assignments/ipv6-unicast-address-assignments/ - def describe_fallback_ipv6_netblocks(self): - # these don't actually tell us anything useful - #self.describe_fallback_ipv6_netblock_mask(12) - #self.describe_fallback_ipv6_netblock_mask(23) - self.describe_fallback_ipv6_netblock_mask(32) - #self.describe_fallback_ipv6_netblock_mask(48) - self.describe_fallback_ipv6_netblock_mask(64) - - # log a message about the proportion of fallbacks in each IPv4 and IPv6 - # netblock - def describe_fallback_netblocks(self): - self.describe_fallback_ipv4_netblocks() - self.describe_fallback_ipv6_netblocks() - - # return a list of fallbacks which are on the IPv4 ORPort port - def fallbacks_on_ipv4_orport(self, port): - return filter(lambda x: x.orport == port, self.fallbacks) - - # return a list of fallbacks which are on the IPv6 ORPort port - def fallbacks_on_ipv6_orport(self, port): - return filter(lambda x: x.ipv6orport == port, self.fallbacks_with_ipv6()) - - # return a list of fallbacks which are on the DirPort port - def fallbacks_on_dirport(self, port): - return filter(lambda x: x.dirport == port, self.fallbacks) - - # log a message about the proportion of fallbacks on IPv4 ORPort port - # and return that count - def describe_fallback_ipv4_orport(self, port): - port_count = len(self.fallbacks_on_ipv4_orport(port)) - fallback_count = len(self.fallbacks) - logging.warning('%s of fallbacks are on IPv4 ORPort %d'%( - CandidateList.describe_percentage(port_count, - fallback_count), - port)) - return port_count - - # log a message about the proportion of IPv6 fallbacks on IPv6 ORPort port - # and return that count - def describe_fallback_ipv6_orport(self, port): - port_count = len(self.fallbacks_on_ipv6_orport(port)) - fallback_count = len(self.fallbacks_with_ipv6()) - logging.warning('%s of IPv6 fallbacks are on IPv6 ORPort %d'%( - CandidateList.describe_percentage(port_count, - fallback_count), - port)) - return port_count - - # log a message about the proportion of fallbacks on DirPort port - # and return that count - def describe_fallback_dirport(self, port): - port_count = len(self.fallbacks_on_dirport(port)) - fallback_count = len(self.fallbacks) - logging.warning('%s of fallbacks are on DirPort %d'%( - CandidateList.describe_percentage(port_count, - fallback_count), - port)) - return port_count - - # log a message about the proportion of fallbacks on each dirport, - # each IPv4 orport, and each IPv6 orport - def describe_fallback_ports(self): - fallback_count = len(self.fallbacks) - ipv4_or_count = fallback_count - ipv4_or_count -= self.describe_fallback_ipv4_orport(443) - ipv4_or_count -= self.describe_fallback_ipv4_orport(9001) - logging.warning('%s of fallbacks are on other IPv4 ORPorts'%( - CandidateList.describe_percentage(ipv4_or_count, - fallback_count))) - ipv6_fallback_count = len(self.fallbacks_with_ipv6()) - ipv6_or_count = ipv6_fallback_count - ipv6_or_count -= self.describe_fallback_ipv6_orport(443) - ipv6_or_count -= self.describe_fallback_ipv6_orport(9001) - logging.warning('%s of IPv6 fallbacks are on other IPv6 ORPorts'%( - CandidateList.describe_percentage(ipv6_or_count, - ipv6_fallback_count))) - dir_count = fallback_count - dir_count -= self.describe_fallback_dirport(80) - dir_count -= self.describe_fallback_dirport(9030) - logging.warning('%s of fallbacks are on other DirPorts'%( - CandidateList.describe_percentage(dir_count, - fallback_count))) - - # return a list of fallbacks which cache extra-info documents - def fallbacks_with_extra_info_cache(self): - return filter(lambda x: x._extra_info_cache, self.fallbacks) - - # log a message about the proportion of fallbacks that cache extra-info docs - def describe_fallback_extra_info_caches(self): - extra_info_falback_count = len(self.fallbacks_with_extra_info_cache()) - fallback_count = len(self.fallbacks) - logging.warning('%s of fallbacks cache extra-info documents'%( - CandidateList.describe_percentage(extra_info_falback_count, - fallback_count))) - - # return a list of fallbacks which have the Exit flag - def fallbacks_with_exit(self): - return filter(lambda x: x.is_exit(), self.fallbacks) - - # log a message about the proportion of fallbacks with an Exit flag - def describe_fallback_exit_flag(self): - exit_falback_count = len(self.fallbacks_with_exit()) - fallback_count = len(self.fallbacks) - logging.warning('%s of fallbacks have the Exit flag'%( - CandidateList.describe_percentage(exit_falback_count, - fallback_count))) - - # return a list of fallbacks which have an IPv6 address - def fallbacks_with_ipv6(self): - return filter(lambda x: x.has_ipv6(), self.fallbacks) - - # log a message about the proportion of fallbacks on IPv6 - def describe_fallback_ip_family(self): - ipv6_falback_count = len(self.fallbacks_with_ipv6()) - fallback_count = len(self.fallbacks) - logging.warning('%s of fallbacks are on IPv6'%( - CandidateList.describe_percentage(ipv6_falback_count, - fallback_count))) - - def summarise_fallbacks(self, eligible_count, operator_count, failed_count, - guard_count, target_count, check_existing): - s = '' - # Report: - # whether we checked consensus download times - # the number of fallback directories (and limits/exclusions, if relevant) - # min & max fallback bandwidths - # #error if below minimum count - if PERFORM_IPV4_DIRPORT_CHECKS or PERFORM_IPV6_DIRPORT_CHECKS: - s += '/* Checked %s%s%s DirPorts served a consensus within %.1fs. */'%( - 'IPv4' if PERFORM_IPV4_DIRPORT_CHECKS else '', - ' and ' if (PERFORM_IPV4_DIRPORT_CHECKS - and PERFORM_IPV6_DIRPORT_CHECKS) else '', - 'IPv6' if PERFORM_IPV6_DIRPORT_CHECKS else '', - CONSENSUS_DOWNLOAD_SPEED_MAX) - else: - s += '/* Did not check IPv4 or IPv6 DirPort consensus downloads. */' - s += '\n' - # Multiline C comment with #error if things go bad - s += '/*' - s += '\n' - # Integers don't need escaping in C comments - fallback_count = len(self.fallbacks) - if FALLBACK_PROPORTION_OF_GUARDS is None: - fallback_proportion = '' - else: - fallback_proportion = ', Target %d (%d * %.2f)'%(target_count, - guard_count, - FALLBACK_PROPORTION_OF_GUARDS) - s += 'Final Count: %d (Eligible %d%s'%(fallback_count, eligible_count, - fallback_proportion) - if MAX_FALLBACK_COUNT is not None: - s += ', Max %d'%(MAX_FALLBACK_COUNT) - s += ')\n' - if eligible_count != fallback_count: - removed_count = eligible_count - fallback_count - excess_to_target_or_max = (eligible_count - operator_count - failed_count - - fallback_count) - # some 'Failed' failed the check, others 'Skipped' the check, - # if we already had enough successful downloads - s += ('Excluded: %d (Same Operator %d, Failed/Skipped Download %d, ' + - 'Excess %d)')%(removed_count, operator_count, failed_count, - excess_to_target_or_max) - s += '\n' - min_fb = self.fallback_min() - min_bw = min_fb._data['measured_bandwidth'] - max_fb = self.fallback_max() - max_bw = max_fb._data['measured_bandwidth'] - s += 'Bandwidth Range: %.1f - %.1f MByte/s'%(min_bw/(1024.0*1024.0), - max_bw/(1024.0*1024.0)) - s += '\n' - s += '*/' - if fallback_count < MIN_FALLBACK_COUNT: - list_type = 'whitelist' - if check_existing: - list_type = 'fallback list' - # We must have a minimum number of fallbacks so they are always - # reachable, and are in diverse locations - s += '\n' - s += '#error Fallback Count %d is too low. '%(fallback_count) - s += 'Must be at least %d for diversity. '%(MIN_FALLBACK_COUNT) - s += 'Try adding entries to %s, '%(list_type) - s += 'or setting INCLUDE_UNLISTED_ENTRIES = True.' - return s - -def process_existing(): - logging.basicConfig(level=logging.INFO) - logging.getLogger('stem').setLevel(logging.INFO) - whitelist = {'data': parse_fallback_file(FALLBACK_FILE_NAME), - 'name': FALLBACK_FILE_NAME, - 'check_existing' : True} - list_fallbacks(whitelist, exact=True) - -def process_default(): - logging.basicConfig(level=logging.WARNING) - logging.getLogger('stem').setLevel(logging.WARNING) - whitelist = {'data': read_from_file(WHITELIST_FILE_NAME, MAX_LIST_FILE_SIZE), - 'name': WHITELIST_FILE_NAME, - 'check_existing': False} - list_fallbacks(whitelist, exact=False) - -## Main Function -def main(): - if get_command() == 'check_existing': - process_existing() - else: - process_default() - -def get_command(): - if len(sys.argv) == 2: - return sys.argv[1] - else: - return None - -def log_excluded(msg, *args): - if get_command() == 'check_existing': - logging.warning(msg, *args) - else: - logging.info(msg, *args) - -def list_fallbacks(whitelist, exact=False): - """ Fetches required onionoo documents and evaluates the - fallback directory criteria for each of the relays, - passing exact to apply_filter_lists(). """ - if whitelist['check_existing']: - print "/* type=fallback */" - else: - print "/* type=whitelist */" - - print ("/* version={} */" - .format(cleanse_c_multiline_comment(FALLBACK_FORMAT_VERSION))) - now = datetime.datetime.utcnow() - timestamp = now.strftime('%Y%m%d%H%M%S') - print ("/* timestamp={} */" - .format(cleanse_c_multiline_comment(timestamp))) - # end the header with a separator, to make it easier for parsers - print SECTION_SEPARATOR_COMMENT - - logging.warning('Downloading and parsing Onionoo data. ' + - 'This may take some time.') - # find relays that could be fallbacks - candidates = CandidateList() - candidates.add_relays() - - # work out how many fallbacks we want - guard_count = candidates.count_guards() - if FALLBACK_PROPORTION_OF_GUARDS is None: - target_count = guard_count - else: - target_count = int(guard_count * FALLBACK_PROPORTION_OF_GUARDS) - # the maximum number of fallbacks is the least of: - # - the target fallback count (FALLBACK_PROPORTION_OF_GUARDS * guard count) - # - the maximum fallback count (MAX_FALLBACK_COUNT) - if MAX_FALLBACK_COUNT is None: - max_count = target_count - else: - max_count = min(target_count, MAX_FALLBACK_COUNT) - - candidates.compute_fallbacks() - prefilter_fallbacks = copy.copy(candidates.fallbacks) - - # filter with the whitelist - # if a relay has changed IPv4 address or ports recently, it will be excluded - # as ineligible before we call apply_filter_lists, and so there will be no - # warning that the details have changed from those in the whitelist. - # instead, there will be an info-level log during the eligibility check. - initial_count = len(candidates.fallbacks) - excluded_count = candidates.apply_filter_lists(whitelist, exact=exact) - print candidates.summarise_filters(initial_count, excluded_count, - whitelist['check_existing']) - eligible_count = len(candidates.fallbacks) - - # calculate the measured bandwidth of each relay, - # then remove low-bandwidth relays - candidates.calculate_measured_bandwidth() - candidates.remove_low_bandwidth_relays() - - # print the raw fallback list - #for x in candidates.fallbacks: - # print x.fallbackdir_line(True) - # print json.dumps(candidates[x]._data, sort_keys=True, indent=4, - # separators=(',', ': '), default=json_util.default) - - # impose mandatory conditions here, like one per contact, family, IP - # in measured bandwidth order - candidates.sort_fallbacks_by_measured_bandwidth() - operator_count = 0 - # only impose these limits on the final list - operators can nominate - # multiple candidate fallbacks, and then we choose the best set - if not OUTPUT_CANDIDATES: - operator_count += candidates.limit_fallbacks_same_ip() - operator_count += candidates.limit_fallbacks_same_contact() - operator_count += candidates.limit_fallbacks_same_family() - - # check if each candidate can serve a consensus - # there's a small risk we've eliminated relays from the same operator that - # can serve a consensus, in favour of one that can't - # but given it takes up to 15 seconds to check each consensus download, - # the risk is worth it - if PERFORM_IPV4_DIRPORT_CHECKS or PERFORM_IPV6_DIRPORT_CHECKS: - logging.warning('Checking consensus download speeds. ' + - 'This may take some time.') - failed_count = candidates.perform_download_consensus_checks(max_count) - - # work out which fallbacks cache extra-infos - candidates.mark_extra_info_caches() - - # analyse and log interesting diversity metrics - # like netblock, ports, exit, IPv4-only - # (we can't easily analyse AS, and it's hard to accurately analyse country) - candidates.describe_fallback_ip_family() - # if we can't import the ipaddress module, we can't do netblock analysis - if HAVE_IPADDRESS: - candidates.describe_fallback_netblocks() - candidates.describe_fallback_ports() - candidates.describe_fallback_extra_info_caches() - candidates.describe_fallback_exit_flag() - - # output C comments summarising the fallback selection process - if len(candidates.fallbacks) > 0: - print candidates.summarise_fallbacks(eligible_count, operator_count, - failed_count, guard_count, - target_count, - whitelist['check_existing']) - else: - print '/* No Fallbacks met criteria */' - - # output C comments specifying the OnionOO data used to create the list - for s in fetch_source_list(): - print describe_fetch_source(s) - - # start the list with a separator, to make it easy for parsers - print SECTION_SEPARATOR_COMMENT - - # sort the list differently depending on why we've created it: - # if we're outputting the final fallback list, sort by fingerprint - # this makes diffs much more stable - # otherwise, if we're trying to find a bandwidth cutoff, or we want to - # contact operators in priority order, sort by bandwidth (not yet - # implemented) - # otherwise, if we're contacting operators, sort by contact - candidates.sort_fallbacks_by(OUTPUT_SORT_FIELD) - - for x in candidates.fallbacks: - print x.fallbackdir_line(candidates.fallbacks, prefilter_fallbacks) - -if __name__ == "__main__": - main() From 139202174bc1df5ba1a9437bb47c7624651e7068 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 16 Jan 2019 08:20:12 -0500 Subject: [PATCH 0409/2557] Remove changes entries that appeared in 0.3.5.7 --- changes/geoip-2019-01-03 | 4 ---- 1 file changed, 4 deletions(-) delete mode 100644 changes/geoip-2019-01-03 diff --git a/changes/geoip-2019-01-03 b/changes/geoip-2019-01-03 deleted file mode 100644 index 27ffb7f460..0000000000 --- a/changes/geoip-2019-01-03 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor features (geoip): - - Update geoip and geoip6 to the January 3 2019 Maxmind GeoLite2 - Country database. Closes ticket 29012. - From 49062d72b5c022957bcb3bbdd116ec32550794fe Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 16 Jan 2019 08:28:07 -0500 Subject: [PATCH 0410/2557] Start on 0.4.0.1-alpha changelog --- ChangeLog | 319 +++++++++++++++++++++++++++++++++++++++ changes/bug21900 | 4 - changes/bug23082 | 4 - changes/bug24393 | 6 - changes/bug24661 | 3 - changes/bug24953 | 4 - changes/bug25885 | 7 - changes/bug27707 | 3 - changes/bug27929 | 5 - changes/bug28518 | 4 - changes/bug28569 | 3 - changes/bug28591 | 4 - changes/bug28654 | 3 - changes/bug28895 | 5 - changes/bug28920 | 6 - changes/bug28938 | 4 - changes/bug28989 | 5 - changes/bug28995 | 5 - changes/doc28560 | 3 - changes/doc28805 | 4 - changes/document_version | 2 - changes/feature27244 | 5 - changes/feature27367 | 4 - changes/prop297 | 7 - changes/subsystems | 6 - changes/ticket24805 | 3 - changes/ticket24838 | 6 - changes/ticket26360 | 4 - changes/ticket26770 | 8 - changes/ticket27167 | 11 -- changes/ticket27225 | 5 - changes/ticket27325 | 4 - changes/ticket27359 | 3 - changes/ticket27402 | 10 -- changes/ticket27490 | 6 - changes/ticket27549 | 3 - changes/ticket27620 | 3 - changes/ticket27625 | 4 - changes/ticket27914 | 4 - changes/ticket27993 | 3 - changes/ticket28006 | 3 - changes/ticket28007 | 3 - changes/ticket28008 | 3 - changes/ticket28009 | 3 - changes/ticket28010 | 3 - changes/ticket28011 | 3 - changes/ticket28012 | 3 - changes/ticket28058 | 2 - changes/ticket28077 | 3 - changes/ticket28100 | 3 - changes/ticket28142 | 10 -- changes/ticket28179 | 5 - changes/ticket28180 | 3 - changes/ticket28266 | 10 -- changes/ticket28335 | 7 - changes/ticket28362 | 6 - changes/ticket28551 | 3 - changes/ticket28624 | 5 - changes/ticket28669 | 6 - changes/ticket28757 | 5 - changes/ticket28768 | 4 - changes/ticket28839 | 3 - changes/ticket28840 | 3 - changes/ticket28843 | 3 - changes/ticket28846 | 3 - changes/ticket28847 | 3 - changes/ticket28852 | 4 - changes/ticket28853 | 3 - changes/ticket28856 | 3 - 69 files changed, 319 insertions(+), 301 deletions(-) delete mode 100644 changes/bug21900 delete mode 100644 changes/bug23082 delete mode 100644 changes/bug24393 delete mode 100644 changes/bug24661 delete mode 100644 changes/bug24953 delete mode 100644 changes/bug25885 delete mode 100644 changes/bug27707 delete mode 100644 changes/bug27929 delete mode 100644 changes/bug28518 delete mode 100644 changes/bug28569 delete mode 100644 changes/bug28591 delete mode 100644 changes/bug28654 delete mode 100644 changes/bug28895 delete mode 100644 changes/bug28920 delete mode 100644 changes/bug28938 delete mode 100644 changes/bug28989 delete mode 100644 changes/bug28995 delete mode 100644 changes/doc28560 delete mode 100644 changes/doc28805 delete mode 100644 changes/document_version delete mode 100644 changes/feature27244 delete mode 100644 changes/feature27367 delete mode 100644 changes/prop297 delete mode 100644 changes/subsystems delete mode 100644 changes/ticket24805 delete mode 100644 changes/ticket24838 delete mode 100644 changes/ticket26360 delete mode 100644 changes/ticket26770 delete mode 100644 changes/ticket27167 delete mode 100644 changes/ticket27225 delete mode 100644 changes/ticket27325 delete mode 100644 changes/ticket27359 delete mode 100644 changes/ticket27402 delete mode 100644 changes/ticket27490 delete mode 100644 changes/ticket27549 delete mode 100644 changes/ticket27620 delete mode 100644 changes/ticket27625 delete mode 100644 changes/ticket27914 delete mode 100644 changes/ticket27993 delete mode 100644 changes/ticket28006 delete mode 100644 changes/ticket28007 delete mode 100644 changes/ticket28008 delete mode 100644 changes/ticket28009 delete mode 100644 changes/ticket28010 delete mode 100644 changes/ticket28011 delete mode 100644 changes/ticket28012 delete mode 100644 changes/ticket28058 delete mode 100644 changes/ticket28077 delete mode 100644 changes/ticket28100 delete mode 100644 changes/ticket28142 delete mode 100644 changes/ticket28179 delete mode 100644 changes/ticket28180 delete mode 100644 changes/ticket28266 delete mode 100644 changes/ticket28335 delete mode 100644 changes/ticket28362 delete mode 100644 changes/ticket28551 delete mode 100644 changes/ticket28624 delete mode 100644 changes/ticket28669 delete mode 100644 changes/ticket28757 delete mode 100644 changes/ticket28768 delete mode 100644 changes/ticket28839 delete mode 100644 changes/ticket28840 delete mode 100644 changes/ticket28843 delete mode 100644 changes/ticket28846 delete mode 100644 changes/ticket28847 delete mode 100644 changes/ticket28852 delete mode 100644 changes/ticket28853 delete mode 100644 changes/ticket28856 diff --git a/ChangeLog b/ChangeLog index 090498c859..d6a72e2efd 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,322 @@ +Changes in version 0.4.0.1-alpha - 2019-01-18 + blurb blurb blurb + + o Major features (battery management, client, dormant mode): + - When Tor is running as a client, and it is unused for a long time, + it can now enter a "dormant" state. When Tor is dormant, it avoids + network activity and CPU wakeups until it is reawoken either by a + user request or by a controller command. For more information, see + the configuration options starting with "Dormant". Implements + tickets 2149 and 28335. + - The client's memory of whether it is "dormant", and how long it + has spend idle, persists across invocations. Implements + ticket 28624. + - There is a DormantOnFirstStartup option that integrators can use + if they expect that in many cases, Tor will be installed but + not used. + + o Major features (bootstrap): + - Report the first connection to a relay as the earliest phases of + bootstrap progress, regardless of whether it's a connection for + building application circuits. This allows finer-grained reporting + of early progress than previously possible with the improvements + of ticket 27169. Closes tickets 27167 and 27103. Addresses + ticket 27308. + - Separately report the intermediate stage of having connected to a + proxy or pluggable transport, versus succesfully using that proxy + or pluggable transport to connect to a relay. Closes tickets 27100 + and 28884. + + o Major features (circuit padding): + - Implement preliminary support for the circuit padding portion of + Proposal 254. The implementation supports Adaptive Padding (aka + WTF-PAD) state machines for use between experimental clients and + relays. Support is also provided for APE-style state machines that + use probability distributions instead of histograms to specify + inter-packet delay. At the moment, Tor does not provide any + padding state machines that are used in normal operation -- this + feature exists solely for experimentation in this release. Closes + ticket 28142. + + o Major features (refactoring): + - Tor now uses an explicit list of its own subsystems when + initializing and shutting down. Previously, these systems were + managed implicitly though various places throughout the codebase. + (There still some subsystems using the old system.) Closes + ticket 28330. + + o Minor feature (bootstrap): + - When reporting bootstrap progress, stop distinguishing between + situations where it seems that only internal paths are available + and situations where it seems that external paths are available. + Previously, tor would often erroneously report that it had only + internal paths. Closes ticket 27402. + + o Minor features (Continuous Integration): + - Log Python version during each Travis CI job. Resolves + issue 28551. + + o Minor features (controller): + - Add a DROPOWNERSHIP command to undo the effects of TAKEOWNERSHIP. + Implements ticket 28843. + + o Minor features (developer tooling): + - Provide a git hook script to prevent "fixup!" and "squash!" + commits from ending up in master. Closes ticket 27993. + + o Minor features (directory authority): + - Directory authorities support a new consensus algorithm, under + which microdescriptor entries are encoded in a canonical form. + This improves their compressibility in transit and on the client. + Closes ticket 28266; implements proposal 298. + + o Minor features (directory authority, relay): + - Authorities now vote on a "StaleDesc" flag to indicate that a + relay's descriptor is so old that the relay should upload again + soon. Relays understand this flag, and treat it as a signal to + upload a new descriptor. This flag will eventually let us remove + the 'published' date from routerstatus entries, and save a great + deal of space in our consensus diffs. Closes ticket 26770; + implements proposal 293. + + o Minor features (fallback directory mirrors): + - Update the fallback whitelist based on operator opt-ins and opt- + outs. Closes ticket 24805, patch by Phoul. + - Accept fallbacks that deliver reasonably live consensuses. + (Consensuses that will become valid less than 24 hours in the + future, or that expired less than 24 hours ago.) Closes + ticket 28768. + - Accept relays that are a fuzzy match to a fallback whitelist + entry. If a relay matches at least one fingerprint, IPv4 address, + or IPv6 address in the fallback whitelist, it can become a + fallback. This reduces the work required to keep the list up to + date. Closes ticket 24838. + + o Minor features (FreeBSD): + - Warn relay operators if the "net.inet.ip.random_id" sysctl (IP ID + randomization) is disabled on their relay if it is running on + FreeBSD based operating systems. Closes ticket 28518. + + o Minor features (HTTP standards compliance): + - Don't send Content-Type: application/octet-stream for transparently + compressed documents, which confused browsers. Closes ticket 28100. + + o Minor features (ipv6): + - We add an option ClientAutoIPv6ORPort which makes clients randomly + prefer a node's IPv4 or IPv6 ORPort. The random preference is set + every time a node is loaded from a new consensus or bridge config. + Closes ticket 27490. Patch by Neel Chauhan. + - When using addrs_in_same_network_family(), check IPv6 subnets as + well as IPv4 ones where possible when a client chooses circuit + paths. Previously, we used this function only for IPv4 subnets. + Closes ticket 24393. Patch by Neel Chauhan. + + o Minor features (log messages): + - Improve log message in HSv3 service that could print out negative + revision counters. Closes ticket 27707. Patch by "ffmancera". + + o Minor features (memory usage): + - Store microdescriptor family lists with a more compact + representation to save memory. Closes ticket 27359. + - Tor clients no longer need to keep the full text of a consensus in + memory in order to parse it, or apply a diff to it. Instead, they + use mmap() to read the consensus files from disk. Closes + ticket 27244. + + o Minor features (parsing): + - Directory authorities now validate that router descriptors and + ExtraInfo documents are in a valid subset of UTF-8, and reject + them if not. Closes ticket 27367. + + o Minor features (performance): + - Avoid parsing the same protocol-versions string over and over in + summarize_protover_flags(). This should save us a huge number of + malloc calls on startup, and may reduce memory fragmentation with + some allocators. Closes ticket 27225. + - Remove a needless memset() call from get_token_arguments, thereby + speeding up the tokenization of directory objects by about 20%. + Closes ticket 28852. + - Replace parse_short_policy() with a faster implementation, to + improve microdescriptor parsing time. Closes ticket 28853. + - Speed up directory parsing a little by avoiding use of the non- + inlined strcmp_len() function. Closes ticket 28856. + - Speed up microdesriptor parsing by about 30%, to help improve + startup time. Closes ticket 28839. + + o Minor features (pluggable transports): + - Add support for emitting STATUS updates to Tor's control port from + a pluggable transport process. Closes ticket 28846. + - Add support for logging to Tor's logging subsystem from a + pluggable transport process. Closes ticket 28180 + + o Minor features (process management): + - Add new Process API for handling child processes. This new API + allows Tor to have bi-directional communication with child + processes on both Unix and Windows. Closes ticket 28179. + - Use the subsystem module to initialize and shut down the process + module. Closes ticket 28847. + + o Minor features (relay): + - When listing relay families, list them in canonical form including + the relay's own identity, and try to give a more useful set of + warnings. Part of ticket 28266 and proposal 298. + + o Minor features (required protocols): + - Tor no longer exits if it is missing a required protocol, if the + consensus that requires the protocol predates the release date of + the version of Tor. This change prevents Tor releases from exiting + because of an old cached consensus, on the theory that a newer + cached consensus might not require the protocol. Implements + proposal 297; closes ticket 27735. + + o Minor features (testing): + - Allow HeartbeatPeriod of less than 30 minutes in testing Tor + networks. Closes ticket 28840, patch by robgjansen + + o Minor bugfixes (client, bootstrap): + - When Tor's clock is behind the clocks on the authorities, allow + Tor to bootstrap successfully. Fixes bug 28591; bugfix + on 0.2.0.9-alpha. + + o Minor bugfixes (client, guard selection): + - When Tor's consensus has expired, but is still reasonably live, + use it to select guards. Fixes bug 24661; bugfix on 0.3.0.1-alpha. + + o Minor bugfixes (compilation): + - Fix missing headers required for proper detection of OpenBSD. Fixes + bug 28938; bugfix on 0.3.5.1-alpha. Patch from Kris Katterjohn. + + o Minor bugfixes (directory clients): + - Mark outdated dirservers when Tor only has a reasonably live + consensus. Fixes bug 28569; bugfix on 0.3.2.5-alpha. + + o Minor bugfixes (directory mirror): + - When Tor's clock is behind the clocks on the authorities, allow + Tor to serve future consensuses. Fixes bug 28654; bugfix + on 0.3.0.1-alpha. + + o Minor bugfixes (DNS): + - Gracefully handle empty or absent resolve.conf file by falling + back to using localhost DNS service and hoping it works. Fixes bug + 21900; bugfix on 0.2.1.10-alpha. + + o Minor bugfixes (fallback scripts): + - In updateFallbackDirs.py, call the filter file a "fallback list" + instead of a "whitelist" in check_existing mode. Fixes bug 24953; + bugfix on 0.3.0.3-alpha. + + o Minor bugfixes (guards): + - In count_acceptable_nodes(), check if we have at least one bridge + or guard node, and two non-guard nodes for a circuit. Previously, + we have added up the sum of all nodes with a descriptor, but that + could cause us to build circuits that fail if we had either too + many bridges, or not enough guard nodes. Fixes bug 25885; bugfix + on 0.3.6.1-alpha. Patch by Neel Chauhan. + + o Minor bugfixes (IPv6): + - Fix tor_ersatz_socketpair on IPv6-only systems. Previously, the + IPv6 socket was bound using an address family of AF_INET instead + of AF_INET6. Fixes bug 28995; bugfix on 0.3.5.1-alpha. Patch from + Kris Katterjohn. + + o Minor bugfixes (logging): + - Rework rep_hist_log_link_protocol_counts() to iterate through all + link protocol versions when logging incoming/outgoing connection + counts. Tor no longer skips version 5 and we don't have to + remember to update this function when new link protocol version is + developed. Fixes bug 28920; bugfix on 0.2.6.10. + + o Minor bugfixes (networking): + - Introduce additional checks into tor_addr_parse() to reject + certain incorrect inputs that previously were not detected. Fixes + bug 23082; bugfix on 0.2.0.10-alpha. + + o Minor bugfixes (onion service v3, client): + - Avoid a BUG() stacktrace in case a SOCKS connection is found + waiting for the descriptor while we do have it in the cache. There + is a rare case when this can happen. Now, tor will recover and + retry the descriptor. Fixes bug 28669; bugfix on 0.3.2.4-alpha. + + o Minor bugfixes (periodic events): + - Refrain from calling routerlist_remove_old_routers() from + check_descriptor_callback(). Instead, create a new periodic event + that will run once every hour even if Tor is not configured as + onion router. Fixes bug 27929; bugfix on 0.2.8.1-alpha. + + o Minor bugfixes (pluggable transports): + - Make sure that data is continously read from standard out and + error of the PT child-process to avoid deadlocking when the pipes' + buffer is full. Fixes bug 26360; bugfix on 0.2.3.6-alpha. + + o Minor bugfixes (unit tests): + - Instead of relying on hs_free_all() to clean up all onion service + objects we created in test_build_descriptors(), deallocate them + one by one. This lets Coverity know that we are not leaking memory + here and fixes CID 1442277. Fixes bug 28989; bugfix + on 0.3.5.1-alpha. + + o Minor bugfixes (usability): + - Stop saying "Your Guard ..." in pathbias_measure_{use,close}_rate() + as that confusingly suggests that mentioned guard node is under + control and responsibility of end user, which it is not. Fixes bug + 28895; bugfix on Tor 0.3.0.1-alpha. + + o Code simplification and refactoring: + - Reimplement NETINFO cell parsing and generation to rely on + trunnel-generated wire format handling code. Closes ticket 27325. + - Remove unnecessarily unsafe code from the rust macro cstr!. Closes + ticket 28077. + - Rework SOCKS wire format handling to rely on trunnel-generated + parsing/generation code. Resolves ticket 27620. + - Split out bootstrap progress reporting from control.c into a + separate file. Part of ticket 27402. + - The .may_include files that we use to describe our directory-by- + directory dependency structure now describe a noncircular + dependency graph over the directories that they cover. Our + checkIncludes.py tool now enforces this. Closes ticket 28362. + + o Documentation: + - Mention that you cannot add new Onion Service if Tor is already + running with Sandbox enabled. Closes ticket 28560. + - Improve ControlPort description in tor manpage to mention that it + accepts address/port pair, and can be used multiple times. Closes + ticket 28805. + - Document the exact output of "tor --version". Closes ticket 28889. + + o Removed features: + - Stop responding to 'GETINFO status/version/num-concurring' and + 'GETINFO status/version/num-versioning' control port commands, as + those were deprecated back in 0.2.0.30. Also stop listing them in + output of 'GETINFO info/names'. Resolves ticket 28757. + - The scripts used to generate and maintain the list of fallback + directories have been extracted into a new "fallback-scripts" + repository. Closes ticket 27914. + + o Testing: + - Run shellcheck for stuff in scripts/ directory. Closes + ticket 28058. + - Write some unit tests for tokenize_string() and get_next_token() + functions. Resolves ticket 27625. + + o Code simplification and refactoring (onion service v3): + - Consolidate the authorized client descriptor cookie computation + code from client and service into one function. Closes + ticket 27549. + + o Code simplification and refactoring (shell scripts): + - Cleanup scan-build.sh to silence shellcheck warnings. Closes + ticket 28007. + - Fix issues that shellcheck found in chutney-git-bisect.sh. + Resolves ticket 28006. + - Fix issues that shellcheck found in updateRustDependencies.sh. + Resolves ticket 28012. + - Fix shellcheck warnings in cov-diff script. Resolves issue 28009. + - Fix shellcheck warnings in run_calltool.sh. Resolves ticket 28011. + - Fix shellcheck warnings in run_trunnel.sh. Resolves issue 28010. + - Fix shellcheck warnings in scripts/test/coverage. Resolves + issue 28008. + + Changes in version 0.3.3.11 - 2018-01-07 Tor 0.3.3.11 backports numerous fixes from later versions of Tor. numerous fixes, including an important fix for anyone using OpenSSL diff --git a/changes/bug21900 b/changes/bug21900 deleted file mode 100644 index 686cb6c584..0000000000 --- a/changes/bug21900 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (DNS): - - Gracefully handle empty or absent resolve.conf file by falling - back to using localhost DNS service and hoping it works. Fixes - bug 21900; bugfix on 0.2.1.10-alpha. diff --git a/changes/bug23082 b/changes/bug23082 deleted file mode 100644 index fc4b52c364..0000000000 --- a/changes/bug23082 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (networking): - - Introduce additional checks into tor_addr_parse() to - reject certain incorrect inputs that previously were - not detected. Fixes bug 23082; bugfix on 0.2.0.10-alpha. diff --git a/changes/bug24393 b/changes/bug24393 deleted file mode 100644 index e190192319..0000000000 --- a/changes/bug24393 +++ /dev/null @@ -1,6 +0,0 @@ - o Minor features (ipv6): - - When using addrs_in_same_network_family(), check IPv6 subnets as well as - IPv4 ones where possible when a client chooses circuit paths. Previously, - we used this function only for IPv4 subnets. Closes ticket 24393. Patch - by Neel Chauhan. - diff --git a/changes/bug24661 b/changes/bug24661 deleted file mode 100644 index a915a93e0e..0000000000 --- a/changes/bug24661 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor bugfixes (client, guard selection): - - When Tor's consensus has expired, but is still reasonably live, use it - to select guards. Fixes bug 24661; bugfix on 0.3.0.1-alpha. diff --git a/changes/bug24953 b/changes/bug24953 deleted file mode 100644 index d142dfd6cc..0000000000 --- a/changes/bug24953 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (fallback scripts): - - In updateFallbackDirs.py, call the filter file a "fallback list" - instead of a "whitelist" in check_existing mode. - Fixes bug 24953; bugfix on 0.3.0.3-alpha. diff --git a/changes/bug25885 b/changes/bug25885 deleted file mode 100644 index 1b89acfe06..0000000000 --- a/changes/bug25885 +++ /dev/null @@ -1,7 +0,0 @@ - o Minor bugfixes (guards): - - In count_acceptable_nodes(), check if we have at least one bridge - or guard node, and two non-guard nodes for a circuit. Previously, - we have added up the sum of all nodes with a descriptor, but that - could cause us to build circuits that fail if we had either too - many bridges, or not enough guard nodes. Fixes bug 25885; bugfix - on 0.3.6.1-alpha. Patch by Neel Chauhan. diff --git a/changes/bug27707 b/changes/bug27707 deleted file mode 100644 index e114222741..0000000000 --- a/changes/bug27707 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor features (log messages): - - Improve log message in HSv3 service that could print out negative - revision counters. Closes ticket 27707. Patch by "ffmancera". \ No newline at end of file diff --git a/changes/bug27929 b/changes/bug27929 deleted file mode 100644 index dab57a2eca..0000000000 --- a/changes/bug27929 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor bugfixes (periodic events): - - Refrain from calling routerlist_remove_old_routers() from - check_descriptor_callback(). Instead, create a new periodic - event that will run once every hour even if Tor is not configured - as onion router. Fixes bug 27929; bugfix on 0.2.8.1-alpha. diff --git a/changes/bug28518 b/changes/bug28518 deleted file mode 100644 index d7ebab29bb..0000000000 --- a/changes/bug28518 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor features (FreeBSD): - - Warn relay operators if the "net.inet.ip.random_id" sysctl (IP ID - randomization) is disabled on their relay if it is running on FreeBSD - based operating systems. Closes ticket 28518. diff --git a/changes/bug28569 b/changes/bug28569 deleted file mode 100644 index 45a57a80ae..0000000000 --- a/changes/bug28569 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor bugfixes (unit tests, directory clients): - - Mark outdated dirservers when Tor only has a reasonably live consensus. - Fixes bug 28569; bugfix on 0.3.2.5-alpha. diff --git a/changes/bug28591 b/changes/bug28591 deleted file mode 100644 index 3a1c96ac16..0000000000 --- a/changes/bug28591 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (client, bootstrap): - - When Tor's clock is behind the clocks on the authorities, allow Tor to - bootstrap successfully. Fixes bug 28591; bugfix on 0.2.0.9-alpha. - diff --git a/changes/bug28654 b/changes/bug28654 deleted file mode 100644 index 4ca843309f..0000000000 --- a/changes/bug28654 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor bugfixes (directory mirror): - - When Tor's clock is behind the clocks on the authorities, allow Tor to - serve future consensuses. Fixes bug 28654; bugfix on 0.3.0.1-alpha. diff --git a/changes/bug28895 b/changes/bug28895 deleted file mode 100644 index 25fb167b2e..0000000000 --- a/changes/bug28895 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor bugfixes (usability): - - Stop saying "Your Guard ..." in pathbias_measure_{use,close}_rate() - as that confusingly suggests that mentioned guard node is under control - and responsibility of end user, which it is not. Fixes bug 28895; - bugfix on Tor 0.3.0.1-alpha. diff --git a/changes/bug28920 b/changes/bug28920 deleted file mode 100644 index e698686a6d..0000000000 --- a/changes/bug28920 +++ /dev/null @@ -1,6 +0,0 @@ - o Minor bugfixes (logging): - - Rework rep_hist_log_link_protocol_counts() to iterate through all link - protocol versions when logging incoming/outgoing connection counts. Tor - no longer skips version 5 and we don't have to remember to update this - function when new link protocol version is developed. Fixes bug 28920; - bugfix on 0.2.6.10. diff --git a/changes/bug28938 b/changes/bug28938 deleted file mode 100644 index de6c5f7b79..0000000000 --- a/changes/bug28938 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (compilation): - - Fix missing headers required for proper detection of - OpenBSD. Fixes bug 28938; bugfix on 0.3.5.1-alpha. - Patch from Kris Katterjohn. diff --git a/changes/bug28989 b/changes/bug28989 deleted file mode 100644 index 3e3ccccaf3..0000000000 --- a/changes/bug28989 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor bugfixes (unit tests): - - Instead of relying on hs_free_all() to clean up all onion service - objects we created in test_build_descriptors(), deallocate - them one by one. This lets Coverity know that we are not leaking memory - here and fixes CID 1442277. Fixes bug 28989; bugfix on 0.3.5.1-alpha. diff --git a/changes/bug28995 b/changes/bug28995 deleted file mode 100644 index f76b6a085a..0000000000 --- a/changes/bug28995 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor bugfix (IPv6): - Fix tor_ersatz_socketpair on IPv6-only systems. Previously, - the IPv6 socket was bound using an address family of AF_INET - instead of AF_INET6. Fixes bug 28995; bugfix on 0.3.5.1-alpha. - Patch from Kris Katterjohn. diff --git a/changes/doc28560 b/changes/doc28560 deleted file mode 100644 index c3356bda0a..0000000000 --- a/changes/doc28560 +++ /dev/null @@ -1,3 +0,0 @@ - o Documentation (hidden service manpage, sandbox): - - Mention that you cannot add new Onion Service if Tor is already - running with Sandbox enabled. Closes ticket 28560. diff --git a/changes/doc28805 b/changes/doc28805 deleted file mode 100644 index 6c9fea44fa..0000000000 --- a/changes/doc28805 +++ /dev/null @@ -1,4 +0,0 @@ - o Documentation (manpage): - - Improve ControlPort description in tor manpage to mention that it - accepts address/port pair, and can be used multiple times. Closes ticket - 28805. diff --git a/changes/document_version b/changes/document_version deleted file mode 100644 index a45992b6b5..0000000000 --- a/changes/document_version +++ /dev/null @@ -1,2 +0,0 @@ - o Documentation: - - Document the exact output of "tor --version". Closes ticket 28889. diff --git a/changes/feature27244 b/changes/feature27244 deleted file mode 100644 index a4debbbe53..0000000000 --- a/changes/feature27244 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor features (memory usage): - - Tor clients no longer need to keep the full text of a consensus in - memory in order to parse it, or apply a diff to it. Instead, they - use mmap() to read the consensus files from disk. Closes ticket - 27244. diff --git a/changes/feature27367 b/changes/feature27367 deleted file mode 100644 index 99c0839621..0000000000 --- a/changes/feature27367 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor features (parsing): - - Directory authorities now validate that router descriptors and ExtraInfo - documents are in a valid subset of UTF-8, and reject them if not. - Closes ticket 27367. diff --git a/changes/prop297 b/changes/prop297 deleted file mode 100644 index 4f93b232d2..0000000000 --- a/changes/prop297 +++ /dev/null @@ -1,7 +0,0 @@ - o Minor features (required protocols): - - Tor no longer exits if it is missing a required protocol, if the - consensus that requires the protocol predates the release date of the - version of Tor. This change prevents Tor releases from exiting because - of an old cached consensus, on the theory that a newer cached - consensus might not require the protocol. Implements proposal 297; - closes ticket 27735. diff --git a/changes/subsystems b/changes/subsystems deleted file mode 100644 index a51fb8e2b1..0000000000 --- a/changes/subsystems +++ /dev/null @@ -1,6 +0,0 @@ - o Major features (refactoring): - - Tor now uses an explicit list of its own subsystems when initializing - and shutting down. Previously, these systems were managed implicitly - though various places throughout the codebase. (There still some - subsystems using the old system.) - Closes ticket 28330. diff --git a/changes/ticket24805 b/changes/ticket24805 deleted file mode 100644 index 4ba6f6ecd4..0000000000 --- a/changes/ticket24805 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor features (fallback directory list): - - Update the fallback whitelist based on operator opt-ins and opt-outs. - Closes ticket 24805, patch by Phoul. diff --git a/changes/ticket24838 b/changes/ticket24838 deleted file mode 100644 index d068e31b91..0000000000 --- a/changes/ticket24838 +++ /dev/null @@ -1,6 +0,0 @@ - o Minor features (fallback directory mirrors): - - Accept relays that are a fuzzy match to a fallback whitelist entry. - If a relay matches at least one fingerprint, IPv4 address, or IPv6 - address in the fallback whitelist, it can become a fallback. This - reduces the work required to keep the list up to date. - Closes ticket 24838. diff --git a/changes/ticket26360 b/changes/ticket26360 deleted file mode 100644 index 80afbd1c17..0000000000 --- a/changes/ticket26360 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (pluggable transports): - - Make sure that data is continously read from standard out and error of the - PT child-process to avoid deadlocking when the pipes' buffer is full. - Fixes bug 26360; bugfix on 0.2.3.6-alpha. diff --git a/changes/ticket26770 b/changes/ticket26770 deleted file mode 100644 index 7f3e92e9dd..0000000000 --- a/changes/ticket26770 +++ /dev/null @@ -1,8 +0,0 @@ - o Minor features (directory authority, relay): - - Authorities now vote on a "StaleDesc" flag to indicate that a relay's - descriptor is so old that the relay should upload again soon. Relays - understand this flag, and treat it as a signal to upload a new - descriptor. This flag will eventually let us remove the 'published' - date from routerstatus entries, and save a great deal of space in our - consensus diffs. Closes ticket 26770; implements proposal 293. - diff --git a/changes/ticket27167 b/changes/ticket27167 deleted file mode 100644 index 81c66630c8..0000000000 --- a/changes/ticket27167 +++ /dev/null @@ -1,11 +0,0 @@ - o Major features (bootstrap): - - Report the first connection to a relay as the earliest phases of - bootstrap progress, regardless of whether it's a connection for - building application circuits. This allows finer-grained - reporting of early progress than previously possible with the - improvements of ticket 27169. Closes tickets 27167 and 27103. - Addresses ticket 27308. - - Separately report the intermediate stage of having connected to - a proxy or pluggable transport, versus succesfully using that - proxy or pluggable transport to connect to a relay. Closes - tickets 27100 and 28884. diff --git a/changes/ticket27225 b/changes/ticket27225 deleted file mode 100644 index 4c05a269d6..0000000000 --- a/changes/ticket27225 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor features (performance): - - Avoid parsing the same protocol-versions string over and over - in summarize_protover_flags(). This should save us a huge number - of malloc calls on startup, and may reduce memory fragmentation with - some allocators. Closes ticket 27225. diff --git a/changes/ticket27325 b/changes/ticket27325 deleted file mode 100644 index 2cf2bb7d69..0000000000 --- a/changes/ticket27325 +++ /dev/null @@ -1,4 +0,0 @@ - o Code simplification and refactoring: - - Reimplement NETINFO cell parsing and generation to rely on - trunnel-generated wire format handling code. Closes ticket - 27325. diff --git a/changes/ticket27359 b/changes/ticket27359 deleted file mode 100644 index bddc90634d..0000000000 --- a/changes/ticket27359 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor features (memory usage): - - Store microdescriptor family lists with a more compact representation - to save memory. Closes ticket 27359. diff --git a/changes/ticket27402 b/changes/ticket27402 deleted file mode 100644 index b79fb56760..0000000000 --- a/changes/ticket27402 +++ /dev/null @@ -1,10 +0,0 @@ - o Minor feature (bootstrap): - - When reporting bootstrap progress, stop distinguishing between - situations where it seems that only internal paths are available - and situations where it seems that external paths are available. - Previously, tor would often erroneously report that it had only - internal paths. Closes ticket 27402. - - o Code simplification and refactoring: - - Split out bootstrap progress reporting from control.c into a - separate file. Part of ticket 27402. diff --git a/changes/ticket27490 b/changes/ticket27490 deleted file mode 100644 index 523477dfea..0000000000 --- a/changes/ticket27490 +++ /dev/null @@ -1,6 +0,0 @@ - o Minor features (ipv6): - - We add an option ClientAutoIPv6ORPort which makes clients randomly - prefer a node's IPv4 or IPv6 ORPort. The random preference is set - every time a node is loaded from a new consensus or bridge config. - Closes ticket 27490. Patch by Neel Chauhan. - diff --git a/changes/ticket27549 b/changes/ticket27549 deleted file mode 100644 index 51d0f24757..0000000000 --- a/changes/ticket27549 +++ /dev/null @@ -1,3 +0,0 @@ - o Code simplification and refactoring (hidden service v3): - - Consolidate the authorized client descriptor cookie computation code - from client and service into one function. Closes ticket 27549. diff --git a/changes/ticket27620 b/changes/ticket27620 deleted file mode 100644 index 6c491696d0..0000000000 --- a/changes/ticket27620 +++ /dev/null @@ -1,3 +0,0 @@ - o Code simplification and refactoring: - - Rework SOCKS wire format handling to rely on trunnel-generated - parsing/generation code. Resolves ticket 27620. diff --git a/changes/ticket27625 b/changes/ticket27625 deleted file mode 100644 index 33d40adf34..0000000000 --- a/changes/ticket27625 +++ /dev/null @@ -1,4 +0,0 @@ - o Testing: - - Write some unit tests for tokenize_string() and - get_next_token() functions. Resolves ticket 27625. - diff --git a/changes/ticket27914 b/changes/ticket27914 deleted file mode 100644 index 433e9657af..0000000000 --- a/changes/ticket27914 +++ /dev/null @@ -1,4 +0,0 @@ - o Removed features: - - The scripts used to generate and maintain the list of fallback - directories have been extracted into a new "fallback-scripts" - repository. Closes ticket 27914. diff --git a/changes/ticket27993 b/changes/ticket27993 deleted file mode 100644 index 78ee7c2054..0000000000 --- a/changes/ticket27993 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor features (developer tooling): - - Provide git hook script to prevent "fixup!" and "squash!" commits from - ending up in master. Closes ticket 27993. diff --git a/changes/ticket28006 b/changes/ticket28006 deleted file mode 100644 index 95a4b2cae4..0000000000 --- a/changes/ticket28006 +++ /dev/null @@ -1,3 +0,0 @@ - o Code simplification and refactoring: - - Fix issues that shellcheck found in chutney-git-bisect.sh. - Resolves ticket 28006. diff --git a/changes/ticket28007 b/changes/ticket28007 deleted file mode 100644 index 1ac87862eb..0000000000 --- a/changes/ticket28007 +++ /dev/null @@ -1,3 +0,0 @@ - o Code simplification and refactoring: - - Cleanup scan-build.sh to silence shellcheck warnings. - Closes ticket 28007. diff --git a/changes/ticket28008 b/changes/ticket28008 deleted file mode 100644 index 1f0de1a14d..0000000000 --- a/changes/ticket28008 +++ /dev/null @@ -1,3 +0,0 @@ - o Code simplification and refactoring: - - Fix shellcheck warnings in scripts/test/coverage. Resolves issue - 28008. diff --git a/changes/ticket28009 b/changes/ticket28009 deleted file mode 100644 index 1d986d4211..0000000000 --- a/changes/ticket28009 +++ /dev/null @@ -1,3 +0,0 @@ - o Code simplification and refactoring: - - Fix shellcheck warnings in cov-diff script. Resolves issue - 28009. diff --git a/changes/ticket28010 b/changes/ticket28010 deleted file mode 100644 index 4fca17d022..0000000000 --- a/changes/ticket28010 +++ /dev/null @@ -1,3 +0,0 @@ - o Code simplification and refactoring: - - Fix shellcheck warnings in run_trunnel.sh. Resolves issue - 28010. diff --git a/changes/ticket28011 b/changes/ticket28011 deleted file mode 100644 index 5efc3c917b..0000000000 --- a/changes/ticket28011 +++ /dev/null @@ -1,3 +0,0 @@ - o Code simplification and refactoring: - - Fix shellcheck warnings in run_calltool.sh. Resolves - ticket 28011. diff --git a/changes/ticket28012 b/changes/ticket28012 deleted file mode 100644 index b2fe83e02a..0000000000 --- a/changes/ticket28012 +++ /dev/null @@ -1,3 +0,0 @@ - o Code simplification and refactoring: - - Fix issues that shellcheck found in updateRustDependencies.sh. - Resolves ticket 28012. diff --git a/changes/ticket28058 b/changes/ticket28058 deleted file mode 100644 index 00ac595864..0000000000 --- a/changes/ticket28058 +++ /dev/null @@ -1,2 +0,0 @@ - o Testing: - - Run shellcheck for stuff in scripts/ directory. Closes ticket 28058. diff --git a/changes/ticket28077 b/changes/ticket28077 deleted file mode 100644 index 2b5afb1678..0000000000 --- a/changes/ticket28077 +++ /dev/null @@ -1,3 +0,0 @@ - o Code simplification and refactoring: - - Remove unnecessarily unsafe code from the rust macro cstr!. Closes - ticket 28077. diff --git a/changes/ticket28100 b/changes/ticket28100 deleted file mode 100644 index b8e3271013..0000000000 --- a/changes/ticket28100 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor features (HTTP standards compliance): - - Don't send Content-Type: application/octet-stream for transparently - compressed documents, which confused browsers. Closes ticket 28100. diff --git a/changes/ticket28142 b/changes/ticket28142 deleted file mode 100644 index b74b2bd47e..0000000000 --- a/changes/ticket28142 +++ /dev/null @@ -1,10 +0,0 @@ - o Major features (circuit padding): - - Implement preliminary support for the circuit padding portion of - Proposal 254. The implementation supports Adaptive Padding (aka - WTF-PAD) state machines for use between experimental clients and - relays. Support is also provided for APE-style state machines that - use probability distributions instead of histograms to specify - inter-packet delay. At the moment, Tor does not provide any padding - state machines that are used in normal operation -- this feature - exists solely for experimentation in this release. Closes - ticket 28142. diff --git a/changes/ticket28179 b/changes/ticket28179 deleted file mode 100644 index f548c4a79a..0000000000 --- a/changes/ticket28179 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor features (process): - - Add new Process API for handling child processes. This - new API allows Tor to have bi-directional - communication with child processes on both Unix and Windows. - Closes ticket 28179. diff --git a/changes/ticket28180 b/changes/ticket28180 deleted file mode 100644 index 59de1c6251..0000000000 --- a/changes/ticket28180 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor features (pluggable transports): - - Add support for logging to Tor's logging subsystem from a pluggable - transport process. Closes ticket 28180 diff --git a/changes/ticket28266 b/changes/ticket28266 deleted file mode 100644 index e0bc171080..0000000000 --- a/changes/ticket28266 +++ /dev/null @@ -1,10 +0,0 @@ - o Minor features (directory authority): - - Directory authorities support a new consensus algorithm, - under which microdescriptor entries are encoded in a canonical - form. This improves their compressibility in transit and on the client. - Closes ticket 28266; implements proposal 298. - - o Minor features (relay): - - When listing relay families, list them in canonical form including the - relay's own identity, and try to give a more useful set of warnings. - Part of ticket 28266 and proposal 298. diff --git a/changes/ticket28335 b/changes/ticket28335 deleted file mode 100644 index eecf7c7fd9..0000000000 --- a/changes/ticket28335 +++ /dev/null @@ -1,7 +0,0 @@ - o Major features (client): - - When Tor is running as a client, and it is unused for a long time, it - can now enter a "dormant" state. When Tor is dormant, it avoids - network activity and CPU wakeups until it is reawoken either by a user - request or by a controller command. For more information, see - the configuration options starting with "Dormant". Implements tickets - 2149 and 28335. diff --git a/changes/ticket28362 b/changes/ticket28362 deleted file mode 100644 index 4ac22d50f2..0000000000 --- a/changes/ticket28362 +++ /dev/null @@ -1,6 +0,0 @@ - o Code simplification and refactoring: - - The .may_include files that we use to describe our - directory-by-directory dependency structure now describe a noncircular - dependency graph over the directories that they cover. - Our checkIncludes.py tool now enforces this. - Closes ticket 28362. diff --git a/changes/ticket28551 b/changes/ticket28551 deleted file mode 100644 index 46ba9d713b..0000000000 --- a/changes/ticket28551 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor features (Continuous Integration): - - Log Python version during each Travis CI job. Resolves issue - 28551. diff --git a/changes/ticket28624 b/changes/ticket28624 deleted file mode 100644 index 353f962be9..0000000000 --- a/changes/ticket28624 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor features (battery management, client, dormant mode): - - The client's memory of whether it is "dormant", and how long it has - spend idle, persists across invocations. Implements ticket 28624. - - There is a DormantOnFirstStartup option that integrators can use if - they expect that in many cases, Tor will be installed but not used. diff --git a/changes/ticket28669 b/changes/ticket28669 deleted file mode 100644 index 32c6114ffc..0000000000 --- a/changes/ticket28669 +++ /dev/null @@ -1,6 +0,0 @@ - o Minor bugfix (hidden service v3, client): - - Avoid a BUG() stacktrace in case a SOCKS connection is found waiting for - the descriptor while we do have it in the cache. There is a rare case - when this can happen. Now, tor will recover and retry the descriptor. - Fixes bug 28669; bugfix on 0.3.2.4-alpha. - diff --git a/changes/ticket28757 b/changes/ticket28757 deleted file mode 100644 index 62c6d099ff..0000000000 --- a/changes/ticket28757 +++ /dev/null @@ -1,5 +0,0 @@ - o Removed features: - - Stop responding to 'GETINFO status/version/num-concurring' and - 'GETINFO status/version/num-versioning' control port commands, as those - were deprecated back in 0.2.0.30. Also stop listing them in output of - 'GETINFO info/names'. Resolves ticket 28757. diff --git a/changes/ticket28768 b/changes/ticket28768 deleted file mode 100644 index 27d90febc8..0000000000 --- a/changes/ticket28768 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor features (fallback directory mirrors): - - Accept fallbacks that deliver reasonably live consensuses. - (Consensuses that will become valid less than 24 hours in the future, - or that expired less than 24 hours ago.) Closes ticket 28768. diff --git a/changes/ticket28839 b/changes/ticket28839 deleted file mode 100644 index e9f81dc405..0000000000 --- a/changes/ticket28839 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor features (performance): - - Speed up microdesriptor parsing by about 30%, to help - improve startup time. Closes ticket 28839. diff --git a/changes/ticket28840 b/changes/ticket28840 deleted file mode 100644 index 05d3fbb94c..0000000000 --- a/changes/ticket28840 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor features (testing): - - Allow HeartbeatPeriod of less than 30 minutes in testing Tor networks. - Closes ticket 28840, patch by robgjansen diff --git a/changes/ticket28843 b/changes/ticket28843 deleted file mode 100644 index 00905293c4..0000000000 --- a/changes/ticket28843 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor features (controller): - - Add a DROPOWNERSHIP command to undo the effects of TAKEOWNERSHIP. - Implements ticket 28843. diff --git a/changes/ticket28846 b/changes/ticket28846 deleted file mode 100644 index efb5b9938e..0000000000 --- a/changes/ticket28846 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor features (pluggable transports): - - Add support for emitting STATUS updates to Tor's control port from a - pluggable transport process. Closes ticket 28846. diff --git a/changes/ticket28847 b/changes/ticket28847 deleted file mode 100644 index 63100c5813..0000000000 --- a/changes/ticket28847 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor features (process): - - Use the subsystem module to initialize and shut down the process module. - Closes ticket 28847. diff --git a/changes/ticket28852 b/changes/ticket28852 deleted file mode 100644 index a58cc3ba0e..0000000000 --- a/changes/ticket28852 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor features (performance): - - Remove a needless memset() call from get_token_arguments, - thereby speeding up the tokenization of directory objects by about - 20%. Closes ticket 28852. diff --git a/changes/ticket28853 b/changes/ticket28853 deleted file mode 100644 index e76f6bd8c9..0000000000 --- a/changes/ticket28853 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor features (performance): - - Replace parse_short_policy() with a faster implementation, to improve - microdescriptor parsing time. Closes ticket 28853. diff --git a/changes/ticket28856 b/changes/ticket28856 deleted file mode 100644 index b136470494..0000000000 --- a/changes/ticket28856 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor features (performance): - - Speed up directory parsing a little by avoiding use of the - non-inlined strcmp_len() function. Closes ticket 28856. From dd524c6335b6d4a554c4bd23e8830c66135a636e Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 16 Jan 2019 08:32:09 -0500 Subject: [PATCH 0411/2557] Bump to 0.4.0.1-alpha --- configure.ac | 4 ++-- contrib/win32build/tor-mingw.nsi.in | 2 +- src/win32/orconfig.h | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/configure.ac b/configure.ac index 76fc12be50..ca1af4ec6e 100644 --- a/configure.ac +++ b/configure.ac @@ -4,7 +4,7 @@ dnl Copyright (c) 2007-2018, The Tor Project, Inc. dnl See LICENSE for licensing information AC_PREREQ([2.63]) -AC_INIT([tor],[0.4.0.0-alpha-dev]) +AC_INIT([tor],[0.4.0.1-alpha]) AC_CONFIG_SRCDIR([src/app/main/tor_main.c]) AC_CONFIG_MACRO_DIR([m4]) @@ -14,7 +14,7 @@ AC_CONFIG_MACRO_DIR([m4]) # version number changes. Tor uses it to make sure that it # only shuts down for missing "required protocols" when those protocols # are listed as required by a consensus after this date. -AC_DEFINE(APPROX_RELEASE_DATE, ["2019-01-15"], # for 0.4.0.0-alpha-dev +AC_DEFINE(APPROX_RELEASE_DATE, ["2019-01-18"], # for 0.4.0.1-alpha [Approximate date when this software was released. (Updated when the version changes.)]) # "foreign" means we don't follow GNU package layout standards diff --git a/contrib/win32build/tor-mingw.nsi.in b/contrib/win32build/tor-mingw.nsi.in index af01a2b499..bc8ab7de94 100644 --- a/contrib/win32build/tor-mingw.nsi.in +++ b/contrib/win32build/tor-mingw.nsi.in @@ -8,7 +8,7 @@ !include "LogicLib.nsh" !include "FileFunc.nsh" !insertmacro GetParameters -!define VERSION "0.4.0.0-alpha-dev" +!define VERSION "0.4.0.1-alpha" !define INSTALLER "tor-${VERSION}-win32.exe" !define WEBSITE "https://www.torproject.org/" !define LICENSE "LICENSE" diff --git a/src/win32/orconfig.h b/src/win32/orconfig.h index cfc3bc9e9e..51b78b0b33 100644 --- a/src/win32/orconfig.h +++ b/src/win32/orconfig.h @@ -218,7 +218,7 @@ #define USING_TWOS_COMPLEMENT /* Version number of package */ -#define VERSION "0.4.0.0-alpha-dev" +#define VERSION "0.4.0.1-alpha" From 554f076f916c6da331e30b84b23f680f89940918 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 16 Jan 2019 11:09:54 -0500 Subject: [PATCH 0412/2557] Give the changelog an edit --- ChangeLog | 240 +++++++++++++++++++++++++++--------------------------- 1 file changed, 119 insertions(+), 121 deletions(-) diff --git a/ChangeLog b/ChangeLog index d6a72e2efd..a98e9999a8 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,13 +1,18 @@ Changes in version 0.4.0.1-alpha - 2019-01-18 - blurb blurb blurb + Tor 0.4.0.1-alpha is the first release in the new 0.4.0.x series. It + introduces improved features for power and bandwidth conservation, + more accurate reporting of bootstrap progress for user interfaces, and + an experimental backend for an exciting new adaptive padding feature. + There is also the usual assortment of bugfixes and minor features, all + described below. o Major features (battery management, client, dormant mode): - When Tor is running as a client, and it is unused for a long time, it can now enter a "dormant" state. When Tor is dormant, it avoids - network activity and CPU wakeups until it is reawoken either by a - user request or by a controller command. For more information, see - the configuration options starting with "Dormant". Implements - tickets 2149 and 28335. + network and CPU activity until it is reawoken either by a user + request or by a controller command. For more information, see the + configuration options starting with "Dormant". Implements tickets + 2149 and 28335. - The client's memory of whether it is "dormant", and how long it has spend idle, persists across invocations. Implements ticket 28624. @@ -15,17 +20,16 @@ Changes in version 0.4.0.1-alpha - 2019-01-18 if they expect that in many cases, Tor will be installed but not used. - o Major features (bootstrap): - - Report the first connection to a relay as the earliest phases of - bootstrap progress, regardless of whether it's a connection for - building application circuits. This allows finer-grained reporting - of early progress than previously possible with the improvements - of ticket 27169. Closes tickets 27167 and 27103. Addresses - ticket 27308. - - Separately report the intermediate stage of having connected to a - proxy or pluggable transport, versus succesfully using that proxy - or pluggable transport to connect to a relay. Closes tickets 27100 - and 28884. + o Major features (bootstrap reporting): + - When reporting bootstrap progress, report the first connection + uniformly, regardless of whether it's a connection for building + application circuits. This allows finer-grained reporting of early + progress than previously possible, with the improvements of ticket + 27169. Closes tickets 27167 and 27103. Addresses ticket 27308. + - When reporting bootstrap progress, treat connecting to a proxy or + pluggable transport as separate from having successfully using + that proxy or pluggable transport to connect to a relay. Closes + tickets 27100 and 28884. o Major features (circuit padding): - Implement preliminary support for the circuit padding portion of @@ -34,8 +38,8 @@ Changes in version 0.4.0.1-alpha - 2019-01-18 relays. Support is also provided for APE-style state machines that use probability distributions instead of histograms to specify inter-packet delay. At the moment, Tor does not provide any - padding state machines that are used in normal operation -- this - feature exists solely for experimentation in this release. Closes + padding state machines that are used in normal operation: for now, + this feature exists solely for experimentation. Closes ticket 28142. o Major features (refactoring): @@ -45,14 +49,14 @@ Changes in version 0.4.0.1-alpha - 2019-01-18 (There still some subsystems using the old system.) Closes ticket 28330. - o Minor feature (bootstrap): + o Minor features (bootstrap reporting): - When reporting bootstrap progress, stop distinguishing between situations where it seems that only internal paths are available and situations where it seems that external paths are available. - Previously, tor would often erroneously report that it had only + Previously, Tor would often erroneously report that it had only internal paths. Closes ticket 27402. - o Minor features (Continuous Integration): + o Minor features (continuous integration): - Log Python version during each Travis CI job. Resolves issue 28551. @@ -62,77 +66,74 @@ Changes in version 0.4.0.1-alpha - 2019-01-18 o Minor features (developer tooling): - Provide a git hook script to prevent "fixup!" and "squash!" - commits from ending up in master. Closes ticket 27993. + commits from ending up in the master branch, as scripts/main/pre- + push.git-hook. Closes ticket 27993. o Minor features (directory authority): - Directory authorities support a new consensus algorithm, under - which microdescriptor entries are encoded in a canonical form. - This improves their compressibility in transit and on the client. - Closes ticket 28266; implements proposal 298. + which the family lines in microdescriptors are encoded in a + canonical form. This change makes family lines more compressible + in transit, and on the client. Closes ticket 28266; implements + proposal 298. o Minor features (directory authority, relay): - Authorities now vote on a "StaleDesc" flag to indicate that a relay's descriptor is so old that the relay should upload again - soon. Relays understand this flag, and treat it as a signal to - upload a new descriptor. This flag will eventually let us remove - the 'published' date from routerstatus entries, and save a great - deal of space in our consensus diffs. Closes ticket 26770; - implements proposal 293. + soon. Relays treat this flag as a signal to upload a new + descriptor. This flag will eventually let us remove the + 'published' date from routerstatus entries, and make our our + consensus diffs much smaller. Closes ticket 26770; implements + proposal 293. o Minor features (fallback directory mirrors): - Update the fallback whitelist based on operator opt-ins and opt- outs. Closes ticket 24805, patch by Phoul. - - Accept fallbacks that deliver reasonably live consensuses. - (Consensuses that will become valid less than 24 hours in the - future, or that expired less than 24 hours ago.) Closes - ticket 28768. - - Accept relays that are a fuzzy match to a fallback whitelist - entry. If a relay matches at least one fingerprint, IPv4 address, - or IPv6 address in the fallback whitelist, it can become a - fallback. This reduces the work required to keep the list up to - date. Closes ticket 24838. o Minor features (FreeBSD): - - Warn relay operators if the "net.inet.ip.random_id" sysctl (IP ID - randomization) is disabled on their relay if it is running on - FreeBSD based operating systems. Closes ticket 28518. + - On FreeBSD-based systems, warn relay operators if the + "net.inet.ip.random_id" sysctl (IP ID randomization) is disabled. + Closes ticket 28518. o Minor features (HTTP standards compliance): - - Don't send Content-Type: application/octet-stream for transparently - compressed documents, which confused browsers. Closes ticket 28100. + - Stop sending the header "Content-type: application/octet-stream" + along with transparently compressed documents: this confused + browsers. Closes ticket 28100. - o Minor features (ipv6): - - We add an option ClientAutoIPv6ORPort which makes clients randomly + o Minor features (IPv6): + - We add an option ClientAutoIPv6ORPort, to make clients randomly prefer a node's IPv4 or IPv6 ORPort. The random preference is set every time a node is loaded from a new consensus or bridge config. - Closes ticket 27490. Patch by Neel Chauhan. - - When using addrs_in_same_network_family(), check IPv6 subnets as - well as IPv4 ones where possible when a client chooses circuit - paths. Previously, we used this function only for IPv4 subnets. - Closes ticket 24393. Patch by Neel Chauhan. + We expect that this option will enable clients to bootstrap more + quickly without having to determine whether they support IPv4, + IPv6, or both. Closes ticket 27490. Patch by Neel Chauhan. + - When using addrs_in_same_network_family(), avoid choosing circuit + paths that pass through the same IPv6 subnet more than once. + Previously, we only checked IPv4 subnets. Closes ticket 24393. + Patch by Neel Chauhan. o Minor features (log messages): - - Improve log message in HSv3 service that could print out negative - revision counters. Closes ticket 27707. Patch by "ffmancera". + - Improve log message in v3 onion services service that could print + out negative revision counters. Closes ticket 27707. Patch + by "ffmancera". o Minor features (memory usage): - - Store microdescriptor family lists with a more compact - representation to save memory. Closes ticket 27359. - - Tor clients no longer need to keep the full text of a consensus in - memory in order to parse it, or apply a diff to it. Instead, they - use mmap() to read the consensus files from disk. Closes - ticket 27244. + - Save memory by storing microdescriptor family lists with a more + compact representation. Closes ticket 27359. + - Tor clients now use mmap() to read consensus files from disk, sot + that they no longer need keep the full text of a consensus in + memory when parsing it or applying a diff. Closes ticket 27244. o Minor features (parsing): - Directory authorities now validate that router descriptors and ExtraInfo documents are in a valid subset of UTF-8, and reject - them if not. Closes ticket 27367. + them if they are not. Closes ticket 27367. o Minor features (performance): - - Avoid parsing the same protocol-versions string over and over in - summarize_protover_flags(). This should save us a huge number of - malloc calls on startup, and may reduce memory fragmentation with - some allocators. Closes ticket 27225. + - Cache the results of summarize_protocol_flags(), so that we don't + have to parse the same Avoid parsing the same protocol-versions + string over and over. This should save us a huge number of malloc + calls on startup, and may reduce memory fragmentation with some + allocators. Closes ticket 27225. - Remove a needless memset() call from get_token_arguments, thereby speeding up the tokenization of directory objects by about 20%. Closes ticket 28852. @@ -140,7 +141,7 @@ Changes in version 0.4.0.1-alpha - 2019-01-18 improve microdescriptor parsing time. Closes ticket 28853. - Speed up directory parsing a little by avoiding use of the non- inlined strcmp_len() function. Closes ticket 28856. - - Speed up microdesriptor parsing by about 30%, to help improve + - Speed up microdescriptor parsing by about 30%, to help improve startup time. Closes ticket 28839. o Minor features (pluggable transports): @@ -150,10 +151,10 @@ Changes in version 0.4.0.1-alpha - 2019-01-18 pluggable transport process. Closes ticket 28180 o Minor features (process management): - - Add new Process API for handling child processes. This new API + - Add a new process API for handling child processes. This new API allows Tor to have bi-directional communication with child processes on both Unix and Windows. Closes ticket 28179. - - Use the subsystem module to initialize and shut down the process + - Use the subsystem manager to initialize and shut down the process module. Closes ticket 28847. o Minor features (relay): @@ -162,56 +163,52 @@ Changes in version 0.4.0.1-alpha - 2019-01-18 warnings. Part of ticket 28266 and proposal 298. o Minor features (required protocols): - - Tor no longer exits if it is missing a required protocol, if the - consensus that requires the protocol predates the release date of - the version of Tor. This change prevents Tor releases from exiting - because of an old cached consensus, on the theory that a newer - cached consensus might not require the protocol. Implements - proposal 297; closes ticket 27735. + - Before exiting because of a missing required protocol, Tor will + now check the publication time of the consensus, and not exit + unless the consensus is newer than the Tor program's own release + date. Previously, Tor would not check the consensus publication + time, and so might exit because of a missing protocol that might + no longer be required in a current consensus. Implements proposal + 297; closes ticket 27735. o Minor features (testing): - - Allow HeartbeatPeriod of less than 30 minutes in testing Tor - networks. Closes ticket 28840, patch by robgjansen + - Allow a HeartbeatPeriod of less than 30 minutes in testing Tor + networks. Closes ticket 28840. Patch by Rob Jansen. - o Minor bugfixes (client, bootstrap): - - When Tor's clock is behind the clocks on the authorities, allow - Tor to bootstrap successfully. Fixes bug 28591; bugfix - on 0.2.0.9-alpha. - - o Minor bugfixes (client, guard selection): - - When Tor's consensus has expired, but is still reasonably live, - use it to select guards. Fixes bug 24661; bugfix on 0.3.0.1-alpha. + o Minor bugfixes (client, clock skew): + - Bootstrap successfully even when Tor's clock is behind the clocks + on the authorities. Fixes bug 28591; bugfix on 0.2.0.9-alpha. + - Select guards even if the consensus has expired, as long as the + consensus is still reasonably live. Fixes bug 24661; bugfix + on 0.3.0.1-alpha. o Minor bugfixes (compilation): - - Fix missing headers required for proper detection of OpenBSD. Fixes - bug 28938; bugfix on 0.3.5.1-alpha. Patch from Kris Katterjohn. + - Compile correctly on OpenBSD; previously, we were missing some + headers required in order to detect it properly. Fixes bug 28938; + bugfix on 0.3.5.1-alpha. Patch from Kris Katterjohn. o Minor bugfixes (directory clients): - Mark outdated dirservers when Tor only has a reasonably live consensus. Fixes bug 28569; bugfix on 0.3.2.5-alpha. - o Minor bugfixes (directory mirror): - - When Tor's clock is behind the clocks on the authorities, allow - Tor to serve future consensuses. Fixes bug 28654; bugfix - on 0.3.0.1-alpha. + o Minor bugfixes (directory mirrors): + - Even when a directory mirror's clock is behind the clocks on the + authorities, we now allow the mirror to serve "future" + consensuses. Fixes bug 28654; bugfix on 0.3.0.1-alpha. o Minor bugfixes (DNS): - - Gracefully handle empty or absent resolve.conf file by falling - back to using localhost DNS service and hoping it works. Fixes bug + - Gracefully handle an empty or absent resolve.conf file by falling + back to using "localhost" as a DNS server (and hoping it works). + Previously, we would just stop running as an exit. Fixes bug 21900; bugfix on 0.2.1.10-alpha. - o Minor bugfixes (fallback scripts): - - In updateFallbackDirs.py, call the filter file a "fallback list" - instead of a "whitelist" in check_existing mode. Fixes bug 24953; - bugfix on 0.3.0.3-alpha. - o Minor bugfixes (guards): - - In count_acceptable_nodes(), check if we have at least one bridge - or guard node, and two non-guard nodes for a circuit. Previously, - we have added up the sum of all nodes with a descriptor, but that - could cause us to build circuits that fail if we had either too - many bridges, or not enough guard nodes. Fixes bug 25885; bugfix - on 0.3.6.1-alpha. Patch by Neel Chauhan. + - In count_acceptable_nodes() we now treat the minimum number of + nodes as at one bridge or guard node, and two non-guard nodes for + a circuit. Previously, we had added up the sum of all nodes with a + descriptor, but that could cause us to build failing circuits when + we had either too many bridges, or not enough guard nodes. Fixes + bug 25885; bugfix on 0.3.6.1-alpha. Patch by Neel Chauhan. o Minor bugfixes (IPv6): - Fix tor_ersatz_socketpair on IPv6-only systems. Previously, the @@ -222,7 +219,7 @@ Changes in version 0.4.0.1-alpha - 2019-01-18 o Minor bugfixes (logging): - Rework rep_hist_log_link_protocol_counts() to iterate through all link protocol versions when logging incoming/outgoing connection - counts. Tor no longer skips version 5 and we don't have to + counts. Tor no longer skips version 5, and we won't have to remember to update this function when new link protocol version is developed. Fixes bug 28920; bugfix on 0.2.6.10. @@ -244,27 +241,27 @@ Changes in version 0.4.0.1-alpha - 2019-01-18 onion router. Fixes bug 27929; bugfix on 0.2.8.1-alpha. o Minor bugfixes (pluggable transports): - - Make sure that data is continously read from standard out and - error of the PT child-process to avoid deadlocking when the pipes' - buffer is full. Fixes bug 26360; bugfix on 0.2.3.6-alpha. + - Make sure that data is continously read from standard output and + standard error pipes of a pluggable transport child-process, to + avoid deadlocking when the a pipe's buffer is full. Fixes bug + 26360; bugfix on 0.2.3.6-alpha. o Minor bugfixes (unit tests): - Instead of relying on hs_free_all() to clean up all onion service - objects we created in test_build_descriptors(), deallocate them - one by one. This lets Coverity know that we are not leaking memory - here and fixes CID 1442277. Fixes bug 28989; bugfix - on 0.3.5.1-alpha. + objects in test_build_descriptors(), we now deallocate them one by + one. This lets Coverity know that we are not leaking memory there + and fixes CID 1442277. Fixes bug 28989; bugfix on 0.3.5.1-alpha. o Minor bugfixes (usability): - - Stop saying "Your Guard ..." in pathbias_measure_{use,close}_rate() - as that confusingly suggests that mentioned guard node is under - control and responsibility of end user, which it is not. Fixes bug + - Stop saying "Your Guard ..." in pathbias_measure_{use,close}_rate(). + Some users took this phrasing to mean that the mentioned guard was + under their control or responsibility, which it is not. Fixes bug 28895; bugfix on Tor 0.3.0.1-alpha. o Code simplification and refactoring: - Reimplement NETINFO cell parsing and generation to rely on trunnel-generated wire format handling code. Closes ticket 27325. - - Remove unnecessarily unsafe code from the rust macro cstr!. Closes + - Remove unnecessarily unsafe code from the Rust macro cstr!. Closes ticket 28077. - Rework SOCKS wire format handling to rely on trunnel-generated parsing/generation code. Resolves ticket 27620. @@ -273,18 +270,19 @@ Changes in version 0.4.0.1-alpha - 2019-01-18 - The .may_include files that we use to describe our directory-by- directory dependency structure now describe a noncircular dependency graph over the directories that they cover. Our - checkIncludes.py tool now enforces this. Closes ticket 28362. + checkIncludes.py tool now enforces this noncircularity. Closes + ticket 28362. o Documentation: - - Mention that you cannot add new Onion Service if Tor is already + - Mention that you cannot add a new onion service if Tor is already running with Sandbox enabled. Closes ticket 28560. - - Improve ControlPort description in tor manpage to mention that it - accepts address/port pair, and can be used multiple times. Closes + - Improve ControlPort deocumentation mention that it accepts + address:port pairs, and can be used multiple times. Closes ticket 28805. - Document the exact output of "tor --version". Closes ticket 28889. o Removed features: - - Stop responding to 'GETINFO status/version/num-concurring' and + - Stop responding to the 'GETINFO status/version/num-concurring' and 'GETINFO status/version/num-versioning' control port commands, as those were deprecated back in 0.2.0.30. Also stop listing them in output of 'GETINFO info/names'. Resolves ticket 28757. @@ -293,9 +291,9 @@ Changes in version 0.4.0.1-alpha - 2019-01-18 repository. Closes ticket 27914. o Testing: - - Run shellcheck for stuff in scripts/ directory. Closes + - Run shellcheck for scripts in the in scripts/ directory. Closes ticket 28058. - - Write some unit tests for tokenize_string() and get_next_token() + - Add unit tests for tokenize_string() and get_next_token() functions. Resolves ticket 27625. o Code simplification and refactoring (onion service v3): From e38e26d3103c4acd73b2aa531fca0fd2db58ce7a Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 16 Jan 2019 11:47:18 -0500 Subject: [PATCH 0413/2557] More line edits to changelog. Thanks, seborn! --- ChangeLog | 75 +++++++++++++++++++++++++++---------------------------- 1 file changed, 37 insertions(+), 38 deletions(-) diff --git a/ChangeLog b/ChangeLog index a98e9999a8..0be9ac5700 100644 --- a/ChangeLog +++ b/ChangeLog @@ -14,7 +14,7 @@ Changes in version 0.4.0.1-alpha - 2019-01-18 configuration options starting with "Dormant". Implements tickets 2149 and 28335. - The client's memory of whether it is "dormant", and how long it - has spend idle, persists across invocations. Implements + has spent idle, persists across invocations. Implements ticket 28624. - There is a DormantOnFirstStartup option that integrators can use if they expect that in many cases, Tor will be installed but @@ -27,9 +27,9 @@ Changes in version 0.4.0.1-alpha - 2019-01-18 progress than previously possible, with the improvements of ticket 27169. Closes tickets 27167 and 27103. Addresses ticket 27308. - When reporting bootstrap progress, treat connecting to a proxy or - pluggable transport as separate from having successfully using - that proxy or pluggable transport to connect to a relay. Closes - tickets 27100 and 28884. + pluggable transport as separate from having successfully used that + proxy or pluggable transport to connect to a relay. Closes tickets + 27100 and 28884. o Major features (circuit padding): - Implement preliminary support for the circuit padding portion of @@ -45,16 +45,16 @@ Changes in version 0.4.0.1-alpha - 2019-01-18 o Major features (refactoring): - Tor now uses an explicit list of its own subsystems when initializing and shutting down. Previously, these systems were - managed implicitly though various places throughout the codebase. - (There still some subsystems using the old system.) Closes + managed implicitly in various places throughout the codebase. + (There may still be some subsystems using the old system.) Closes ticket 28330. o Minor features (bootstrap reporting): - When reporting bootstrap progress, stop distinguishing between - situations where it seems that only internal paths are available - and situations where it seems that external paths are available. - Previously, Tor would often erroneously report that it had only - internal paths. Closes ticket 27402. + situations where only internal paths are available and situations + where external paths are available. Previously, Tor would often + erroneously report that it had only internal paths. Closes + ticket 27402. o Minor features (continuous integration): - Log Python version during each Travis CI job. Resolves @@ -81,9 +81,8 @@ Changes in version 0.4.0.1-alpha - 2019-01-18 relay's descriptor is so old that the relay should upload again soon. Relays treat this flag as a signal to upload a new descriptor. This flag will eventually let us remove the - 'published' date from routerstatus entries, and make our our - consensus diffs much smaller. Closes ticket 26770; implements - proposal 293. + 'published' date from routerstatus entries, and make our consensus + diffs much smaller. Closes ticket 26770; implements proposal 293. o Minor features (fallback directory mirrors): - Update the fallback whitelist based on operator opt-ins and opt- @@ -112,14 +111,14 @@ Changes in version 0.4.0.1-alpha - 2019-01-18 Patch by Neel Chauhan. o Minor features (log messages): - - Improve log message in v3 onion services service that could print - out negative revision counters. Closes ticket 27707. Patch + - Improve log message in v3 onion services that could print out + negative revision counters. Closes ticket 27707. Patch by "ffmancera". o Minor features (memory usage): - Save memory by storing microdescriptor family lists with a more compact representation. Closes ticket 27359. - - Tor clients now use mmap() to read consensus files from disk, sot + - Tor clients now use mmap() to read consensus files from disk, so that they no longer need keep the full text of a consensus in memory when parsing it or applying a diff. Closes ticket 27244. @@ -130,10 +129,10 @@ Changes in version 0.4.0.1-alpha - 2019-01-18 o Minor features (performance): - Cache the results of summarize_protocol_flags(), so that we don't - have to parse the same Avoid parsing the same protocol-versions - string over and over. This should save us a huge number of malloc - calls on startup, and may reduce memory fragmentation with some - allocators. Closes ticket 27225. + have to parse the same protocol-versions string over and over. + This should save us a huge number of malloc calls on startup, and + may reduce memory fragmentation with some allocators. Closes + ticket 27225. - Remove a needless memset() call from get_token_arguments, thereby speeding up the tokenization of directory objects by about 20%. Closes ticket 28852. @@ -148,7 +147,7 @@ Changes in version 0.4.0.1-alpha - 2019-01-18 - Add support for emitting STATUS updates to Tor's control port from a pluggable transport process. Closes ticket 28846. - Add support for logging to Tor's logging subsystem from a - pluggable transport process. Closes ticket 28180 + pluggable transport process. Closes ticket 28180. o Minor features (process management): - Add a new process API for handling child processes. This new API @@ -203,12 +202,12 @@ Changes in version 0.4.0.1-alpha - 2019-01-18 21900; bugfix on 0.2.1.10-alpha. o Minor bugfixes (guards): - - In count_acceptable_nodes() we now treat the minimum number of - nodes as at one bridge or guard node, and two non-guard nodes for - a circuit. Previously, we had added up the sum of all nodes with a - descriptor, but that could cause us to build failing circuits when - we had either too many bridges, or not enough guard nodes. Fixes - bug 25885; bugfix on 0.3.6.1-alpha. Patch by Neel Chauhan. + - In count_acceptable_nodes(), the minimum number is now one bridge + or guard node, and two non-guard nodes for a circuit. Previously, + we had added up the sum of all nodes with a descriptor, but that + could cause us to build failing circuits when we had either too + many bridges or not enough guard nodes. Fixes bug 25885; bugfix on + 0.3.6.1-alpha. Patch by Neel Chauhan. o Minor bugfixes (IPv6): - Fix tor_ersatz_socketpair on IPv6-only systems. Previously, the @@ -229,22 +228,22 @@ Changes in version 0.4.0.1-alpha - 2019-01-18 bug 23082; bugfix on 0.2.0.10-alpha. o Minor bugfixes (onion service v3, client): - - Avoid a BUG() stacktrace in case a SOCKS connection is found - waiting for the descriptor while we do have it in the cache. There - is a rare case when this can happen. Now, tor will recover and - retry the descriptor. Fixes bug 28669; bugfix on 0.3.2.4-alpha. + - Stop logging a "BUG()" warning and stacktrace when we find a SOCKS + connection waiting for a descriptor that we actually have in the + cache. It turns out that this can actually happen, though it is + rare. Now, tor will recover and retry the descriptor. Fixes bug + 28669; bugfix on 0.3.2.4-alpha. o Minor bugfixes (periodic events): - Refrain from calling routerlist_remove_old_routers() from - check_descriptor_callback(). Instead, create a new periodic event - that will run once every hour even if Tor is not configured as - onion router. Fixes bug 27929; bugfix on 0.2.8.1-alpha. + check_descriptor_callback(). Instead, create a new hourly periodic + event. Fixes bug 27929; bugfix on 0.2.8.1-alpha. o Minor bugfixes (pluggable transports): - Make sure that data is continously read from standard output and standard error pipes of a pluggable transport child-process, to - avoid deadlocking when the a pipe's buffer is full. Fixes bug - 26360; bugfix on 0.2.3.6-alpha. + avoid deadlocking when a pipe's buffer is full. Fixes bug 26360; + bugfix on 0.2.3.6-alpha. o Minor bugfixes (unit tests): - Instead of relying on hs_free_all() to clean up all onion service @@ -261,7 +260,7 @@ Changes in version 0.4.0.1-alpha - 2019-01-18 o Code simplification and refactoring: - Reimplement NETINFO cell parsing and generation to rely on trunnel-generated wire format handling code. Closes ticket 27325. - - Remove unnecessarily unsafe code from the Rust macro cstr!. Closes + - Remove unnecessary unsafe code from the Rust macro "cstr!". Closes ticket 28077. - Rework SOCKS wire format handling to rely on trunnel-generated parsing/generation code. Resolves ticket 27620. @@ -276,7 +275,7 @@ Changes in version 0.4.0.1-alpha - 2019-01-18 o Documentation: - Mention that you cannot add a new onion service if Tor is already running with Sandbox enabled. Closes ticket 28560. - - Improve ControlPort deocumentation mention that it accepts + - Improve ControlPort documentation. Mention that it accepts address:port pairs, and can be used multiple times. Closes ticket 28805. - Document the exact output of "tor --version". Closes ticket 28889. From 9473a386c46a1f57061d1128e7e73835e7eadf21 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 16 Jan 2019 12:33:09 -0500 Subject: [PATCH 0414/2557] Update copyright script for 2019 --- scripts/maint/updateCopyright.pl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/maint/updateCopyright.pl b/scripts/maint/updateCopyright.pl index bd24377d38..36894b1baf 100755 --- a/scripts/maint/updateCopyright.pl +++ b/scripts/maint/updateCopyright.pl @@ -1,7 +1,7 @@ #!/usr/bin/perl -i -w -p -$NEWYEAR=2018; +$NEWYEAR=2019; -s/Copyright(.*) (201[^8]), The Tor Project/Copyright$1 $2-${NEWYEAR}, The Tor Project/; +s/Copyright(.*) (201[^9]), The Tor Project/Copyright$1 $2-${NEWYEAR}, The Tor Project/; s/Copyright(.*)-(20..), The Tor Project/Copyright$1-${NEWYEAR}, The Tor Project/; From 2f683465d4b666c5d8f84fb3b234ad539d8511cd Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 16 Jan 2019 12:33:22 -0500 Subject: [PATCH 0415/2557] Bump copyright date to 2019 --- LICENSE | 2 +- Makefile.am | 2 +- acinclude.m4 | 2 +- configure.ac | 2 +- scripts/codegen/gen_server_ciphers.py | 2 +- scripts/codegen/get_mozilla_ciphers.py | 2 +- scripts/codegen/makedesc.py | 2 +- scripts/maint/annotate_ifdef_directives | 2 +- scripts/maint/format_changelog.py | 2 +- scripts/maint/redox.py | 2 +- scripts/maint/sortChanges.py | 2 +- src/app/config/config.c | 4 ++-- src/app/config/config.h | 2 +- src/app/config/confparse.c | 2 +- src/app/config/confparse.h | 2 +- src/app/config/or_options_st.h | 2 +- src/app/config/or_state_st.h | 2 +- src/app/config/statefile.c | 2 +- src/app/config/statefile.h | 2 +- src/app/main/main.c | 2 +- src/app/main/main.h | 2 +- src/app/main/ntmain.c | 2 +- src/app/main/ntmain.h | 2 +- src/app/main/subsysmgr.c | 2 +- src/app/main/subsysmgr.h | 2 +- src/app/main/subsystem_list.c | 2 +- src/app/main/tor_main.c | 2 +- src/core/crypto/hs_ntor.c | 2 +- src/core/crypto/hs_ntor.h | 2 +- src/core/crypto/onion_crypto.c | 2 +- src/core/crypto/onion_crypto.h | 2 +- src/core/crypto/onion_fast.c | 2 +- src/core/crypto/onion_fast.h | 2 +- src/core/crypto/onion_ntor.c | 2 +- src/core/crypto/onion_ntor.h | 2 +- src/core/crypto/onion_tap.c | 2 +- src/core/crypto/onion_tap.h | 2 +- src/core/crypto/relay_crypto.c | 2 +- src/core/crypto/relay_crypto.h | 2 +- src/core/mainloop/connection.c | 2 +- src/core/mainloop/connection.h | 2 +- src/core/mainloop/cpuworker.c | 2 +- src/core/mainloop/cpuworker.h | 2 +- src/core/mainloop/mainloop.c | 2 +- src/core/mainloop/mainloop.h | 2 +- src/core/mainloop/netstatus.c | 2 +- src/core/mainloop/netstatus.h | 2 +- src/core/mainloop/periodic.c | 2 +- src/core/mainloop/periodic.h | 2 +- src/core/or/addr_policy_st.h | 2 +- src/core/or/address_set.c | 2 +- src/core/or/address_set.h | 2 +- src/core/or/cell_queue_st.h | 2 +- src/core/or/cell_st.h | 2 +- src/core/or/channel.c | 2 +- src/core/or/channel.h | 2 +- src/core/or/channelpadding.c | 2 +- src/core/or/channelpadding.h | 2 +- src/core/or/channeltls.c | 2 +- src/core/or/channeltls.h | 2 +- src/core/or/circuit_st.h | 2 +- src/core/or/circuitbuild.c | 2 +- src/core/or/circuitbuild.h | 2 +- src/core/or/circuitlist.c | 2 +- src/core/or/circuitlist.h | 2 +- src/core/or/circuitmux.c | 2 +- src/core/or/circuitmux.h | 2 +- src/core/or/circuitmux_ewma.c | 2 +- src/core/or/circuitmux_ewma.h | 2 +- src/core/or/circuitpadding.h | 2 +- src/core/or/circuitstats.c | 2 +- src/core/or/circuitstats.h | 2 +- src/core/or/circuituse.c | 2 +- src/core/or/circuituse.h | 2 +- src/core/or/command.c | 2 +- src/core/or/command.h | 2 +- src/core/or/connection_edge.c | 2 +- src/core/or/connection_edge.h | 2 +- src/core/or/connection_or.c | 2 +- src/core/or/connection_or.h | 2 +- src/core/or/connection_st.h | 2 +- src/core/or/cpath_build_state_st.h | 2 +- src/core/or/crypt_path_reference_st.h | 2 +- src/core/or/crypt_path_st.h | 2 +- src/core/or/destroy_cell_queue_st.h | 2 +- src/core/or/dos.c | 2 +- src/core/or/dos.h | 2 +- src/core/or/edge_connection_st.h | 2 +- src/core/or/entry_connection_st.h | 2 +- src/core/or/entry_port_cfg_st.h | 2 +- src/core/or/extend_info_st.h | 2 +- src/core/or/half_edge_st.h | 2 +- src/core/or/listener_connection_st.h | 2 +- src/core/or/ocirc_event.c | 2 +- src/core/or/ocirc_event.h | 2 +- src/core/or/ocirc_event_sys.h | 2 +- src/core/or/onion.c | 2 +- src/core/or/onion.h | 2 +- src/core/or/or.h | 2 +- src/core/or/or_circuit_st.h | 2 +- src/core/or/or_connection_st.h | 2 +- src/core/or/or_handshake_certs_st.h | 2 +- src/core/or/or_handshake_state_st.h | 2 +- src/core/or/orconn_event.c | 2 +- src/core/or/orconn_event.h | 2 +- src/core/or/orconn_event_sys.h | 2 +- src/core/or/origin_circuit_st.h | 2 +- src/core/or/policies.c | 2 +- src/core/or/policies.h | 2 +- src/core/or/port_cfg_st.h | 2 +- src/core/or/protover.c | 2 +- src/core/or/protover.h | 2 +- src/core/or/protover_rust.c | 2 +- src/core/or/reasons.c | 2 +- src/core/or/reasons.h | 2 +- src/core/or/relay.c | 2 +- src/core/or/relay.h | 2 +- src/core/or/relay_crypto_st.h | 2 +- src/core/or/scheduler.c | 2 +- src/core/or/scheduler.h | 2 +- src/core/or/scheduler_kist.c | 2 +- src/core/or/scheduler_vanilla.c | 2 +- src/core/or/server_port_cfg_st.h | 2 +- src/core/or/socks_request_st.h | 2 +- src/core/or/status.c | 2 +- src/core/or/status.h | 2 +- src/core/or/tor_version_st.h | 2 +- src/core/or/var_cell_st.h | 2 +- src/core/or/versions.c | 2 +- src/core/or/versions.h | 2 +- src/core/proto/proto_cell.c | 2 +- src/core/proto/proto_cell.h | 2 +- src/core/proto/proto_control0.c | 2 +- src/core/proto/proto_control0.h | 2 +- src/core/proto/proto_ext_or.c | 2 +- src/core/proto/proto_ext_or.h | 2 +- src/core/proto/proto_http.c | 2 +- src/core/proto/proto_http.h | 2 +- src/core/proto/proto_socks.c | 2 +- src/core/proto/proto_socks.h | 2 +- src/ext/ht.h | 2 +- src/ext/trunnel/trunnel-impl.h | 2 +- src/ext/trunnel/trunnel.c | 2 +- src/ext/trunnel/trunnel.h | 2 +- src/feature/api/tor_api.c | 2 +- src/feature/api/tor_api.h | 2 +- src/feature/api/tor_api_internal.h | 2 +- src/feature/client/addressmap.c | 2 +- src/feature/client/addressmap.h | 2 +- src/feature/client/bridges.c | 2 +- src/feature/client/bridges.h | 2 +- src/feature/client/circpathbias.c | 2 +- src/feature/client/circpathbias.h | 2 +- src/feature/client/dnsserv.c | 2 +- src/feature/client/dnsserv.h | 2 +- src/feature/client/entrynodes.c | 2 +- src/feature/client/entrynodes.h | 2 +- src/feature/client/transports.c | 2 +- src/feature/client/transports.h | 2 +- src/feature/control/btrack.c | 2 +- src/feature/control/btrack_circuit.c | 2 +- src/feature/control/btrack_circuit.h | 2 +- src/feature/control/btrack_orconn.c | 2 +- src/feature/control/btrack_orconn.h | 2 +- src/feature/control/btrack_orconn_cevent.c | 2 +- src/feature/control/btrack_orconn_cevent.h | 2 +- src/feature/control/btrack_orconn_maps.c | 2 +- src/feature/control/btrack_orconn_maps.h | 2 +- src/feature/control/btrack_sys.h | 2 +- src/feature/control/control.c | 2 +- src/feature/control/control.h | 2 +- src/feature/control/control_bootstrap.c | 2 +- src/feature/control/control_connection_st.h | 2 +- src/feature/control/fmt_serverstatus.c | 2 +- src/feature/control/fmt_serverstatus.h | 2 +- src/feature/control/getinfo_geoip.h | 2 +- src/feature/dirauth/authmode.c | 2 +- src/feature/dirauth/authmode.h | 2 +- src/feature/dirauth/bwauth.c | 2 +- src/feature/dirauth/bwauth.h | 2 +- src/feature/dirauth/dircollate.c | 2 +- src/feature/dirauth/dircollate.h | 2 +- src/feature/dirauth/dirvote.c | 2 +- src/feature/dirauth/dirvote.h | 2 +- src/feature/dirauth/dsigs_parse.c | 2 +- src/feature/dirauth/dsigs_parse.h | 2 +- src/feature/dirauth/guardfraction.c | 2 +- src/feature/dirauth/guardfraction.h | 2 +- src/feature/dirauth/keypin.c | 2 +- src/feature/dirauth/keypin.h | 2 +- src/feature/dirauth/ns_detached_signatures_st.h | 2 +- src/feature/dirauth/process_descs.c | 2 +- src/feature/dirauth/process_descs.h | 2 +- src/feature/dirauth/reachability.c | 2 +- src/feature/dirauth/reachability.h | 2 +- src/feature/dirauth/recommend_pkg.c | 2 +- src/feature/dirauth/recommend_pkg.h | 2 +- src/feature/dirauth/shared_random.c | 2 +- src/feature/dirauth/shared_random.h | 2 +- src/feature/dirauth/shared_random_state.c | 2 +- src/feature/dirauth/shared_random_state.h | 2 +- src/feature/dirauth/vote_microdesc_hash_st.h | 2 +- src/feature/dirauth/voteflags.c | 2 +- src/feature/dirauth/voteflags.h | 2 +- src/feature/dircache/cached_dir_st.h | 2 +- src/feature/dircache/conscache.c | 2 +- src/feature/dircache/conscache.h | 2 +- src/feature/dircache/consdiffmgr.c | 2 +- src/feature/dircache/consdiffmgr.h | 2 +- src/feature/dircache/dircache.c | 2 +- src/feature/dircache/dircache.h | 2 +- src/feature/dircache/dirserv.c | 2 +- src/feature/dircache/dirserv.h | 2 +- src/feature/dirclient/dir_server_st.h | 2 +- src/feature/dirclient/dirclient.c | 2 +- src/feature/dirclient/dirclient.h | 2 +- src/feature/dirclient/dlstatus.c | 2 +- src/feature/dirclient/dlstatus.h | 2 +- src/feature/dirclient/download_status_st.h | 2 +- src/feature/dircommon/consdiff.c | 2 +- src/feature/dircommon/consdiff.h | 2 +- src/feature/dircommon/dir_connection_st.h | 2 +- src/feature/dircommon/directory.c | 2 +- src/feature/dircommon/directory.h | 2 +- src/feature/dircommon/fp_pair.c | 2 +- src/feature/dircommon/fp_pair.h | 2 +- src/feature/dircommon/vote_timing_st.h | 2 +- src/feature/dircommon/voting_schedule.c | 2 +- src/feature/dircommon/voting_schedule.h | 2 +- src/feature/dirparse/authcert_parse.c | 2 +- src/feature/dirparse/authcert_parse.h | 2 +- src/feature/dirparse/microdesc_parse.c | 2 +- src/feature/dirparse/microdesc_parse.h | 2 +- src/feature/dirparse/ns_parse.c | 2 +- src/feature/dirparse/ns_parse.h | 2 +- src/feature/dirparse/parsecommon.c | 2 +- src/feature/dirparse/parsecommon.h | 2 +- src/feature/dirparse/policy_parse.c | 2 +- src/feature/dirparse/policy_parse.h | 2 +- src/feature/dirparse/routerparse.c | 2 +- src/feature/dirparse/routerparse.h | 2 +- src/feature/dirparse/sigcommon.c | 2 +- src/feature/dirparse/sigcommon.h | 2 +- src/feature/dirparse/signing.c | 2 +- src/feature/dirparse/signing.h | 2 +- src/feature/dirparse/unparseable.c | 2 +- src/feature/dirparse/unparseable.h | 2 +- src/feature/hibernate/hibernate.c | 2 +- src/feature/hibernate/hibernate.h | 2 +- src/feature/hs/hs_cache.c | 2 +- src/feature/hs/hs_cache.h | 2 +- src/feature/hs/hs_cell.c | 2 +- src/feature/hs/hs_cell.h | 2 +- src/feature/hs/hs_circuit.c | 2 +- src/feature/hs/hs_circuit.h | 2 +- src/feature/hs/hs_circuitmap.c | 2 +- src/feature/hs/hs_circuitmap.h | 2 +- src/feature/hs/hs_client.c | 2 +- src/feature/hs/hs_client.h | 2 +- src/feature/hs/hs_common.c | 2 +- src/feature/hs/hs_common.h | 2 +- src/feature/hs/hs_config.c | 2 +- src/feature/hs/hs_config.h | 2 +- src/feature/hs/hs_control.c | 2 +- src/feature/hs/hs_control.h | 2 +- src/feature/hs/hs_descriptor.c | 2 +- src/feature/hs/hs_descriptor.h | 2 +- src/feature/hs/hs_ident.c | 2 +- src/feature/hs/hs_ident.h | 2 +- src/feature/hs/hs_intropoint.c | 2 +- src/feature/hs/hs_intropoint.h | 2 +- src/feature/hs/hs_service.c | 2 +- src/feature/hs/hs_service.h | 2 +- src/feature/hs/hs_stats.c | 2 +- src/feature/hs/hs_stats.h | 2 +- src/feature/hs/hsdir_index_st.h | 2 +- src/feature/hs_common/replaycache.c | 2 +- src/feature/hs_common/replaycache.h | 2 +- src/feature/hs_common/shared_random_client.c | 2 +- src/feature/hs_common/shared_random_client.h | 2 +- src/feature/keymgt/loadkey.c | 2 +- src/feature/keymgt/loadkey.h | 2 +- src/feature/nodelist/authcert.c | 2 +- src/feature/nodelist/authcert.h | 2 +- src/feature/nodelist/authority_cert_st.h | 2 +- src/feature/nodelist/desc_store_st.h | 2 +- src/feature/nodelist/describe.c | 2 +- src/feature/nodelist/describe.h | 2 +- src/feature/nodelist/dirlist.c | 2 +- src/feature/nodelist/dirlist.h | 2 +- src/feature/nodelist/document_signature_st.h | 2 +- src/feature/nodelist/extrainfo_st.h | 2 +- src/feature/nodelist/fmt_routerstatus.c | 2 +- src/feature/nodelist/fmt_routerstatus.h | 2 +- src/feature/nodelist/microdesc.c | 2 +- src/feature/nodelist/microdesc.h | 2 +- src/feature/nodelist/microdesc_st.h | 2 +- src/feature/nodelist/networkstatus.c | 2 +- src/feature/nodelist/networkstatus.h | 2 +- src/feature/nodelist/networkstatus_sr_info_st.h | 2 +- src/feature/nodelist/networkstatus_st.h | 2 +- src/feature/nodelist/networkstatus_voter_info_st.h | 2 +- src/feature/nodelist/nickname.c | 2 +- src/feature/nodelist/nickname.h | 2 +- src/feature/nodelist/node_select.c | 2 +- src/feature/nodelist/node_select.h | 2 +- src/feature/nodelist/node_st.h | 2 +- src/feature/nodelist/nodefamily.c | 2 +- src/feature/nodelist/nodefamily.h | 2 +- src/feature/nodelist/nodefamily_st.h | 2 +- src/feature/nodelist/nodelist.c | 2 +- src/feature/nodelist/nodelist.h | 2 +- src/feature/nodelist/routerinfo.c | 2 +- src/feature/nodelist/routerinfo.h | 2 +- src/feature/nodelist/routerinfo_st.h | 2 +- src/feature/nodelist/routerlist.c | 2 +- src/feature/nodelist/routerlist.h | 2 +- src/feature/nodelist/routerlist_st.h | 2 +- src/feature/nodelist/routerset.c | 2 +- src/feature/nodelist/routerset.h | 2 +- src/feature/nodelist/routerstatus_st.h | 2 +- src/feature/nodelist/signed_descriptor_st.h | 2 +- src/feature/nodelist/torcert.c | 2 +- src/feature/nodelist/torcert.h | 2 +- src/feature/nodelist/vote_routerstatus_st.h | 2 +- src/feature/relay/dns.c | 2 +- src/feature/relay/dns.h | 2 +- src/feature/relay/dns_structs.h | 2 +- src/feature/relay/ext_orport.c | 2 +- src/feature/relay/ext_orport.h | 2 +- src/feature/relay/onion_queue.c | 2 +- src/feature/relay/onion_queue.h | 2 +- src/feature/relay/router.c | 2 +- src/feature/relay/router.h | 2 +- src/feature/relay/routerkeys.c | 2 +- src/feature/relay/routerkeys.h | 2 +- src/feature/relay/routermode.c | 2 +- src/feature/relay/routermode.h | 2 +- src/feature/relay/selftest.c | 2 +- src/feature/relay/selftest.h | 2 +- src/feature/rend/rend_authorized_client_st.h | 2 +- src/feature/rend/rend_encoded_v2_service_descriptor_st.h | 2 +- src/feature/rend/rend_intro_point_st.h | 2 +- src/feature/rend/rend_service_descriptor_st.h | 2 +- src/feature/rend/rendcache.c | 2 +- src/feature/rend/rendcache.h | 2 +- src/feature/rend/rendclient.c | 2 +- src/feature/rend/rendclient.h | 2 +- src/feature/rend/rendcommon.c | 2 +- src/feature/rend/rendcommon.h | 2 +- src/feature/rend/rendmid.c | 2 +- src/feature/rend/rendmid.h | 2 +- src/feature/rend/rendparse.c | 2 +- src/feature/rend/rendparse.h | 2 +- src/feature/rend/rendservice.c | 2 +- src/feature/rend/rendservice.h | 2 +- src/feature/stats/geoip_stats.c | 2 +- src/feature/stats/geoip_stats.h | 2 +- src/feature/stats/predict_ports.c | 2 +- src/feature/stats/predict_ports.h | 2 +- src/feature/stats/rephist.c | 2 +- src/feature/stats/rephist.h | 2 +- src/lib/arch/bytes.h | 2 +- src/lib/buf/buffers.c | 2 +- src/lib/buf/buffers.h | 2 +- src/lib/cc/compat_compiler.h | 2 +- src/lib/cc/torint.h | 2 +- src/lib/compress/compress.c | 2 +- src/lib/compress/compress.h | 2 +- src/lib/compress/compress_buf.c | 2 +- src/lib/compress/compress_lzma.c | 2 +- src/lib/compress/compress_lzma.h | 2 +- src/lib/compress/compress_none.c | 2 +- src/lib/compress/compress_none.h | 2 +- src/lib/compress/compress_sys.h | 2 +- src/lib/compress/compress_zlib.c | 2 +- src/lib/compress/compress_zlib.h | 2 +- src/lib/compress/compress_zstd.c | 2 +- src/lib/compress/compress_zstd.h | 2 +- src/lib/container/bitarray.h | 2 +- src/lib/container/bloomfilt.c | 2 +- src/lib/container/bloomfilt.h | 2 +- src/lib/container/handles.h | 2 +- src/lib/container/map.c | 2 +- src/lib/container/map.h | 2 +- src/lib/container/order.c | 2 +- src/lib/container/order.h | 2 +- src/lib/container/smartlist.c | 2 +- src/lib/container/smartlist.h | 2 +- src/lib/crypt_ops/aes.h | 2 +- src/lib/crypt_ops/aes_nss.c | 2 +- src/lib/crypt_ops/aes_openssl.c | 2 +- src/lib/crypt_ops/compat_openssl.h | 2 +- src/lib/crypt_ops/crypto_cipher.c | 2 +- src/lib/crypt_ops/crypto_cipher.h | 2 +- src/lib/crypt_ops/crypto_curve25519.c | 2 +- src/lib/crypt_ops/crypto_curve25519.h | 2 +- src/lib/crypt_ops/crypto_dh.c | 2 +- src/lib/crypt_ops/crypto_dh.h | 2 +- src/lib/crypt_ops/crypto_dh_nss.c | 2 +- src/lib/crypt_ops/crypto_dh_openssl.c | 2 +- src/lib/crypt_ops/crypto_digest.c | 2 +- src/lib/crypt_ops/crypto_digest.h | 2 +- src/lib/crypt_ops/crypto_ed25519.c | 2 +- src/lib/crypt_ops/crypto_ed25519.h | 2 +- src/lib/crypt_ops/crypto_format.c | 2 +- src/lib/crypt_ops/crypto_format.h | 2 +- src/lib/crypt_ops/crypto_hkdf.c | 2 +- src/lib/crypt_ops/crypto_hkdf.h | 2 +- src/lib/crypt_ops/crypto_init.c | 2 +- src/lib/crypt_ops/crypto_init.h | 2 +- src/lib/crypt_ops/crypto_nss_mgt.c | 2 +- src/lib/crypt_ops/crypto_nss_mgt.h | 2 +- src/lib/crypt_ops/crypto_ope.c | 2 +- src/lib/crypt_ops/crypto_ope.h | 2 +- src/lib/crypt_ops/crypto_openssl_mgt.c | 2 +- src/lib/crypt_ops/crypto_openssl_mgt.h | 2 +- src/lib/crypt_ops/crypto_pwbox.c | 2 +- src/lib/crypt_ops/crypto_pwbox.h | 2 +- src/lib/crypt_ops/crypto_rand.c | 2 +- src/lib/crypt_ops/crypto_rand.h | 2 +- src/lib/crypt_ops/crypto_rsa.c | 2 +- src/lib/crypt_ops/crypto_rsa.h | 2 +- src/lib/crypt_ops/crypto_rsa_nss.c | 2 +- src/lib/crypt_ops/crypto_rsa_openssl.c | 2 +- src/lib/crypt_ops/crypto_s2k.c | 2 +- src/lib/crypt_ops/crypto_s2k.h | 2 +- src/lib/crypt_ops/crypto_sys.h | 2 +- src/lib/crypt_ops/crypto_util.c | 2 +- src/lib/crypt_ops/crypto_util.h | 2 +- src/lib/crypt_ops/digestset.c | 2 +- src/lib/crypt_ops/digestset.h | 2 +- src/lib/ctime/di_ops.c | 2 +- src/lib/ctime/di_ops.h | 2 +- src/lib/defs/dh_sizes.h | 2 +- src/lib/defs/digest_sizes.h | 2 +- src/lib/defs/time.h | 2 +- src/lib/defs/x25519_sizes.h | 2 +- src/lib/encoding/binascii.c | 2 +- src/lib/encoding/binascii.h | 2 +- src/lib/encoding/confline.c | 2 +- src/lib/encoding/confline.h | 2 +- src/lib/encoding/cstring.c | 2 +- src/lib/encoding/cstring.h | 2 +- src/lib/encoding/keyval.c | 2 +- src/lib/encoding/keyval.h | 2 +- src/lib/encoding/kvline.c | 2 +- src/lib/encoding/kvline.h | 2 +- src/lib/encoding/pem.c | 2 +- src/lib/encoding/pem.h | 2 +- src/lib/encoding/time_fmt.c | 2 +- src/lib/encoding/time_fmt.h | 2 +- src/lib/err/backtrace.c | 2 +- src/lib/err/backtrace.h | 2 +- src/lib/err/torerr.c | 2 +- src/lib/err/torerr.h | 2 +- src/lib/err/torerr_sys.c | 2 +- src/lib/err/torerr_sys.h | 2 +- src/lib/evloop/compat_libevent.c | 2 +- src/lib/evloop/compat_libevent.h | 2 +- src/lib/evloop/procmon.c | 2 +- src/lib/evloop/procmon.h | 2 +- src/lib/evloop/timers.c | 2 +- src/lib/evloop/timers.h | 2 +- src/lib/evloop/token_bucket.c | 2 +- src/lib/evloop/token_bucket.h | 2 +- src/lib/evloop/workqueue.h | 2 +- src/lib/fdio/fdio.c | 2 +- src/lib/fdio/fdio.h | 2 +- src/lib/fs/conffile.c | 2 +- src/lib/fs/conffile.h | 2 +- src/lib/fs/dir.c | 2 +- src/lib/fs/dir.h | 2 +- src/lib/fs/files.c | 2 +- src/lib/fs/files.h | 2 +- src/lib/fs/freespace.c | 2 +- src/lib/fs/lockfile.c | 2 +- src/lib/fs/lockfile.h | 2 +- src/lib/fs/mmap.c | 2 +- src/lib/fs/mmap.h | 2 +- src/lib/fs/path.c | 2 +- src/lib/fs/path.h | 2 +- src/lib/fs/storagedir.c | 2 +- src/lib/fs/storagedir.h | 2 +- src/lib/fs/userdb.c | 2 +- src/lib/fs/userdb.h | 2 +- src/lib/fs/winlib.c | 2 +- src/lib/fs/winlib.h | 2 +- src/lib/geoip/country.h | 2 +- src/lib/geoip/geoip.c | 2 +- src/lib/geoip/geoip.h | 2 +- src/lib/intmath/addsub.c | 2 +- src/lib/intmath/addsub.h | 2 +- src/lib/intmath/bits.c | 2 +- src/lib/intmath/bits.h | 2 +- src/lib/intmath/cmp.h | 2 +- src/lib/intmath/logic.h | 2 +- src/lib/intmath/muldiv.c | 2 +- src/lib/intmath/muldiv.h | 2 +- src/lib/intmath/weakrng.c | 2 +- src/lib/intmath/weakrng.h | 2 +- src/lib/lock/compat_mutex.c | 2 +- src/lib/lock/compat_mutex.h | 2 +- src/lib/lock/compat_mutex_pthreads.c | 2 +- src/lib/lock/compat_mutex_winthreads.c | 2 +- src/lib/log/escape.c | 2 +- src/lib/log/escape.h | 2 +- src/lib/log/log.c | 2 +- src/lib/log/log.h | 2 +- src/lib/log/log_sys.c | 2 +- src/lib/log/log_sys.h | 2 +- src/lib/log/ratelim.c | 2 +- src/lib/log/ratelim.h | 2 +- src/lib/log/util_bug.c | 2 +- src/lib/log/util_bug.h | 2 +- src/lib/log/win32err.c | 2 +- src/lib/log/win32err.h | 2 +- src/lib/malloc/malloc.c | 2 +- src/lib/malloc/malloc.h | 2 +- src/lib/math/fp.c | 2 +- src/lib/math/fp.h | 2 +- src/lib/math/laplace.c | 2 +- src/lib/math/laplace.h | 2 +- src/lib/math/prob_distr.c | 2 +- src/lib/memarea/memarea.c | 2 +- src/lib/memarea/memarea.h | 2 +- src/lib/meminfo/meminfo.c | 2 +- src/lib/meminfo/meminfo.h | 2 +- src/lib/net/address.c | 2 +- src/lib/net/address.h | 2 +- src/lib/net/alertsock.c | 2 +- src/lib/net/alertsock.h | 2 +- src/lib/net/buffers_net.c | 2 +- src/lib/net/buffers_net.h | 2 +- src/lib/net/gethostname.c | 2 +- src/lib/net/gethostname.h | 2 +- src/lib/net/inaddr.c | 2 +- src/lib/net/inaddr.h | 2 +- src/lib/net/inaddr_st.h | 2 +- src/lib/net/nettypes.h | 2 +- src/lib/net/network_sys.c | 2 +- src/lib/net/network_sys.h | 2 +- src/lib/net/resolve.c | 2 +- src/lib/net/resolve.h | 2 +- src/lib/net/socket.c | 2 +- src/lib/net/socket.h | 2 +- src/lib/net/socketpair.c | 2 +- src/lib/net/socketpair.h | 2 +- src/lib/net/socks5_status.h | 2 +- src/lib/osinfo/uname.c | 2 +- src/lib/osinfo/uname.h | 2 +- src/lib/process/daemon.c | 2 +- src/lib/process/daemon.h | 2 +- src/lib/process/env.c | 2 +- src/lib/process/env.h | 2 +- src/lib/process/pidfile.c | 2 +- src/lib/process/pidfile.h | 2 +- src/lib/process/process.c | 2 +- src/lib/process/process.h | 2 +- src/lib/process/process_sys.c | 2 +- src/lib/process/process_sys.h | 2 +- src/lib/process/process_unix.c | 2 +- src/lib/process/process_unix.h | 2 +- src/lib/process/process_win32.c | 2 +- src/lib/process/process_win32.h | 2 +- src/lib/process/restrict.c | 2 +- src/lib/process/restrict.h | 2 +- src/lib/process/setuid.c | 2 +- src/lib/process/setuid.h | 2 +- src/lib/process/waitpid.c | 2 +- src/lib/process/waitpid.h | 2 +- src/lib/process/winprocess_sys.c | 2 +- src/lib/process/winprocess_sys.h | 2 +- src/lib/sandbox/sandbox.c | 2 +- src/lib/sandbox/sandbox.h | 2 +- src/lib/smartlist_core/smartlist_core.c | 2 +- src/lib/smartlist_core/smartlist_core.h | 2 +- src/lib/smartlist_core/smartlist_foreach.h | 2 +- src/lib/smartlist_core/smartlist_split.c | 2 +- src/lib/smartlist_core/smartlist_split.h | 2 +- src/lib/string/compat_ctype.c | 2 +- src/lib/string/compat_ctype.h | 2 +- src/lib/string/compat_string.c | 2 +- src/lib/string/compat_string.h | 2 +- src/lib/string/parse_int.c | 2 +- src/lib/string/parse_int.h | 2 +- src/lib/string/printf.c | 2 +- src/lib/string/printf.h | 2 +- src/lib/string/scanf.c | 2 +- src/lib/string/scanf.h | 2 +- src/lib/string/util_string.c | 2 +- src/lib/string/util_string.h | 2 +- src/lib/subsys/subsys.h | 2 +- src/lib/term/getpass.c | 2 +- src/lib/term/getpass.h | 2 +- src/lib/testsupport/testsupport.h | 2 +- src/lib/thread/compat_pthreads.c | 2 +- src/lib/thread/compat_threads.c | 2 +- src/lib/thread/compat_winthreads.c | 2 +- src/lib/thread/numcpus.c | 2 +- src/lib/thread/numcpus.h | 2 +- src/lib/thread/thread_sys.h | 2 +- src/lib/thread/threads.h | 2 +- src/lib/time/compat_time.c | 2 +- src/lib/time/compat_time.h | 2 +- src/lib/time/time_sys.c | 2 +- src/lib/time/time_sys.h | 2 +- src/lib/time/tvdiff.c | 2 +- src/lib/time/tvdiff.h | 2 +- src/lib/tls/buffers_tls.c | 2 +- src/lib/tls/buffers_tls.h | 2 +- src/lib/tls/nss_countbytes.c | 2 +- src/lib/tls/nss_countbytes.h | 2 +- src/lib/tls/tortls.c | 2 +- src/lib/tls/tortls.h | 2 +- src/lib/tls/tortls_internal.h | 2 +- src/lib/tls/tortls_nss.c | 2 +- src/lib/tls/tortls_openssl.c | 2 +- src/lib/tls/tortls_st.h | 2 +- src/lib/tls/tortls_sys.h | 2 +- src/lib/tls/x509.c | 2 +- src/lib/tls/x509.h | 2 +- src/lib/tls/x509_internal.h | 2 +- src/lib/tls/x509_nss.c | 2 +- src/lib/tls/x509_openssl.c | 2 +- src/lib/trace/debug.h | 2 +- src/lib/trace/events.h | 2 +- src/lib/trace/trace.c | 2 +- src/lib/trace/trace.h | 2 +- src/lib/version/git_revision.c | 2 +- src/lib/version/git_revision.h | 2 +- src/lib/version/torversion.h | 2 +- src/lib/version/version.c | 2 +- src/lib/wallclock/approx_time.c | 2 +- src/lib/wallclock/approx_time.h | 2 +- src/lib/wallclock/time_to_tm.c | 2 +- src/lib/wallclock/time_to_tm.h | 2 +- src/lib/wallclock/timeval.h | 2 +- src/lib/wallclock/tor_gettimeofday.c | 2 +- src/lib/wallclock/tor_gettimeofday.h | 2 +- src/lib/wallclock/wallclock_sys.h | 2 +- src/rust/crypto/digests/mod.rs | 2 +- src/rust/crypto/digests/sha2.rs | 2 +- src/rust/crypto/lib.rs | 2 +- src/rust/crypto/rand/mod.rs | 2 +- src/rust/crypto/rand/rng.rs | 2 +- src/rust/external/crypto_digest.rs | 2 +- src/rust/external/crypto_rand.rs | 2 +- src/rust/external/external.rs | 2 +- src/rust/external/lib.rs | 2 +- src/rust/protover/errors.rs | 2 +- src/rust/protover/ffi.rs | 2 +- src/rust/protover/lib.rs | 2 +- src/rust/protover/protoset.rs | 2 +- src/rust/protover/protover.rs | 2 +- src/rust/protover/tests/protover.rs | 2 +- src/rust/smartlist/lib.rs | 2 +- src/rust/smartlist/smartlist.rs | 2 +- src/rust/tor_allocate/lib.rs | 2 +- src/rust/tor_allocate/tor_allocate.rs | 2 +- src/rust/tor_log/lib.rs | 2 +- src/rust/tor_log/tor_log.rs | 2 +- src/rust/tor_util/ffi.rs | 2 +- src/rust/tor_util/lib.rs | 2 +- src/rust/tor_util/strings.rs | 2 +- src/test/bench.c | 2 +- src/test/bt_test.py | 2 +- src/test/ed25519_exts_ref.py | 2 +- src/test/fakechans.h | 2 +- src/test/fuzz/dict/http | 2 +- src/test/fuzz/fuzz_consensus.c | 2 +- src/test/fuzz/fuzz_descriptor.c | 2 +- src/test/fuzz/fuzz_diff.c | 2 +- src/test/fuzz/fuzz_diff_apply.c | 2 +- src/test/fuzz/fuzz_extrainfo.c | 2 +- src/test/fuzz/fuzz_hsdescv2.c | 2 +- src/test/fuzz/fuzz_hsdescv3.c | 2 +- src/test/fuzz/fuzz_http.c | 2 +- src/test/fuzz/fuzz_http_connect.c | 2 +- src/test/fuzz/fuzz_iptsv2.c | 2 +- src/test/fuzz/fuzz_microdesc.c | 2 +- src/test/fuzz/fuzz_socks.c | 2 +- src/test/fuzz/fuzz_strops.c | 2 +- src/test/fuzz/fuzz_vrs.c | 2 +- src/test/fuzz/fuzzing.h | 2 +- src/test/fuzz/fuzzing_common.c | 2 +- src/test/fuzz_static_testcases.sh | 2 +- src/test/hs_ntor_ref.py | 2 +- src/test/hs_test_helpers.c | 2 +- src/test/hs_test_helpers.h | 2 +- src/test/log_test_helpers.c | 2 +- src/test/log_test_helpers.h | 2 +- src/test/ntor_ref.py | 2 +- src/test/ope_ref.py | 2 +- src/test/prob_distr_mpfr_ref.c | 2 +- src/test/rend_test_helpers.c | 2 +- src/test/rend_test_helpers.h | 2 +- src/test/test-memwipe.c | 2 +- src/test/test-process.c | 2 +- src/test/test-timers.c | 2 +- src/test/test.c | 2 +- src/test/test.h | 2 +- src/test/test_accounting.c | 2 +- src/test/test_addr.c | 2 +- src/test/test_address.c | 2 +- src/test/test_address_set.c | 2 +- src/test/test_bridges.c | 2 +- src/test/test_bt_cl.c | 2 +- src/test/test_btrack.c | 2 +- src/test/test_buffers.c | 2 +- src/test/test_bwmgt.c | 2 +- src/test/test_cell_formats.c | 2 +- src/test/test_cell_queue.c | 2 +- src/test/test_channel.c | 2 +- src/test/test_channelpadding.c | 2 +- src/test/test_channeltls.c | 2 +- src/test/test_checkdir.c | 2 +- src/test/test_circuitbuild.c | 2 +- src/test/test_circuitlist.c | 2 +- src/test/test_circuitmux.c | 2 +- src/test/test_circuitstats.c | 2 +- src/test/test_circuituse.c | 2 +- src/test/test_compat_libevent.c | 2 +- src/test/test_config.c | 2 +- src/test/test_connection.c | 2 +- src/test/test_connection.h | 2 +- src/test/test_conscache.c | 2 +- src/test/test_consdiff.c | 2 +- src/test/test_consdiffmgr.c | 2 +- src/test/test_containers.c | 2 +- src/test/test_controller.c | 2 +- src/test/test_controller_events.c | 2 +- src/test/test_crypto.c | 2 +- src/test/test_crypto_ope.c | 2 +- src/test/test_crypto_openssl.c | 2 +- src/test/test_crypto_slow.c | 2 +- src/test/test_data.c | 2 +- src/test/test_dir.c | 2 +- src/test/test_dir_common.c | 2 +- src/test/test_dir_common.h | 2 +- src/test/test_dir_handle_get.c | 2 +- src/test/test_dns.c | 2 +- src/test/test_dos.c | 2 +- src/test/test_entryconn.c | 2 +- src/test/test_entrynodes.c | 2 +- src/test/test_extorport.c | 2 +- src/test/test_geoip.c | 2 +- src/test/test_guardfraction.c | 2 +- src/test/test_handles.c | 2 +- src/test/test_helpers.c | 2 +- src/test/test_helpers.h | 2 +- src/test/test_hs.c | 2 +- src/test/test_hs_cache.c | 2 +- src/test/test_hs_cell.c | 2 +- src/test/test_hs_client.c | 2 +- src/test/test_hs_common.c | 2 +- src/test/test_hs_config.c | 2 +- src/test/test_hs_control.c | 2 +- src/test/test_hs_descriptor.c | 2 +- src/test/test_hs_intropoint.c | 2 +- src/test/test_hs_ntor.c | 2 +- src/test/test_hs_ntor_cl.c | 2 +- src/test/test_hs_service.c | 2 +- src/test/test_introduce.c | 2 +- src/test/test_keypin.c | 2 +- src/test/test_link_handshake.c | 2 +- src/test/test_logging.c | 2 +- src/test/test_mainloop.c | 2 +- src/test/test_microdesc.c | 2 +- src/test/test_netinfo.c | 2 +- src/test/test_nodelist.c | 2 +- src/test/test_ntor_cl.c | 2 +- src/test/test_oom.c | 2 +- src/test/test_oos.c | 2 +- src/test/test_options.c | 2 +- src/test/test_parsecommon.c | 2 +- src/test/test_pem.c | 2 +- src/test/test_periodic_event.c | 2 +- src/test/test_policy.c | 2 +- src/test/test_prob_distr.c | 2 +- src/test/test_process.c | 2 +- src/test/test_process_slow.c | 2 +- src/test/test_procmon.c | 2 +- src/test/test_proto_http.c | 2 +- src/test/test_proto_misc.c | 2 +- src/test/test_protover.c | 2 +- src/test/test_pt.c | 2 +- src/test/test_relay.c | 2 +- src/test/test_relaycell.c | 2 +- src/test/test_relaycrypt.c | 2 +- src/test/test_rendcache.c | 2 +- src/test/test_replay.c | 2 +- src/test/test_router.c | 2 +- src/test/test_routerkeys.c | 2 +- src/test/test_routerlist.c | 2 +- src/test/test_routerset.c | 2 +- src/test/test_scheduler.c | 2 +- src/test/test_shared_random.c | 2 +- src/test/test_slow.c | 2 +- src/test/test_socks.c | 2 +- src/test/test_status.c | 2 +- src/test/test_storagedir.c | 2 +- src/test/test_switch_id.c | 2 +- src/test/test_threads.c | 2 +- src/test/test_tortls.c | 2 +- src/test/test_tortls.h | 2 +- src/test/test_tortls_openssl.c | 2 +- src/test/test_util.c | 2 +- src/test/test_util_format.c | 2 +- src/test/test_util_process.c | 2 +- src/test/test_voting_flags.c | 2 +- src/test/test_voting_schedule.c | 2 +- src/test/test_workqueue.c | 2 +- src/test/test_x509.c | 2 +- src/test/testing_common.c | 2 +- src/test/testing_rsakeys.c | 2 +- src/tools/tor-gencert.c | 2 +- src/tools/tor-print-ed-signing-cert.c | 2 +- src/tools/tor-resolve.c | 2 +- src/tools/tor_runner.c | 2 +- 820 files changed, 821 insertions(+), 821 deletions(-) diff --git a/LICENSE b/LICENSE index 9e2709aea6..cc27668427 100644 --- a/LICENSE +++ b/LICENSE @@ -13,7 +13,7 @@ Tor is distributed under this license: Copyright (c) 2001-2004, Roger Dingledine Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson -Copyright (c) 2007-2018, The Tor Project, Inc. +Copyright (c) 2007-2019, The Tor Project, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/Makefile.am b/Makefile.am index a5086b3035..3ac9bc3fec 100644 --- a/Makefile.am +++ b/Makefile.am @@ -1,6 +1,6 @@ # Copyright (c) 2001-2004, Roger Dingledine # Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson -# Copyright (c) 2007-2018, The Tor Project, Inc. +# Copyright (c) 2007-2019, The Tor Project, Inc. # See LICENSE for licensing information ACLOCAL_AMFLAGS = -I m4 diff --git a/acinclude.m4 b/acinclude.m4 index c9cfc3f014..5ecdf1d5c2 100644 --- a/acinclude.m4 +++ b/acinclude.m4 @@ -2,7 +2,7 @@ dnl Helper macros for Tor configure.ac dnl Copyright (c) 2001-2004, Roger Dingledine dnl Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson dnl Copyright (c) 2007-2008, Roger Dingledine, Nick Mathewson -dnl Copyright (c) 2007-2018, The Tor Project, Inc. +dnl Copyright (c) 2007-2019, The Tor Project, Inc. dnl See LICENSE for licensing information AC_DEFUN([TOR_EXTEND_CODEPATH], diff --git a/configure.ac b/configure.ac index ca1af4ec6e..6d1b58923d 100644 --- a/configure.ac +++ b/configure.ac @@ -1,6 +1,6 @@ dnl Copyright (c) 2001-2004, Roger Dingledine dnl Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson -dnl Copyright (c) 2007-2018, The Tor Project, Inc. +dnl Copyright (c) 2007-2019, The Tor Project, Inc. dnl See LICENSE for licensing information AC_PREREQ([2.63]) diff --git a/scripts/codegen/gen_server_ciphers.py b/scripts/codegen/gen_server_ciphers.py index 5b2eef07ef..5d326f8b9e 100755 --- a/scripts/codegen/gen_server_ciphers.py +++ b/scripts/codegen/gen_server_ciphers.py @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright 2014-2018, The Tor Project, Inc +# Copyright 2014-2019, The Tor Project, Inc # See LICENSE for licensing information # This script parses openssl headers to find ciphersuite names, determines diff --git a/scripts/codegen/get_mozilla_ciphers.py b/scripts/codegen/get_mozilla_ciphers.py index 4f986daba9..f23f2f1e6f 100755 --- a/scripts/codegen/get_mozilla_ciphers.py +++ b/scripts/codegen/get_mozilla_ciphers.py @@ -1,6 +1,6 @@ #!/usr/bin/python # coding=utf-8 -# Copyright 2011-2018, The Tor Project, Inc +# Copyright 2011-2019, The Tor Project, Inc # original version by Arturo Filastò # See LICENSE for licensing information diff --git a/scripts/codegen/makedesc.py b/scripts/codegen/makedesc.py index 4ee8106f03..efca4dda9a 100644 --- a/scripts/codegen/makedesc.py +++ b/scripts/codegen/makedesc.py @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright 2014-2018, The Tor Project, Inc. +# Copyright 2014-2019, The Tor Project, Inc. # See LICENSE for license information # This is a kludgey python script that uses ctypes and openssl to sign diff --git a/scripts/maint/annotate_ifdef_directives b/scripts/maint/annotate_ifdef_directives index 368d842e2e..ca267a865e 100755 --- a/scripts/maint/annotate_ifdef_directives +++ b/scripts/maint/annotate_ifdef_directives @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (c) 2017, The Tor Project, Inc. +# Copyright (c) 2017-2019, The Tor Project, Inc. # See LICENSE for licensing information import re diff --git a/scripts/maint/format_changelog.py b/scripts/maint/format_changelog.py index 98fbbfb516..8dce4b6e51 100755 --- a/scripts/maint/format_changelog.py +++ b/scripts/maint/format_changelog.py @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (c) 2014-2018, The Tor Project, Inc. +# Copyright (c) 2014-2019, The Tor Project, Inc. # See LICENSE for licensing information # # This script reformats a section of the changelog to wrap everything to diff --git a/scripts/maint/redox.py b/scripts/maint/redox.py index e8b2622ab9..203cce0107 100755 --- a/scripts/maint/redox.py +++ b/scripts/maint/redox.py @@ -1,6 +1,6 @@ #!/usr/bin/python # -# Copyright (c) 2008-2018, The Tor Project, Inc. +# Copyright (c) 2008-2019, The Tor Project, Inc. # See LICENSE for licensing information. # # Hi! diff --git a/scripts/maint/sortChanges.py b/scripts/maint/sortChanges.py index c85e6563b8..986b94b025 100755 --- a/scripts/maint/sortChanges.py +++ b/scripts/maint/sortChanges.py @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (c) 2014-2018, The Tor Project, Inc. +# Copyright (c) 2014-2019, The Tor Project, Inc. # See LICENSE for licensing information """This script sorts a bunch of changes files listed on its command diff --git a/src/app/config/config.c b/src/app/config/config.c index ecf4c21545..952b9cd301 100644 --- a/src/app/config/config.c +++ b/src/app/config/config.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -2626,7 +2626,7 @@ print_usage(void) printf( "Copyright (c) 2001-2004, Roger Dingledine\n" "Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson\n" -"Copyright (c) 2007-2018, The Tor Project, Inc.\n\n" +"Copyright (c) 2007-2019, The Tor Project, Inc.\n\n" "tor -f [args]\n" "See man page for options, or https://www.torproject.org/ for " "documentation.\n"); diff --git a/src/app/config/config.h b/src/app/config/config.h index 4c497b83a6..b3b3150825 100644 --- a/src/app/config/config.h +++ b/src/app/config/config.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/app/config/confparse.c b/src/app/config/confparse.c index 35897935f3..8681f648da 100644 --- a/src/app/config/confparse.c +++ b/src/app/config/confparse.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/app/config/confparse.h b/src/app/config/confparse.h index efaa3e480a..57f1ec1762 100644 --- a/src/app/config/confparse.h +++ b/src/app/config/confparse.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/app/config/or_options_st.h b/src/app/config/or_options_st.h index 63a17c9771..06e11d3c75 100644 --- a/src/app/config/or_options_st.h +++ b/src/app/config/or_options_st.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/app/config/or_state_st.h b/src/app/config/or_state_st.h index 00968d3731..cdb9b38287 100644 --- a/src/app/config/or_state_st.h +++ b/src/app/config/or_state_st.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/app/config/statefile.c b/src/app/config/statefile.c index 97b96f1149..9681f6f8b3 100644 --- a/src/app/config/statefile.c +++ b/src/app/config/statefile.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/app/config/statefile.h b/src/app/config/statefile.h index 6433affa62..1950078450 100644 --- a/src/app/config/statefile.h +++ b/src/app/config/statefile.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/app/main/main.c b/src/app/main/main.c index ba2dfebd77..0ffc27d456 100644 --- a/src/app/main/main.c +++ b/src/app/main/main.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/app/main/main.h b/src/app/main/main.h index 23a436703e..bbbbf984fb 100644 --- a/src/app/main/main.h +++ b/src/app/main/main.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/app/main/ntmain.c b/src/app/main/ntmain.c index 8d2135a587..05d203b0be 100644 --- a/src/app/main/ntmain.c +++ b/src/app/main/ntmain.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/app/main/ntmain.h b/src/app/main/ntmain.h index 223d9e318b..c39386c054 100644 --- a/src/app/main/ntmain.h +++ b/src/app/main/ntmain.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/app/main/subsysmgr.c b/src/app/main/subsysmgr.c index abd2edd10b..e0ca3ce4df 100644 --- a/src/app/main/subsysmgr.c +++ b/src/app/main/subsysmgr.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "orconfig.h" diff --git a/src/app/main/subsysmgr.h b/src/app/main/subsysmgr.h index 4b3cad62ad..a5e62f71d9 100644 --- a/src/app/main/subsysmgr.h +++ b/src/app/main/subsysmgr.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_SUBSYSMGR_T diff --git a/src/app/main/subsystem_list.c b/src/app/main/subsystem_list.c index 9f9bf10866..3834176182 100644 --- a/src/app/main/subsystem_list.c +++ b/src/app/main/subsystem_list.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "orconfig.h" diff --git a/src/app/main/tor_main.c b/src/app/main/tor_main.c index 8c497fff8a..8a887ed269 100644 --- a/src/app/main/tor_main.c +++ b/src/app/main/tor_main.c @@ -1,6 +1,6 @@ /* Copyright 2001-2004 Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "orconfig.h" diff --git a/src/core/crypto/hs_ntor.c b/src/core/crypto/hs_ntor.c index d98d16f7f6..c34073690e 100644 --- a/src/core/crypto/hs_ntor.c +++ b/src/core/crypto/hs_ntor.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2017-2018, The Tor Project, Inc. */ +/* Copyright (c) 2017-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** \file hs_ntor.c diff --git a/src/core/crypto/hs_ntor.h b/src/core/crypto/hs_ntor.h index 30738c4ae0..e5a5171915 100644 --- a/src/core/crypto/hs_ntor.h +++ b/src/core/crypto/hs_ntor.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2017-2018, The Tor Project, Inc. */ +/* Copyright (c) 2017-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_HS_NTOR_H diff --git a/src/core/crypto/onion_crypto.c b/src/core/crypto/onion_crypto.c index 4978e0d46c..56b02e2996 100644 --- a/src/core/crypto/onion_crypto.c +++ b/src/core/crypto/onion_crypto.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/core/crypto/onion_crypto.h b/src/core/crypto/onion_crypto.h index ed410ef252..1cddde3610 100644 --- a/src/core/crypto/onion_crypto.h +++ b/src/core/crypto/onion_crypto.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/core/crypto/onion_fast.c b/src/core/crypto/onion_fast.c index 292aeca402..31bd20235f 100644 --- a/src/core/crypto/onion_fast.c +++ b/src/core/crypto/onion_fast.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/core/crypto/onion_fast.h b/src/core/crypto/onion_fast.h index a7b6ec53f4..0ba8cbbc35 100644 --- a/src/core/crypto/onion_fast.h +++ b/src/core/crypto/onion_fast.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/core/crypto/onion_ntor.c b/src/core/crypto/onion_ntor.c index 3614e0c9b1..7087fe1bd7 100644 --- a/src/core/crypto/onion_ntor.c +++ b/src/core/crypto/onion_ntor.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2012-2018, The Tor Project, Inc. */ +/* Copyright (c) 2012-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/core/crypto/onion_ntor.h b/src/core/crypto/onion_ntor.h index 0ba4abe49e..51e72b4083 100644 --- a/src/core/crypto/onion_ntor.h +++ b/src/core/crypto/onion_ntor.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2012-2018, The Tor Project, Inc. */ +/* Copyright (c) 2012-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_ONION_NTOR_H diff --git a/src/core/crypto/onion_tap.c b/src/core/crypto/onion_tap.c index 7ffe0ea5c5..854889d88d 100644 --- a/src/core/crypto/onion_tap.c +++ b/src/core/crypto/onion_tap.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/core/crypto/onion_tap.h b/src/core/crypto/onion_tap.h index 9a3df684d6..0e43b9c8ba 100644 --- a/src/core/crypto/onion_tap.h +++ b/src/core/crypto/onion_tap.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/core/crypto/relay_crypto.c b/src/core/crypto/relay_crypto.c index 311160a669..0b83b2d0a5 100644 --- a/src/core/crypto/relay_crypto.c +++ b/src/core/crypto/relay_crypto.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "core/or/or.h" diff --git a/src/core/crypto/relay_crypto.h b/src/core/crypto/relay_crypto.h index 67da93344f..45a21d14ab 100644 --- a/src/core/crypto/relay_crypto.h +++ b/src/core/crypto/relay_crypto.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/core/mainloop/connection.c b/src/core/mainloop/connection.c index 6cc6122702..37f35c8b8d 100644 --- a/src/core/mainloop/connection.c +++ b/src/core/mainloop/connection.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/core/mainloop/connection.h b/src/core/mainloop/connection.h index 07b8df4138..f4f0e839ae 100644 --- a/src/core/mainloop/connection.h +++ b/src/core/mainloop/connection.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/core/mainloop/cpuworker.c b/src/core/mainloop/cpuworker.c index 8d0d23ab91..e704d55642 100644 --- a/src/core/mainloop/cpuworker.c +++ b/src/core/mainloop/cpuworker.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/core/mainloop/cpuworker.h b/src/core/mainloop/cpuworker.h index 50812b2dab..77e2c42508 100644 --- a/src/core/mainloop/cpuworker.h +++ b/src/core/mainloop/cpuworker.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/core/mainloop/mainloop.c b/src/core/mainloop/mainloop.c index ca0fefca71..18e87fa87a 100644 --- a/src/core/mainloop/mainloop.c +++ b/src/core/mainloop/mainloop.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/core/mainloop/mainloop.h b/src/core/mainloop/mainloop.h index 730234857d..6ed93fa900 100644 --- a/src/core/mainloop/mainloop.h +++ b/src/core/mainloop/mainloop.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/core/mainloop/netstatus.c b/src/core/mainloop/netstatus.c index d1989cb839..fc5a465ff7 100644 --- a/src/core/mainloop/netstatus.c +++ b/src/core/mainloop/netstatus.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "core/or/or.h" diff --git a/src/core/mainloop/netstatus.h b/src/core/mainloop/netstatus.h index 9a0fa410fd..aba631e2fb 100644 --- a/src/core/mainloop/netstatus.h +++ b/src/core/mainloop/netstatus.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_NETSTATUS_H diff --git a/src/core/mainloop/periodic.c b/src/core/mainloop/periodic.c index 9f9b178e43..c0363b15ea 100644 --- a/src/core/mainloop/periodic.c +++ b/src/core/mainloop/periodic.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2015-2018, The Tor Project, Inc. */ +/* Copyright (c) 2015-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/core/mainloop/periodic.h b/src/core/mainloop/periodic.h index 05ba4297f3..344fc9ad25 100644 --- a/src/core/mainloop/periodic.h +++ b/src/core/mainloop/periodic.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2015-2018, The Tor Project, Inc. */ +/* Copyright (c) 2015-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_PERIODIC_H diff --git a/src/core/or/addr_policy_st.h b/src/core/or/addr_policy_st.h index 222a067252..a75f1a731d 100644 --- a/src/core/or/addr_policy_st.h +++ b/src/core/or/addr_policy_st.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_ADDR_POLICY_ST_H diff --git a/src/core/or/address_set.c b/src/core/or/address_set.c index 014e650d2d..758fba4aac 100644 --- a/src/core/or/address_set.c +++ b/src/core/or/address_set.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2018-2018, The Tor Project, Inc. */ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/core/or/address_set.h b/src/core/or/address_set.h index 2efa1cb03b..7a9e71628e 100644 --- a/src/core/or/address_set.h +++ b/src/core/or/address_set.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2018-2018, The Tor Project, Inc. */ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/core/or/cell_queue_st.h b/src/core/or/cell_queue_st.h index 40110019bc..130b95a011 100644 --- a/src/core/or/cell_queue_st.h +++ b/src/core/or/cell_queue_st.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef PACKED_CELL_ST_H diff --git a/src/core/or/cell_st.h b/src/core/or/cell_st.h index 6728e783b9..7ab7eceb50 100644 --- a/src/core/or/cell_st.h +++ b/src/core/or/cell_st.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef CELL_ST_H diff --git a/src/core/or/channel.c b/src/core/or/channel.c index 2af0e5944f..fd7bf62789 100644 --- a/src/core/or/channel.c +++ b/src/core/or/channel.c @@ -1,5 +1,5 @@ -/* * Copyright (c) 2012-2018, The Tor Project, Inc. */ +/* * Copyright (c) 2012-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/core/or/channel.h b/src/core/or/channel.h index b3d97a9add..97aa000337 100644 --- a/src/core/or/channel.h +++ b/src/core/or/channel.h @@ -1,4 +1,4 @@ -/* * Copyright (c) 2012-2018, The Tor Project, Inc. */ +/* * Copyright (c) 2012-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/core/or/channelpadding.c b/src/core/or/channelpadding.c index fcc3b8f03d..4a0f0e00da 100644 --- a/src/core/or/channelpadding.c +++ b/src/core/or/channelpadding.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /* TOR_CHANNEL_INTERNAL_ define needed for an O(1) implementation of diff --git a/src/core/or/channelpadding.h b/src/core/or/channelpadding.h index 7f8cfd4cc7..48002eedb7 100644 --- a/src/core/or/channelpadding.h +++ b/src/core/or/channelpadding.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/core/or/channeltls.c b/src/core/or/channeltls.c index cf33feec01..f552b20770 100644 --- a/src/core/or/channeltls.c +++ b/src/core/or/channeltls.c @@ -1,4 +1,4 @@ -/* * Copyright (c) 2012-2018, The Tor Project, Inc. */ +/* * Copyright (c) 2012-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/core/or/channeltls.h b/src/core/or/channeltls.h index 2ec7fe5453..634a2a00e9 100644 --- a/src/core/or/channeltls.h +++ b/src/core/or/channeltls.h @@ -1,4 +1,4 @@ -/* * Copyright (c) 2012-2018, The Tor Project, Inc. */ +/* * Copyright (c) 2012-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/core/or/circuit_st.h b/src/core/or/circuit_st.h index 29bcaa098f..af343f082e 100644 --- a/src/core/or/circuit_st.h +++ b/src/core/or/circuit_st.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef CIRCUIT_ST_H diff --git a/src/core/or/circuitbuild.c b/src/core/or/circuitbuild.c index 22e4cf96d8..3ec1e01f11 100644 --- a/src/core/or/circuitbuild.c +++ b/src/core/or/circuitbuild.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/core/or/circuitbuild.h b/src/core/or/circuitbuild.h index 93f903f060..b19bc41235 100644 --- a/src/core/or/circuitbuild.h +++ b/src/core/or/circuitbuild.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/core/or/circuitlist.c b/src/core/or/circuitlist.c index 71f8becddc..6b5f30e418 100644 --- a/src/core/or/circuitlist.c +++ b/src/core/or/circuitlist.c @@ -1,7 +1,7 @@ /* Copyright 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/core/or/circuitlist.h b/src/core/or/circuitlist.h index 37d37a089d..f34f4ed6b7 100644 --- a/src/core/or/circuitlist.h +++ b/src/core/or/circuitlist.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/core/or/circuitmux.c b/src/core/or/circuitmux.c index f55386df23..88f9ac7923 100644 --- a/src/core/or/circuitmux.c +++ b/src/core/or/circuitmux.c @@ -1,4 +1,4 @@ -/* * Copyright (c) 2012-2018, The Tor Project, Inc. */ +/* * Copyright (c) 2012-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/core/or/circuitmux.h b/src/core/or/circuitmux.h index 31bf81463b..67cd9bcdd8 100644 --- a/src/core/or/circuitmux.h +++ b/src/core/or/circuitmux.h @@ -1,4 +1,4 @@ -/* * Copyright (c) 2012-2018, The Tor Project, Inc. */ +/* * Copyright (c) 2012-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/core/or/circuitmux_ewma.c b/src/core/or/circuitmux_ewma.c index deca81f0ac..3f83c3fd5a 100644 --- a/src/core/or/circuitmux_ewma.c +++ b/src/core/or/circuitmux_ewma.c @@ -1,4 +1,4 @@ -/* * Copyright (c) 2012-2018, The Tor Project, Inc. */ +/* * Copyright (c) 2012-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/core/or/circuitmux_ewma.h b/src/core/or/circuitmux_ewma.h index 8e87830a80..b45ce1f916 100644 --- a/src/core/or/circuitmux_ewma.h +++ b/src/core/or/circuitmux_ewma.h @@ -1,4 +1,4 @@ -/* * Copyright (c) 2012-2018, The Tor Project, Inc. */ +/* * Copyright (c) 2012-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/core/or/circuitpadding.h b/src/core/or/circuitpadding.h index 628f27ec11..92fd4fc2d5 100644 --- a/src/core/or/circuitpadding.h +++ b/src/core/or/circuitpadding.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, The Tor Project, Inc. */ + * Copyright (c) 2017-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/core/or/circuitstats.c b/src/core/or/circuitstats.c index 61d5e18a45..c6ea2fff97 100644 --- a/src/core/or/circuitstats.c +++ b/src/core/or/circuitstats.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/core/or/circuitstats.h b/src/core/or/circuitstats.h index 174730d035..845d7b6722 100644 --- a/src/core/or/circuitstats.h +++ b/src/core/or/circuitstats.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/core/or/circuituse.c b/src/core/or/circuituse.c index 70e3e97ff7..07b7db93f6 100644 --- a/src/core/or/circuituse.c +++ b/src/core/or/circuituse.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/core/or/circuituse.h b/src/core/or/circuituse.h index b65e85d170..25588dbb11 100644 --- a/src/core/or/circuituse.h +++ b/src/core/or/circuituse.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/core/or/command.c b/src/core/or/command.c index cbe7f622e7..5fb6640c22 100644 --- a/src/core/or/command.c +++ b/src/core/or/command.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/core/or/command.h b/src/core/or/command.h index 83ffd8dccd..8c90e1de6f 100644 --- a/src/core/or/command.h +++ b/src/core/or/command.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/core/or/connection_edge.c b/src/core/or/connection_edge.c index 6b9ed0f211..cc240bdc98 100644 --- a/src/core/or/connection_edge.c +++ b/src/core/or/connection_edge.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/core/or/connection_edge.h b/src/core/or/connection_edge.h index b8a7365a05..68d8b19a11 100644 --- a/src/core/or/connection_edge.h +++ b/src/core/or/connection_edge.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/core/or/connection_or.c b/src/core/or/connection_or.c index c8667fc399..55047da167 100644 --- a/src/core/or/connection_or.c +++ b/src/core/or/connection_or.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/core/or/connection_or.h b/src/core/or/connection_or.h index 5f4856d51f..272f536b83 100644 --- a/src/core/or/connection_or.h +++ b/src/core/or/connection_or.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/core/or/connection_st.h b/src/core/or/connection_st.h index 6c22478689..d1430eda14 100644 --- a/src/core/or/connection_st.h +++ b/src/core/or/connection_st.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef CONNECTION_ST_H diff --git a/src/core/or/cpath_build_state_st.h b/src/core/or/cpath_build_state_st.h index 1db7251132..dbe596d851 100644 --- a/src/core/or/cpath_build_state_st.h +++ b/src/core/or/cpath_build_state_st.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef CIRCUIT_BUILD_STATE_ST_ST_H diff --git a/src/core/or/crypt_path_reference_st.h b/src/core/or/crypt_path_reference_st.h index bb0e519233..3d79f26c1c 100644 --- a/src/core/or/crypt_path_reference_st.h +++ b/src/core/or/crypt_path_reference_st.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef CRYPT_PATH_REFERENCE_ST_H diff --git a/src/core/or/crypt_path_st.h b/src/core/or/crypt_path_st.h index 1380913360..429480f8ab 100644 --- a/src/core/or/crypt_path_st.h +++ b/src/core/or/crypt_path_st.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef CRYPT_PATH_ST_H diff --git a/src/core/or/destroy_cell_queue_st.h b/src/core/or/destroy_cell_queue_st.h index 2839b0bd11..56630670ba 100644 --- a/src/core/or/destroy_cell_queue_st.h +++ b/src/core/or/destroy_cell_queue_st.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef DESTROY_CELL_QUEUE_ST_H diff --git a/src/core/or/dos.c b/src/core/or/dos.c index 4303b8fd59..5f9bbf90ab 100644 --- a/src/core/or/dos.c +++ b/src/core/or/dos.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2018-2018, The Tor Project, Inc. */ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /* diff --git a/src/core/or/dos.h b/src/core/or/dos.h index 760ef11057..95448d0530 100644 --- a/src/core/or/dos.h +++ b/src/core/or/dos.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2018-2018, The Tor Project, Inc. */ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /* diff --git a/src/core/or/edge_connection_st.h b/src/core/or/edge_connection_st.h index f4388c10e6..1665b8589f 100644 --- a/src/core/or/edge_connection_st.h +++ b/src/core/or/edge_connection_st.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef EDGE_CONNECTION_ST_H diff --git a/src/core/or/entry_connection_st.h b/src/core/or/entry_connection_st.h index ebaee2f1a6..45621fadbf 100644 --- a/src/core/or/entry_connection_st.h +++ b/src/core/or/entry_connection_st.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef ENTRY_CONNECTION_ST_H diff --git a/src/core/or/entry_port_cfg_st.h b/src/core/or/entry_port_cfg_st.h index 492fafbd6d..87dfb331e5 100644 --- a/src/core/or/entry_port_cfg_st.h +++ b/src/core/or/entry_port_cfg_st.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef ENTRY_PORT_CFG_ST_H diff --git a/src/core/or/extend_info_st.h b/src/core/or/extend_info_st.h index 277766c4d6..bc7a77b1b2 100644 --- a/src/core/or/extend_info_st.h +++ b/src/core/or/extend_info_st.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef EXTEND_INFO_ST_H diff --git a/src/core/or/half_edge_st.h b/src/core/or/half_edge_st.h index 5ed24dabeb..d4617be108 100644 --- a/src/core/or/half_edge_st.h +++ b/src/core/or/half_edge_st.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef HALF_EDGE_ST_H diff --git a/src/core/or/listener_connection_st.h b/src/core/or/listener_connection_st.h index ec350c1b0d..8989a39dc8 100644 --- a/src/core/or/listener_connection_st.h +++ b/src/core/or/listener_connection_st.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef LISTENER_CONNECTION_ST_H diff --git a/src/core/or/ocirc_event.c b/src/core/or/ocirc_event.c index b400022bb7..4a6fc748c9 100644 --- a/src/core/or/ocirc_event.c +++ b/src/core/or/ocirc_event.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/core/or/ocirc_event.h b/src/core/or/ocirc_event.h index 19a237d7df..0b125c2898 100644 --- a/src/core/or/ocirc_event.h +++ b/src/core/or/ocirc_event.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/core/or/ocirc_event_sys.h b/src/core/or/ocirc_event_sys.h index 0bc135ffaf..9d4bfe5333 100644 --- a/src/core/or/ocirc_event_sys.h +++ b/src/core/or/ocirc_event_sys.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* Copyright (c) 2007-2019, The Tor Project, Inc. */ /** * \file ocirc_event_sys.h diff --git a/src/core/or/onion.c b/src/core/or/onion.c index 5c29441947..aa77465b96 100644 --- a/src/core/or/onion.c +++ b/src/core/or/onion.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/core/or/onion.h b/src/core/or/onion.h index 2049fdf419..bb0b5b8dfd 100644 --- a/src/core/or/onion.h +++ b/src/core/or/onion.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/core/or/or.h b/src/core/or/or.h index bf5e3957ad..db6d089582 100644 --- a/src/core/or/or.h +++ b/src/core/or/or.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/core/or/or_circuit_st.h b/src/core/or/or_circuit_st.h index b5e21d9867..6b6feb9d89 100644 --- a/src/core/or/or_circuit_st.h +++ b/src/core/or/or_circuit_st.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef OR_CIRCUIT_ST_H diff --git a/src/core/or/or_connection_st.h b/src/core/or/or_connection_st.h index 020a717c11..d5db5e8694 100644 --- a/src/core/or/or_connection_st.h +++ b/src/core/or/or_connection_st.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef OR_CONNECTION_ST_H diff --git a/src/core/or/or_handshake_certs_st.h b/src/core/or/or_handshake_certs_st.h index 38e798b5e2..a93b7104aa 100644 --- a/src/core/or/or_handshake_certs_st.h +++ b/src/core/or/or_handshake_certs_st.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef OR_HANDSHAKE_CERTS_ST diff --git a/src/core/or/or_handshake_state_st.h b/src/core/or/or_handshake_state_st.h index 4ee095d9af..09a8a34179 100644 --- a/src/core/or/or_handshake_state_st.h +++ b/src/core/or/or_handshake_state_st.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef OR_HANDSHAKE_STATE_ST diff --git a/src/core/or/orconn_event.c b/src/core/or/orconn_event.c index d81f7b5a0c..9fb34bd1ff 100644 --- a/src/core/or/orconn_event.c +++ b/src/core/or/orconn_event.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/core/or/orconn_event.h b/src/core/or/orconn_event.h index 4c999e53be..80289d53e6 100644 --- a/src/core/or/orconn_event.h +++ b/src/core/or/orconn_event.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/core/or/orconn_event_sys.h b/src/core/or/orconn_event_sys.h index 7639023386..bfb0a3ac4a 100644 --- a/src/core/or/orconn_event_sys.h +++ b/src/core/or/orconn_event_sys.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* Copyright (c) 2007-2019, The Tor Project, Inc. */ /** * \file orconn_event_sys.h diff --git a/src/core/or/origin_circuit_st.h b/src/core/or/origin_circuit_st.h index 921076c1b9..daa5f41dad 100644 --- a/src/core/or/origin_circuit_st.h +++ b/src/core/or/origin_circuit_st.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef ORIGIN_CIRCUIT_ST_H diff --git a/src/core/or/policies.c b/src/core/or/policies.c index bffdb1fddd..a6d66d36de 100644 --- a/src/core/or/policies.c +++ b/src/core/or/policies.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/core/or/policies.h b/src/core/or/policies.h index 0c64ecf378..324c1c2dd1 100644 --- a/src/core/or/policies.h +++ b/src/core/or/policies.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/core/or/port_cfg_st.h b/src/core/or/port_cfg_st.h index 19410871ed..b67091ce32 100644 --- a/src/core/or/port_cfg_st.h +++ b/src/core/or/port_cfg_st.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef PORT_CFG_ST_H diff --git a/src/core/or/protover.c b/src/core/or/protover.c index c0c09c9d17..53709ad002 100644 --- a/src/core/or/protover.c +++ b/src/core/or/protover.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2016-2018, The Tor Project, Inc. */ +/* Copyright (c) 2016-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/core/or/protover.h b/src/core/or/protover.h index ffd4f2c18e..27106d4bec 100644 --- a/src/core/or/protover.h +++ b/src/core/or/protover.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2016-2018, The Tor Project, Inc. */ +/* Copyright (c) 2016-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/core/or/protover_rust.c b/src/core/or/protover_rust.c index dbdd5c8237..bc56ea11d0 100644 --- a/src/core/or/protover_rust.c +++ b/src/core/or/protover_rust.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2016-2018, The Tor Project, Inc. */ +/* Copyright (c) 2016-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /* diff --git a/src/core/or/reasons.c b/src/core/or/reasons.c index 610338ee1d..a7952279ba 100644 --- a/src/core/or/reasons.c +++ b/src/core/or/reasons.c @@ -1,5 +1,5 @@ /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/core/or/reasons.h b/src/core/or/reasons.h index 837b4a0f1a..c45a8bc38d 100644 --- a/src/core/or/reasons.h +++ b/src/core/or/reasons.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/core/or/relay.c b/src/core/or/relay.c index 00c2111955..706a6e05cb 100644 --- a/src/core/or/relay.c +++ b/src/core/or/relay.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/core/or/relay.h b/src/core/or/relay.h index e84727e373..044f6be156 100644 --- a/src/core/or/relay.h +++ b/src/core/or/relay.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/core/or/relay_crypto_st.h b/src/core/or/relay_crypto_st.h index f186e182f0..dafce257c7 100644 --- a/src/core/or/relay_crypto_st.h +++ b/src/core/or/relay_crypto_st.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef RELAY_CRYPTO_ST_H diff --git a/src/core/or/scheduler.c b/src/core/or/scheduler.c index 937e7e45db..ee22a38142 100644 --- a/src/core/or/scheduler.c +++ b/src/core/or/scheduler.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2018, The Tor Project, Inc. */ +/* Copyright (c) 2013-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "core/or/or.h" diff --git a/src/core/or/scheduler.h b/src/core/or/scheduler.h index 05a888365b..843be2603c 100644 --- a/src/core/or/scheduler.h +++ b/src/core/or/scheduler.h @@ -1,4 +1,4 @@ -/* * Copyright (c) 2017-2018, The Tor Project, Inc. */ +/* * Copyright (c) 2017-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/core/or/scheduler_kist.c b/src/core/or/scheduler_kist.c index 3ed0f1a5e2..34e5672074 100644 --- a/src/core/or/scheduler_kist.c +++ b/src/core/or/scheduler_kist.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2017-2018, The Tor Project, Inc. */ +/* Copyright (c) 2017-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define SCHEDULER_KIST_PRIVATE diff --git a/src/core/or/scheduler_vanilla.c b/src/core/or/scheduler_vanilla.c index db8374eadb..33536ae04b 100644 --- a/src/core/or/scheduler_vanilla.c +++ b/src/core/or/scheduler_vanilla.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2017-2018, The Tor Project, Inc. */ +/* Copyright (c) 2017-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "core/or/or.h" diff --git a/src/core/or/server_port_cfg_st.h b/src/core/or/server_port_cfg_st.h index e1a9ca496a..bd026af7ee 100644 --- a/src/core/or/server_port_cfg_st.h +++ b/src/core/or/server_port_cfg_st.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef SERVER_PORT_CFG_ST_H diff --git a/src/core/or/socks_request_st.h b/src/core/or/socks_request_st.h index 17b668e179..5922870c61 100644 --- a/src/core/or/socks_request_st.h +++ b/src/core/or/socks_request_st.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef SOCKS_REQUEST_ST_H diff --git a/src/core/or/status.c b/src/core/or/status.c index cdb6df1230..46494ca76c 100644 --- a/src/core/or/status.c +++ b/src/core/or/status.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2010-2018, The Tor Project, Inc. */ +/* Copyright (c) 2010-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/core/or/status.h b/src/core/or/status.h index 7258ed5939..3467501ebb 100644 --- a/src/core/or/status.h +++ b/src/core/or/status.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2010-2018, The Tor Project, Inc. */ +/* Copyright (c) 2010-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_STATUS_H diff --git a/src/core/or/tor_version_st.h b/src/core/or/tor_version_st.h index 5950c5d5c4..716429bd32 100644 --- a/src/core/or/tor_version_st.h +++ b/src/core/or/tor_version_st.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_VERSION_ST_H diff --git a/src/core/or/var_cell_st.h b/src/core/or/var_cell_st.h index 514afc44b1..4287c83f6d 100644 --- a/src/core/or/var_cell_st.h +++ b/src/core/or/var_cell_st.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef VAR_CELL_ST_H diff --git a/src/core/or/versions.c b/src/core/or/versions.c index 736313a9cd..2a572d4704 100644 --- a/src/core/or/versions.c +++ b/src/core/or/versions.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/core/or/versions.h b/src/core/or/versions.h index acd8998918..9aa7a0db87 100644 --- a/src/core/or/versions.h +++ b/src/core/or/versions.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/core/proto/proto_cell.c b/src/core/proto/proto_cell.c index 70278cd488..697fed29e1 100644 --- a/src/core/proto/proto_cell.c +++ b/src/core/proto/proto_cell.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "core/or/or.h" diff --git a/src/core/proto/proto_cell.h b/src/core/proto/proto_cell.h index b29645e41d..4f3982ea43 100644 --- a/src/core/proto/proto_cell.h +++ b/src/core/proto/proto_cell.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_PROTO_CELL_H diff --git a/src/core/proto/proto_control0.c b/src/core/proto/proto_control0.c index a770a061a7..d741f28f09 100644 --- a/src/core/proto/proto_control0.c +++ b/src/core/proto/proto_control0.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "core/or/or.h" diff --git a/src/core/proto/proto_control0.h b/src/core/proto/proto_control0.h index b80dc6c8f8..162e513a1b 100644 --- a/src/core/proto/proto_control0.h +++ b/src/core/proto/proto_control0.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_PROTO_CONTROL0_H diff --git a/src/core/proto/proto_ext_or.c b/src/core/proto/proto_ext_or.c index fe36f6b396..4213bc14dd 100644 --- a/src/core/proto/proto_ext_or.c +++ b/src/core/proto/proto_ext_or.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "core/or/or.h" diff --git a/src/core/proto/proto_ext_or.h b/src/core/proto/proto_ext_or.h index 2ff6ad45ca..b2bc64af85 100644 --- a/src/core/proto/proto_ext_or.h +++ b/src/core/proto/proto_ext_or.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_PROTO_EXT_OR_H diff --git a/src/core/proto/proto_http.c b/src/core/proto/proto_http.c index 4ce9ba02f5..88c59ef561 100644 --- a/src/core/proto/proto_http.c +++ b/src/core/proto/proto_http.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define PROTO_HTTP_PRIVATE diff --git a/src/core/proto/proto_http.h b/src/core/proto/proto_http.h index 587e435ede..cd70050205 100644 --- a/src/core/proto/proto_http.h +++ b/src/core/proto/proto_http.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_PROTO_HTTP_H diff --git a/src/core/proto/proto_socks.c b/src/core/proto/proto_socks.c index 4071f34f0d..86c656e39d 100644 --- a/src/core/proto/proto_socks.c +++ b/src/core/proto/proto_socks.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "core/or/or.h" diff --git a/src/core/proto/proto_socks.h b/src/core/proto/proto_socks.h index 53de288f65..2a387bf848 100644 --- a/src/core/proto/proto_socks.h +++ b/src/core/proto/proto_socks.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_PROTO_SOCKS_H diff --git a/src/ext/ht.h b/src/ext/ht.h index df9f60ba1d..54e5eb7cba 100644 --- a/src/ext/ht.h +++ b/src/ext/ht.h @@ -1,6 +1,6 @@ /* Copyright (c) 2002, Christopher Clark. * Copyright (c) 2005-2006, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See license at end. */ /* Based on ideas by Christopher Clark and interfaces from Niels Provos. */ diff --git a/src/ext/trunnel/trunnel-impl.h b/src/ext/trunnel/trunnel-impl.h index b233cf7631..15d1c8633e 100644 --- a/src/ext/trunnel/trunnel-impl.h +++ b/src/ext/trunnel/trunnel-impl.h @@ -5,7 +5,7 @@ /* trunnel-impl.h -- Implementation helpers for trunnel, included by * generated trunnel files * - * Copyright 2014-2017, The Tor Project, Inc. + * Copyright 2014-2019, The Tor Project, Inc. * See license at the end of this file for copying information. */ diff --git a/src/ext/trunnel/trunnel.c b/src/ext/trunnel/trunnel.c index 2442bc3909..3ae3fe02c8 100644 --- a/src/ext/trunnel/trunnel.c +++ b/src/ext/trunnel/trunnel.c @@ -4,7 +4,7 @@ */ /* trunnel.c -- Helper functions to implement trunnel. * - * Copyright 2014-2017, The Tor Project, Inc. + * Copyright 2014-2019, The Tor Project, Inc. * See license at the end of this file for copying information. * * See trunnel-impl.h for documentation of these functions. diff --git a/src/ext/trunnel/trunnel.h b/src/ext/trunnel/trunnel.h index 32c80bac23..9b708437b8 100644 --- a/src/ext/trunnel/trunnel.h +++ b/src/ext/trunnel/trunnel.h @@ -5,7 +5,7 @@ /* trunnel.h -- Public declarations for trunnel, to be included * in trunnel header files. - * Copyright 2014-2017, The Tor Project, Inc. + * Copyright 2014-2019, The Tor Project, Inc. * See license at the end of this file for copying information. */ diff --git a/src/feature/api/tor_api.c b/src/feature/api/tor_api.c index 5d194f6d8c..697397d46b 100644 --- a/src/feature/api/tor_api.c +++ b/src/feature/api/tor_api.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/api/tor_api.h b/src/feature/api/tor_api.h index 1ac9d892f2..2bf130c376 100644 --- a/src/feature/api/tor_api.h +++ b/src/feature/api/tor_api.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/api/tor_api_internal.h b/src/feature/api/tor_api_internal.h index 1e32012d01..60e0f3aa59 100644 --- a/src/feature/api/tor_api_internal.h +++ b/src/feature/api/tor_api_internal.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_API_INTERNAL_H diff --git a/src/feature/client/addressmap.c b/src/feature/client/addressmap.c index e62d82b7f3..bbe786a6a2 100644 --- a/src/feature/client/addressmap.c +++ b/src/feature/client/addressmap.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/client/addressmap.h b/src/feature/client/addressmap.h index b0db5c8b4e..9179aef1d0 100644 --- a/src/feature/client/addressmap.h +++ b/src/feature/client/addressmap.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_ADDRESSMAP_H diff --git a/src/feature/client/bridges.c b/src/feature/client/bridges.c index 5b537eb16b..05f89ad36c 100644 --- a/src/feature/client/bridges.c +++ b/src/feature/client/bridges.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/client/bridges.h b/src/feature/client/bridges.h index 70588c1b91..27b2750a45 100644 --- a/src/feature/client/bridges.h +++ b/src/feature/client/bridges.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/client/circpathbias.c b/src/feature/client/circpathbias.c index eaeef5fe5d..1743ab5a81 100644 --- a/src/feature/client/circpathbias.c +++ b/src/feature/client/circpathbias.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/client/circpathbias.h b/src/feature/client/circpathbias.h index 9ce4a6b23a..a9a8d18df2 100644 --- a/src/feature/client/circpathbias.h +++ b/src/feature/client/circpathbias.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/client/dnsserv.c b/src/feature/client/dnsserv.c index e5abe5c6aa..44e0caaafa 100644 --- a/src/feature/client/dnsserv.c +++ b/src/feature/client/dnsserv.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/client/dnsserv.h b/src/feature/client/dnsserv.h index afdde3a342..fff1ed2adb 100644 --- a/src/feature/client/dnsserv.h +++ b/src/feature/client/dnsserv.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/client/entrynodes.c b/src/feature/client/entrynodes.c index e0fe1b9a58..e543289ce0 100644 --- a/src/feature/client/entrynodes.c +++ b/src/feature/client/entrynodes.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/client/entrynodes.h b/src/feature/client/entrynodes.h index 5f9b5bdcba..4e5eb4e960 100644 --- a/src/feature/client/entrynodes.h +++ b/src/feature/client/entrynodes.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/client/transports.c b/src/feature/client/transports.c index 8a8bcd9f7f..72f1b7d9c8 100644 --- a/src/feature/client/transports.c +++ b/src/feature/client/transports.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2011-2018, The Tor Project, Inc. */ +/* Copyright (c) 2011-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/client/transports.h b/src/feature/client/transports.h index 1a910ae82c..900dd9288e 100644 --- a/src/feature/client/transports.h +++ b/src/feature/client/transports.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/control/btrack.c b/src/feature/control/btrack.c index 14220faad1..d3d12cb2b7 100644 --- a/src/feature/control/btrack.c +++ b/src/feature/control/btrack.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/control/btrack_circuit.c b/src/feature/control/btrack_circuit.c index bf09e0b99c..dcee9e460e 100644 --- a/src/feature/control/btrack_circuit.c +++ b/src/feature/control/btrack_circuit.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/control/btrack_circuit.h b/src/feature/control/btrack_circuit.h index ab8b8b652c..c40822f1f1 100644 --- a/src/feature/control/btrack_circuit.h +++ b/src/feature/control/btrack_circuit.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/control/btrack_orconn.c b/src/feature/control/btrack_orconn.c index 0fbf521000..93ebe8d9cc 100644 --- a/src/feature/control/btrack_orconn.c +++ b/src/feature/control/btrack_orconn.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/control/btrack_orconn.h b/src/feature/control/btrack_orconn.h index 4e514d4b04..6ab4892a78 100644 --- a/src/feature/control/btrack_orconn.h +++ b/src/feature/control/btrack_orconn.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/control/btrack_orconn_cevent.c b/src/feature/control/btrack_orconn_cevent.c index c7970dca4d..ee142f2873 100644 --- a/src/feature/control/btrack_orconn_cevent.c +++ b/src/feature/control/btrack_orconn_cevent.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/control/btrack_orconn_cevent.h b/src/feature/control/btrack_orconn_cevent.h index 165ff69cdb..f9d24633aa 100644 --- a/src/feature/control/btrack_orconn_cevent.h +++ b/src/feature/control/btrack_orconn_cevent.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/control/btrack_orconn_maps.c b/src/feature/control/btrack_orconn_maps.c index b6bb23804c..e64bd3f0fe 100644 --- a/src/feature/control/btrack_orconn_maps.c +++ b/src/feature/control/btrack_orconn_maps.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/control/btrack_orconn_maps.h b/src/feature/control/btrack_orconn_maps.h index b1c2c7aa08..3ead40984c 100644 --- a/src/feature/control/btrack_orconn_maps.h +++ b/src/feature/control/btrack_orconn_maps.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/control/btrack_sys.h b/src/feature/control/btrack_sys.h index f80cf342e7..fad35b41db 100644 --- a/src/feature/control/btrack_sys.h +++ b/src/feature/control/btrack_sys.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/control/control.c b/src/feature/control/control.c index 170037e060..6f8cd8f0aa 100644 --- a/src/feature/control/control.c +++ b/src/feature/control/control.c @@ -1,6 +1,6 @@ /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/control/control.h b/src/feature/control/control.h index 08d8924e2d..b2ab4c1997 100644 --- a/src/feature/control/control.h +++ b/src/feature/control/control.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/control/control_bootstrap.c b/src/feature/control/control_bootstrap.c index d7a0438513..a20a94ba89 100644 --- a/src/feature/control/control_bootstrap.c +++ b/src/feature/control/control_bootstrap.c @@ -1,5 +1,5 @@ /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/control/control_connection_st.h b/src/feature/control/control_connection_st.h index ff6264a9a5..177a916257 100644 --- a/src/feature/control/control_connection_st.h +++ b/src/feature/control/control_connection_st.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef CONTROL_CONNECTION_ST_H diff --git a/src/feature/control/fmt_serverstatus.c b/src/feature/control/fmt_serverstatus.c index eef85d356c..a1ddd2119a 100644 --- a/src/feature/control/fmt_serverstatus.c +++ b/src/feature/control/fmt_serverstatus.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "core/or/or.h" diff --git a/src/feature/control/fmt_serverstatus.h b/src/feature/control/fmt_serverstatus.h index 2ae9c1778a..4b95e5b59f 100644 --- a/src/feature/control/fmt_serverstatus.h +++ b/src/feature/control/fmt_serverstatus.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/control/getinfo_geoip.h b/src/feature/control/getinfo_geoip.h index ff77cefecd..fe22137859 100644 --- a/src/feature/control/getinfo_geoip.h +++ b/src/feature/control/getinfo_geoip.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_GETINFO_GEOIP_H diff --git a/src/feature/dirauth/authmode.c b/src/feature/dirauth/authmode.c index 7c900ea7bf..29fcc6d1a9 100644 --- a/src/feature/dirauth/authmode.c +++ b/src/feature/dirauth/authmode.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/dirauth/authmode.h b/src/feature/dirauth/authmode.h index 3ca127b829..40a89c7397 100644 --- a/src/feature/dirauth/authmode.h +++ b/src/feature/dirauth/authmode.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2018-2018, The Tor Project, Inc. */ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/dirauth/bwauth.c b/src/feature/dirauth/bwauth.c index 29e0c90962..a31050ff9c 100644 --- a/src/feature/dirauth/bwauth.c +++ b/src/feature/dirauth/bwauth.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/dirauth/bwauth.h b/src/feature/dirauth/bwauth.h index f10f8227af..4507728458 100644 --- a/src/feature/dirauth/bwauth.h +++ b/src/feature/dirauth/bwauth.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/dirauth/dircollate.c b/src/feature/dirauth/dircollate.c index ca8e5b7873..7992e3a85f 100644 --- a/src/feature/dirauth/dircollate.c +++ b/src/feature/dirauth/dircollate.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/dirauth/dircollate.h b/src/feature/dirauth/dircollate.h index 0e84c66e6f..754a094817 100644 --- a/src/feature/dirauth/dircollate.h +++ b/src/feature/dirauth/dircollate.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/dirauth/dirvote.c b/src/feature/dirauth/dirvote.c index d8a18835f2..9587f36550 100644 --- a/src/feature/dirauth/dirvote.c +++ b/src/feature/dirauth/dirvote.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define DIRVOTE_PRIVATE diff --git a/src/feature/dirauth/dirvote.h b/src/feature/dirauth/dirvote.h index 85f4800700..f9de5ebc41 100644 --- a/src/feature/dirauth/dirvote.h +++ b/src/feature/dirauth/dirvote.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/dirauth/dsigs_parse.c b/src/feature/dirauth/dsigs_parse.c index b0c407567b..d88176fee9 100644 --- a/src/feature/dirauth/dsigs_parse.c +++ b/src/feature/dirauth/dsigs_parse.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/dirauth/dsigs_parse.h b/src/feature/dirauth/dsigs_parse.h index ffb2ac0361..fec51ba488 100644 --- a/src/feature/dirauth/dsigs_parse.h +++ b/src/feature/dirauth/dsigs_parse.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/dirauth/guardfraction.c b/src/feature/dirauth/guardfraction.c index 1734d0a13c..d1a7f194d4 100644 --- a/src/feature/dirauth/guardfraction.c +++ b/src/feature/dirauth/guardfraction.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/dirauth/guardfraction.h b/src/feature/dirauth/guardfraction.h index 38a0781dbb..72404907a4 100644 --- a/src/feature/dirauth/guardfraction.h +++ b/src/feature/dirauth/guardfraction.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/dirauth/keypin.c b/src/feature/dirauth/keypin.c index fd281377df..667feb2c03 100644 --- a/src/feature/dirauth/keypin.c +++ b/src/feature/dirauth/keypin.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2014-2018, The Tor Project, Inc. */ +/* Copyright (c) 2014-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/dirauth/keypin.h b/src/feature/dirauth/keypin.h index 73a76be563..722b6ca5fc 100644 --- a/src/feature/dirauth/keypin.h +++ b/src/feature/dirauth/keypin.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2014-2018, The Tor Project, Inc. */ +/* Copyright (c) 2014-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_KEYPIN_H diff --git a/src/feature/dirauth/ns_detached_signatures_st.h b/src/feature/dirauth/ns_detached_signatures_st.h index 26ceec84b9..0f92be2f0d 100644 --- a/src/feature/dirauth/ns_detached_signatures_st.h +++ b/src/feature/dirauth/ns_detached_signatures_st.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef NS_DETACHED_SIGNATURES_ST_H diff --git a/src/feature/dirauth/process_descs.c b/src/feature/dirauth/process_descs.c index dca87b3eaf..656922233e 100644 --- a/src/feature/dirauth/process_descs.c +++ b/src/feature/dirauth/process_descs.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/dirauth/process_descs.h b/src/feature/dirauth/process_descs.h index 5a0914acd8..510e54f813 100644 --- a/src/feature/dirauth/process_descs.h +++ b/src/feature/dirauth/process_descs.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/dirauth/reachability.c b/src/feature/dirauth/reachability.c index 122c239f9a..883b692cbb 100644 --- a/src/feature/dirauth/reachability.c +++ b/src/feature/dirauth/reachability.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/dirauth/reachability.h b/src/feature/dirauth/reachability.h index 6e4bf28ca9..5a938673ff 100644 --- a/src/feature/dirauth/reachability.h +++ b/src/feature/dirauth/reachability.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/dirauth/recommend_pkg.c b/src/feature/dirauth/recommend_pkg.c index 41c091455e..0456ff8463 100644 --- a/src/feature/dirauth/recommend_pkg.c +++ b/src/feature/dirauth/recommend_pkg.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/dirauth/recommend_pkg.h b/src/feature/dirauth/recommend_pkg.h index 29a41d6dff..8200d78f72 100644 --- a/src/feature/dirauth/recommend_pkg.h +++ b/src/feature/dirauth/recommend_pkg.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/dirauth/shared_random.c b/src/feature/dirauth/shared_random.c index b027d9e375..34b2283250 100644 --- a/src/feature/dirauth/shared_random.c +++ b/src/feature/dirauth/shared_random.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2016-2018, The Tor Project, Inc. */ +/* Copyright (c) 2016-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/dirauth/shared_random.h b/src/feature/dirauth/shared_random.h index 68ece9aec0..25d95ebbc7 100644 --- a/src/feature/dirauth/shared_random.h +++ b/src/feature/dirauth/shared_random.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2016-2018, The Tor Project, Inc. */ +/* Copyright (c) 2016-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_SHARED_RANDOM_H diff --git a/src/feature/dirauth/shared_random_state.c b/src/feature/dirauth/shared_random_state.c index 8c5b28b7cd..92f0b3e737 100644 --- a/src/feature/dirauth/shared_random_state.c +++ b/src/feature/dirauth/shared_random_state.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2016-2018, The Tor Project, Inc. */ +/* Copyright (c) 2016-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/dirauth/shared_random_state.h b/src/feature/dirauth/shared_random_state.h index 83edfaf103..35626be3f6 100644 --- a/src/feature/dirauth/shared_random_state.h +++ b/src/feature/dirauth/shared_random_state.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2016-2018, The Tor Project, Inc. */ +/* Copyright (c) 2016-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_SHARED_RANDOM_STATE_H diff --git a/src/feature/dirauth/vote_microdesc_hash_st.h b/src/feature/dirauth/vote_microdesc_hash_st.h index 31fc98040e..92acdf1157 100644 --- a/src/feature/dirauth/vote_microdesc_hash_st.h +++ b/src/feature/dirauth/vote_microdesc_hash_st.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef VOTE_MICRODESC_HASH_ST_H diff --git a/src/feature/dirauth/voteflags.c b/src/feature/dirauth/voteflags.c index aab322d96f..4f7593a3e1 100644 --- a/src/feature/dirauth/voteflags.c +++ b/src/feature/dirauth/voteflags.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/dirauth/voteflags.h b/src/feature/dirauth/voteflags.h index 8dce9fbb04..cca6f53746 100644 --- a/src/feature/dirauth/voteflags.h +++ b/src/feature/dirauth/voteflags.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/dircache/cached_dir_st.h b/src/feature/dircache/cached_dir_st.h index 38ae86d975..71dca8c3a2 100644 --- a/src/feature/dircache/cached_dir_st.h +++ b/src/feature/dircache/cached_dir_st.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef CACHED_DIR_ST_H diff --git a/src/feature/dircache/conscache.c b/src/feature/dircache/conscache.c index e9bf58a180..cf4fe8701d 100644 --- a/src/feature/dircache/conscache.c +++ b/src/feature/dircache/conscache.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2017-2018, The Tor Project, Inc. */ +/* Copyright (c) 2017-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "core/or/or.h" diff --git a/src/feature/dircache/conscache.h b/src/feature/dircache/conscache.h index c274a60393..d848e57617 100644 --- a/src/feature/dircache/conscache.h +++ b/src/feature/dircache/conscache.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2017-2018, The Tor Project, Inc. */ +/* Copyright (c) 2017-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_CONSCACHE_H diff --git a/src/feature/dircache/consdiffmgr.c b/src/feature/dircache/consdiffmgr.c index 8ecab5ca69..6b16307e3c 100644 --- a/src/feature/dircache/consdiffmgr.c +++ b/src/feature/dircache/consdiffmgr.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2017-2018, The Tor Project, Inc. */ +/* Copyright (c) 2017-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/dircache/consdiffmgr.h b/src/feature/dircache/consdiffmgr.h index 011c8799d6..b1b3323b6c 100644 --- a/src/feature/dircache/consdiffmgr.h +++ b/src/feature/dircache/consdiffmgr.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2017-2018, The Tor Project, Inc. */ +/* Copyright (c) 2017-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_CONSDIFFMGR_H diff --git a/src/feature/dircache/dircache.c b/src/feature/dircache/dircache.c index 8b89d05e98..f6e57c5064 100644 --- a/src/feature/dircache/dircache.c +++ b/src/feature/dircache/dircache.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define DIRCACHE_PRIVATE diff --git a/src/feature/dircache/dircache.h b/src/feature/dircache/dircache.h index f05780375a..236ea649ef 100644 --- a/src/feature/dircache/dircache.h +++ b/src/feature/dircache/dircache.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/dircache/dirserv.c b/src/feature/dircache/dirserv.c index 4366000e2e..4be6836fe1 100644 --- a/src/feature/dircache/dirserv.c +++ b/src/feature/dircache/dirserv.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define DIRSERV_PRIVATE diff --git a/src/feature/dircache/dirserv.h b/src/feature/dircache/dirserv.h index aa1e2494ca..7f944459da 100644 --- a/src/feature/dircache/dirserv.h +++ b/src/feature/dircache/dirserv.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/dirclient/dir_server_st.h b/src/feature/dirclient/dir_server_st.h index 0a6d8155ae..2f5706cdd9 100644 --- a/src/feature/dirclient/dir_server_st.h +++ b/src/feature/dirclient/dir_server_st.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef DIR_SERVER_ST_H diff --git a/src/feature/dirclient/dirclient.c b/src/feature/dirclient/dirclient.c index cd88fa5ebf..70b6a20028 100644 --- a/src/feature/dirclient/dirclient.c +++ b/src/feature/dirclient/dirclient.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define DIRCLIENT_PRIVATE diff --git a/src/feature/dirclient/dirclient.h b/src/feature/dirclient/dirclient.h index 6bbff2f846..1a93265dc3 100644 --- a/src/feature/dirclient/dirclient.h +++ b/src/feature/dirclient/dirclient.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/dirclient/dlstatus.c b/src/feature/dirclient/dlstatus.c index aea17bdacb..0842a2c676 100644 --- a/src/feature/dirclient/dlstatus.c +++ b/src/feature/dirclient/dlstatus.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define DLSTATUS_PRIVATE diff --git a/src/feature/dirclient/dlstatus.h b/src/feature/dirclient/dlstatus.h index aeceb8cb22..99e0d0225b 100644 --- a/src/feature/dirclient/dlstatus.h +++ b/src/feature/dirclient/dlstatus.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/dirclient/download_status_st.h b/src/feature/dirclient/download_status_st.h index 3f18f754a1..11555a1dcc 100644 --- a/src/feature/dirclient/download_status_st.h +++ b/src/feature/dirclient/download_status_st.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef DOWNLOAD_STATUS_ST_H diff --git a/src/feature/dircommon/consdiff.c b/src/feature/dircommon/consdiff.c index 15266f77bf..698f548013 100644 --- a/src/feature/dircommon/consdiff.c +++ b/src/feature/dircommon/consdiff.c @@ -1,5 +1,5 @@ /* Copyright (c) 2014, Daniel Martí - * Copyright (c) 2014-2018, The Tor Project, Inc. */ + * Copyright (c) 2014-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/dircommon/consdiff.h b/src/feature/dircommon/consdiff.h index eb7c9f9fe0..b63fcb2cc6 100644 --- a/src/feature/dircommon/consdiff.h +++ b/src/feature/dircommon/consdiff.h @@ -1,5 +1,5 @@ /* Copyright (c) 2014, Daniel Martí - * Copyright (c) 2014-2018, The Tor Project, Inc. */ + * Copyright (c) 2014-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_CONSDIFF_H diff --git a/src/feature/dircommon/dir_connection_st.h b/src/feature/dircommon/dir_connection_st.h index 768f6ba81e..8c59cc7a46 100644 --- a/src/feature/dircommon/dir_connection_st.h +++ b/src/feature/dircommon/dir_connection_st.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef DIR_CONNECTION_ST_H diff --git a/src/feature/dircommon/directory.c b/src/feature/dircommon/directory.c index dff68fcf0e..9e6f72e9ac 100644 --- a/src/feature/dircommon/directory.c +++ b/src/feature/dircommon/directory.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "core/or/or.h" diff --git a/src/feature/dircommon/directory.h b/src/feature/dircommon/directory.h index ec95573f51..ba3f8c1b0e 100644 --- a/src/feature/dircommon/directory.h +++ b/src/feature/dircommon/directory.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/dircommon/fp_pair.c b/src/feature/dircommon/fp_pair.c index 0544145284..284600df77 100644 --- a/src/feature/dircommon/fp_pair.c +++ b/src/feature/dircommon/fp_pair.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2018, The Tor Project, Inc. */ +/* Copyright (c) 2013-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/dircommon/fp_pair.h b/src/feature/dircommon/fp_pair.h index 500c7c9928..5041583e88 100644 --- a/src/feature/dircommon/fp_pair.h +++ b/src/feature/dircommon/fp_pair.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2018, The Tor Project, Inc. */ +/* Copyright (c) 2013-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/dircommon/vote_timing_st.h b/src/feature/dircommon/vote_timing_st.h index 14c13eed28..47b90ab009 100644 --- a/src/feature/dircommon/vote_timing_st.h +++ b/src/feature/dircommon/vote_timing_st.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef VOTE_TIMING_ST_H diff --git a/src/feature/dircommon/voting_schedule.c b/src/feature/dircommon/voting_schedule.c index 07e65ef06d..0a7476eda7 100644 --- a/src/feature/dircommon/voting_schedule.c +++ b/src/feature/dircommon/voting_schedule.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2018-2018, The Tor Project, Inc. */ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/dircommon/voting_schedule.h b/src/feature/dircommon/voting_schedule.h index 0e0b0cc988..bafd81184e 100644 --- a/src/feature/dircommon/voting_schedule.h +++ b/src/feature/dircommon/voting_schedule.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2018-2018, The Tor Project, Inc. */ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/dirparse/authcert_parse.c b/src/feature/dirparse/authcert_parse.c index 334baf8b1a..8ba5a53981 100644 --- a/src/feature/dirparse/authcert_parse.c +++ b/src/feature/dirparse/authcert_parse.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "core/or/or.h" diff --git a/src/feature/dirparse/authcert_parse.h b/src/feature/dirparse/authcert_parse.h index e4e9fec993..800631c3de 100644 --- a/src/feature/dirparse/authcert_parse.h +++ b/src/feature/dirparse/authcert_parse.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/dirparse/microdesc_parse.c b/src/feature/dirparse/microdesc_parse.c index 165945e392..3b11e65ca0 100644 --- a/src/feature/dirparse/microdesc_parse.c +++ b/src/feature/dirparse/microdesc_parse.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/dirparse/microdesc_parse.h b/src/feature/dirparse/microdesc_parse.h index e8f8b2e17e..23a90084b1 100644 --- a/src/feature/dirparse/microdesc_parse.h +++ b/src/feature/dirparse/microdesc_parse.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/dirparse/ns_parse.c b/src/feature/dirparse/ns_parse.c index e0cdb2d46d..d653a59826 100644 --- a/src/feature/dirparse/ns_parse.c +++ b/src/feature/dirparse/ns_parse.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/dirparse/ns_parse.h b/src/feature/dirparse/ns_parse.h index 85d9ded685..dedfa6fc88 100644 --- a/src/feature/dirparse/ns_parse.h +++ b/src/feature/dirparse/ns_parse.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/dirparse/parsecommon.c b/src/feature/dirparse/parsecommon.c index 88df5eec6f..036a51689c 100644 --- a/src/feature/dirparse/parsecommon.c +++ b/src/feature/dirparse/parsecommon.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2016-2018, The Tor Project, Inc. */ +/* Copyright (c) 2016-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/dirparse/parsecommon.h b/src/feature/dirparse/parsecommon.h index f14862f04a..ef74925b26 100644 --- a/src/feature/dirparse/parsecommon.h +++ b/src/feature/dirparse/parsecommon.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2016-2018, The Tor Project, Inc. */ +/* Copyright (c) 2016-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/dirparse/policy_parse.c b/src/feature/dirparse/policy_parse.c index f9102dd873..7562ae409b 100644 --- a/src/feature/dirparse/policy_parse.c +++ b/src/feature/dirparse/policy_parse.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/dirparse/policy_parse.h b/src/feature/dirparse/policy_parse.h index 887aa9261b..e09ee5559f 100644 --- a/src/feature/dirparse/policy_parse.h +++ b/src/feature/dirparse/policy_parse.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/dirparse/routerparse.c b/src/feature/dirparse/routerparse.c index 358f6e44e8..ff7e15f1f2 100644 --- a/src/feature/dirparse/routerparse.c +++ b/src/feature/dirparse/routerparse.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/dirparse/routerparse.h b/src/feature/dirparse/routerparse.h index 6568dce0a4..f9a13f2168 100644 --- a/src/feature/dirparse/routerparse.h +++ b/src/feature/dirparse/routerparse.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/dirparse/sigcommon.c b/src/feature/dirparse/sigcommon.c index 28e6ff56ed..2019e09918 100644 --- a/src/feature/dirparse/sigcommon.c +++ b/src/feature/dirparse/sigcommon.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/dirparse/sigcommon.h b/src/feature/dirparse/sigcommon.h index 5f25817cdb..fdd8e839a9 100644 --- a/src/feature/dirparse/sigcommon.h +++ b/src/feature/dirparse/sigcommon.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/dirparse/signing.c b/src/feature/dirparse/signing.c index 8d6a40605b..3ab40c3807 100644 --- a/src/feature/dirparse/signing.c +++ b/src/feature/dirparse/signing.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/dirparse/signing.h b/src/feature/dirparse/signing.h index 2b547a185f..2e3699baf8 100644 --- a/src/feature/dirparse/signing.h +++ b/src/feature/dirparse/signing.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/dirparse/unparseable.c b/src/feature/dirparse/unparseable.c index 80e38d0703..941b5a1f6d 100644 --- a/src/feature/dirparse/unparseable.c +++ b/src/feature/dirparse/unparseable.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define UNPARSEABLE_PRIVATE diff --git a/src/feature/dirparse/unparseable.h b/src/feature/dirparse/unparseable.h index 2e48c6a9a0..853fe8cb0f 100644 --- a/src/feature/dirparse/unparseable.h +++ b/src/feature/dirparse/unparseable.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/hibernate/hibernate.c b/src/feature/hibernate/hibernate.c index f10a45f4ae..70c2b4f69f 100644 --- a/src/feature/hibernate/hibernate.c +++ b/src/feature/hibernate/hibernate.c @@ -1,5 +1,5 @@ /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/hibernate/hibernate.h b/src/feature/hibernate/hibernate.h index bfd8571cd6..3309ef0ce3 100644 --- a/src/feature/hibernate/hibernate.h +++ b/src/feature/hibernate/hibernate.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/hs/hs_cache.c b/src/feature/hs/hs_cache.c index afd69e1bec..82ce686424 100644 --- a/src/feature/hs/hs_cache.c +++ b/src/feature/hs/hs_cache.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2016-2018, The Tor Project, Inc. */ +/* Copyright (c) 2016-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/hs/hs_cache.h b/src/feature/hs/hs_cache.h index 7cd4995d2c..079d31d437 100644 --- a/src/feature/hs/hs_cache.h +++ b/src/feature/hs/hs_cache.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2016-2018, The Tor Project, Inc. */ +/* Copyright (c) 2016-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/hs/hs_cell.c b/src/feature/hs/hs_cell.c index 9bbae6d325..597982b34e 100644 --- a/src/feature/hs/hs_cell.c +++ b/src/feature/hs/hs_cell.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2017-2018, The Tor Project, Inc. */ +/* Copyright (c) 2017-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/hs/hs_cell.h b/src/feature/hs/hs_cell.h index 7b9d7e5792..abdaba4fba 100644 --- a/src/feature/hs/hs_cell.h +++ b/src/feature/hs/hs_cell.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2017-2018, The Tor Project, Inc. */ +/* Copyright (c) 2017-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/hs/hs_circuit.c b/src/feature/hs/hs_circuit.c index 092781d7ed..e3873d2f18 100644 --- a/src/feature/hs/hs_circuit.c +++ b/src/feature/hs/hs_circuit.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2017-2018, The Tor Project, Inc. */ +/* Copyright (c) 2017-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/hs/hs_circuit.h b/src/feature/hs/hs_circuit.h index 54f28a39ab..b8d8b25add 100644 --- a/src/feature/hs/hs_circuit.h +++ b/src/feature/hs/hs_circuit.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2017-2018, The Tor Project, Inc. */ +/* Copyright (c) 2017-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/hs/hs_circuitmap.c b/src/feature/hs/hs_circuitmap.c index 962a421a00..5480d5eb84 100644 --- a/src/feature/hs/hs_circuitmap.c +++ b/src/feature/hs/hs_circuitmap.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2016-2018, The Tor Project, Inc. */ +/* Copyright (c) 2016-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/hs/hs_circuitmap.h b/src/feature/hs/hs_circuitmap.h index c39a37c052..c1bbb1ff1c 100644 --- a/src/feature/hs/hs_circuitmap.h +++ b/src/feature/hs/hs_circuitmap.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2016-2018, The Tor Project, Inc. */ +/* Copyright (c) 2016-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/hs/hs_client.c b/src/feature/hs/hs_client.c index 5fded92fe3..d4eee50bb7 100644 --- a/src/feature/hs/hs_client.c +++ b/src/feature/hs/hs_client.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2016-2018, The Tor Project, Inc. */ +/* Copyright (c) 2016-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/hs/hs_client.h b/src/feature/hs/hs_client.h index f6fb167ea2..dadfa024b8 100644 --- a/src/feature/hs/hs_client.h +++ b/src/feature/hs/hs_client.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2017-2018, The Tor Project, Inc. */ +/* Copyright (c) 2017-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/hs/hs_common.c b/src/feature/hs/hs_common.c index 8dbd9485ea..ebe49f09a5 100644 --- a/src/feature/hs/hs_common.c +++ b/src/feature/hs/hs_common.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2016-2018, The Tor Project, Inc. */ +/* Copyright (c) 2016-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/hs/hs_common.h b/src/feature/hs/hs_common.h index 888eb0a4ec..a44505930a 100644 --- a/src/feature/hs/hs_common.h +++ b/src/feature/hs/hs_common.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2016-2018, The Tor Project, Inc. */ +/* Copyright (c) 2016-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/hs/hs_config.c b/src/feature/hs/hs_config.c index 497e31fbb4..ee4499ef5b 100644 --- a/src/feature/hs/hs_config.c +++ b/src/feature/hs/hs_config.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2017-2018, The Tor Project, Inc. */ +/* Copyright (c) 2017-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/hs/hs_config.h b/src/feature/hs/hs_config.h index f443e814c4..040e451f13 100644 --- a/src/feature/hs/hs_config.h +++ b/src/feature/hs/hs_config.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2016-2018, The Tor Project, Inc. */ +/* Copyright (c) 2016-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/hs/hs_control.c b/src/feature/hs/hs_control.c index a21788ecd7..9970fdd123 100644 --- a/src/feature/hs/hs_control.c +++ b/src/feature/hs/hs_control.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2017-2018, The Tor Project, Inc. */ +/* Copyright (c) 2017-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/hs/hs_control.h b/src/feature/hs/hs_control.h index 63e3fe13d6..f7ab642652 100644 --- a/src/feature/hs/hs_control.h +++ b/src/feature/hs/hs_control.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2017-2018, The Tor Project, Inc. */ +/* Copyright (c) 2017-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/hs/hs_descriptor.c b/src/feature/hs/hs_descriptor.c index 1b2008c804..b09d50e010 100644 --- a/src/feature/hs/hs_descriptor.c +++ b/src/feature/hs/hs_descriptor.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2016-2018, The Tor Project, Inc. */ +/* Copyright (c) 2016-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/hs/hs_descriptor.h b/src/feature/hs/hs_descriptor.h index adfb94deaa..04a8e16d63 100644 --- a/src/feature/hs/hs_descriptor.h +++ b/src/feature/hs/hs_descriptor.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2016-2018, The Tor Project, Inc. */ +/* Copyright (c) 2016-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/hs/hs_ident.c b/src/feature/hs/hs_ident.c index c6ef8c2ce3..8fd0013941 100644 --- a/src/feature/hs/hs_ident.c +++ b/src/feature/hs/hs_ident.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2017-2018, The Tor Project, Inc. */ +/* Copyright (c) 2017-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/hs/hs_ident.h b/src/feature/hs/hs_ident.h index ab87d16d17..8c46936a1e 100644 --- a/src/feature/hs/hs_ident.h +++ b/src/feature/hs/hs_ident.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2017-2018, The Tor Project, Inc. */ +/* Copyright (c) 2017-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/hs/hs_intropoint.c b/src/feature/hs/hs_intropoint.c index 2ea53af6a0..b28a5c2b80 100644 --- a/src/feature/hs/hs_intropoint.c +++ b/src/feature/hs/hs_intropoint.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2016-2018, The Tor Project, Inc. */ +/* Copyright (c) 2016-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/hs/hs_intropoint.h b/src/feature/hs/hs_intropoint.h index 562836fb07..659a9ad052 100644 --- a/src/feature/hs/hs_intropoint.h +++ b/src/feature/hs/hs_intropoint.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2016-2018, The Tor Project, Inc. */ +/* Copyright (c) 2016-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/hs/hs_service.c b/src/feature/hs/hs_service.c index 623a239d50..b94dd9a481 100644 --- a/src/feature/hs/hs_service.c +++ b/src/feature/hs/hs_service.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2016-2018, The Tor Project, Inc. */ +/* Copyright (c) 2016-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/hs/hs_service.h b/src/feature/hs/hs_service.h index be1155bcd1..ec53f2f23b 100644 --- a/src/feature/hs/hs_service.h +++ b/src/feature/hs/hs_service.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2016-2018, The Tor Project, Inc. */ +/* Copyright (c) 2016-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/hs/hs_stats.c b/src/feature/hs/hs_stats.c index b109a37cc1..f24b731328 100644 --- a/src/feature/hs/hs_stats.c +++ b/src/feature/hs/hs_stats.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2016-2018, The Tor Project, Inc. */ +/* Copyright (c) 2016-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/hs/hs_stats.h b/src/feature/hs/hs_stats.h index a946ad75e5..d89440faca 100644 --- a/src/feature/hs/hs_stats.h +++ b/src/feature/hs/hs_stats.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2016-2018, The Tor Project, Inc. */ +/* Copyright (c) 2016-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/hs/hsdir_index_st.h b/src/feature/hs/hsdir_index_st.h index de5cc9bd16..7d4116d8bb 100644 --- a/src/feature/hs/hsdir_index_st.h +++ b/src/feature/hs/hsdir_index_st.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef HSDIR_INDEX_ST_H diff --git a/src/feature/hs_common/replaycache.c b/src/feature/hs_common/replaycache.c index 1d3f20e819..9e8c13b1c5 100644 --- a/src/feature/hs_common/replaycache.c +++ b/src/feature/hs_common/replaycache.c @@ -1,4 +1,4 @@ - /* Copyright (c) 2012-2018, The Tor Project, Inc. */ + /* Copyright (c) 2012-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/hs_common/replaycache.h b/src/feature/hs_common/replaycache.h index 3118a88a1a..01f5e600c2 100644 --- a/src/feature/hs_common/replaycache.h +++ b/src/feature/hs_common/replaycache.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2012-2018, The Tor Project, Inc. */ +/* Copyright (c) 2012-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/hs_common/shared_random_client.c b/src/feature/hs_common/shared_random_client.c index a13404a329..5772034c6d 100644 --- a/src/feature/hs_common/shared_random_client.c +++ b/src/feature/hs_common/shared_random_client.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2018-2018, The Tor Project, Inc. */ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/hs_common/shared_random_client.h b/src/feature/hs_common/shared_random_client.h index 0e26f530a4..95fe2c65ab 100644 --- a/src/feature/hs_common/shared_random_client.h +++ b/src/feature/hs_common/shared_random_client.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2018-2018, The Tor Project, Inc. */ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/keymgt/loadkey.c b/src/feature/keymgt/loadkey.c index 4621e39c54..a8cbf0e582 100644 --- a/src/feature/keymgt/loadkey.c +++ b/src/feature/keymgt/loadkey.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/keymgt/loadkey.h b/src/feature/keymgt/loadkey.h index 7717bda29e..8beee57a20 100644 --- a/src/feature/keymgt/loadkey.h +++ b/src/feature/keymgt/loadkey.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/nodelist/authcert.c b/src/feature/nodelist/authcert.c index 2c4915e913..9fc3b62525 100644 --- a/src/feature/nodelist/authcert.c +++ b/src/feature/nodelist/authcert.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/nodelist/authcert.h b/src/feature/nodelist/authcert.h index 48326d7bd0..2effdb06e6 100644 --- a/src/feature/nodelist/authcert.h +++ b/src/feature/nodelist/authcert.h @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/nodelist/authority_cert_st.h b/src/feature/nodelist/authority_cert_st.h index c2846548c4..68a84bc452 100644 --- a/src/feature/nodelist/authority_cert_st.h +++ b/src/feature/nodelist/authority_cert_st.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef AUTHORITY_CERT_ST_H diff --git a/src/feature/nodelist/desc_store_st.h b/src/feature/nodelist/desc_store_st.h index 168a83b230..b04a1abc7d 100644 --- a/src/feature/nodelist/desc_store_st.h +++ b/src/feature/nodelist/desc_store_st.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef DESC_STORE_ST_H diff --git a/src/feature/nodelist/describe.c b/src/feature/nodelist/describe.c index 6df3da1965..5c376408c0 100644 --- a/src/feature/nodelist/describe.c +++ b/src/feature/nodelist/describe.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/nodelist/describe.h b/src/feature/nodelist/describe.h index e5723bb933..018af6470e 100644 --- a/src/feature/nodelist/describe.h +++ b/src/feature/nodelist/describe.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/nodelist/dirlist.c b/src/feature/nodelist/dirlist.c index c14d7df0f0..93baa6e4e0 100644 --- a/src/feature/nodelist/dirlist.c +++ b/src/feature/nodelist/dirlist.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/nodelist/dirlist.h b/src/feature/nodelist/dirlist.h index 6baa5686c5..9fabd0a44a 100644 --- a/src/feature/nodelist/dirlist.h +++ b/src/feature/nodelist/dirlist.h @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/nodelist/document_signature_st.h b/src/feature/nodelist/document_signature_st.h index 0291e099bf..66e32c422f 100644 --- a/src/feature/nodelist/document_signature_st.h +++ b/src/feature/nodelist/document_signature_st.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef DOCUMENT_SIGNATURE_ST_H diff --git a/src/feature/nodelist/extrainfo_st.h b/src/feature/nodelist/extrainfo_st.h index f5d977e751..c54277b05e 100644 --- a/src/feature/nodelist/extrainfo_st.h +++ b/src/feature/nodelist/extrainfo_st.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef EXTRAINFO_ST_H diff --git a/src/feature/nodelist/fmt_routerstatus.c b/src/feature/nodelist/fmt_routerstatus.c index b1d4a48038..8c9212e05c 100644 --- a/src/feature/nodelist/fmt_routerstatus.c +++ b/src/feature/nodelist/fmt_routerstatus.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/nodelist/fmt_routerstatus.h b/src/feature/nodelist/fmt_routerstatus.h index 1a6630d266..ddd7a7cf37 100644 --- a/src/feature/nodelist/fmt_routerstatus.h +++ b/src/feature/nodelist/fmt_routerstatus.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/nodelist/microdesc.c b/src/feature/nodelist/microdesc.c index 82070afb98..b4f05b63a0 100644 --- a/src/feature/nodelist/microdesc.c +++ b/src/feature/nodelist/microdesc.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2009-2018, The Tor Project, Inc. */ +/* Copyright (c) 2009-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/nodelist/microdesc.h b/src/feature/nodelist/microdesc.h index f11b841cf1..c18099d540 100644 --- a/src/feature/nodelist/microdesc.h +++ b/src/feature/nodelist/microdesc.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/nodelist/microdesc_st.h b/src/feature/nodelist/microdesc_st.h index 30c896181d..367e6a3ef6 100644 --- a/src/feature/nodelist/microdesc_st.h +++ b/src/feature/nodelist/microdesc_st.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef MICRODESC_ST_H diff --git a/src/feature/nodelist/networkstatus.c b/src/feature/nodelist/networkstatus.c index b9c142787a..d9659b67c0 100644 --- a/src/feature/nodelist/networkstatus.c +++ b/src/feature/nodelist/networkstatus.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/nodelist/networkstatus.h b/src/feature/nodelist/networkstatus.h index 572b42cc5a..8269fc6182 100644 --- a/src/feature/nodelist/networkstatus.h +++ b/src/feature/nodelist/networkstatus.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/nodelist/networkstatus_sr_info_st.h b/src/feature/nodelist/networkstatus_sr_info_st.h index 6c937a75f5..677d8ed811 100644 --- a/src/feature/nodelist/networkstatus_sr_info_st.h +++ b/src/feature/nodelist/networkstatus_sr_info_st.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef NETWORKSTATUS_SR_INFO_ST_H diff --git a/src/feature/nodelist/networkstatus_st.h b/src/feature/nodelist/networkstatus_st.h index 2bb0e3ae35..6160f12361 100644 --- a/src/feature/nodelist/networkstatus_st.h +++ b/src/feature/nodelist/networkstatus_st.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef NETWORKSTATUS_ST_H diff --git a/src/feature/nodelist/networkstatus_voter_info_st.h b/src/feature/nodelist/networkstatus_voter_info_st.h index 93ff3cd418..4037fcdeca 100644 --- a/src/feature/nodelist/networkstatus_voter_info_st.h +++ b/src/feature/nodelist/networkstatus_voter_info_st.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef NETWORKSTATUS_VOTER_INFO_ST_H diff --git a/src/feature/nodelist/nickname.c b/src/feature/nodelist/nickname.c index 7b0b29a934..5378b749ca 100644 --- a/src/feature/nodelist/nickname.c +++ b/src/feature/nodelist/nickname.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/nodelist/nickname.h b/src/feature/nodelist/nickname.h index 86d4309918..9bdc6b50e8 100644 --- a/src/feature/nodelist/nickname.h +++ b/src/feature/nodelist/nickname.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/nodelist/node_select.c b/src/feature/nodelist/node_select.c index 04a24de9a1..e31abb247f 100644 --- a/src/feature/nodelist/node_select.c +++ b/src/feature/nodelist/node_select.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/nodelist/node_select.h b/src/feature/nodelist/node_select.h index 05dabd1234..ed7450b92c 100644 --- a/src/feature/nodelist/node_select.h +++ b/src/feature/nodelist/node_select.h @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/nodelist/node_st.h b/src/feature/nodelist/node_st.h index 8d182050ac..53ffde29e4 100644 --- a/src/feature/nodelist/node_st.h +++ b/src/feature/nodelist/node_st.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef NODE_ST_H diff --git a/src/feature/nodelist/nodefamily.c b/src/feature/nodelist/nodefamily.c index 944ad54755..2ec9d5fa40 100644 --- a/src/feature/nodelist/nodefamily.c +++ b/src/feature/nodelist/nodefamily.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/nodelist/nodefamily.h b/src/feature/nodelist/nodefamily.h index ea1076876d..bc5dafce03 100644 --- a/src/feature/nodelist/nodefamily.h +++ b/src/feature/nodelist/nodefamily.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/nodelist/nodefamily_st.h b/src/feature/nodelist/nodefamily_st.h index a498b4b3b9..be533da824 100644 --- a/src/feature/nodelist/nodefamily_st.h +++ b/src/feature/nodelist/nodefamily_st.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_NODEFAMILY_ST_H diff --git a/src/feature/nodelist/nodelist.c b/src/feature/nodelist/nodelist.c index 33601fe1fa..9a27701803 100644 --- a/src/feature/nodelist/nodelist.c +++ b/src/feature/nodelist/nodelist.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/nodelist/nodelist.h b/src/feature/nodelist/nodelist.h index 32300eb00c..3420959618 100644 --- a/src/feature/nodelist/nodelist.h +++ b/src/feature/nodelist/nodelist.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/nodelist/routerinfo.c b/src/feature/nodelist/routerinfo.c index 601de78d60..975b503615 100644 --- a/src/feature/nodelist/routerinfo.c +++ b/src/feature/nodelist/routerinfo.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "core/or/or.h" diff --git a/src/feature/nodelist/routerinfo.h b/src/feature/nodelist/routerinfo.h index b4b245bb23..bfa28c7754 100644 --- a/src/feature/nodelist/routerinfo.h +++ b/src/feature/nodelist/routerinfo.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/nodelist/routerinfo_st.h b/src/feature/nodelist/routerinfo_st.h index 6d4d118ad2..59656818c1 100644 --- a/src/feature/nodelist/routerinfo_st.h +++ b/src/feature/nodelist/routerinfo_st.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef ROUTERINFO_ST_H diff --git a/src/feature/nodelist/routerlist.c b/src/feature/nodelist/routerlist.c index c8a658414b..d1220f553a 100644 --- a/src/feature/nodelist/routerlist.c +++ b/src/feature/nodelist/routerlist.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/nodelist/routerlist.h b/src/feature/nodelist/routerlist.h index c3e97d9dd2..5771ebb1ab 100644 --- a/src/feature/nodelist/routerlist.h +++ b/src/feature/nodelist/routerlist.h @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/nodelist/routerlist_st.h b/src/feature/nodelist/routerlist_st.h index 26cc66138c..7446ead3cb 100644 --- a/src/feature/nodelist/routerlist_st.h +++ b/src/feature/nodelist/routerlist_st.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef ROUTERLIST_ST_H diff --git a/src/feature/nodelist/routerset.c b/src/feature/nodelist/routerset.c index 45863de6e0..55e2756959 100644 --- a/src/feature/nodelist/routerset.c +++ b/src/feature/nodelist/routerset.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. n * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/nodelist/routerset.h b/src/feature/nodelist/routerset.h index 8a13ca042a..ca8b6fed93 100644 --- a/src/feature/nodelist/routerset.h +++ b/src/feature/nodelist/routerset.h @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/nodelist/routerstatus_st.h b/src/feature/nodelist/routerstatus_st.h index ea06587799..8d91b45e11 100644 --- a/src/feature/nodelist/routerstatus_st.h +++ b/src/feature/nodelist/routerstatus_st.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef ROUTERSTATUS_ST_H diff --git a/src/feature/nodelist/signed_descriptor_st.h b/src/feature/nodelist/signed_descriptor_st.h index bffad62895..bdcebf184a 100644 --- a/src/feature/nodelist/signed_descriptor_st.h +++ b/src/feature/nodelist/signed_descriptor_st.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef SIGNED_DESCRIPTOR_ST_H diff --git a/src/feature/nodelist/torcert.c b/src/feature/nodelist/torcert.c index 938d7d09f6..b0197e9f13 100644 --- a/src/feature/nodelist/torcert.c +++ b/src/feature/nodelist/torcert.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2014-2018, The Tor Project, Inc. */ +/* Copyright (c) 2014-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/nodelist/torcert.h b/src/feature/nodelist/torcert.h index cb5e23cc33..492275b514 100644 --- a/src/feature/nodelist/torcert.h +++ b/src/feature/nodelist/torcert.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2014-2018, The Tor Project, Inc. */ +/* Copyright (c) 2014-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TORCERT_H_INCLUDED diff --git a/src/feature/nodelist/vote_routerstatus_st.h b/src/feature/nodelist/vote_routerstatus_st.h index ad0d35b4e6..366754c166 100644 --- a/src/feature/nodelist/vote_routerstatus_st.h +++ b/src/feature/nodelist/vote_routerstatus_st.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef VOTE_ROUTERSTATUS_ST_H diff --git a/src/feature/relay/dns.c b/src/feature/relay/dns.c index 5213296992..fa0a1b5910 100644 --- a/src/feature/relay/dns.c +++ b/src/feature/relay/dns.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/relay/dns.h b/src/feature/relay/dns.h index 5758ea4363..7b2a31a311 100644 --- a/src/feature/relay/dns.h +++ b/src/feature/relay/dns.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/relay/dns_structs.h b/src/feature/relay/dns_structs.h index 28c48ca0bc..e128746f81 100644 --- a/src/feature/relay/dns_structs.h +++ b/src/feature/relay/dns_structs.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/relay/ext_orport.c b/src/feature/relay/ext_orport.c index 0a649f2743..8589efb48d 100644 --- a/src/feature/relay/ext_orport.c +++ b/src/feature/relay/ext_orport.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2012-2018, The Tor Project, Inc. */ +/* Copyright (c) 2012-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/relay/ext_orport.h b/src/feature/relay/ext_orport.h index 7eebfdb25b..7313ebd03d 100644 --- a/src/feature/relay/ext_orport.h +++ b/src/feature/relay/ext_orport.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef EXT_ORPORT_H diff --git a/src/feature/relay/onion_queue.c b/src/feature/relay/onion_queue.c index 13142bb053..696905cf5e 100644 --- a/src/feature/relay/onion_queue.c +++ b/src/feature/relay/onion_queue.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/relay/onion_queue.h b/src/feature/relay/onion_queue.h index a71f497e34..0df921e057 100644 --- a/src/feature/relay/onion_queue.h +++ b/src/feature/relay/onion_queue.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/relay/router.c b/src/feature/relay/router.c index be9ef3d2f3..e796815409 100644 --- a/src/feature/relay/router.c +++ b/src/feature/relay/router.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define ROUTER_PRIVATE diff --git a/src/feature/relay/router.h b/src/feature/relay/router.h index 7a63c33d32..60bc857ceb 100644 --- a/src/feature/relay/router.h +++ b/src/feature/relay/router.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/relay/routerkeys.c b/src/feature/relay/routerkeys.c index 2499d7c8ff..876f908d41 100644 --- a/src/feature/relay/routerkeys.c +++ b/src/feature/relay/routerkeys.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2014-2018, The Tor Project, Inc. */ +/* Copyright (c) 2014-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/relay/routerkeys.h b/src/feature/relay/routerkeys.h index c5a58e553d..0badd34191 100644 --- a/src/feature/relay/routerkeys.h +++ b/src/feature/relay/routerkeys.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2014-2018, The Tor Project, Inc. */ +/* Copyright (c) 2014-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_ROUTERKEYS_H diff --git a/src/feature/relay/routermode.c b/src/feature/relay/routermode.c index 3f87cda505..2a9ddeac4d 100644 --- a/src/feature/relay/routermode.c +++ b/src/feature/relay/routermode.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "core/or/or.h" diff --git a/src/feature/relay/routermode.h b/src/feature/relay/routermode.h index 1442d706dd..be535af478 100644 --- a/src/feature/relay/routermode.h +++ b/src/feature/relay/routermode.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/relay/selftest.c b/src/feature/relay/selftest.c index 26205aad0f..064eea6c46 100644 --- a/src/feature/relay/selftest.c +++ b/src/feature/relay/selftest.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/relay/selftest.h b/src/feature/relay/selftest.h index 26034c9e8e..a80ec8936e 100644 --- a/src/feature/relay/selftest.h +++ b/src/feature/relay/selftest.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/rend/rend_authorized_client_st.h b/src/feature/rend/rend_authorized_client_st.h index 7ccf9771e1..7bd4f2fe8c 100644 --- a/src/feature/rend/rend_authorized_client_st.h +++ b/src/feature/rend/rend_authorized_client_st.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef REND_AUTHORIZED_CLIENT_ST_H diff --git a/src/feature/rend/rend_encoded_v2_service_descriptor_st.h b/src/feature/rend/rend_encoded_v2_service_descriptor_st.h index 0555ef6728..05ff145d53 100644 --- a/src/feature/rend/rend_encoded_v2_service_descriptor_st.h +++ b/src/feature/rend/rend_encoded_v2_service_descriptor_st.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef REND_ENCODED_V2_SERVICE_DESCRIPTOR_ST_H diff --git a/src/feature/rend/rend_intro_point_st.h b/src/feature/rend/rend_intro_point_st.h index 89fe5ef2b3..de6987e569 100644 --- a/src/feature/rend/rend_intro_point_st.h +++ b/src/feature/rend/rend_intro_point_st.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef REND_INTRO_POINT_ST_H diff --git a/src/feature/rend/rend_service_descriptor_st.h b/src/feature/rend/rend_service_descriptor_st.h index 8ea8a62305..aeb3178064 100644 --- a/src/feature/rend/rend_service_descriptor_st.h +++ b/src/feature/rend/rend_service_descriptor_st.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef REND_SERVICE_DESCRIPTOR_ST_H diff --git a/src/feature/rend/rendcache.c b/src/feature/rend/rendcache.c index b851e71959..fadfb43883 100644 --- a/src/feature/rend/rendcache.c +++ b/src/feature/rend/rendcache.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2015-2018, The Tor Project, Inc. */ +/* Copyright (c) 2015-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/rend/rendcache.h b/src/feature/rend/rendcache.h index 455e51645c..aec97eabb8 100644 --- a/src/feature/rend/rendcache.h +++ b/src/feature/rend/rendcache.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2015-2018, The Tor Project, Inc. */ +/* Copyright (c) 2015-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/rend/rendclient.c b/src/feature/rend/rendclient.c index 6ecb3eb3c6..4ca783c7c3 100644 --- a/src/feature/rend/rendclient.c +++ b/src/feature/rend/rendclient.c @@ -1,5 +1,5 @@ /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/rend/rendclient.h b/src/feature/rend/rendclient.h index 0d27d63e65..e5f333238e 100644 --- a/src/feature/rend/rendclient.h +++ b/src/feature/rend/rendclient.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/rend/rendcommon.c b/src/feature/rend/rendcommon.c index 15e4534fca..de48af795f 100644 --- a/src/feature/rend/rendcommon.c +++ b/src/feature/rend/rendcommon.c @@ -1,5 +1,5 @@ /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/rend/rendcommon.h b/src/feature/rend/rendcommon.h index 4ea35f88c2..f136863c7a 100644 --- a/src/feature/rend/rendcommon.h +++ b/src/feature/rend/rendcommon.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/rend/rendmid.c b/src/feature/rend/rendmid.c index 67a8b5f8f4..7c96f23629 100644 --- a/src/feature/rend/rendmid.c +++ b/src/feature/rend/rendmid.c @@ -1,5 +1,5 @@ /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/rend/rendmid.h b/src/feature/rend/rendmid.h index 907a0c6a73..8ae1fa16b8 100644 --- a/src/feature/rend/rendmid.h +++ b/src/feature/rend/rendmid.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/rend/rendparse.c b/src/feature/rend/rendparse.c index e2378e340f..abd0feb448 100644 --- a/src/feature/rend/rendparse.c +++ b/src/feature/rend/rendparse.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/rend/rendparse.h b/src/feature/rend/rendparse.h index 337d3e3b1f..0cef931e90 100644 --- a/src/feature/rend/rendparse.h +++ b/src/feature/rend/rendparse.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/rend/rendservice.c b/src/feature/rend/rendservice.c index d135581061..5ee084b0b7 100644 --- a/src/feature/rend/rendservice.c +++ b/src/feature/rend/rendservice.c @@ -1,5 +1,5 @@ /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/rend/rendservice.h b/src/feature/rend/rendservice.h index 7186289fc7..a8eb28bee2 100644 --- a/src/feature/rend/rendservice.h +++ b/src/feature/rend/rendservice.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/stats/geoip_stats.c b/src/feature/stats/geoip_stats.c index 3106c6c82c..5119da19a0 100644 --- a/src/feature/stats/geoip_stats.c +++ b/src/feature/stats/geoip_stats.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/stats/geoip_stats.h b/src/feature/stats/geoip_stats.h index 97011d62ce..2fc62b5466 100644 --- a/src/feature/stats/geoip_stats.h +++ b/src/feature/stats/geoip_stats.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/stats/predict_ports.c b/src/feature/stats/predict_ports.c index ebf4a42468..3cbba2c831 100644 --- a/src/feature/stats/predict_ports.c +++ b/src/feature/stats/predict_ports.c @@ -1,5 +1,5 @@ /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/stats/predict_ports.h b/src/feature/stats/predict_ports.h index ecb2e9caf2..272344da2f 100644 --- a/src/feature/stats/predict_ports.h +++ b/src/feature/stats/predict_ports.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/stats/rephist.c b/src/feature/stats/rephist.c index 3967a548b5..3f560fbce7 100644 --- a/src/feature/stats/rephist.c +++ b/src/feature/stats/rephist.c @@ -1,5 +1,5 @@ /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/feature/stats/rephist.h b/src/feature/stats/rephist.h index 0584b4684a..3accc8c610 100644 --- a/src/feature/stats/rephist.h +++ b/src/feature/stats/rephist.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/arch/bytes.h b/src/lib/arch/bytes.h index a2e2224d3c..fa82241b28 100644 --- a/src/lib/arch/bytes.h +++ b/src/lib/arch/bytes.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_BYTES_H diff --git a/src/lib/buf/buffers.c b/src/lib/buf/buffers.c index 495c5ec453..e7a3b87df0 100644 --- a/src/lib/buf/buffers.c +++ b/src/lib/buf/buffers.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/buf/buffers.h b/src/lib/buf/buffers.h index c48f83cfc7..c103b93a82 100644 --- a/src/lib/buf/buffers.h +++ b/src/lib/buf/buffers.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/cc/compat_compiler.h b/src/lib/cc/compat_compiler.h index d45316b241..3a0f307186 100644 --- a/src/lib/cc/compat_compiler.h +++ b/src/lib/cc/compat_compiler.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/cc/torint.h b/src/lib/cc/torint.h index 5097724726..c9b2d329f2 100644 --- a/src/lib/cc/torint.h +++ b/src/lib/cc/torint.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/compress/compress.c b/src/lib/compress/compress.c index 6cb9bd492b..51591410a2 100644 --- a/src/lib/compress/compress.c +++ b/src/lib/compress/compress.c @@ -1,6 +1,6 @@ /* Copyright (c) 2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/compress/compress.h b/src/lib/compress/compress.h index 4dd6506238..8cea4ead60 100644 --- a/src/lib/compress/compress.h +++ b/src/lib/compress/compress.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/compress/compress_buf.c b/src/lib/compress/compress_buf.c index ecf76ee078..2e704466f2 100644 --- a/src/lib/compress/compress_buf.c +++ b/src/lib/compress/compress_buf.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/compress/compress_lzma.c b/src/lib/compress/compress_lzma.c index e6c8368f6a..2dab37e433 100644 --- a/src/lib/compress/compress_lzma.c +++ b/src/lib/compress/compress_lzma.c @@ -1,6 +1,6 @@ /* Copyright (c) 2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/compress/compress_lzma.h b/src/lib/compress/compress_lzma.h index 9ef3382a25..556ab437dc 100644 --- a/src/lib/compress/compress_lzma.h +++ b/src/lib/compress/compress_lzma.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/compress/compress_none.c b/src/lib/compress/compress_none.c index a0e82d5a99..0b5760773a 100644 --- a/src/lib/compress/compress_none.c +++ b/src/lib/compress/compress_none.c @@ -1,6 +1,6 @@ /* Copyright (c) 2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/compress/compress_none.h b/src/lib/compress/compress_none.h index 5c395bbb30..2bb9c3d66c 100644 --- a/src/lib/compress/compress_none.h +++ b/src/lib/compress/compress_none.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/compress/compress_sys.h b/src/lib/compress/compress_sys.h index a162140cfb..6181072315 100644 --- a/src/lib/compress/compress_sys.h +++ b/src/lib/compress/compress_sys.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2018, The Tor Project, Inc. */ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/compress/compress_zlib.c b/src/lib/compress/compress_zlib.c index 64be1d4256..df0d1bff5f 100644 --- a/src/lib/compress/compress_zlib.c +++ b/src/lib/compress/compress_zlib.c @@ -1,6 +1,6 @@ /* Copyright (c) 2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/compress/compress_zlib.h b/src/lib/compress/compress_zlib.h index 7af68044de..e4f248cd9b 100644 --- a/src/lib/compress/compress_zlib.h +++ b/src/lib/compress/compress_zlib.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/compress/compress_zstd.c b/src/lib/compress/compress_zstd.c index fe88d4a544..45d0d4d602 100644 --- a/src/lib/compress/compress_zstd.c +++ b/src/lib/compress/compress_zstd.c @@ -1,6 +1,6 @@ /* Copyright (c) 2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/compress/compress_zstd.h b/src/lib/compress/compress_zstd.h index 1177537a9e..47f950b9e0 100644 --- a/src/lib/compress/compress_zstd.h +++ b/src/lib/compress/compress_zstd.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/container/bitarray.h b/src/lib/container/bitarray.h index ae82a4ab4d..910d5fea65 100644 --- a/src/lib/container/bitarray.h +++ b/src/lib/container/bitarray.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_BITARRAY_H diff --git a/src/lib/container/bloomfilt.c b/src/lib/container/bloomfilt.c index a64fcb9300..8c61db81d6 100644 --- a/src/lib/container/bloomfilt.c +++ b/src/lib/container/bloomfilt.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/container/bloomfilt.h b/src/lib/container/bloomfilt.h index 14f909cb19..0ce18bd3ec 100644 --- a/src/lib/container/bloomfilt.h +++ b/src/lib/container/bloomfilt.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_BLOOMFILT_H diff --git a/src/lib/container/handles.h b/src/lib/container/handles.h index 7144e1720d..ca7c94559e 100644 --- a/src/lib/container/handles.h +++ b/src/lib/container/handles.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2016-2018, The Tor Project, Inc. */ +/* Copyright (c) 2016-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/container/map.c b/src/lib/container/map.c index 137e316920..fde33d6ace 100644 --- a/src/lib/container/map.c +++ b/src/lib/container/map.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/container/map.h b/src/lib/container/map.h index 4f427fe767..d61b1ec18f 100644 --- a/src/lib/container/map.h +++ b/src/lib/container/map.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_MAP_H diff --git a/src/lib/container/order.c b/src/lib/container/order.c index 1efef2c734..f6503a124e 100644 --- a/src/lib/container/order.c +++ b/src/lib/container/order.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/container/order.h b/src/lib/container/order.h index f0675f347b..a176d6d8a6 100644 --- a/src/lib/container/order.h +++ b/src/lib/container/order.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_ORDER_H diff --git a/src/lib/container/smartlist.c b/src/lib/container/smartlist.c index 64cabfcc6f..3ab2797d68 100644 --- a/src/lib/container/smartlist.c +++ b/src/lib/container/smartlist.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/container/smartlist.h b/src/lib/container/smartlist.h index 0f5af3a923..77682db03e 100644 --- a/src/lib/container/smartlist.h +++ b/src/lib/container/smartlist.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_SMARTLIST_H diff --git a/src/lib/crypt_ops/aes.h b/src/lib/crypt_ops/aes.h index 578a1a65a2..7c774062d9 100644 --- a/src/lib/crypt_ops/aes.h +++ b/src/lib/crypt_ops/aes.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /* Implements a minimal interface to counter-mode AES. */ diff --git a/src/lib/crypt_ops/aes_nss.c b/src/lib/crypt_ops/aes_nss.c index 272edc5592..4eda5e5902 100644 --- a/src/lib/crypt_ops/aes_nss.c +++ b/src/lib/crypt_ops/aes_nss.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/crypt_ops/aes_openssl.c b/src/lib/crypt_ops/aes_openssl.c index ac275af33c..42ee924a8a 100644 --- a/src/lib/crypt_ops/aes_openssl.c +++ b/src/lib/crypt_ops/aes_openssl.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/crypt_ops/compat_openssl.h b/src/lib/crypt_ops/compat_openssl.h index f2f632ab40..9c10386c34 100644 --- a/src/lib/crypt_ops/compat_openssl.h +++ b/src/lib/crypt_ops/compat_openssl.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_COMPAT_OPENSSL_H diff --git a/src/lib/crypt_ops/crypto_cipher.c b/src/lib/crypt_ops/crypto_cipher.c index 6b762e374d..7bc2edad54 100644 --- a/src/lib/crypt_ops/crypto_cipher.c +++ b/src/lib/crypt_ops/crypto_cipher.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/crypt_ops/crypto_cipher.h b/src/lib/crypt_ops/crypto_cipher.h index f9444d03fc..cc4fbf7a41 100644 --- a/src/lib/crypt_ops/crypto_cipher.h +++ b/src/lib/crypt_ops/crypto_cipher.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/crypt_ops/crypto_curve25519.c b/src/lib/crypt_ops/crypto_curve25519.c index f3a9de9fc5..de4e17a296 100644 --- a/src/lib/crypt_ops/crypto_curve25519.c +++ b/src/lib/crypt_ops/crypto_curve25519.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2012-2018, The Tor Project, Inc. */ +/* Copyright (c) 2012-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/crypt_ops/crypto_curve25519.h b/src/lib/crypt_ops/crypto_curve25519.h index 1bab4a4197..061a7a3505 100644 --- a/src/lib/crypt_ops/crypto_curve25519.h +++ b/src/lib/crypt_ops/crypto_curve25519.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2012-2018, The Tor Project, Inc. */ +/* Copyright (c) 2012-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/crypt_ops/crypto_dh.c b/src/lib/crypt_ops/crypto_dh.c index 673ef311f9..4be7948761 100644 --- a/src/lib/crypt_ops/crypto_dh.c +++ b/src/lib/crypt_ops/crypto_dh.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/crypt_ops/crypto_dh.h b/src/lib/crypt_ops/crypto_dh.h index 3ee343a278..850d50c7ae 100644 --- a/src/lib/crypt_ops/crypto_dh.h +++ b/src/lib/crypt_ops/crypto_dh.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/crypt_ops/crypto_dh_nss.c b/src/lib/crypt_ops/crypto_dh_nss.c index e2d9040f5e..379eb84a4f 100644 --- a/src/lib/crypt_ops/crypto_dh_nss.c +++ b/src/lib/crypt_ops/crypto_dh_nss.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/crypt_ops/crypto_dh_openssl.c b/src/lib/crypt_ops/crypto_dh_openssl.c index 0d9bd513cf..8c6388fd5d 100644 --- a/src/lib/crypt_ops/crypto_dh_openssl.c +++ b/src/lib/crypt_ops/crypto_dh_openssl.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/crypt_ops/crypto_digest.c b/src/lib/crypt_ops/crypto_digest.c index 77cf18dca9..26f06c6c79 100644 --- a/src/lib/crypt_ops/crypto_digest.c +++ b/src/lib/crypt_ops/crypto_digest.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/crypt_ops/crypto_digest.h b/src/lib/crypt_ops/crypto_digest.h index 204f1aaff3..47e60ce617 100644 --- a/src/lib/crypt_ops/crypto_digest.h +++ b/src/lib/crypt_ops/crypto_digest.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/crypt_ops/crypto_ed25519.c b/src/lib/crypt_ops/crypto_ed25519.c index 11c1f56aef..400f963898 100644 --- a/src/lib/crypt_ops/crypto_ed25519.c +++ b/src/lib/crypt_ops/crypto_ed25519.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2018, The Tor Project, Inc. */ +/* Copyright (c) 2013-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/crypt_ops/crypto_ed25519.h b/src/lib/crypt_ops/crypto_ed25519.h index 03b3afe206..325b28244d 100644 --- a/src/lib/crypt_ops/crypto_ed25519.h +++ b/src/lib/crypt_ops/crypto_ed25519.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2012-2018, The Tor Project, Inc. */ +/* Copyright (c) 2012-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/crypt_ops/crypto_format.c b/src/lib/crypt_ops/crypto_format.c index 09ec753a00..84f73e5272 100644 --- a/src/lib/crypt_ops/crypto_format.c +++ b/src/lib/crypt_ops/crypto_format.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/crypt_ops/crypto_format.h b/src/lib/crypt_ops/crypto_format.h index a246071458..fe852e6a61 100644 --- a/src/lib/crypt_ops/crypto_format.h +++ b/src/lib/crypt_ops/crypto_format.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/crypt_ops/crypto_hkdf.c b/src/lib/crypt_ops/crypto_hkdf.c index a63d9131d9..6c82fa14f6 100644 --- a/src/lib/crypt_ops/crypto_hkdf.c +++ b/src/lib/crypt_ops/crypto_hkdf.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/crypt_ops/crypto_hkdf.h b/src/lib/crypt_ops/crypto_hkdf.h index 4c42584277..2994d18e3d 100644 --- a/src/lib/crypt_ops/crypto_hkdf.h +++ b/src/lib/crypt_ops/crypto_hkdf.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/crypt_ops/crypto_init.c b/src/lib/crypt_ops/crypto_init.c index f9943939fb..4040085c76 100644 --- a/src/lib/crypt_ops/crypto_init.c +++ b/src/lib/crypt_ops/crypto_init.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/crypt_ops/crypto_init.h b/src/lib/crypt_ops/crypto_init.h index b71f144276..540d08eb56 100644 --- a/src/lib/crypt_ops/crypto_init.h +++ b/src/lib/crypt_ops/crypto_init.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/crypt_ops/crypto_nss_mgt.c b/src/lib/crypt_ops/crypto_nss_mgt.c index a1da74aff5..0179126e38 100644 --- a/src/lib/crypt_ops/crypto_nss_mgt.c +++ b/src/lib/crypt_ops/crypto_nss_mgt.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/crypt_ops/crypto_nss_mgt.h b/src/lib/crypt_ops/crypto_nss_mgt.h index 27793dcc45..72fd2a1229 100644 --- a/src/lib/crypt_ops/crypto_nss_mgt.h +++ b/src/lib/crypt_ops/crypto_nss_mgt.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/crypt_ops/crypto_ope.c b/src/lib/crypt_ops/crypto_ope.c index 789517eba2..2186d2a939 100644 --- a/src/lib/crypt_ops/crypto_ope.c +++ b/src/lib/crypt_ops/crypto_ope.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2018, The Tor Project, Inc. */ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/crypt_ops/crypto_ope.h b/src/lib/crypt_ops/crypto_ope.h index c62ed2a942..610d956335 100644 --- a/src/lib/crypt_ops/crypto_ope.h +++ b/src/lib/crypt_ops/crypto_ope.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2018, The Tor Project, Inc. */ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef CRYPTO_OPE_H diff --git a/src/lib/crypt_ops/crypto_openssl_mgt.c b/src/lib/crypt_ops/crypto_openssl_mgt.c index 125da0786b..60e4ea795e 100644 --- a/src/lib/crypt_ops/crypto_openssl_mgt.c +++ b/src/lib/crypt_ops/crypto_openssl_mgt.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/crypt_ops/crypto_openssl_mgt.h b/src/lib/crypt_ops/crypto_openssl_mgt.h index 3b288fb9d8..83fb44cadf 100644 --- a/src/lib/crypt_ops/crypto_openssl_mgt.h +++ b/src/lib/crypt_ops/crypto_openssl_mgt.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/crypt_ops/crypto_pwbox.c b/src/lib/crypt_ops/crypto_pwbox.c index 91536e891b..a8db08f7b7 100644 --- a/src/lib/crypt_ops/crypto_pwbox.c +++ b/src/lib/crypt_ops/crypto_pwbox.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2014-2018, The Tor Project, Inc. */ +/* Copyright (c) 2014-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/crypt_ops/crypto_pwbox.h b/src/lib/crypt_ops/crypto_pwbox.h index 00fabd0913..5a26889fb2 100644 --- a/src/lib/crypt_ops/crypto_pwbox.h +++ b/src/lib/crypt_ops/crypto_pwbox.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2014-2018, The Tor Project, Inc. */ +/* Copyright (c) 2014-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/crypt_ops/crypto_rand.c b/src/lib/crypt_ops/crypto_rand.c index d148dfb3a8..f45ff719d3 100644 --- a/src/lib/crypt_ops/crypto_rand.c +++ b/src/lib/crypt_ops/crypto_rand.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/crypt_ops/crypto_rand.h b/src/lib/crypt_ops/crypto_rand.h index 874fcd4d08..cc2762842a 100644 --- a/src/lib/crypt_ops/crypto_rand.h +++ b/src/lib/crypt_ops/crypto_rand.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/crypt_ops/crypto_rsa.c b/src/lib/crypt_ops/crypto_rsa.c index a510e12964..8cc62bc180 100644 --- a/src/lib/crypt_ops/crypto_rsa.c +++ b/src/lib/crypt_ops/crypto_rsa.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/crypt_ops/crypto_rsa.h b/src/lib/crypt_ops/crypto_rsa.h index 007964b268..c1ea767f85 100644 --- a/src/lib/crypt_ops/crypto_rsa.h +++ b/src/lib/crypt_ops/crypto_rsa.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/crypt_ops/crypto_rsa_nss.c b/src/lib/crypt_ops/crypto_rsa_nss.c index dc282d7c9d..ad2ad38b66 100644 --- a/src/lib/crypt_ops/crypto_rsa_nss.c +++ b/src/lib/crypt_ops/crypto_rsa_nss.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/crypt_ops/crypto_rsa_openssl.c b/src/lib/crypt_ops/crypto_rsa_openssl.c index df81c963eb..fbdc76ccd6 100644 --- a/src/lib/crypt_ops/crypto_rsa_openssl.c +++ b/src/lib/crypt_ops/crypto_rsa_openssl.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/crypt_ops/crypto_s2k.c b/src/lib/crypt_ops/crypto_s2k.c index e0b2f40bb3..42276597d4 100644 --- a/src/lib/crypt_ops/crypto_s2k.c +++ b/src/lib/crypt_ops/crypto_s2k.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/crypt_ops/crypto_s2k.h b/src/lib/crypt_ops/crypto_s2k.h index 2429185b52..a16a3d781e 100644 --- a/src/lib/crypt_ops/crypto_s2k.h +++ b/src/lib/crypt_ops/crypto_s2k.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/crypt_ops/crypto_sys.h b/src/lib/crypt_ops/crypto_sys.h index 31644d088b..894243b175 100644 --- a/src/lib/crypt_ops/crypto_sys.h +++ b/src/lib/crypt_ops/crypto_sys.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2018, The Tor Project, Inc. */ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/crypt_ops/crypto_util.c b/src/lib/crypt_ops/crypto_util.c index 64b4e6b71b..67a1a9eb92 100644 --- a/src/lib/crypt_ops/crypto_util.c +++ b/src/lib/crypt_ops/crypto_util.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/crypt_ops/crypto_util.h b/src/lib/crypt_ops/crypto_util.h index e032263225..613a1bd0dd 100644 --- a/src/lib/crypt_ops/crypto_util.h +++ b/src/lib/crypt_ops/crypto_util.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/crypt_ops/digestset.c b/src/lib/crypt_ops/digestset.c index 84516e0172..c931b58369 100644 --- a/src/lib/crypt_ops/digestset.c +++ b/src/lib/crypt_ops/digestset.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2018-2018, The Tor Project, Inc. */ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/crypt_ops/digestset.h b/src/lib/crypt_ops/digestset.h index 328979ae0d..91d53a0542 100644 --- a/src/lib/crypt_ops/digestset.h +++ b/src/lib/crypt_ops/digestset.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2018, The Tor Project, Inc. */ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/ctime/di_ops.c b/src/lib/ctime/di_ops.c index 73441f84f8..89e0837ae9 100644 --- a/src/lib/ctime/di_ops.c +++ b/src/lib/ctime/di_ops.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2011-2018, The Tor Project, Inc. */ +/* Copyright (c) 2011-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/ctime/di_ops.h b/src/lib/ctime/di_ops.h index 92af7ae278..264b56a8c1 100644 --- a/src/lib/ctime/di_ops.h +++ b/src/lib/ctime/di_ops.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/defs/dh_sizes.h b/src/lib/defs/dh_sizes.h index a02ffc5281..a2ffbc51c2 100644 --- a/src/lib/defs/dh_sizes.h +++ b/src/lib/defs/dh_sizes.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/defs/digest_sizes.h b/src/lib/defs/digest_sizes.h index dd772cae07..525e5209d6 100644 --- a/src/lib/defs/digest_sizes.h +++ b/src/lib/defs/digest_sizes.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_DIGEST_SIZES_H diff --git a/src/lib/defs/time.h b/src/lib/defs/time.h index 762b23feab..c25f5022c5 100644 --- a/src/lib/defs/time.h +++ b/src/lib/defs/time.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_TIME_DEFS_H diff --git a/src/lib/defs/x25519_sizes.h b/src/lib/defs/x25519_sizes.h index d8ada46b97..8933a8866b 100644 --- a/src/lib/defs/x25519_sizes.h +++ b/src/lib/defs/x25519_sizes.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/encoding/binascii.c b/src/lib/encoding/binascii.c index 067db075ad..de4d1648bb 100644 --- a/src/lib/encoding/binascii.c +++ b/src/lib/encoding/binascii.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/encoding/binascii.h b/src/lib/encoding/binascii.h index c71ba65dfb..44998bb85b 100644 --- a/src/lib/encoding/binascii.h +++ b/src/lib/encoding/binascii.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/encoding/confline.c b/src/lib/encoding/confline.c index 71ce5b8424..8110f3dd9c 100644 --- a/src/lib/encoding/confline.c +++ b/src/lib/encoding/confline.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/encoding/confline.h b/src/lib/encoding/confline.h index 41f1200947..3d9ae8a662 100644 --- a/src/lib/encoding/confline.h +++ b/src/lib/encoding/confline.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/encoding/cstring.c b/src/lib/encoding/cstring.c index 69aa7f3da5..29d3714126 100644 --- a/src/lib/encoding/cstring.c +++ b/src/lib/encoding/cstring.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/encoding/cstring.h b/src/lib/encoding/cstring.h index 2da109d958..904a2c9c1c 100644 --- a/src/lib/encoding/cstring.h +++ b/src/lib/encoding/cstring.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/encoding/keyval.c b/src/lib/encoding/keyval.c index d5e4e4e438..c5da5a0bfc 100644 --- a/src/lib/encoding/keyval.c +++ b/src/lib/encoding/keyval.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/encoding/keyval.h b/src/lib/encoding/keyval.h index 8bf0797627..cd327b7a82 100644 --- a/src/lib/encoding/keyval.h +++ b/src/lib/encoding/keyval.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/encoding/kvline.c b/src/lib/encoding/kvline.c index 11ff4f0f96..307adc3f12 100644 --- a/src/lib/encoding/kvline.c +++ b/src/lib/encoding/kvline.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/encoding/kvline.h b/src/lib/encoding/kvline.h index 3272cc1754..4eed30a223 100644 --- a/src/lib/encoding/kvline.h +++ b/src/lib/encoding/kvline.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/encoding/pem.c b/src/lib/encoding/pem.c index 0d4a814f6f..24b238b130 100644 --- a/src/lib/encoding/pem.c +++ b/src/lib/encoding/pem.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/encoding/pem.h b/src/lib/encoding/pem.h index ba21228848..0bbb06a794 100644 --- a/src/lib/encoding/pem.h +++ b/src/lib/encoding/pem.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/encoding/time_fmt.c b/src/lib/encoding/time_fmt.c index 233d0dddff..5b2440d1ab 100644 --- a/src/lib/encoding/time_fmt.c +++ b/src/lib/encoding/time_fmt.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/encoding/time_fmt.h b/src/lib/encoding/time_fmt.h index 2892442adf..0ddeca57fc 100644 --- a/src/lib/encoding/time_fmt.h +++ b/src/lib/encoding/time_fmt.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/err/backtrace.c b/src/lib/err/backtrace.c index b568c888c5..1d1b3bcfa3 100644 --- a/src/lib/err/backtrace.c +++ b/src/lib/err/backtrace.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2018, The Tor Project, Inc. */ +/* Copyright (c) 2013-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/err/backtrace.h b/src/lib/err/backtrace.h index 70c43484f5..9b313261e6 100644 --- a/src/lib/err/backtrace.h +++ b/src/lib/err/backtrace.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2018, The Tor Project, Inc. */ +/* Copyright (c) 2013-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_BACKTRACE_H diff --git a/src/lib/err/torerr.c b/src/lib/err/torerr.c index e9de86837f..ecffb7f7bb 100644 --- a/src/lib/err/torerr.c +++ b/src/lib/err/torerr.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/err/torerr.h b/src/lib/err/torerr.h index b415ef73ef..0badaf7c6d 100644 --- a/src/lib/err/torerr.h +++ b/src/lib/err/torerr.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/err/torerr_sys.c b/src/lib/err/torerr_sys.c index 96bb1308a4..3ab1b3c4e1 100644 --- a/src/lib/err/torerr_sys.c +++ b/src/lib/err/torerr_sys.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2018, The Tor Project, Inc. */ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/err/torerr_sys.h b/src/lib/err/torerr_sys.h index b56270d538..c947695689 100644 --- a/src/lib/err/torerr_sys.h +++ b/src/lib/err/torerr_sys.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2018, The Tor Project, Inc. */ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/evloop/compat_libevent.c b/src/lib/evloop/compat_libevent.c index 2ee71c5598..91eacb9938 100644 --- a/src/lib/evloop/compat_libevent.c +++ b/src/lib/evloop/compat_libevent.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2009-2018, The Tor Project, Inc. */ +/* Copyright (c) 2009-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/evloop/compat_libevent.h b/src/lib/evloop/compat_libevent.h index 5fda49f741..afe887a013 100644 --- a/src/lib/evloop/compat_libevent.h +++ b/src/lib/evloop/compat_libevent.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2009-2018, The Tor Project, Inc. */ +/* Copyright (c) 2009-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/evloop/procmon.c b/src/lib/evloop/procmon.c index 02e167377f..52469fa5fc 100644 --- a/src/lib/evloop/procmon.c +++ b/src/lib/evloop/procmon.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2011-2018, The Tor Project, Inc. */ +/* Copyright (c) 2011-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/evloop/procmon.h b/src/lib/evloop/procmon.h index 42c3788692..6caae5be86 100644 --- a/src/lib/evloop/procmon.h +++ b/src/lib/evloop/procmon.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2011-2018, The Tor Project, Inc. */ +/* Copyright (c) 2011-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/evloop/timers.c b/src/lib/evloop/timers.c index 6743b6af51..4b2a96ef7d 100644 --- a/src/lib/evloop/timers.c +++ b/src/lib/evloop/timers.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2016-2018, The Tor Project, Inc. */ +/* Copyright (c) 2016-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/evloop/timers.h b/src/lib/evloop/timers.h index 4ffed1b458..7595554204 100644 --- a/src/lib/evloop/timers.h +++ b/src/lib/evloop/timers.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2016-2018, The Tor Project, Inc. */ +/* Copyright (c) 2016-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/evloop/token_bucket.c b/src/lib/evloop/token_bucket.c index f7cd05c6c5..ee6d631e3b 100644 --- a/src/lib/evloop/token_bucket.c +++ b/src/lib/evloop/token_bucket.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2018-2018, The Tor Project, Inc. */ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/evloop/token_bucket.h b/src/lib/evloop/token_bucket.h index f004358f47..9398d2baa3 100644 --- a/src/lib/evloop/token_bucket.h +++ b/src/lib/evloop/token_bucket.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2018-2018, The Tor Project, Inc. */ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/evloop/workqueue.h b/src/lib/evloop/workqueue.h index 10d5d47464..d0ee8f2be2 100644 --- a/src/lib/evloop/workqueue.h +++ b/src/lib/evloop/workqueue.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2018, The Tor Project, Inc. */ +/* Copyright (c) 2013-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/fdio/fdio.c b/src/lib/fdio/fdio.c index afb57e03dd..6c87af791d 100644 --- a/src/lib/fdio/fdio.c +++ b/src/lib/fdio/fdio.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/fdio/fdio.h b/src/lib/fdio/fdio.h index c8f05455b7..8395af353b 100644 --- a/src/lib/fdio/fdio.h +++ b/src/lib/fdio/fdio.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/fs/conffile.c b/src/lib/fs/conffile.c index d8622bff6b..7bb2f23931 100644 --- a/src/lib/fs/conffile.c +++ b/src/lib/fs/conffile.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/fs/conffile.h b/src/lib/fs/conffile.h index a926f4ac05..7af9119dbb 100644 --- a/src/lib/fs/conffile.h +++ b/src/lib/fs/conffile.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_CONFFILE_H diff --git a/src/lib/fs/dir.c b/src/lib/fs/dir.c index 6c24460005..3c31e00d99 100644 --- a/src/lib/fs/dir.c +++ b/src/lib/fs/dir.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/fs/dir.h b/src/lib/fs/dir.h index 61a04e6d58..826bc2dfc5 100644 --- a/src/lib/fs/dir.h +++ b/src/lib/fs/dir.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_DIR_H diff --git a/src/lib/fs/files.c b/src/lib/fs/files.c index 43dcbad333..b98a51a287 100644 --- a/src/lib/fs/files.c +++ b/src/lib/fs/files.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/fs/files.h b/src/lib/fs/files.h index 2ee1b20149..52c94c914f 100644 --- a/src/lib/fs/files.h +++ b/src/lib/fs/files.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/fs/freespace.c b/src/lib/fs/freespace.c index c18b1e0234..ee0f93073d 100644 --- a/src/lib/fs/freespace.c +++ b/src/lib/fs/freespace.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/fs/lockfile.c b/src/lib/fs/lockfile.c index ca1711c344..933ff1e02f 100644 --- a/src/lib/fs/lockfile.c +++ b/src/lib/fs/lockfile.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/fs/lockfile.h b/src/lib/fs/lockfile.h index e26349811c..8aeee4cc7f 100644 --- a/src/lib/fs/lockfile.h +++ b/src/lib/fs/lockfile.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/fs/mmap.c b/src/lib/fs/mmap.c index e7da6fc63a..daaee1f9b1 100644 --- a/src/lib/fs/mmap.c +++ b/src/lib/fs/mmap.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/fs/mmap.h b/src/lib/fs/mmap.h index 8d6ca9a0e2..18fb18a13c 100644 --- a/src/lib/fs/mmap.h +++ b/src/lib/fs/mmap.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/fs/path.c b/src/lib/fs/path.c index eb5170920c..b3ef61979d 100644 --- a/src/lib/fs/path.c +++ b/src/lib/fs/path.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/fs/path.h b/src/lib/fs/path.h index 384d1f514f..4675ac84e8 100644 --- a/src/lib/fs/path.h +++ b/src/lib/fs/path.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/fs/storagedir.c b/src/lib/fs/storagedir.c index 014eb4d9eb..2caddf1ad9 100644 --- a/src/lib/fs/storagedir.c +++ b/src/lib/fs/storagedir.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2017-2018, The Tor Project, Inc. */ +/* Copyright (c) 2017-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/fs/storagedir.h b/src/lib/fs/storagedir.h index 58594b4634..7e6633a0bb 100644 --- a/src/lib/fs/storagedir.h +++ b/src/lib/fs/storagedir.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2017-2018, The Tor Project, Inc. */ +/* Copyright (c) 2017-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/fs/userdb.c b/src/lib/fs/userdb.c index df2935baa8..95205c670e 100644 --- a/src/lib/fs/userdb.c +++ b/src/lib/fs/userdb.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/fs/userdb.h b/src/lib/fs/userdb.h index 3b3ab6ed2b..5c39794873 100644 --- a/src/lib/fs/userdb.h +++ b/src/lib/fs/userdb.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/fs/winlib.c b/src/lib/fs/winlib.c index 532807c03f..b7302bd4ca 100644 --- a/src/lib/fs/winlib.c +++ b/src/lib/fs/winlib.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/fs/winlib.h b/src/lib/fs/winlib.h index 5b10b9b78d..64a22439e5 100644 --- a/src/lib/fs/winlib.h +++ b/src/lib/fs/winlib.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/geoip/country.h b/src/lib/geoip/country.h index 080c156023..9a8911d494 100644 --- a/src/lib/geoip/country.h +++ b/src/lib/geoip/country.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_COUNTRY_H diff --git a/src/lib/geoip/geoip.c b/src/lib/geoip/geoip.c index b1c0973d03..70b1c2dc8c 100644 --- a/src/lib/geoip/geoip.c +++ b/src/lib/geoip/geoip.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/geoip/geoip.h b/src/lib/geoip/geoip.h index 6ef27d66d0..f872ebd25f 100644 --- a/src/lib/geoip/geoip.h +++ b/src/lib/geoip/geoip.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/intmath/addsub.c b/src/lib/intmath/addsub.c index fcfdca6822..12146f4e72 100644 --- a/src/lib/intmath/addsub.c +++ b/src/lib/intmath/addsub.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/intmath/addsub.h b/src/lib/intmath/addsub.h index 5bbc32e4a9..83efa82919 100644 --- a/src/lib/intmath/addsub.h +++ b/src/lib/intmath/addsub.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/intmath/bits.c b/src/lib/intmath/bits.c index 7da524449d..2158790e3f 100644 --- a/src/lib/intmath/bits.c +++ b/src/lib/intmath/bits.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/intmath/bits.h b/src/lib/intmath/bits.h index 80eebe9358..c1483a18b8 100644 --- a/src/lib/intmath/bits.h +++ b/src/lib/intmath/bits.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/intmath/cmp.h b/src/lib/intmath/cmp.h index 11b6fdf98e..67a738861b 100644 --- a/src/lib/intmath/cmp.h +++ b/src/lib/intmath/cmp.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/intmath/logic.h b/src/lib/intmath/logic.h index b3eabc652e..a4cecd69cc 100644 --- a/src/lib/intmath/logic.h +++ b/src/lib/intmath/logic.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/intmath/muldiv.c b/src/lib/intmath/muldiv.c index c5fc689e2d..6a292db7ba 100644 --- a/src/lib/intmath/muldiv.c +++ b/src/lib/intmath/muldiv.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/intmath/muldiv.h b/src/lib/intmath/muldiv.h index 45b896922f..64500b6dce 100644 --- a/src/lib/intmath/muldiv.h +++ b/src/lib/intmath/muldiv.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/intmath/weakrng.c b/src/lib/intmath/weakrng.c index 36cf5fb0aa..99c9252c2b 100644 --- a/src/lib/intmath/weakrng.c +++ b/src/lib/intmath/weakrng.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/intmath/weakrng.h b/src/lib/intmath/weakrng.h index 679bf2449c..e26bf58cbb 100644 --- a/src/lib/intmath/weakrng.h +++ b/src/lib/intmath/weakrng.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/lock/compat_mutex.c b/src/lib/lock/compat_mutex.c index b53676cf49..4ad5929715 100644 --- a/src/lib/lock/compat_mutex.c +++ b/src/lib/lock/compat_mutex.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/lock/compat_mutex.h b/src/lib/lock/compat_mutex.h index f8689422b1..b63ce24024 100644 --- a/src/lib/lock/compat_mutex.h +++ b/src/lib/lock/compat_mutex.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/lock/compat_mutex_pthreads.c b/src/lib/lock/compat_mutex_pthreads.c index 983abf5ae5..ee5f520cd0 100644 --- a/src/lib/lock/compat_mutex_pthreads.c +++ b/src/lib/lock/compat_mutex_pthreads.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/lock/compat_mutex_winthreads.c b/src/lib/lock/compat_mutex_winthreads.c index 22c1edeed4..b0f5999e42 100644 --- a/src/lib/lock/compat_mutex_winthreads.c +++ b/src/lib/lock/compat_mutex_winthreads.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/log/escape.c b/src/lib/log/escape.c index 37d7e0fdc4..6ca01c6963 100644 --- a/src/lib/log/escape.c +++ b/src/lib/log/escape.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/log/escape.h b/src/lib/log/escape.h index f47e7e004d..2f726186c5 100644 --- a/src/lib/log/escape.h +++ b/src/lib/log/escape.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/log/log.c b/src/lib/log/log.c index d032f57add..d21d8d1d41 100644 --- a/src/lib/log/log.c +++ b/src/lib/log/log.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/log/log.h b/src/lib/log/log.h index 423e58e11b..dbc1c47021 100644 --- a/src/lib/log/log.h +++ b/src/lib/log/log.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/log/log_sys.c b/src/lib/log/log_sys.c index e20f3156ca..d1080f2264 100644 --- a/src/lib/log/log_sys.c +++ b/src/lib/log/log_sys.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2018, The Tor Project, Inc. */ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/log/log_sys.h b/src/lib/log/log_sys.h index f7afbb279d..7043253066 100644 --- a/src/lib/log/log_sys.h +++ b/src/lib/log/log_sys.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2018, The Tor Project, Inc. */ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/log/ratelim.c b/src/lib/log/ratelim.c index 2d0e8b17ca..5eec742aa7 100644 --- a/src/lib/log/ratelim.c +++ b/src/lib/log/ratelim.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/log/ratelim.h b/src/lib/log/ratelim.h index d423e10b85..48edd7c849 100644 --- a/src/lib/log/ratelim.h +++ b/src/lib/log/ratelim.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/log/util_bug.c b/src/lib/log/util_bug.c index b23f4edc97..f42d2d2ab4 100644 --- a/src/lib/log/util_bug.c +++ b/src/lib/log/util_bug.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/log/util_bug.h b/src/lib/log/util_bug.h index 557d932ac3..18d40bbf39 100644 --- a/src/lib/log/util_bug.h +++ b/src/lib/log/util_bug.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/log/win32err.c b/src/lib/log/win32err.c index 41f563bfa5..dc45cb4c3d 100644 --- a/src/lib/log/win32err.c +++ b/src/lib/log/win32err.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/log/win32err.h b/src/lib/log/win32err.h index 92958c9879..33413dfd15 100644 --- a/src/lib/log/win32err.h +++ b/src/lib/log/win32err.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/malloc/malloc.c b/src/lib/malloc/malloc.c index 271e84071a..8628acfc97 100644 --- a/src/lib/malloc/malloc.c +++ b/src/lib/malloc/malloc.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/malloc/malloc.h b/src/lib/malloc/malloc.h index 26ee30742e..ef6b509ca4 100644 --- a/src/lib/malloc/malloc.h +++ b/src/lib/malloc/malloc.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/math/fp.c b/src/lib/math/fp.c index 57082fa468..52c57c1d7f 100644 --- a/src/lib/math/fp.c +++ b/src/lib/math/fp.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/math/fp.h b/src/lib/math/fp.h index ddf3ed24d6..cb24649e6c 100644 --- a/src/lib/math/fp.h +++ b/src/lib/math/fp.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/math/laplace.c b/src/lib/math/laplace.c index 6b33b46902..302edb20b8 100644 --- a/src/lib/math/laplace.c +++ b/src/lib/math/laplace.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/math/laplace.h b/src/lib/math/laplace.h index 62d698e369..e8651e5197 100644 --- a/src/lib/math/laplace.h +++ b/src/lib/math/laplace.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/math/prob_distr.c b/src/lib/math/prob_distr.c index 4263ba2074..c952dadc06 100644 --- a/src/lib/math/prob_distr.c +++ b/src/lib/math/prob_distr.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2018, The Tor Project, Inc. */ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/memarea/memarea.c b/src/lib/memarea/memarea.c index 96d94c89d9..84c73b0b95 100644 --- a/src/lib/memarea/memarea.c +++ b/src/lib/memarea/memarea.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2008-2018, The Tor Project, Inc. */ +/* Copyright (c) 2008-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/memarea/memarea.h b/src/lib/memarea/memarea.h index 4978b54162..9c23cf62e9 100644 --- a/src/lib/memarea/memarea.h +++ b/src/lib/memarea/memarea.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2008-2018, The Tor Project, Inc. */ +/* Copyright (c) 2008-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/meminfo/meminfo.c b/src/lib/meminfo/meminfo.c index 648f54e0c5..790d698910 100644 --- a/src/lib/meminfo/meminfo.c +++ b/src/lib/meminfo/meminfo.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/meminfo/meminfo.h b/src/lib/meminfo/meminfo.h index b67d235559..2d64e1ab06 100644 --- a/src/lib/meminfo/meminfo.h +++ b/src/lib/meminfo/meminfo.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/net/address.c b/src/lib/net/address.c index 240201d7b6..e1c9e1310f 100644 --- a/src/lib/net/address.c +++ b/src/lib/net/address.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/net/address.h b/src/lib/net/address.h index e857b4068b..9b826c8359 100644 --- a/src/lib/net/address.h +++ b/src/lib/net/address.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/net/alertsock.c b/src/lib/net/alertsock.c index 340f9513fb..cc59d7d893 100644 --- a/src/lib/net/alertsock.c +++ b/src/lib/net/alertsock.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/net/alertsock.h b/src/lib/net/alertsock.h index 5dfe53a2a0..c45f42be81 100644 --- a/src/lib/net/alertsock.h +++ b/src/lib/net/alertsock.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/net/buffers_net.c b/src/lib/net/buffers_net.c index a4c0aca3b5..cfe1a7dc26 100644 --- a/src/lib/net/buffers_net.c +++ b/src/lib/net/buffers_net.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/net/buffers_net.h b/src/lib/net/buffers_net.h index 8911b082a2..a3a90172a1 100644 --- a/src/lib/net/buffers_net.h +++ b/src/lib/net/buffers_net.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/net/gethostname.c b/src/lib/net/gethostname.c index 1c4431af29..e54a1ea16e 100644 --- a/src/lib/net/gethostname.c +++ b/src/lib/net/gethostname.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/net/gethostname.h b/src/lib/net/gethostname.h index 7bf0ce5920..69b0528bc0 100644 --- a/src/lib/net/gethostname.h +++ b/src/lib/net/gethostname.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/net/inaddr.c b/src/lib/net/inaddr.c index 0960d323c5..d9ae7cd562 100644 --- a/src/lib/net/inaddr.c +++ b/src/lib/net/inaddr.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/net/inaddr.h b/src/lib/net/inaddr.h index 121025a126..36352b65ea 100644 --- a/src/lib/net/inaddr.h +++ b/src/lib/net/inaddr.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/net/inaddr_st.h b/src/lib/net/inaddr_st.h index a6b7796268..806f2c096a 100644 --- a/src/lib/net/inaddr_st.h +++ b/src/lib/net/inaddr_st.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/net/nettypes.h b/src/lib/net/nettypes.h index f7f2ec7d6a..6209bbe18a 100644 --- a/src/lib/net/nettypes.h +++ b/src/lib/net/nettypes.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/net/network_sys.c b/src/lib/net/network_sys.c index ac49288ee6..9dfdb2b45a 100644 --- a/src/lib/net/network_sys.c +++ b/src/lib/net/network_sys.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2018, The Tor Project, Inc. */ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/net/network_sys.h b/src/lib/net/network_sys.h index 62b778bb66..43e62592ca 100644 --- a/src/lib/net/network_sys.h +++ b/src/lib/net/network_sys.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2018, The Tor Project, Inc. */ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/net/resolve.c b/src/lib/net/resolve.c index 95c1b171b5..49c263faa2 100644 --- a/src/lib/net/resolve.c +++ b/src/lib/net/resolve.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/net/resolve.h b/src/lib/net/resolve.h index 39157aaa67..0fb77f1661 100644 --- a/src/lib/net/resolve.h +++ b/src/lib/net/resolve.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/net/socket.c b/src/lib/net/socket.c index 385c987e68..f978deeab8 100644 --- a/src/lib/net/socket.c +++ b/src/lib/net/socket.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/net/socket.h b/src/lib/net/socket.h index 822b9975e6..86ae336dfb 100644 --- a/src/lib/net/socket.h +++ b/src/lib/net/socket.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/net/socketpair.c b/src/lib/net/socketpair.c index b4aeb6eba7..15c706bec7 100644 --- a/src/lib/net/socketpair.c +++ b/src/lib/net/socketpair.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ #include "lib/cc/torint.h" #include "lib/net/socketpair.h" diff --git a/src/lib/net/socketpair.h b/src/lib/net/socketpair.h index 6eecc0737a..6be0803881 100644 --- a/src/lib/net/socketpair.h +++ b/src/lib/net/socketpair.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_SOCKETPAIR_H diff --git a/src/lib/net/socks5_status.h b/src/lib/net/socks5_status.h index 0f31132545..e55242ce66 100644 --- a/src/lib/net/socks5_status.h +++ b/src/lib/net/socks5_status.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/osinfo/uname.c b/src/lib/osinfo/uname.c index 7111ae31d2..2b37ff136c 100644 --- a/src/lib/osinfo/uname.c +++ b/src/lib/osinfo/uname.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/osinfo/uname.h b/src/lib/osinfo/uname.h index ef8cd078ee..fcce629074 100644 --- a/src/lib/osinfo/uname.h +++ b/src/lib/osinfo/uname.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/process/daemon.c b/src/lib/process/daemon.c index ab3ac73ad5..3b90bef671 100644 --- a/src/lib/process/daemon.c +++ b/src/lib/process/daemon.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/process/daemon.h b/src/lib/process/daemon.h index e33bd56701..20920e0aae 100644 --- a/src/lib/process/daemon.h +++ b/src/lib/process/daemon.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/process/env.c b/src/lib/process/env.c index dc0b63b499..0060200ba1 100644 --- a/src/lib/process/env.c +++ b/src/lib/process/env.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/process/env.h b/src/lib/process/env.h index 288b923ace..15d59351e0 100644 --- a/src/lib/process/env.h +++ b/src/lib/process/env.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/process/pidfile.c b/src/lib/process/pidfile.c index ea008168ab..1b9d1c6d25 100644 --- a/src/lib/process/pidfile.c +++ b/src/lib/process/pidfile.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/process/pidfile.h b/src/lib/process/pidfile.h index 945edee990..dfeb39e046 100644 --- a/src/lib/process/pidfile.h +++ b/src/lib/process/pidfile.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/process/process.c b/src/lib/process/process.c index ae345ceeae..422942dc83 100644 --- a/src/lib/process/process.c +++ b/src/lib/process/process.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/process/process.h b/src/lib/process/process.h index 956d34ab29..14069923a0 100644 --- a/src/lib/process/process.h +++ b/src/lib/process/process.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/process/process_sys.c b/src/lib/process/process_sys.c index d0a94f35f7..3c809a00e8 100644 --- a/src/lib/process/process_sys.c +++ b/src/lib/process/process_sys.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2018, The Tor Project, Inc. */ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/process/process_sys.h b/src/lib/process/process_sys.h index b299334b6b..b7a116d838 100644 --- a/src/lib/process/process_sys.h +++ b/src/lib/process/process_sys.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2018, The Tor Project, Inc. */ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/process/process_unix.c b/src/lib/process/process_unix.c index dd4ccbf603..790ab897e9 100644 --- a/src/lib/process/process_unix.c +++ b/src/lib/process/process_unix.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/process/process_unix.h b/src/lib/process/process_unix.h index 86c10d7449..a1d8f72993 100644 --- a/src/lib/process/process_unix.h +++ b/src/lib/process/process_unix.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/process/process_win32.c b/src/lib/process/process_win32.c index d8a895875c..21d0b23476 100644 --- a/src/lib/process/process_win32.c +++ b/src/lib/process/process_win32.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/process/process_win32.h b/src/lib/process/process_win32.h index 8ab4880fbd..d79dde157e 100644 --- a/src/lib/process/process_win32.h +++ b/src/lib/process/process_win32.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/process/restrict.c b/src/lib/process/restrict.c index bca325e5bc..534b39d101 100644 --- a/src/lib/process/restrict.c +++ b/src/lib/process/restrict.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/process/restrict.h b/src/lib/process/restrict.h index 2e78dc468c..8491c99044 100644 --- a/src/lib/process/restrict.h +++ b/src/lib/process/restrict.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/process/setuid.c b/src/lib/process/setuid.c index d038039ff4..6e8258f279 100644 --- a/src/lib/process/setuid.c +++ b/src/lib/process/setuid.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/process/setuid.h b/src/lib/process/setuid.h index 49751c97c2..7d03e1f025 100644 --- a/src/lib/process/setuid.h +++ b/src/lib/process/setuid.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/process/waitpid.c b/src/lib/process/waitpid.c index 46d30bf50e..2b38481aeb 100644 --- a/src/lib/process/waitpid.c +++ b/src/lib/process/waitpid.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/process/waitpid.h b/src/lib/process/waitpid.h index 85905da6bf..5faef468c1 100644 --- a/src/lib/process/waitpid.h +++ b/src/lib/process/waitpid.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2011-2018, The Tor Project, Inc. */ +/* Copyright (c) 2011-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/process/winprocess_sys.c b/src/lib/process/winprocess_sys.c index ef66f8bfb1..1266babca8 100644 --- a/src/lib/process/winprocess_sys.c +++ b/src/lib/process/winprocess_sys.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2018, The Tor Project, Inc. */ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/process/winprocess_sys.h b/src/lib/process/winprocess_sys.h index cb096e0c92..7ab2aa04a6 100644 --- a/src/lib/process/winprocess_sys.h +++ b/src/lib/process/winprocess_sys.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2018, The Tor Project, Inc. */ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/sandbox/sandbox.c b/src/lib/sandbox/sandbox.c index ea738b273e..1f0f5d858f 100644 --- a/src/lib/sandbox/sandbox.c +++ b/src/lib/sandbox/sandbox.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/sandbox/sandbox.h b/src/lib/sandbox/sandbox.h index 60d8e8816a..5bec09a36a 100644 --- a/src/lib/sandbox/sandbox.h +++ b/src/lib/sandbox/sandbox.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/smartlist_core/smartlist_core.c b/src/lib/smartlist_core/smartlist_core.c index 8364a8180b..ac85a6cc84 100644 --- a/src/lib/smartlist_core/smartlist_core.c +++ b/src/lib/smartlist_core/smartlist_core.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/smartlist_core/smartlist_core.h b/src/lib/smartlist_core/smartlist_core.h index 974fb01758..a7fbaa099b 100644 --- a/src/lib/smartlist_core/smartlist_core.h +++ b/src/lib/smartlist_core/smartlist_core.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/smartlist_core/smartlist_foreach.h b/src/lib/smartlist_core/smartlist_foreach.h index c9afebd6a2..a1fbcd444c 100644 --- a/src/lib/smartlist_core/smartlist_foreach.h +++ b/src/lib/smartlist_core/smartlist_foreach.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/smartlist_core/smartlist_split.c b/src/lib/smartlist_core/smartlist_split.c index f63894869c..c9cf59851f 100644 --- a/src/lib/smartlist_core/smartlist_split.c +++ b/src/lib/smartlist_core/smartlist_split.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/smartlist_core/smartlist_split.h b/src/lib/smartlist_core/smartlist_split.h index 4dd48295ce..4f72376125 100644 --- a/src/lib/smartlist_core/smartlist_split.h +++ b/src/lib/smartlist_core/smartlist_split.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/string/compat_ctype.c b/src/lib/string/compat_ctype.c index 35f4ec6534..f5d82be3ae 100644 --- a/src/lib/string/compat_ctype.c +++ b/src/lib/string/compat_ctype.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/string/compat_ctype.h b/src/lib/string/compat_ctype.h index 210c460c31..dbddd356c1 100644 --- a/src/lib/string/compat_ctype.h +++ b/src/lib/string/compat_ctype.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/string/compat_string.c b/src/lib/string/compat_string.c index b3f1e0fd96..187f784be5 100644 --- a/src/lib/string/compat_string.c +++ b/src/lib/string/compat_string.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/string/compat_string.h b/src/lib/string/compat_string.h index 9292717337..a0e37bb6dc 100644 --- a/src/lib/string/compat_string.h +++ b/src/lib/string/compat_string.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/string/parse_int.c b/src/lib/string/parse_int.c index 52ff49ef1e..fbdd554a47 100644 --- a/src/lib/string/parse_int.c +++ b/src/lib/string/parse_int.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/string/parse_int.h b/src/lib/string/parse_int.h index 663a5acd74..925547942e 100644 --- a/src/lib/string/parse_int.h +++ b/src/lib/string/parse_int.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/string/printf.c b/src/lib/string/printf.c index e23da69d0e..415d4ac4a7 100644 --- a/src/lib/string/printf.c +++ b/src/lib/string/printf.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/string/printf.h b/src/lib/string/printf.h index 49c37d43e0..2cc13d6bee 100644 --- a/src/lib/string/printf.h +++ b/src/lib/string/printf.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/string/scanf.c b/src/lib/string/scanf.c index 7b08442148..1bc39b5182 100644 --- a/src/lib/string/scanf.c +++ b/src/lib/string/scanf.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/string/scanf.h b/src/lib/string/scanf.h index ada2322bb1..6673173de5 100644 --- a/src/lib/string/scanf.h +++ b/src/lib/string/scanf.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/string/util_string.c b/src/lib/string/util_string.c index 36e19d029c..0c4e399008 100644 --- a/src/lib/string/util_string.c +++ b/src/lib/string/util_string.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/string/util_string.h b/src/lib/string/util_string.h index 6541afa4cb..da4fab159c 100644 --- a/src/lib/string/util_string.h +++ b/src/lib/string/util_string.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/subsys/subsys.h b/src/lib/subsys/subsys.h index 2452ec6e2f..241ad7829c 100644 --- a/src/lib/subsys/subsys.h +++ b/src/lib/subsys/subsys.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_SUBSYS_T diff --git a/src/lib/term/getpass.c b/src/lib/term/getpass.c index a473fb765b..8741344acf 100644 --- a/src/lib/term/getpass.c +++ b/src/lib/term/getpass.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/term/getpass.h b/src/lib/term/getpass.h index e8347e7fe8..a9c146ea8f 100644 --- a/src/lib/term/getpass.h +++ b/src/lib/term/getpass.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/testsupport/testsupport.h b/src/lib/testsupport/testsupport.h index 3ae1b48f87..9363a9ba66 100644 --- a/src/lib/testsupport/testsupport.h +++ b/src/lib/testsupport/testsupport.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2018, The Tor Project, Inc. */ +/* Copyright (c) 2013-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/thread/compat_pthreads.c b/src/lib/thread/compat_pthreads.c index e5c8805ddc..05efe9cfd0 100644 --- a/src/lib/thread/compat_pthreads.c +++ b/src/lib/thread/compat_pthreads.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/thread/compat_threads.c b/src/lib/thread/compat_threads.c index 0b466da212..35cfeba64c 100644 --- a/src/lib/thread/compat_threads.c +++ b/src/lib/thread/compat_threads.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/thread/compat_winthreads.c b/src/lib/thread/compat_winthreads.c index 799eeda1b4..f0b1430e84 100644 --- a/src/lib/thread/compat_winthreads.c +++ b/src/lib/thread/compat_winthreads.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/thread/numcpus.c b/src/lib/thread/numcpus.c index cca15eb7aa..b293d965d2 100644 --- a/src/lib/thread/numcpus.c +++ b/src/lib/thread/numcpus.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/thread/numcpus.h b/src/lib/thread/numcpus.h index 0b026e4249..3f0a29ce7c 100644 --- a/src/lib/thread/numcpus.h +++ b/src/lib/thread/numcpus.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/thread/thread_sys.h b/src/lib/thread/thread_sys.h index 984abe88e8..c0daf2b5e9 100644 --- a/src/lib/thread/thread_sys.h +++ b/src/lib/thread/thread_sys.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2018, The Tor Project, Inc. */ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/thread/threads.h b/src/lib/thread/threads.h index 4d5191124c..ecf60641b5 100644 --- a/src/lib/thread/threads.h +++ b/src/lib/thread/threads.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/time/compat_time.c b/src/lib/time/compat_time.c index 387b0fad22..33e077a587 100644 --- a/src/lib/time/compat_time.c +++ b/src/lib/time/compat_time.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/time/compat_time.h b/src/lib/time/compat_time.h index 3c8797c450..2cd4b3bee3 100644 --- a/src/lib/time/compat_time.h +++ b/src/lib/time/compat_time.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/time/time_sys.c b/src/lib/time/time_sys.c index b29ca35e69..b3feb7b46a 100644 --- a/src/lib/time/time_sys.c +++ b/src/lib/time/time_sys.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2018, The Tor Project, Inc. */ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/time/time_sys.h b/src/lib/time/time_sys.h index 5f5982a33b..6a860ffd08 100644 --- a/src/lib/time/time_sys.h +++ b/src/lib/time/time_sys.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2018, The Tor Project, Inc. */ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/time/tvdiff.c b/src/lib/time/tvdiff.c index 9dfb63c26f..d7c245f57a 100644 --- a/src/lib/time/tvdiff.c +++ b/src/lib/time/tvdiff.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/time/tvdiff.h b/src/lib/time/tvdiff.h index a15ce52ad6..724af1528a 100644 --- a/src/lib/time/tvdiff.h +++ b/src/lib/time/tvdiff.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/tls/buffers_tls.c b/src/lib/tls/buffers_tls.c index b4059292ea..3c18cc7e43 100644 --- a/src/lib/tls/buffers_tls.c +++ b/src/lib/tls/buffers_tls.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/tls/buffers_tls.h b/src/lib/tls/buffers_tls.h index 7a1ca6d16c..65788c3f34 100644 --- a/src/lib/tls/buffers_tls.h +++ b/src/lib/tls/buffers_tls.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/tls/nss_countbytes.c b/src/lib/tls/nss_countbytes.c index c727684529..7761727acd 100644 --- a/src/lib/tls/nss_countbytes.c +++ b/src/lib/tls/nss_countbytes.c @@ -1,4 +1,4 @@ -/* Copyright 2018, The Tor Project Inc. */ +/* Copyright 2018-2019, The Tor Project Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/tls/nss_countbytes.h b/src/lib/tls/nss_countbytes.h index f26280edf2..8b31603923 100644 --- a/src/lib/tls/nss_countbytes.h +++ b/src/lib/tls/nss_countbytes.h @@ -1,4 +1,4 @@ -/* Copyright 2018, The Tor Project, Inc. */ +/* Copyright 2018-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/tls/tortls.c b/src/lib/tls/tortls.c index 654cacacf7..1aff40c437 100644 --- a/src/lib/tls/tortls.c +++ b/src/lib/tls/tortls.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define TORTLS_PRIVATE diff --git a/src/lib/tls/tortls.h b/src/lib/tls/tortls.h index 81db5ce5a9..8efc7a1c98 100644 --- a/src/lib/tls/tortls.h +++ b/src/lib/tls/tortls.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_TORTLS_H diff --git a/src/lib/tls/tortls_internal.h b/src/lib/tls/tortls_internal.h index 2bf2212104..071c506561 100644 --- a/src/lib/tls/tortls_internal.h +++ b/src/lib/tls/tortls_internal.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TORTLS_INTERNAL_H diff --git a/src/lib/tls/tortls_nss.c b/src/lib/tls/tortls_nss.c index 462cd5b0ff..00c4af0e97 100644 --- a/src/lib/tls/tortls_nss.c +++ b/src/lib/tls/tortls_nss.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/tls/tortls_openssl.c b/src/lib/tls/tortls_openssl.c index c0ad74d908..f9f7a7a179 100644 --- a/src/lib/tls/tortls_openssl.c +++ b/src/lib/tls/tortls_openssl.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/tls/tortls_st.h b/src/lib/tls/tortls_st.h index 549443a4e7..3f7ea8ac6a 100644 --- a/src/lib/tls/tortls_st.h +++ b/src/lib/tls/tortls_st.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_TORTLS_ST_H diff --git a/src/lib/tls/tortls_sys.h b/src/lib/tls/tortls_sys.h index fd909f6019..4b04f85f0c 100644 --- a/src/lib/tls/tortls_sys.h +++ b/src/lib/tls/tortls_sys.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2018, The Tor Project, Inc. */ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/tls/x509.c b/src/lib/tls/x509.c index 0d62ee41af..67a8b49b9d 100644 --- a/src/lib/tls/x509.c +++ b/src/lib/tls/x509.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/tls/x509.h b/src/lib/tls/x509.h index e7440a192f..5e6660de5c 100644 --- a/src/lib/tls/x509.h +++ b/src/lib/tls/x509.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_X509_H diff --git a/src/lib/tls/x509_internal.h b/src/lib/tls/x509_internal.h index 139ecedd23..bf2bec9689 100644 --- a/src/lib/tls/x509_internal.h +++ b/src/lib/tls/x509_internal.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_X509_INTERNAL_H diff --git a/src/lib/tls/x509_nss.c b/src/lib/tls/x509_nss.c index a0a9c559cf..fb4af54c52 100644 --- a/src/lib/tls/x509_nss.c +++ b/src/lib/tls/x509_nss.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/tls/x509_openssl.c b/src/lib/tls/x509_openssl.c index dd74e84418..cf276c4240 100644 --- a/src/lib/tls/x509_openssl.c +++ b/src/lib/tls/x509_openssl.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/trace/debug.h b/src/lib/trace/debug.h index 191d94226d..e35616cf50 100644 --- a/src/lib/trace/debug.h +++ b/src/lib/trace/debug.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2017-2018, The Tor Project, Inc. */ +/* Copyright (c) 2017-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/trace/events.h b/src/lib/trace/events.h index 6d4269aaed..1e1e7b9d16 100644 --- a/src/lib/trace/events.h +++ b/src/lib/trace/events.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2017-2018, The Tor Project, Inc. */ +/* Copyright (c) 2017-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/trace/trace.c b/src/lib/trace/trace.c index 535ffde183..18be63c5a8 100644 --- a/src/lib/trace/trace.c +++ b/src/lib/trace/trace.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2017-2018, The Tor Project, Inc. */ +/* Copyright (c) 2017-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/trace/trace.h b/src/lib/trace/trace.h index 5f7b0ee7cd..606d435568 100644 --- a/src/lib/trace/trace.h +++ b/src/lib/trace/trace.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2017-2018, The Tor Project, Inc. */ +/* Copyright (c) 2017-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/version/git_revision.c b/src/lib/version/git_revision.c index e5b2ff534e..900a1e12a0 100644 --- a/src/lib/version/git_revision.c +++ b/src/lib/version/git_revision.c @@ -1,6 +1,6 @@ /* Copyright 2001-2004 Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "orconfig.h" diff --git a/src/lib/version/git_revision.h b/src/lib/version/git_revision.h index 0ce1190795..79e3c6684b 100644 --- a/src/lib/version/git_revision.h +++ b/src/lib/version/git_revision.h @@ -1,6 +1,6 @@ /* Copyright 2001-2004 Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_GIT_REVISION_H diff --git a/src/lib/version/torversion.h b/src/lib/version/torversion.h index 761d6f25ab..7b0fb66ec0 100644 --- a/src/lib/version/torversion.h +++ b/src/lib/version/torversion.h @@ -1,6 +1,6 @@ /* Copyright 2001-2004 Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_VERSION_H diff --git a/src/lib/version/version.c b/src/lib/version/version.c index 29ada39c9d..434e6fb424 100644 --- a/src/lib/version/version.c +++ b/src/lib/version/version.c @@ -1,6 +1,6 @@ /* Copyright 2001-2004 Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "orconfig.h" diff --git a/src/lib/wallclock/approx_time.c b/src/lib/wallclock/approx_time.c index 0b0ef382c2..7b32804026 100644 --- a/src/lib/wallclock/approx_time.c +++ b/src/lib/wallclock/approx_time.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/wallclock/approx_time.h b/src/lib/wallclock/approx_time.h index becc632fe3..e6b53f2c27 100644 --- a/src/lib/wallclock/approx_time.h +++ b/src/lib/wallclock/approx_time.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/wallclock/time_to_tm.c b/src/lib/wallclock/time_to_tm.c index 6543b97e37..f7cb21827b 100644 --- a/src/lib/wallclock/time_to_tm.c +++ b/src/lib/wallclock/time_to_tm.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/wallclock/time_to_tm.h b/src/lib/wallclock/time_to_tm.h index 0527a97b34..abe78c0efe 100644 --- a/src/lib/wallclock/time_to_tm.h +++ b/src/lib/wallclock/time_to_tm.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/wallclock/timeval.h b/src/lib/wallclock/timeval.h index b34277cda6..4967e939bf 100644 --- a/src/lib/wallclock/timeval.h +++ b/src/lib/wallclock/timeval.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/wallclock/tor_gettimeofday.c b/src/lib/wallclock/tor_gettimeofday.c index aefe796ad2..63538f3b81 100644 --- a/src/lib/wallclock/tor_gettimeofday.c +++ b/src/lib/wallclock/tor_gettimeofday.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/wallclock/tor_gettimeofday.h b/src/lib/wallclock/tor_gettimeofday.h index aac6366a65..c7fff9747a 100644 --- a/src/lib/wallclock/tor_gettimeofday.h +++ b/src/lib/wallclock/tor_gettimeofday.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/wallclock/wallclock_sys.h b/src/lib/wallclock/wallclock_sys.h index e009578a83..a30912b8fb 100644 --- a/src/lib/wallclock/wallclock_sys.h +++ b/src/lib/wallclock/wallclock_sys.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2018, The Tor Project, Inc. */ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/rust/crypto/digests/mod.rs b/src/rust/crypto/digests/mod.rs index a2463b89eb..58343b9ca7 100644 --- a/src/rust/crypto/digests/mod.rs +++ b/src/rust/crypto/digests/mod.rs @@ -1,4 +1,4 @@ -// Copyright (c) 2018, The Tor Project, Inc. +// Copyright (c) 2018-2019, The Tor Project, Inc. // Copyright (c) 2018, isis agora lovecruft // See LICENSE for licensing information diff --git a/src/rust/crypto/digests/sha2.rs b/src/rust/crypto/digests/sha2.rs index 55d0027665..91e8b2b3c9 100644 --- a/src/rust/crypto/digests/sha2.rs +++ b/src/rust/crypto/digests/sha2.rs @@ -1,4 +1,4 @@ -// Copyright (c) 2018, The Tor Project, Inc. +// Copyright (c) 2018-2019, The Tor Project, Inc. // Copyright (c) 2018, isis agora lovecruft // See LICENSE for licensing information diff --git a/src/rust/crypto/lib.rs b/src/rust/crypto/lib.rs index 4eceb4cbd1..866ea93547 100644 --- a/src/rust/crypto/lib.rs +++ b/src/rust/crypto/lib.rs @@ -1,4 +1,4 @@ -// Copyright (c) 2018, The Tor Project, Inc. +// Copyright (c) 2018-2019, The Tor Project, Inc. // Copyright (c) 2018, isis agora lovecruft // See LICENSE for licensing information diff --git a/src/rust/crypto/rand/mod.rs b/src/rust/crypto/rand/mod.rs index 82d02a70bb..da8b3bd8a5 100644 --- a/src/rust/crypto/rand/mod.rs +++ b/src/rust/crypto/rand/mod.rs @@ -1,4 +1,4 @@ -// Copyright (c) 2018, The Tor Project, Inc. +// Copyright (c) 2018-2019, The Tor Project, Inc. // Copyright (c) 2018, isis agora lovecruft // See LICENSE for licensing information diff --git a/src/rust/crypto/rand/rng.rs b/src/rust/crypto/rand/rng.rs index 64ceb22424..96e112799e 100644 --- a/src/rust/crypto/rand/rng.rs +++ b/src/rust/crypto/rand/rng.rs @@ -1,4 +1,4 @@ -// Copyright (c) 2018, The Tor Project, Inc. +// Copyright (c) 2018-2019, The Tor Project, Inc. // Copyright (c) 2018, isis agora lovecruft // See LICENSE for licensing information diff --git a/src/rust/external/crypto_digest.rs b/src/rust/external/crypto_digest.rs index ebcf2e88a9..454f836bad 100644 --- a/src/rust/external/crypto_digest.rs +++ b/src/rust/external/crypto_digest.rs @@ -1,4 +1,4 @@ -// Copyright (c) 2018, The Tor Project, Inc. +// Copyright (c) 2018-2019, The Tor Project, Inc. // Copyright (c) 2018, isis agora lovecruft // See LICENSE for licensing information diff --git a/src/rust/external/crypto_rand.rs b/src/rust/external/crypto_rand.rs index b68f98b358..703382093c 100644 --- a/src/rust/external/crypto_rand.rs +++ b/src/rust/external/crypto_rand.rs @@ -1,4 +1,4 @@ -// Copyright (c) 2018, The Tor Project, Inc. +// Copyright (c) 2018-2019, The Tor Project, Inc. // Copyright (c) 2018, isis agora lovecruft // See LICENSE for licensing information diff --git a/src/rust/external/external.rs b/src/rust/external/external.rs index aa43d2a928..0d324c8820 100644 --- a/src/rust/external/external.rs +++ b/src/rust/external/external.rs @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018, The Tor Project, Inc. */ +// Copyright (c) 2016-2019, The Tor Project, Inc. */ // See LICENSE for licensing information */ use libc::{c_char, c_int}; diff --git a/src/rust/external/lib.rs b/src/rust/external/lib.rs index d68036fcad..2f50610a4d 100644 --- a/src/rust/external/lib.rs +++ b/src/rust/external/lib.rs @@ -1,4 +1,4 @@ -//! Copyright (c) 2016-2018, The Tor Project, Inc. */ +//! Copyright (c) 2016-2019, The Tor Project, Inc. */ //! See LICENSE for licensing information */ //! Interface for external calls to tor C ABI diff --git a/src/rust/protover/errors.rs b/src/rust/protover/errors.rs index f26a48b019..dc0d8735f4 100644 --- a/src/rust/protover/errors.rs +++ b/src/rust/protover/errors.rs @@ -1,4 +1,4 @@ -// Copyright (c) 2018, The Tor Project, Inc. +// Copyright (c) 2018-2019, The Tor Project, Inc. // Copyright (c) 2018, isis agora lovecruft // See LICENSE for licensing information diff --git a/src/rust/protover/ffi.rs b/src/rust/protover/ffi.rs index 5c29adf045..6ee63adb10 100644 --- a/src/rust/protover/ffi.rs +++ b/src/rust/protover/ffi.rs @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018, The Tor Project, Inc. */ +// Copyright (c) 2016-2019, The Tor Project, Inc. */ // See LICENSE for licensing information */ //! FFI functions, only to be called from C. diff --git a/src/rust/protover/lib.rs b/src/rust/protover/lib.rs index 9625cb58ad..35c4106ae5 100644 --- a/src/rust/protover/lib.rs +++ b/src/rust/protover/lib.rs @@ -1,4 +1,4 @@ -//! Copyright (c) 2016-2018, The Tor Project, Inc. */ +//! Copyright (c) 2016-2019, The Tor Project, Inc. */ //! See LICENSE for licensing information */ //! Versioning information for different pieces of the Tor protocol. diff --git a/src/rust/protover/protoset.rs b/src/rust/protover/protoset.rs index aa8d243bad..3b283983c8 100644 --- a/src/rust/protover/protoset.rs +++ b/src/rust/protover/protoset.rs @@ -1,4 +1,4 @@ -// Copyright (c) 2018, The Tor Project, Inc. +// Copyright (c) 2018-2019, The Tor Project, Inc. // Copyright (c) 2018, isis agora lovecruft // See LICENSE for licensing information diff --git a/src/rust/protover/protover.rs b/src/rust/protover/protover.rs index 0b2a78c210..74158d9f6d 100644 --- a/src/rust/protover/protover.rs +++ b/src/rust/protover/protover.rs @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018, The Tor Project, Inc. */ +// Copyright (c) 2016-2019, The Tor Project, Inc. */ // See LICENSE for licensing information */ use std::collections::hash_map; diff --git a/src/rust/protover/tests/protover.rs b/src/rust/protover/tests/protover.rs index 86e276cf73..942fe3c6ab 100644 --- a/src/rust/protover/tests/protover.rs +++ b/src/rust/protover/tests/protover.rs @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018, The Tor Project, Inc. */ +// Copyright (c) 2016-2019, The Tor Project, Inc. */ // See LICENSE for licensing information */ extern crate protover; diff --git a/src/rust/smartlist/lib.rs b/src/rust/smartlist/lib.rs index 34d0b907ed..23301f88c3 100644 --- a/src/rust/smartlist/lib.rs +++ b/src/rust/smartlist/lib.rs @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018, The Tor Project, Inc. */ +// Copyright (c) 2016-2019, The Tor Project, Inc. */ // See LICENSE for licensing information */ extern crate libc; diff --git a/src/rust/smartlist/smartlist.rs b/src/rust/smartlist/smartlist.rs index bce58c0ef9..d8f8083dff 100644 --- a/src/rust/smartlist/smartlist.rs +++ b/src/rust/smartlist/smartlist.rs @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018, The Tor Project, Inc. */ +// Copyright (c) 2016-2019, The Tor Project, Inc. */ // See LICENSE for licensing information */ use libc::{c_char, c_int}; diff --git a/src/rust/tor_allocate/lib.rs b/src/rust/tor_allocate/lib.rs index 1cfa0b5178..fff8a08006 100644 --- a/src/rust/tor_allocate/lib.rs +++ b/src/rust/tor_allocate/lib.rs @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018, The Tor Project, Inc. */ +// Copyright (c) 2016-2019, The Tor Project, Inc. */ // See LICENSE for licensing information */ //! Allocation helper functions that allow data to be allocated in Rust diff --git a/src/rust/tor_allocate/tor_allocate.rs b/src/rust/tor_allocate/tor_allocate.rs index 48351d8482..682a524ee7 100644 --- a/src/rust/tor_allocate/tor_allocate.rs +++ b/src/rust/tor_allocate/tor_allocate.rs @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018, The Tor Project, Inc. */ +// Copyright (c) 2016-2019, The Tor Project, Inc. */ // See LICENSE for licensing information */ // No-op defined purely for testing at the module level use libc::c_char; diff --git a/src/rust/tor_log/lib.rs b/src/rust/tor_log/lib.rs index 21855ae73b..4aa658e35b 100644 --- a/src/rust/tor_log/lib.rs +++ b/src/rust/tor_log/lib.rs @@ -1,4 +1,4 @@ -//! Copyright (c) 2016-2018, The Tor Project, Inc. */ +//! Copyright (c) 2016-2019, The Tor Project, Inc. */ //! See LICENSE for licensing information */ //! Logging wrapper for Rust to utilize Tor's logger, found at diff --git a/src/rust/tor_log/tor_log.rs b/src/rust/tor_log/tor_log.rs index 757c74ff49..98fccba5a9 100644 --- a/src/rust/tor_log/tor_log.rs +++ b/src/rust/tor_log/tor_log.rs @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018, The Tor Project, Inc. */ +// Copyright (c) 2016-2019, The Tor Project, Inc. */ // See LICENSE for licensing information */ // Note that these functions are untested due to the fact that there are no diff --git a/src/rust/tor_util/ffi.rs b/src/rust/tor_util/ffi.rs index f015590178..b71b2bd093 100644 --- a/src/rust/tor_util/ffi.rs +++ b/src/rust/tor_util/ffi.rs @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018, The Tor Project, Inc. */ +// Copyright (c) 2016-2019, The Tor Project, Inc. */ // See LICENSE for licensing information */ //! FFI functions to announce Rust support during tor startup, only to be diff --git a/src/rust/tor_util/lib.rs b/src/rust/tor_util/lib.rs index 4ce5fc9374..8886767ede 100644 --- a/src/rust/tor_util/lib.rs +++ b/src/rust/tor_util/lib.rs @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018, The Tor Project, Inc. */ +// Copyright (c) 2016-2019, The Tor Project, Inc. */ // See LICENSE for licensing information */ //! Small module to announce Rust support during startup for demonstration diff --git a/src/rust/tor_util/strings.rs b/src/rust/tor_util/strings.rs index 71a908a58c..ede42c6ea8 100644 --- a/src/rust/tor_util/strings.rs +++ b/src/rust/tor_util/strings.rs @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018, The Tor Project, Inc. */ +// Copyright (c) 2016-2019, The Tor Project, Inc. */ // See LICENSE for licensing information */ //! Utilities for working with static strings. diff --git a/src/test/bench.c b/src/test/bench.c index f8680c3ab6..0713eb6719 100644 --- a/src/test/bench.c +++ b/src/test/bench.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/test/bt_test.py b/src/test/bt_test.py index f8894aac0b..f9ca79efde 100755 --- a/src/test/bt_test.py +++ b/src/test/bt_test.py @@ -1,4 +1,4 @@ -# Copyright 2013-2018, The Tor Project, Inc +# Copyright 2013-2019, The Tor Project, Inc # See LICENSE for licensing information """ diff --git a/src/test/ed25519_exts_ref.py b/src/test/ed25519_exts_ref.py index a9090c9ed2..75562184b5 100644 --- a/src/test/ed25519_exts_ref.py +++ b/src/test/ed25519_exts_ref.py @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright 2014-2018, The Tor Project, Inc +# Copyright 2014-2019, The Tor Project, Inc # See LICENSE for licensing information """ diff --git a/src/test/fakechans.h b/src/test/fakechans.h index 0770be8e04..4006e1bec4 100644 --- a/src/test/fakechans.h +++ b/src/test/fakechans.h @@ -1,4 +1,4 @@ - /* Copyright (c) 2014-2018, The Tor Project, Inc. */ + /* Copyright (c) 2014-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_FAKECHANS_H diff --git a/src/test/fuzz/dict/http b/src/test/fuzz/dict/http index 63627ac380..aa3dec990d 100644 --- a/src/test/fuzz/dict/http +++ b/src/test/fuzz/dict/http @@ -4,7 +4,7 @@ # # Extracted from directory_handle_command() in the tor source code # -# Copyright (c) 2016-2018, The Tor Project, Inc. +# Copyright (c) 2016-2019, The Tor Project, Inc. # See LICENSE for licensing information # # Usage: diff --git a/src/test/fuzz/fuzz_consensus.c b/src/test/fuzz/fuzz_consensus.c index 1a4195b418..656ef0bdb2 100644 --- a/src/test/fuzz/fuzz_consensus.c +++ b/src/test/fuzz/fuzz_consensus.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2016-2018, The Tor Project, Inc. */ +/* Copyright (c) 2016-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define SIGCOMMON_PRIVATE #include "core/or/or.h" diff --git a/src/test/fuzz/fuzz_descriptor.c b/src/test/fuzz/fuzz_descriptor.c index 3420113717..58ee3dbc35 100644 --- a/src/test/fuzz/fuzz_descriptor.c +++ b/src/test/fuzz/fuzz_descriptor.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2016-2018, The Tor Project, Inc. */ +/* Copyright (c) 2016-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define SIGCOMMON_PRIVATE #include "core/or/or.h" diff --git a/src/test/fuzz/fuzz_diff.c b/src/test/fuzz/fuzz_diff.c index 64aecc8a64..a31445666c 100644 --- a/src/test/fuzz/fuzz_diff.c +++ b/src/test/fuzz/fuzz_diff.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2016-2018, The Tor Project, Inc. */ +/* Copyright (c) 2016-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define CONSDIFF_PRIVATE diff --git a/src/test/fuzz/fuzz_diff_apply.c b/src/test/fuzz/fuzz_diff_apply.c index 9b25185225..d8a0f9e590 100644 --- a/src/test/fuzz/fuzz_diff_apply.c +++ b/src/test/fuzz/fuzz_diff_apply.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2016-2018, The Tor Project, Inc. */ +/* Copyright (c) 2016-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define CONSDIFF_PRIVATE diff --git a/src/test/fuzz/fuzz_extrainfo.c b/src/test/fuzz/fuzz_extrainfo.c index da0fe80838..f18bd68d65 100644 --- a/src/test/fuzz/fuzz_extrainfo.c +++ b/src/test/fuzz/fuzz_extrainfo.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2016-2018, The Tor Project, Inc. */ +/* Copyright (c) 2016-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define SIGCOMMON_PRIVATE #include "core/or/or.h" diff --git a/src/test/fuzz/fuzz_hsdescv2.c b/src/test/fuzz/fuzz_hsdescv2.c index 667b58b3aa..34639b237c 100644 --- a/src/test/fuzz/fuzz_hsdescv2.c +++ b/src/test/fuzz/fuzz_hsdescv2.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2016-2018, The Tor Project, Inc. */ +/* Copyright (c) 2016-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "core/or/or.h" #include "feature/dirparse/unparseable.h" diff --git a/src/test/fuzz/fuzz_hsdescv3.c b/src/test/fuzz/fuzz_hsdescv3.c index d5ddcc2e27..2cbd655898 100644 --- a/src/test/fuzz/fuzz_hsdescv3.c +++ b/src/test/fuzz/fuzz_hsdescv3.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2017-2018, The Tor Project, Inc. */ +/* Copyright (c) 2017-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define HS_DESCRIPTOR_PRIVATE diff --git a/src/test/fuzz/fuzz_http.c b/src/test/fuzz/fuzz_http.c index 4341bfabae..44393b3a10 100644 --- a/src/test/fuzz/fuzz_http.c +++ b/src/test/fuzz/fuzz_http.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2016-2018, The Tor Project, Inc. */ +/* Copyright (c) 2016-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "orconfig.h" diff --git a/src/test/fuzz/fuzz_http_connect.c b/src/test/fuzz/fuzz_http_connect.c index e03d9e29d8..2a597cae74 100644 --- a/src/test/fuzz/fuzz_http_connect.c +++ b/src/test/fuzz/fuzz_http_connect.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2016-2018, The Tor Project, Inc. */ +/* Copyright (c) 2016-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "orconfig.h" diff --git a/src/test/fuzz/fuzz_iptsv2.c b/src/test/fuzz/fuzz_iptsv2.c index 265677eebe..76fa3c164e 100644 --- a/src/test/fuzz/fuzz_iptsv2.c +++ b/src/test/fuzz/fuzz_iptsv2.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2016-2018, The Tor Project, Inc. */ +/* Copyright (c) 2016-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "core/or/or.h" diff --git a/src/test/fuzz/fuzz_microdesc.c b/src/test/fuzz/fuzz_microdesc.c index ab54cf2a34..28fdc5e24d 100644 --- a/src/test/fuzz/fuzz_microdesc.c +++ b/src/test/fuzz/fuzz_microdesc.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2016-2018, The Tor Project, Inc. */ +/* Copyright (c) 2016-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "core/or/or.h" diff --git a/src/test/fuzz/fuzz_socks.c b/src/test/fuzz/fuzz_socks.c index 2d93bea924..d6c416a0f9 100644 --- a/src/test/fuzz/fuzz_socks.c +++ b/src/test/fuzz/fuzz_socks.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2016-2018, The Tor Project, Inc. */ +/* Copyright (c) 2016-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "orconfig.h" diff --git a/src/test/fuzz/fuzz_strops.c b/src/test/fuzz/fuzz_strops.c index 5da590acfa..64a6453050 100644 --- a/src/test/fuzz/fuzz_strops.c +++ b/src/test/fuzz/fuzz_strops.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2018, The Tor Project, Inc. */ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/test/fuzz/fuzz_vrs.c b/src/test/fuzz/fuzz_vrs.c index f0d90d7cc6..7b61b8df2d 100644 --- a/src/test/fuzz/fuzz_vrs.c +++ b/src/test/fuzz/fuzz_vrs.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2016-2018, The Tor Project, Inc. */ +/* Copyright (c) 2016-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define NS_PARSE_PRIVATE #define NETWORKSTATUS_PRIVATE diff --git a/src/test/fuzz/fuzzing.h b/src/test/fuzz/fuzzing.h index e90e5d58e0..150ac4aa7d 100644 --- a/src/test/fuzz/fuzzing.h +++ b/src/test/fuzz/fuzzing.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2016-2018, The Tor Project, Inc. */ +/* Copyright (c) 2016-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef FUZZING_H #define FUZZING_H diff --git a/src/test/fuzz/fuzzing_common.c b/src/test/fuzz/fuzzing_common.c index 21aa07cfe2..387c865a9b 100644 --- a/src/test/fuzz/fuzzing_common.c +++ b/src/test/fuzz/fuzzing_common.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2016-2018, The Tor Project, Inc. */ +/* Copyright (c) 2016-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define CRYPTO_ED25519_PRIVATE #include "orconfig.h" diff --git a/src/test/fuzz_static_testcases.sh b/src/test/fuzz_static_testcases.sh index 138f85b106..f7b3adffb1 100755 --- a/src/test/fuzz_static_testcases.sh +++ b/src/test/fuzz_static_testcases.sh @@ -1,6 +1,6 @@ #!/bin/sh -# Copyright (c) 2016-2018, The Tor Project, Inc. +# Copyright (c) 2016-2019, The Tor Project, Inc. # See LICENSE for licensing information set -e diff --git a/src/test/hs_ntor_ref.py b/src/test/hs_ntor_ref.py index 0c5756ad73..1b9772a5d6 100644 --- a/src/test/hs_ntor_ref.py +++ b/src/test/hs_ntor_ref.py @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright 2017-2018, The Tor Project, Inc +# Copyright 2017-2019, The Tor Project, Inc # See LICENSE for licensing information """ diff --git a/src/test/hs_test_helpers.c b/src/test/hs_test_helpers.c index dcec1b9d48..f2ae8398df 100644 --- a/src/test/hs_test_helpers.c +++ b/src/test/hs_test_helpers.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2017-2018, The Tor Project, Inc. */ +/* Copyright (c) 2017-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "core/or/or.h" diff --git a/src/test/hs_test_helpers.h b/src/test/hs_test_helpers.h index b7c2714769..9662a83ba8 100644 --- a/src/test/hs_test_helpers.h +++ b/src/test/hs_test_helpers.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2017-2018, The Tor Project, Inc. */ +/* Copyright (c) 2017-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_HS_TEST_HELPERS_H diff --git a/src/test/log_test_helpers.c b/src/test/log_test_helpers.c index 2e91b1ecdc..03c52dd6bd 100644 --- a/src/test/log_test_helpers.c +++ b/src/test/log_test_helpers.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2015-2018, The Tor Project, Inc. */ +/* Copyright (c) 2015-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define LOG_PRIVATE #include "lib/log/log.h" diff --git a/src/test/log_test_helpers.h b/src/test/log_test_helpers.h index 6a774cdfc7..5d1c3c1914 100644 --- a/src/test/log_test_helpers.h +++ b/src/test/log_test_helpers.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2014-2018, The Tor Project, Inc. */ +/* Copyright (c) 2014-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "core/or/or.h" diff --git a/src/test/ntor_ref.py b/src/test/ntor_ref.py index 56e97ece36..204f05e2ad 100755 --- a/src/test/ntor_ref.py +++ b/src/test/ntor_ref.py @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright 2012-2018, The Tor Project, Inc +# Copyright 2012-2019, The Tor Project, Inc # See LICENSE for licensing information """ diff --git a/src/test/ope_ref.py b/src/test/ope_ref.py index 3677e57a61..f9bd97c546 100644 --- a/src/test/ope_ref.py +++ b/src/test/ope_ref.py @@ -1,5 +1,5 @@ #!/usr/bin/python3 -# Copyright 2018, The Tor Project, Inc. See LICENSE for licensing info. +# Copyright 2018-2019, The Tor Project, Inc. See LICENSE for licensing info. # Reference implementation for our rudimentary OPE code, used to # generate test vectors. See crypto_ope.c for more details. diff --git a/src/test/prob_distr_mpfr_ref.c b/src/test/prob_distr_mpfr_ref.c index 4e64d731cd..425733dc1b 100644 --- a/src/test/prob_distr_mpfr_ref.c +++ b/src/test/prob_distr_mpfr_ref.c @@ -1,4 +1,4 @@ -/* Copyright 2012-2018, The Tor Project, Inc +/* Copyright 2012-2019, The Tor Project, Inc * See LICENSE for licensing information */ /** prob_distr_mpfr_ref.c diff --git a/src/test/rend_test_helpers.c b/src/test/rend_test_helpers.c index 85a679a967..f12d193cc5 100644 --- a/src/test/rend_test_helpers.c +++ b/src/test/rend_test_helpers.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2014-2018, The Tor Project, Inc. */ +/* Copyright (c) 2014-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "core/or/or.h" diff --git a/src/test/rend_test_helpers.h b/src/test/rend_test_helpers.h index 103e143ec6..c10da52cd7 100644 --- a/src/test/rend_test_helpers.h +++ b/src/test/rend_test_helpers.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2014-2018, The Tor Project, Inc. */ +/* Copyright (c) 2014-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "core/or/or.h" diff --git a/src/test/test-memwipe.c b/src/test/test-memwipe.c index b00e854a1c..15e6a03d8a 100644 --- a/src/test/test-memwipe.c +++ b/src/test/test-memwipe.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2015-2018, The Tor Project, Inc. */ +/* Copyright (c) 2015-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "orconfig.h" diff --git a/src/test/test-process.c b/src/test/test-process.c index ec1b395002..eb28ad90e9 100644 --- a/src/test/test-process.c +++ b/src/test/test-process.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2011-2018, The Tor Project, Inc. */ +/* Copyright (c) 2011-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "orconfig.h" diff --git a/src/test/test-timers.c b/src/test/test-timers.c index 923f51ecce..c80fb1e305 100644 --- a/src/test/test-timers.c +++ b/src/test/test-timers.c @@ -1,4 +1,4 @@ -/* Copyright 2016-2018, The Tor Project, Inc. */ +/* Copyright 2016-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "orconfig.h" diff --git a/src/test/test.c b/src/test/test.c index 902565dfbe..82bdba676e 100644 --- a/src/test/test.c +++ b/src/test/test.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/test/test.h b/src/test/test.h index 39953e9f7e..5f549e5421 100644 --- a/src/test/test.h +++ b/src/test/test.h @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2003, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_TEST_H diff --git a/src/test/test_accounting.c b/src/test/test_accounting.c index 7721a9eb99..8ae8fe4343 100644 --- a/src/test/test_accounting.c +++ b/src/test/test_accounting.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2014-2018, The Tor Project, Inc. */ +/* Copyright (c) 2014-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "core/or/or.h" diff --git a/src/test/test_addr.c b/src/test/test_addr.c index 1d97db52a6..1aa7a3dcd6 100644 --- a/src/test/test_addr.c +++ b/src/test/test_addr.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define ADDRESSMAP_PRIVATE diff --git a/src/test/test_address.c b/src/test/test_address.c index a823fd9cd5..6cfe461b65 100644 --- a/src/test/test_address.c +++ b/src/test/test_address.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2014-2018, The Tor Project, Inc. */ +/* Copyright (c) 2014-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define ADDRESS_PRIVATE diff --git a/src/test/test_address_set.c b/src/test/test_address_set.c index f231740011..fb8408b3c3 100644 --- a/src/test/test_address_set.c +++ b/src/test/test_address_set.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2017-2018, The Tor Project, Inc. */ +/* Copyright (c) 2017-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "core/or/or.h" diff --git a/src/test/test_bridges.c b/src/test/test_bridges.c index 1cad5445f4..879ae6636b 100644 --- a/src/test/test_bridges.c +++ b/src/test/test_bridges.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2018, The Tor Project, Inc. */ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/test/test_bt_cl.c b/src/test/test_bt_cl.c index 89cbca2066..0c15a02ee4 100644 --- a/src/test/test_bt_cl.c +++ b/src/test/test_bt_cl.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2012-2018, The Tor Project, Inc. */ +/* Copyright (c) 2012-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "orconfig.h" diff --git a/src/test/test_btrack.c b/src/test/test_btrack.c index 7b5d108f98..48486fb5a1 100644 --- a/src/test/test_btrack.c +++ b/src/test/test_btrack.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2018, The Tor Project, Inc. */ +/* Copyright (c) 2013-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "core/or/or.h" diff --git a/src/test/test_buffers.c b/src/test/test_buffers.c index 85e7b8d90a..97311c85cc 100644 --- a/src/test/test_buffers.c +++ b/src/test/test_buffers.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define BUFFERS_PRIVATE diff --git a/src/test/test_bwmgt.c b/src/test/test_bwmgt.c index 7a1782c2c9..5a013aa268 100644 --- a/src/test/test_bwmgt.c +++ b/src/test/test_bwmgt.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2018, The Tor Project, Inc. */ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/test/test_cell_formats.c b/src/test/test_cell_formats.c index daf0296e2a..fc5367557d 100644 --- a/src/test/test_cell_formats.c +++ b/src/test/test_cell_formats.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "orconfig.h" diff --git a/src/test/test_cell_queue.c b/src/test/test_cell_queue.c index d74bb9c622..8fc1da031e 100644 --- a/src/test/test_cell_queue.c +++ b/src/test/test_cell_queue.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2018, The Tor Project, Inc. */ +/* Copyright (c) 2013-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define CIRCUITLIST_PRIVATE diff --git a/src/test/test_channel.c b/src/test/test_channel.c index 26af8de917..e55b9b0750 100644 --- a/src/test/test_channel.c +++ b/src/test/test_channel.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2018, The Tor Project, Inc. */ +/* Copyright (c) 2013-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define TOR_CHANNEL_INTERNAL_ diff --git a/src/test/test_channelpadding.c b/src/test/test_channelpadding.c index bdd7c5f0a6..5d012e462b 100644 --- a/src/test/test_channelpadding.c +++ b/src/test/test_channelpadding.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2016-2018, The Tor Project, Inc. */ +/* Copyright (c) 2016-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define TOR_CHANNEL_INTERNAL_ diff --git a/src/test/test_channeltls.c b/src/test/test_channeltls.c index 44d623561b..054d3910e4 100644 --- a/src/test/test_channeltls.c +++ b/src/test/test_channeltls.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2014-2018, The Tor Project, Inc. */ +/* Copyright (c) 2014-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "orconfig.h" diff --git a/src/test/test_checkdir.c b/src/test/test_checkdir.c index 652e308ed8..1df74c390a 100644 --- a/src/test/test_checkdir.c +++ b/src/test/test_checkdir.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2014-2018, The Tor Project, Inc. */ +/* Copyright (c) 2014-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "orconfig.h" diff --git a/src/test/test_circuitbuild.c b/src/test/test_circuitbuild.c index dd47ad7689..27f2cd1ca5 100644 --- a/src/test/test_circuitbuild.c +++ b/src/test/test_circuitbuild.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define CIRCUITBUILD_PRIVATE diff --git a/src/test/test_circuitlist.c b/src/test/test_circuitlist.c index 8dd7f5f5a9..5cebdbeda0 100644 --- a/src/test/test_circuitlist.c +++ b/src/test/test_circuitlist.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2018, The Tor Project, Inc. */ +/* Copyright (c) 2013-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define TOR_CHANNEL_INTERNAL_ diff --git a/src/test/test_circuitmux.c b/src/test/test_circuitmux.c index 1be2ff5281..a2b3e62fe8 100644 --- a/src/test/test_circuitmux.c +++ b/src/test/test_circuitmux.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2018, The Tor Project, Inc. */ +/* Copyright (c) 2013-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define TOR_CHANNEL_INTERNAL_ diff --git a/src/test/test_circuitstats.c b/src/test/test_circuitstats.c index c3cfad88da..1cbcb14f2b 100644 --- a/src/test/test_circuitstats.c +++ b/src/test/test_circuitstats.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2017-2018, The Tor Project, Inc. */ +/* Copyright (c) 2017-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define CIRCUITBUILD_PRIVATE diff --git a/src/test/test_circuituse.c b/src/test/test_circuituse.c index 720adeac84..3acfc12044 100644 --- a/src/test/test_circuituse.c +++ b/src/test/test_circuituse.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define CIRCUITLIST_PRIVATE diff --git a/src/test/test_compat_libevent.c b/src/test/test_compat_libevent.c index ade76bdb07..5d625483da 100644 --- a/src/test/test_compat_libevent.c +++ b/src/test/test_compat_libevent.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2010-2018, The Tor Project, Inc. */ +/* Copyright (c) 2010-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define COMPAT_LIBEVENT_PRIVATE diff --git a/src/test/test_config.c b/src/test/test_config.c index 67a43d669e..72649dd9b1 100644 --- a/src/test/test_config.c +++ b/src/test/test_config.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "orconfig.h" diff --git a/src/test/test_connection.c b/src/test/test_connection.c index 0013f47fbc..ebe7c6d36f 100644 --- a/src/test/test_connection.c +++ b/src/test/test_connection.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2015-2018, The Tor Project, Inc. */ +/* Copyright (c) 2015-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "orconfig.h" diff --git a/src/test/test_connection.h b/src/test/test_connection.h index 27c296504a..47a5599e5f 100644 --- a/src/test/test_connection.h +++ b/src/test/test_connection.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2014-2018, The Tor Project, Inc. */ +/* Copyright (c) 2014-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** Some constants used by test_connection and helpers */ diff --git a/src/test/test_conscache.c b/src/test/test_conscache.c index b5cbd72515..095ff09350 100644 --- a/src/test/test_conscache.c +++ b/src/test/test_conscache.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2017-2018, The Tor Project, Inc. */ +/* Copyright (c) 2017-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "core/or/or.h" diff --git a/src/test/test_consdiff.c b/src/test/test_consdiff.c index 23e8f7167e..7c4c92ea42 100644 --- a/src/test/test_consdiff.c +++ b/src/test/test_consdiff.c @@ -1,5 +1,5 @@ /* Copyright (c) 2014, Daniel Martí - * Copyright (c) 2014-2018, The Tor Project, Inc. */ + * Copyright (c) 2014-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define CONSDIFF_PRIVATE diff --git a/src/test/test_consdiffmgr.c b/src/test/test_consdiffmgr.c index b84753ff83..74226b8c52 100644 --- a/src/test/test_consdiffmgr.c +++ b/src/test/test_consdiffmgr.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2017-2018, The Tor Project, Inc. */ +/* Copyright (c) 2017-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define CONSDIFFMGR_PRIVATE diff --git a/src/test/test_containers.c b/src/test/test_containers.c index ad0edf4aa3..a0832f868e 100644 --- a/src/test/test_containers.c +++ b/src/test/test_containers.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "orconfig.h" diff --git a/src/test/test_controller.c b/src/test/test_controller.c index 4f5a9f58d5..5b406e159b 100644 --- a/src/test/test_controller.c +++ b/src/test/test_controller.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2015-2018, The Tor Project, Inc. */ +/* Copyright (c) 2015-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define CONTROL_PRIVATE diff --git a/src/test/test_controller_events.c b/src/test/test_controller_events.c index 99e1eb7cb0..647eac43c7 100644 --- a/src/test/test_controller_events.c +++ b/src/test/test_controller_events.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2018, The Tor Project, Inc. */ +/* Copyright (c) 2013-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define CONNECTION_PRIVATE diff --git a/src/test/test_crypto.c b/src/test/test_crypto.c index 81d2fa6f33..fa79f4cc47 100644 --- a/src/test/test_crypto.c +++ b/src/test/test_crypto.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "orconfig.h" diff --git a/src/test/test_crypto_ope.c b/src/test/test_crypto_ope.c index 4e7b952327..dc67c02676 100644 --- a/src/test/test_crypto_ope.c +++ b/src/test/test_crypto_ope.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2017, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "orconfig.h" diff --git a/src/test/test_crypto_openssl.c b/src/test/test_crypto_openssl.c index 92f9cbab2f..42dc3f6be2 100644 --- a/src/test/test_crypto_openssl.c +++ b/src/test/test_crypto_openssl.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "orconfig.h" diff --git a/src/test/test_crypto_slow.c b/src/test/test_crypto_slow.c index ca6b7b8d4d..e24aee8930 100644 --- a/src/test/test_crypto_slow.c +++ b/src/test/test_crypto_slow.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "orconfig.h" diff --git a/src/test/test_data.c b/src/test/test_data.c index be8153258b..fe1190ea77 100644 --- a/src/test/test_data.c +++ b/src/test/test_data.c @@ -1,6 +1,6 @@ /* Copyright 2001-2004 Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "test/test.h" diff --git a/src/test/test_dir.c b/src/test/test_dir.c index 5cdbd877ce..dab0b7444a 100644 --- a/src/test/test_dir.c +++ b/src/test/test_dir.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "orconfig.h" diff --git a/src/test/test_dir_common.c b/src/test/test_dir_common.c index eadeb11921..0b87e29873 100644 --- a/src/test/test_dir_common.c +++ b/src/test/test_dir_common.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "orconfig.h" diff --git a/src/test/test_dir_common.h b/src/test/test_dir_common.h index 1e958a21ff..d6c5241b14 100644 --- a/src/test/test_dir_common.h +++ b/src/test/test_dir_common.h @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "core/or/or.h" diff --git a/src/test/test_dir_handle_get.c b/src/test/test_dir_handle_get.c index 2ce98769af..841ac77916 100644 --- a/src/test/test_dir_handle_get.c +++ b/src/test/test_dir_handle_get.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define RENDCOMMON_PRIVATE diff --git a/src/test/test_dns.c b/src/test/test_dns.c index ea0fcf8c5e..231e6965f7 100644 --- a/src/test/test_dns.c +++ b/src/test/test_dns.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2015-2018, The Tor Project, Inc. */ +/* Copyright (c) 2015-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "orconfig.h" diff --git a/src/test/test_dos.c b/src/test/test_dos.c index 40a4c6ba29..4756c5014e 100644 --- a/src/test/test_dos.c +++ b/src/test/test_dos.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2018, The Tor Project, Inc. */ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define DOS_PRIVATE diff --git a/src/test/test_entryconn.c b/src/test/test_entryconn.c index bec70090e6..fc7c5d5800 100644 --- a/src/test/test_entryconn.c +++ b/src/test/test_entryconn.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2014-2018, The Tor Project, Inc. */ +/* Copyright (c) 2014-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "orconfig.h" diff --git a/src/test/test_entrynodes.c b/src/test/test_entrynodes.c index 069440a8ce..729795b674 100644 --- a/src/test/test_entrynodes.c +++ b/src/test/test_entrynodes.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2014-2018, The Tor Project, Inc. */ +/* Copyright (c) 2014-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "orconfig.h" diff --git a/src/test/test_extorport.c b/src/test/test_extorport.c index 432a9ea5e3..aeb71ec583 100644 --- a/src/test/test_extorport.c +++ b/src/test/test_extorport.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2018, The Tor Project, Inc. */ +/* Copyright (c) 2013-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define CONNECTION_PRIVATE diff --git a/src/test/test_geoip.c b/src/test/test_geoip.c index 6f9c39063b..16c566bdbc 100644 --- a/src/test/test_geoip.c +++ b/src/test/test_geoip.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "orconfig.h" diff --git a/src/test/test_guardfraction.c b/src/test/test_guardfraction.c index d6f4cd63f2..ac8bfbfded 100644 --- a/src/test/test_guardfraction.c +++ b/src/test/test_guardfraction.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2014-2018, The Tor Project, Inc. */ +/* Copyright (c) 2014-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define GUARDFRACTION_PRIVATE diff --git a/src/test/test_handles.c b/src/test/test_handles.c index 2910d7e18f..7f1d6e1898 100644 --- a/src/test/test_handles.c +++ b/src/test/test_handles.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2016-2018, The Tor Project, Inc. */ +/* Copyright (c) 2016-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "orconfig.h" diff --git a/src/test/test_helpers.c b/src/test/test_helpers.c index b7bda16494..13de1e154b 100644 --- a/src/test/test_helpers.c +++ b/src/test/test_helpers.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2014-2018, The Tor Project, Inc. */ +/* Copyright (c) 2014-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/test/test_helpers.h b/src/test/test_helpers.h index 72bf7f2f71..9e376a563d 100644 --- a/src/test/test_helpers.h +++ b/src/test/test_helpers.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2017-2018, The Tor Project, Inc. */ +/* Copyright (c) 2017-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_TEST_HELPERS_H diff --git a/src/test/test_hs.c b/src/test/test_hs.c index e3599d5720..a611b46ca6 100644 --- a/src/test/test_hs.c +++ b/src/test/test_hs.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/test/test_hs_cache.c b/src/test/test_hs_cache.c index 203a1d7039..9182829116 100644 --- a/src/test/test_hs_cache.c +++ b/src/test/test_hs_cache.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2016-2018, The Tor Project, Inc. */ +/* Copyright (c) 2016-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/test/test_hs_cell.c b/src/test/test_hs_cell.c index 5b48dd3785..0c93f593ce 100644 --- a/src/test/test_hs_cell.c +++ b/src/test/test_hs_cell.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2017-2018, The Tor Project, Inc. */ +/* Copyright (c) 2017-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/test/test_hs_client.c b/src/test/test_hs_client.c index 91b3ed1ec4..2f2bb45581 100644 --- a/src/test/test_hs_client.c +++ b/src/test/test_hs_client.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2016-2018, The Tor Project, Inc. */ +/* Copyright (c) 2016-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/test/test_hs_common.c b/src/test/test_hs_common.c index 6198573f22..eb7f3bfbb0 100644 --- a/src/test/test_hs_common.c +++ b/src/test/test_hs_common.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2017-2018, The Tor Project, Inc. */ +/* Copyright (c) 2017-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/test/test_hs_config.c b/src/test/test_hs_config.c index b6ab0c21f9..c2c556307d 100644 --- a/src/test/test_hs_config.c +++ b/src/test/test_hs_config.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2016-2018, The Tor Project, Inc. */ +/* Copyright (c) 2016-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/test/test_hs_control.c b/src/test/test_hs_control.c index 48402030bf..ba67712f1b 100644 --- a/src/test/test_hs_control.c +++ b/src/test/test_hs_control.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2017-2018, The Tor Project, Inc. */ +/* Copyright (c) 2017-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/test/test_hs_descriptor.c b/src/test/test_hs_descriptor.c index 428ca1024b..de584ed47a 100644 --- a/src/test/test_hs_descriptor.c +++ b/src/test/test_hs_descriptor.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2016-2018, The Tor Project, Inc. */ +/* Copyright (c) 2016-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/test/test_hs_intropoint.c b/src/test/test_hs_intropoint.c index 628d99bfde..660f21ffd8 100644 --- a/src/test/test_hs_intropoint.c +++ b/src/test/test_hs_intropoint.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2016-2018, The Tor Project, Inc. */ +/* Copyright (c) 2016-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/test/test_hs_ntor.c b/src/test/test_hs_ntor.c index eeb0491657..1c694e6040 100644 --- a/src/test/test_hs_ntor.c +++ b/src/test/test_hs_ntor.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2017-2018, The Tor Project, Inc. */ +/* Copyright (c) 2017-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/test/test_hs_ntor_cl.c b/src/test/test_hs_ntor_cl.c index a4915c4f8a..6341b96d84 100644 --- a/src/test/test_hs_ntor_cl.c +++ b/src/test/test_hs_ntor_cl.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2017-2018, The Tor Project, Inc. */ +/* Copyright (c) 2017-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** This is a wrapper over the little-t-tor HS ntor functions. The wrapper is diff --git a/src/test/test_hs_service.c b/src/test/test_hs_service.c index ccb4d93feb..43bf894383 100644 --- a/src/test/test_hs_service.c +++ b/src/test/test_hs_service.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2016-2018, The Tor Project, Inc. */ +/* Copyright (c) 2016-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/test/test_introduce.c b/src/test/test_introduce.c index 4d2d909945..4a6d90d97e 100644 --- a/src/test/test_introduce.c +++ b/src/test/test_introduce.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2012-2018, The Tor Project, Inc. */ +/* Copyright (c) 2012-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "orconfig.h" diff --git a/src/test/test_keypin.c b/src/test/test_keypin.c index 9af12ff548..e7beef8609 100644 --- a/src/test/test_keypin.c +++ b/src/test/test_keypin.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2014-2018, The Tor Project, Inc. */ +/* Copyright (c) 2014-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "orconfig.h" diff --git a/src/test/test_link_handshake.c b/src/test/test_link_handshake.c index 82a91a9ae2..34f59f26cd 100644 --- a/src/test/test_link_handshake.c +++ b/src/test/test_link_handshake.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2014-2018, The Tor Project, Inc. */ +/* Copyright (c) 2014-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "orconfig.h" diff --git a/src/test/test_logging.c b/src/test/test_logging.c index 2ecae461a3..6416e98a4e 100644 --- a/src/test/test_logging.c +++ b/src/test/test_logging.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2018, The Tor Project, Inc. */ +/* Copyright (c) 2013-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define CONFIG_PRIVATE diff --git a/src/test/test_mainloop.c b/src/test/test_mainloop.c index d797417912..2c3449305a 100644 --- a/src/test/test_mainloop.c +++ b/src/test/test_mainloop.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2018, The Tor Project, Inc. */ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/test/test_microdesc.c b/src/test/test_microdesc.c index fd79aee6be..53ee799185 100644 --- a/src/test/test_microdesc.c +++ b/src/test/test_microdesc.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2010-2018, The Tor Project, Inc. */ +/* Copyright (c) 2010-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "orconfig.h" diff --git a/src/test/test_netinfo.c b/src/test/test_netinfo.c index 8fc5330a46..27d276d42f 100644 --- a/src/test/test_netinfo.c +++ b/src/test/test_netinfo.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "orconfig.h" diff --git a/src/test/test_nodelist.c b/src/test/test_nodelist.c index ed919f4edf..8d6d3cb974 100644 --- a/src/test/test_nodelist.c +++ b/src/test/test_nodelist.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/test/test_ntor_cl.c b/src/test/test_ntor_cl.c index 3f914523a3..68b6927f56 100644 --- a/src/test/test_ntor_cl.c +++ b/src/test/test_ntor_cl.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2012-2018, The Tor Project, Inc. */ +/* Copyright (c) 2012-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "orconfig.h" diff --git a/src/test/test_oom.c b/src/test/test_oom.c index f84dc0764b..da6b2ee14d 100644 --- a/src/test/test_oom.c +++ b/src/test/test_oom.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2014-2018, The Tor Project, Inc. */ +/* Copyright (c) 2014-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /* Unit tests for OOM handling logic */ diff --git a/src/test/test_oos.c b/src/test/test_oos.c index fb0daa7a8d..815feda7ce 100644 --- a/src/test/test_oos.c +++ b/src/test/test_oos.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2016-2018, The Tor Project, Inc. */ +/* Copyright (c) 2016-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /* Unit tests for OOS handler */ diff --git a/src/test/test_options.c b/src/test/test_options.c index 6506731823..f12e6b6763 100644 --- a/src/test/test_options.c +++ b/src/test/test_options.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define CONFIG_PRIVATE diff --git a/src/test/test_parsecommon.c b/src/test/test_parsecommon.c index 8e74fcdb4d..0c8f467a45 100644 --- a/src/test/test_parsecommon.c +++ b/src/test/test_parsecommon.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "core/or/or.h" diff --git a/src/test/test_pem.c b/src/test/test_pem.c index 2bae286e25..865688b1a7 100644 --- a/src/test/test_pem.c +++ b/src/test/test_pem.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "orconfig.h" diff --git a/src/test/test_periodic_event.c b/src/test/test_periodic_event.c index f3d518eb7b..ebac20838f 100644 --- a/src/test/test_periodic_event.c +++ b/src/test/test_periodic_event.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2018, The Tor Project, Inc. */ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/test/test_policy.c b/src/test/test_policy.c index 3820c6c1db..46d4a1b94a 100644 --- a/src/test/test_policy.c +++ b/src/test/test_policy.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2018, The Tor Project, Inc. */ +/* Copyright (c) 2013-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define CONFIG_PRIVATE diff --git a/src/test/test_prob_distr.c b/src/test/test_prob_distr.c index ff23f01033..aac9d23e46 100644 --- a/src/test/test_prob_distr.c +++ b/src/test/test_prob_distr.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2018, The Tor Project, Inc. */ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/test/test_process.c b/src/test/test_process.c index d85cb831b4..7cc01d2442 100644 --- a/src/test/test_process.c +++ b/src/test/test_process.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2018, The Tor Project, Inc. */ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/test/test_process_slow.c b/src/test/test_process_slow.c index c28ade033c..d7cce59493 100644 --- a/src/test/test_process_slow.c +++ b/src/test/test_process_slow.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2018, The Tor Project, Inc. */ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/test/test_procmon.c b/src/test/test_procmon.c index 2c7918f580..e23578f4fd 100644 --- a/src/test/test_procmon.c +++ b/src/test/test_procmon.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2010-2018, The Tor Project, Inc. */ +/* Copyright (c) 2010-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define PROCMON_PRIVATE diff --git a/src/test/test_proto_http.c b/src/test/test_proto_http.c index b4e8278423..f9339e8dd3 100644 --- a/src/test/test_proto_http.c +++ b/src/test/test_proto_http.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2017-2018, The Tor Project, Inc. */ +/* Copyright (c) 2017-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/test/test_proto_misc.c b/src/test/test_proto_misc.c index f7f6f69667..18669a7772 100644 --- a/src/test/test_proto_misc.c +++ b/src/test/test_proto_misc.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2017-2018, The Tor Project, Inc. */ +/* Copyright (c) 2017-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/test/test_protover.c b/src/test/test_protover.c index 5f9a8b7937..63c508bd13 100644 --- a/src/test/test_protover.c +++ b/src/test/test_protover.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2016-2018, The Tor Project, Inc. */ +/* Copyright (c) 2016-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define PROTOVER_PRIVATE diff --git a/src/test/test_pt.c b/src/test/test_pt.c index 60a044aeca..bacb6d5a36 100644 --- a/src/test/test_pt.c +++ b/src/test/test_pt.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "orconfig.h" diff --git a/src/test/test_relay.c b/src/test/test_relay.c index 4311392be8..0b7a7be332 100644 --- a/src/test/test_relay.c +++ b/src/test/test_relay.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2014-2018, The Tor Project, Inc. */ +/* Copyright (c) 2014-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define CIRCUITBUILD_PRIVATE diff --git a/src/test/test_relaycell.c b/src/test/test_relaycell.c index 3d3addfb9e..0623583511 100644 --- a/src/test/test_relaycell.c +++ b/src/test/test_relaycell.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2014-2018, The Tor Project, Inc. */ +/* Copyright (c) 2014-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /* Unit tests for handling different kinds of relay cell */ diff --git a/src/test/test_relaycrypt.c b/src/test/test_relaycrypt.c index c3cfb7d10b..fe6889e521 100644 --- a/src/test/test_relaycrypt.c +++ b/src/test/test_relaycrypt.c @@ -1,6 +1,6 @@ /* Copyright 2001-2004 Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "core/or/or.h" diff --git a/src/test/test_rendcache.c b/src/test/test_rendcache.c index 2ace45d085..4f544cf21c 100644 --- a/src/test/test_rendcache.c +++ b/src/test/test_rendcache.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2010-2018, The Tor Project, Inc. */ +/* Copyright (c) 2010-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "orconfig.h" diff --git a/src/test/test_replay.c b/src/test/test_replay.c index bca3a6660a..28a508bf4d 100644 --- a/src/test/test_replay.c +++ b/src/test/test_replay.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2012-2018, The Tor Project, Inc. */ +/* Copyright (c) 2012-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define REPLAYCACHE_PRIVATE diff --git a/src/test/test_router.c b/src/test/test_router.c index 91cdd2c064..ea0ee3e84c 100644 --- a/src/test/test_router.c +++ b/src/test/test_router.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2017-2018, The Tor Project, Inc. */ +/* Copyright (c) 2017-2019, The Tor Project, Inc. */ /* Copyright (c) 2017, isis agora lovecruft */ /* See LICENSE for licensing information */ diff --git a/src/test/test_routerkeys.c b/src/test/test_routerkeys.c index f05401ba0d..727fa5660f 100644 --- a/src/test/test_routerkeys.c +++ b/src/test/test_routerkeys.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "orconfig.h" diff --git a/src/test/test_routerlist.c b/src/test/test_routerlist.c index 3e1e727429..84ec8cc462 100644 --- a/src/test/test_routerlist.c +++ b/src/test/test_routerlist.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2014-2018, The Tor Project, Inc. */ +/* Copyright (c) 2014-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "orconfig.h" diff --git a/src/test/test_routerset.c b/src/test/test_routerset.c index 86f9c0fa82..c45f0e1595 100644 --- a/src/test/test_routerset.c +++ b/src/test/test_routerset.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2014-2018, The Tor Project, Inc. */ +/* Copyright (c) 2014-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define ROUTERSET_PRIVATE diff --git a/src/test/test_scheduler.c b/src/test/test_scheduler.c index 2d562299ab..bf9c6a49cd 100644 --- a/src/test/test_scheduler.c +++ b/src/test/test_scheduler.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2014-2018, The Tor Project, Inc. */ +/* Copyright (c) 2014-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "orconfig.h" diff --git a/src/test/test_shared_random.c b/src/test/test_shared_random.c index 16e28afafe..617c76542e 100644 --- a/src/test/test_shared_random.c +++ b/src/test/test_shared_random.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2016-2018, The Tor Project, Inc. */ +/* Copyright (c) 2016-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define SHARED_RANDOM_PRIVATE diff --git a/src/test/test_slow.c b/src/test/test_slow.c index 39a203c726..c3e7edd408 100644 --- a/src/test/test_slow.c +++ b/src/test/test_slow.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/test/test_socks.c b/src/test/test_socks.c index d430f4329b..aa5136de04 100644 --- a/src/test/test_socks.c +++ b/src/test/test_socks.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "core/or/or.h" diff --git a/src/test/test_status.c b/src/test/test_status.c index 3ceba77a84..9c47469975 100644 --- a/src/test/test_status.c +++ b/src/test/test_status.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2014-2018, The Tor Project, Inc. */ +/* Copyright (c) 2014-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define STATUS_PRIVATE diff --git a/src/test/test_storagedir.c b/src/test/test_storagedir.c index 76aae7e033..24e45c7428 100644 --- a/src/test/test_storagedir.c +++ b/src/test/test_storagedir.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2017-2018, The Tor Project, Inc. */ +/* Copyright (c) 2017-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "core/or/or.h" diff --git a/src/test/test_switch_id.c b/src/test/test_switch_id.c index d8a1d15e4e..baddf8d66e 100644 --- a/src/test/test_switch_id.c +++ b/src/test/test_switch_id.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2015-2018, The Tor Project, Inc. */ +/* Copyright (c) 2015-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "core/or/or.h" diff --git a/src/test/test_threads.c b/src/test/test_threads.c index 2bf5026061..4a5ecc6fae 100644 --- a/src/test/test_threads.c +++ b/src/test/test_threads.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "orconfig.h" diff --git a/src/test/test_tortls.c b/src/test/test_tortls.c index 79b52437f8..11e35be2fa 100644 --- a/src/test/test_tortls.c +++ b/src/test/test_tortls.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2010-2018, The Tor Project, Inc. */ +/* Copyright (c) 2010-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define TORTLS_PRIVATE diff --git a/src/test/test_tortls.h b/src/test/test_tortls.h index c997934ebc..1a8b117d0f 100644 --- a/src/test/test_tortls.h +++ b/src/test/test_tortls.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2010-2018, The Tor Project, Inc. */ +/* Copyright (c) 2010-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TEST_TORTLS_H diff --git a/src/test/test_tortls_openssl.c b/src/test/test_tortls_openssl.c index abe1fb7889..73041a871c 100644 --- a/src/test/test_tortls_openssl.c +++ b/src/test/test_tortls_openssl.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2010-2018, The Tor Project, Inc. */ +/* Copyright (c) 2010-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define TORTLS_PRIVATE diff --git a/src/test/test_util.c b/src/test/test_util.c index 2fd9d24498..4e95303f2e 100644 --- a/src/test/test_util.c +++ b/src/test/test_util.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "orconfig.h" diff --git a/src/test/test_util_format.c b/src/test/test_util_format.c index fd57125b86..3a0b41faa5 100644 --- a/src/test/test_util_format.c +++ b/src/test/test_util_format.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2010-2018, The Tor Project, Inc. */ +/* Copyright (c) 2010-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "orconfig.h" diff --git a/src/test/test_util_process.c b/src/test/test_util_process.c index 44c4da9169..4d04eb6dfc 100644 --- a/src/test/test_util_process.c +++ b/src/test/test_util_process.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2010-2018, The Tor Project, Inc. */ +/* Copyright (c) 2010-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define UTIL_PROCESS_PRIVATE diff --git a/src/test/test_voting_flags.c b/src/test/test_voting_flags.c index 740b96b4c0..5c9eebd00e 100644 --- a/src/test/test_voting_flags.c +++ b/src/test/test_voting_flags.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2018, The Tor Project, Inc. */ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "orconfig.h" diff --git a/src/test/test_voting_schedule.c b/src/test/test_voting_schedule.c index c3a581cf21..ba4d53a4ae 100644 --- a/src/test/test_voting_schedule.c +++ b/src/test/test_voting_schedule.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2018, The Tor Project, Inc. */ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "orconfig.h" diff --git a/src/test/test_workqueue.c b/src/test/test_workqueue.c index 28fbd6fb9f..c58634da5c 100644 --- a/src/test/test_workqueue.c +++ b/src/test/test_workqueue.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "core/or/or.h" diff --git a/src/test/test_x509.c b/src/test/test_x509.c index 9128958492..792849ae4b 100644 --- a/src/test/test_x509.c +++ b/src/test/test_x509.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2010-2018, The Tor Project, Inc. */ +/* Copyright (c) 2010-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define TOR_X509_PRIVATE diff --git a/src/test/testing_common.c b/src/test/testing_common.c index 6d2db28f15..4e603b529a 100644 --- a/src/test/testing_common.c +++ b/src/test/testing_common.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/test/testing_rsakeys.c b/src/test/testing_rsakeys.c index c8062b82d5..0f22d4e01b 100644 --- a/src/test/testing_rsakeys.c +++ b/src/test/testing_rsakeys.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "lib/crypt_ops/crypto_rand.h" diff --git a/src/tools/tor-gencert.c b/src/tools/tor-gencert.c index 238564125a..25113420df 100644 --- a/src/tools/tor-gencert.c +++ b/src/tools/tor-gencert.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "orconfig.h" diff --git a/src/tools/tor-print-ed-signing-cert.c b/src/tools/tor-print-ed-signing-cert.c index 9bb4db0a6e..1f1a01ab5c 100644 --- a/src/tools/tor-print-ed-signing-cert.c +++ b/src/tools/tor-print-ed-signing-cert.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include diff --git a/src/tools/tor-resolve.c b/src/tools/tor-resolve.c index 803ed26b3b..98b3a4a74c 100644 --- a/src/tools/tor-resolve.c +++ b/src/tools/tor-resolve.c @@ -1,5 +1,5 @@ /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson - * Copyright (c) 2007-2018, The Tor Project, Inc. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ diff --git a/src/tools/tor_runner.c b/src/tools/tor_runner.c index d962e5abc7..3c6ade91d9 100644 --- a/src/tools/tor_runner.c +++ b/src/tools/tor_runner.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ + * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** From 3da3aca08f022f6c1e33a06afe4e30804754d005 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 16 Jan 2019 15:54:35 -0500 Subject: [PATCH 0416/2557] Fix a LOG_ERR message from test_pt.c Fixes an instance of ticket28668. --- src/test/test_pt.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/test/test_pt.c b/src/test/test_pt.c index bacb6d5a36..8759f05e80 100644 --- a/src/test/test_pt.c +++ b/src/test/test_pt.c @@ -23,6 +23,8 @@ #include "app/config/or_state_st.h" +#include "test/log_test_helpers.h" + static void reset_mp(managed_proxy_t *mp) { @@ -414,7 +416,10 @@ test_pt_configure_proxy(void *arg) "650 TRANSPORT_LAUNCHED server mock5 127.0.0.1 5555\r\n"); /* Get the log message out. */ + setup_full_capture_of_logs(LOG_ERR); process_notify_event_stdout(mp->process); + expect_single_log_msg_containing("Oh noes, something bad happened"); + teardown_capture_of_logs(); tt_int_op(controlevent_n, OP_EQ, 10); tt_int_op(controlevent_event, OP_EQ, EVENT_PT_LOG); @@ -475,6 +480,7 @@ test_pt_configure_proxy(void *arg) } done: + teardown_capture_of_logs(); or_state_free(dummy_state); UNMOCK(process_read_stdout); UNMOCK(get_or_state); From 7e1f89346022a4e90d416dad84ffdda483dbd3b5 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 17 Jan 2019 08:46:06 -0500 Subject: [PATCH 0417/2557] lintChanges.py: Two python re usage fixes --- scripts/maint/lintChanges.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/maint/lintChanges.py b/scripts/maint/lintChanges.py index 39fa08bb4a..82c118f07e 100755 --- a/scripts/maint/lintChanges.py +++ b/scripts/maint/lintChanges.py @@ -43,7 +43,7 @@ def split_tor_version(version): If the version is malformed, returns None. ''' - version_match = re.search('([0-9]+)\.([0-9]+)\.([0-9]+)(\.([0-9]+))?', version) + version_match = re.match('([0-9]+)\.([0-9]+)\.([0-9]+)(\.([0-9]+))?', version) if version_match is None: return None @@ -125,7 +125,7 @@ def lintfile(fname): warn("Versions must have at least 3 digits. ('0.1.2', '0.3.4.8', or '0.3.5.1-alpha'.)") else: bugfix_match = re.search('bugfix on ([0-9a-z][-.0-9a-z]+[0-9a-z])', contents) - bugfix_group = bugfix_match.groups(0) if bugfix_match is not None else None + bugfix_group = bugfix_match.groups() if bugfix_match is not None else None bugfix_version = bugfix_group[0] if bugfix_group is not None else None package_version = os.environ.get('PACKAGE_VERSION', None) if bugfix_version is None: From c3931714038e422972b80a893db0807d4d8ae937 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 13 Dec 2018 13:25:30 -0500 Subject: [PATCH 0418/2557] Use openssl's version of sha3 when available. Part of 28837. --- configure.ac | 32 ++++--- src/ext/include.am | 2 + src/lib/crypt_ops/crypto_digest.c | 135 +++++++++++++++++++++++++++++- src/test/test_crypto.c | 3 +- 4 files changed, 158 insertions(+), 14 deletions(-) diff --git a/configure.ac b/configure.ac index 6d1b58923d..5c75ed0bf4 100644 --- a/configure.ac +++ b/configure.ac @@ -947,21 +947,24 @@ AC_CHECK_MEMBERS([struct ssl_method_st.get_cipher_by_char], , , [#include ]) +dnl OpenSSL functions which we might not have. In theory, we could just +dnl check the openssl version number, but in practice that gets pretty +dnl confusing with LibreSSL, OpenSSL, and various distributions' patches +dnl to them. AC_CHECK_FUNCS([ \ ERR_load_KDF_strings \ - SSL_SESSION_get_master_key \ - SSL_get_server_random \ - SSL_get_client_ciphers \ - SSL_get_client_random \ - SSL_CTX_set1_groups_list \ + EVP_PBE_scrypt \ + EVP_sha3_256 \ SSL_CIPHER_find \ - SSL_CTX_set_security_level \ - TLS_method + SSL_CTX_set1_groups_list \ + SSL_CTX_set_security_level \ + SSL_SESSION_get_master_key \ + SSL_get_client_ciphers \ + SSL_get_client_random \ + SSL_get_server_random \ + TLS_method \ ]) -dnl Check if OpenSSL has scrypt implementation. -AC_CHECK_FUNCS([ EVP_PBE_scrypt ]) - dnl Check if OpenSSL structures are opaque AC_CHECK_MEMBERS([SSL.state], , , [#include @@ -973,6 +976,15 @@ AC_CHECK_SIZEOF(SHA_CTX, , [AC_INCLUDES_DEFAULT() fi # enable_nss +dnl We will someday make KECCAK_TINY optional, but for now we still need +dnl it for SHAKE, since OpenSSL's SHAKE can't be squeezed more than +dnl once. See comment in the definition of crypto_xof_t. + +dnl AM_CONDITIONAL(BUILD_KECCAK_TINY, +dnl test "x$ac_cv_func_EVP_sha3_256" != "xyes") + +AM_CONDITIONAL(BUILD_KECCAK_TINY, true) + dnl ====================================================================== dnl Can we use KIST? diff --git a/src/ext/include.am b/src/ext/include.am index 6bdce2d79e..317e25d78e 100644 --- a/src/ext/include.am +++ b/src/ext/include.am @@ -143,6 +143,7 @@ noinst_HEADERS += $(ED25519_DONNA_HDRS) LIBED25519_DONNA=src/ext/ed25519/donna/libed25519_donna.a noinst_LIBRARIES += $(LIBED25519_DONNA) +if BUILD_KECCAK_TINY src_ext_keccak_tiny_libkeccak_tiny_a_CFLAGS=\ @CFLAGS_CONSTTIME@ @@ -156,6 +157,7 @@ noinst_HEADERS += $(LIBKECCAK_TINY_HDRS) LIBKECCAK_TINY=src/ext/keccak-tiny/libkeccak-tiny.a noinst_LIBRARIES += $(LIBKECCAK_TINY) +endif EXTRA_DIST += \ src/ext/timeouts/bench/bench-add.lua \ diff --git a/src/lib/crypt_ops/crypto_digest.c b/src/lib/crypt_ops/crypto_digest.c index 26f06c6c79..2cbd3d1124 100644 --- a/src/lib/crypt_ops/crypto_digest.c +++ b/src/lib/crypt_ops/crypto_digest.c @@ -37,6 +37,12 @@ DISABLE_GCC_WARNING(redundant-decls) #include ENABLE_GCC_WARNING(redundant-decls) + +#ifdef HAVE_EVP_SHA3_256 +#define OPENSSL_HAS_SHA3 +#include +#endif + #endif #ifdef ENABLE_NSS @@ -150,8 +156,13 @@ crypto_digest256(char *digest, const char *m, size_t len, ret = (SHA256((const uint8_t*)m,len,(uint8_t*)digest) != NULL); #endif } else { +#ifdef OPENSSL_HAS_SHA3 + unsigned int dlen = DIGEST256_LEN; + ret = EVP_Digest(m, len, (uint8_t*)digest, &dlen, EVP_sha3_256(), NULL); +#else ret = (sha3_256((uint8_t *)digest, DIGEST256_LEN,(const uint8_t *)m, len) > -1); +#endif } if (!ret) @@ -179,8 +190,13 @@ crypto_digest512(char *digest, const char *m, size_t len, != NULL); #endif } else { +#ifdef OPENSSL_HAS_SHA3 + unsigned int dlen = DIGEST512_LEN; + ret = EVP_Digest(m, len, (uint8_t*)digest, &dlen, EVP_sha3_512(), NULL); +#else ret = (sha3_512((uint8_t*)digest, DIGEST512_LEN, (const uint8_t*)m, len) > -1); +#endif } if (!ret) @@ -282,7 +298,11 @@ struct crypto_digest_t { SHA256_CTX sha2; /**< state for SHA256 */ SHA512_CTX sha512; /**< state for SHA512 */ #endif +#ifdef OPENSSL_HAS_SHA3 + EVP_MD_CTX *md; +#else keccak_state sha3; /**< state for SHA3-[256,512] */ +#endif } d; }; @@ -325,9 +345,15 @@ crypto_digest_alloc_bytes(digest_algorithm_t alg) case DIGEST_SHA512: return END_OF_FIELD(d.sha512); #endif - case DIGEST_SHA3_256: +#ifdef OPENSSL_HAS_SHA3 + case DIGEST_SHA3_256: /* Fall through */ + case DIGEST_SHA3_512: + return END_OF_FIELD(d.md); +#else + case DIGEST_SHA3_256: /* Fall through */ case DIGEST_SHA3_512: return END_OF_FIELD(d.sha3); +#endif default: tor_assert(0); // LCOV_EXCL_LINE return 0; // LCOV_EXCL_LINE @@ -373,12 +399,29 @@ crypto_digest_new_internal(digest_algorithm_t algorithm) SHA512_Init(&r->d.sha512); break; #endif +#ifdef OPENSSL_HAS_SHA3 + case DIGEST_SHA3_256: + r->d.md = EVP_MD_CTX_new(); + if (!EVP_DigestInit(r->d.md, EVP_sha3_256())) { + crypto_digest_free(r); + return NULL; + } + break; + case DIGEST_SHA3_512: + r->d.md = EVP_MD_CTX_new(); + if (!EVP_DigestInit(r->d.md, EVP_sha3_512())) { + crypto_digest_free(r); + return NULL; + } + break; +#else case DIGEST_SHA3_256: keccak_digest_init(&r->d.sha3, 256); break; case DIGEST_SHA3_512: keccak_digest_init(&r->d.sha3, 512); break; +#endif default: tor_assert_unreached(); } @@ -427,6 +470,14 @@ crypto_digest_free_(crypto_digest_t *digest) if (library_supports_digest(digest->algorithm)) { PK11_DestroyContext(digest->d.ctx, PR_TRUE); } +#endif +#ifdef OPENSSL_HAS_SHA3 + if (digest->algorithm == DIGEST_SHA3_256 || + digest->algorithm == DIGEST_SHA3_512) { + if (digest->d.md) { + EVP_MD_CTX_free(digest->d.md); + } + } #endif size_t bytes = crypto_digest_alloc_bytes(digest->algorithm); memwipe(digest, 0, bytes); @@ -471,10 +522,19 @@ crypto_digest_add_bytes(crypto_digest_t *digest, const char *data, SHA512_Update(&digest->d.sha512, (void*)data, len); break; #endif +#ifdef OPENSSL_HAS_SHA3 + case DIGEST_SHA3_256: /* FALLSTHROUGH */ + case DIGEST_SHA3_512: { + int r = EVP_DigestUpdate(digest->d.md, data, len); + tor_assert(r); + } + break; +#else case DIGEST_SHA3_256: /* FALLSTHROUGH */ case DIGEST_SHA3_512: keccak_digest_update(&digest->d.sha3, (const uint8_t *)data, len); break; +#endif default: /* LCOV_EXCL_START */ tor_fragile_assert(); @@ -499,12 +559,24 @@ crypto_digest_get_digest(crypto_digest_t *digest, tor_assert(out); tor_assert(out_len <= crypto_digest_algorithm_get_length(digest->algorithm)); - /* The SHA-3 code handles copying into a temporary ctx, and also can handle - * short output buffers by truncating appropriately. */ if (digest->algorithm == DIGEST_SHA3_256 || digest->algorithm == DIGEST_SHA3_512) { +#ifdef OPENSSL_HAS_SHA3 + unsigned dlen = (unsigned) + crypto_digest_algorithm_get_length(digest->algorithm); + EVP_MD_CTX *tmp = EVP_MD_CTX_new(); + EVP_MD_CTX_copy(tmp, digest->d.md); + memset(r, 0xff, sizeof(r)); + int res = EVP_DigestFinal(tmp, r, &dlen); + EVP_MD_CTX_free(tmp); + tor_assert(res == 1); + goto done; +#else + /* Tiny-Keccak handles copying into a temporary ctx, and also can handle + * short output buffers by truncating appropriately. */ keccak_digest_sum(&digest->d.sha3, (uint8_t *)out, out_len); return; +#endif } #ifdef ENABLE_NSS @@ -550,6 +622,10 @@ crypto_digest_get_digest(crypto_digest_t *digest, //LCOV_EXCL_STOP } #endif + +#ifdef OPENSSL_HAS_SHA3 + done: +#endif memcpy(out, r, out_len); memwipe(r, 0, sizeof(r)); } @@ -570,6 +646,13 @@ crypto_digest_dup(const crypto_digest_t *digest) if (library_supports_digest(digest->algorithm)) { result->d.ctx = PK11_CloneContext(digest->d.ctx); } +#endif +#ifdef OPENSSL_HAS_SHA3 + if (digest->algorithm == DIGEST_SHA3_256 || + digest->algorithm == DIGEST_SHA3_512) { + result->d.md = EVP_MD_CTX_new(); + EVP_MD_CTX_copy(result->d.md, digest->d.md); + } #endif return result; } @@ -637,6 +720,15 @@ crypto_digest_assign(crypto_digest_t *into, return; } #endif + +#ifdef OPENSSL_HAS_SHA3 + if (from->algorithm == DIGEST_SHA3_256 || + from->algorithm == DIGEST_SHA3_512) { + EVP_MD_CTX_copy(into->d.md, from->d.md); + return; + } +#endif + memcpy(into,from,alloc_bytes); } @@ -779,7 +871,23 @@ crypto_mac_sha3_256(uint8_t *mac_out, size_t len_out, /** Internal state for a eXtendable-Output Function (XOF). */ struct crypto_xof_t { +#ifdef OPENSSL_HAS_SHAKE3_EVP + /* XXXX We can't enable this yet, because OpenSSL's + * DigestFinalXOF function can't be called repeatedly on the same + * XOF. + * + * We could in theory use the undocumented SHA3_absorb and SHA3_squeeze + * functions, but let's not mess with undocumented OpenSSL internals any + * more than we have to. + * + * We could also revise our XOF code so that it only allows a single + * squeeze operation; we don't require streaming squeeze operations + * outside the tests yet. + */ + EVP_MD_CTX *ctx; +#else keccak_state s; +#endif }; /** Allocate a new XOF object backed by SHAKE-256. The security level @@ -792,7 +900,14 @@ crypto_xof_new(void) { crypto_xof_t *xof; xof = tor_malloc(sizeof(crypto_xof_t)); +#ifdef OPENSSL_HAS_SHAKE256 + xof->ctx = EVP_MD_CTX_new(); + tor_assert(xof->ctx); + int r = EVP_DigestInit(xof->ctx, EVP_shake256()); + tor_assert(r == 1); +#else keccak_xof_init(&xof->s, 256); +#endif return xof; } @@ -803,8 +918,13 @@ crypto_xof_new(void) void crypto_xof_add_bytes(crypto_xof_t *xof, const uint8_t *data, size_t len) { +#ifdef OPENSSL_HAS_SHAKE256 + int r = EVP_DigestUpdate(xof->ctx, data, len); + tor_assert(r == 1); +#else int i = keccak_xof_absorb(&xof->s, data, len); tor_assert(i == 0); +#endif } /** Squeeze bytes out of a XOF object. Calling this routine will render @@ -813,8 +933,13 @@ crypto_xof_add_bytes(crypto_xof_t *xof, const uint8_t *data, size_t len) void crypto_xof_squeeze_bytes(crypto_xof_t *xof, uint8_t *out, size_t len) { +#ifdef OPENSSL_HAS_SHAKE256 + int r = EVP_DigestFinalXOF(xof->ctx, out, len); + tor_assert(r == 1); +#else int i = keccak_xof_squeeze(&xof->s, out, len); tor_assert(i == 0); +#endif } /** Cleanse and deallocate a XOF object. */ @@ -823,6 +948,10 @@ crypto_xof_free_(crypto_xof_t *xof) { if (!xof) return; +#ifdef OPENSSL_HAS_SHAKE256 + if (xof->ctx) + EVP_MD_CTX_free(xof->ctx); +#endif memwipe(xof, 0, sizeof(crypto_xof_t)); tor_free(xof); } diff --git a/src/test/test_crypto.c b/src/test/test_crypto.c index fa79f4cc47..ec0c3cae64 100644 --- a/src/test/test_crypto.c +++ b/src/test/test_crypto.c @@ -1178,8 +1178,9 @@ test_crypto_sha3_xof(void *arg) tt_assert(xof); for (size_t i = 0; i < sizeof(msg); i++) crypto_xof_add_bytes(xof, msg + i, 1); - for (size_t i = 0; i < sizeof(out); i++) + for (size_t i = 0; i < sizeof(out); i++) { crypto_xof_squeeze_bytes(xof, out + i, 1); + } test_memeq_hex(out, squeezed_hex); done: From 9b0dd1ae04bb4bb3bf86fe8dd629f84a07bf5cbf Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 13 Dec 2018 16:59:04 -0500 Subject: [PATCH 0419/2557] Add a function to compute an XOF in one shot. Motivation: 1. It's convenient. 2. It's all that openssl supports. Part of 28837. --- src/lib/crypt_ops/crypto_digest.c | 24 ++++++++++++++++++++++++ src/lib/crypt_ops/crypto_digest.h | 2 ++ src/test/test_crypto.c | 5 +++++ 3 files changed, 31 insertions(+) diff --git a/src/lib/crypt_ops/crypto_digest.c b/src/lib/crypt_ops/crypto_digest.c index 2cbd3d1124..1e64100f2f 100644 --- a/src/lib/crypt_ops/crypto_digest.c +++ b/src/lib/crypt_ops/crypto_digest.c @@ -955,3 +955,27 @@ crypto_xof_free_(crypto_xof_t *xof) memwipe(xof, 0, sizeof(crypto_xof_t)); tor_free(xof); } + +/** Compute the XOF (SHAKE256) of a input_len bytes at input, + * putting output_len bytes at output. */ +void +crypto_xof(uint8_t *output, size_t output_len, + const uint8_t *input, size_t input_len) +{ +#ifdef OPENSSL_HAS_SHA3 + EVP_MD_CTX *ctx = EVP_MD_CTX_new(); + tor_assert(ctx); + int r = EVP_DigestInit(ctx, EVP_shake256()); + tor_assert(r == 1); + r = EVP_DigestUpdate(ctx, input, input_len); + tor_assert(r == 1); + r = EVP_DigestFinalXOF(ctx, output, output_len); + tor_assert(r == 1); + EVP_MD_CTX_free(ctx); +#else + crypto_xof_t *xof = crypto_xof_new(); + crypto_xof_add_bytes(xof, input, input_len); + crypto_xof_squeeze_bytes(xof, output, output_len); + crypto_xof_free(xof); +#endif +} diff --git a/src/lib/crypt_ops/crypto_digest.h b/src/lib/crypt_ops/crypto_digest.h index 47e60ce617..5869db7800 100644 --- a/src/lib/crypt_ops/crypto_digest.h +++ b/src/lib/crypt_ops/crypto_digest.h @@ -124,6 +124,8 @@ void crypto_xof_squeeze_bytes(crypto_xof_t *xof, uint8_t *out, size_t len); void crypto_xof_free_(crypto_xof_t *xof); #define crypto_xof_free(xof) \ FREE_AND_NULL(crypto_xof_t, crypto_xof_free_, (xof)) +void crypto_xof(uint8_t *output, size_t output_len, + const uint8_t *input, size_t input_len); #ifdef TOR_UNIT_TESTS digest_algorithm_t crypto_digest_get_algorithm(crypto_digest_t *digest); diff --git a/src/test/test_crypto.c b/src/test/test_crypto.c index ec0c3cae64..556666df8d 100644 --- a/src/test/test_crypto.c +++ b/src/test/test_crypto.c @@ -1173,6 +1173,11 @@ test_crypto_sha3_xof(void *arg) crypto_xof_free(xof); memset(out, 0, sizeof(out)); + /* Test one-function absorb/squeeze. */ + crypto_xof(out, sizeof(out), msg, sizeof(msg)); + test_memeq_hex(out, squeezed_hex); + memset(out, 0, sizeof(out)); + /* Test incremental absorb/squeeze. */ xof = crypto_xof_new(); tt_assert(xof); From b770adbd035c33b12db1c442286eab1461a6dbe6 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 13 Dec 2018 17:15:03 -0500 Subject: [PATCH 0420/2557] Use crypto_xof() in hs_ntor.c. --- src/core/crypto/hs_ntor.c | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/src/core/crypto/hs_ntor.c b/src/core/crypto/hs_ntor.c index c34073690e..add8a2b8f2 100644 --- a/src/core/crypto/hs_ntor.c +++ b/src/core/crypto/hs_ntor.c @@ -176,7 +176,6 @@ get_introduce1_key_material(const uint8_t *secret_input, uint8_t keystream[CIPHER256_KEY_LEN + DIGEST256_LEN]; uint8_t info_blob[INFO_BLOB_LEN]; uint8_t kdf_input[KDF_INPUT_LEN]; - crypto_xof_t *xof; uint8_t *ptr; /* Let's build info */ @@ -193,10 +192,8 @@ get_introduce1_key_material(const uint8_t *secret_input, tor_assert(ptr == kdf_input + sizeof(kdf_input)); /* Now we need to run kdf_input over SHAKE-256 */ - xof = crypto_xof_new(); - crypto_xof_add_bytes(xof, kdf_input, sizeof(kdf_input)); - crypto_xof_squeeze_bytes(xof, keystream, sizeof(keystream)) ; - crypto_xof_free(xof); + crypto_xof(keystream, sizeof(keystream), + kdf_input, sizeof(kdf_input)); { /* Get the keys */ memcpy(&hs_ntor_intro_cell_keys_out->enc_key, keystream,CIPHER256_KEY_LEN); @@ -594,7 +591,6 @@ hs_ntor_circuit_key_expansion(const uint8_t *ntor_key_seed, size_t seed_len, { uint8_t *ptr; uint8_t kdf_input[NTOR_KEY_EXPANSION_KDF_INPUT_LEN]; - crypto_xof_t *xof; /* Sanity checks on lengths to make sure we are good */ if (BUG(seed_len != DIGEST256_LEN)) { @@ -611,10 +607,8 @@ hs_ntor_circuit_key_expansion(const uint8_t *ntor_key_seed, size_t seed_len, tor_assert(ptr == kdf_input + sizeof(kdf_input)); /* Generate the keys */ - xof = crypto_xof_new(); - crypto_xof_add_bytes(xof, kdf_input, sizeof(kdf_input)); - crypto_xof_squeeze_bytes(xof, keys_out, HS_NTOR_KEY_EXPANSION_KDF_OUT_LEN); - crypto_xof_free(xof); + crypto_xof(keys_out, HS_NTOR_KEY_EXPANSION_KDF_OUT_LEN, + kdf_input, sizeof(kdf_input)); return 0; } From d3b122485e09302f7befd43bef59e0e8f09142f0 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 13 Dec 2018 17:22:43 -0500 Subject: [PATCH 0421/2557] Add a changes file for 28837 (OpenSSL sha3) --- changes/ticket28837 | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 changes/ticket28837 diff --git a/changes/ticket28837 b/changes/ticket28837 new file mode 100644 index 0000000000..3bc8f12597 --- /dev/null +++ b/changes/ticket28837 @@ -0,0 +1,4 @@ + o Minor features (performance): + - Use OpenSSL's implementations of SHA3 when available (in OpenSSL 1.1.1 + and later), since they tend to be faster than tiny-keccak. Closes + ticket 28837. From f632335feb27b45a3ee5eb64690826bda52467bd Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 17 Jan 2019 13:32:19 -0500 Subject: [PATCH 0422/2557] Fix users of base32_decode to check for expected length in return. Also, when we log about a failure from base32_decode(), we now say that the length is wrong or that the characters were invalid: previously we would just say that there were invalid characters. Follow-up on 28913 work. --- src/feature/hs/hs_client.c | 5 ++++- src/feature/hs/hs_common.c | 3 ++- src/feature/hs/hs_service.c | 3 ++- src/feature/rend/rendcache.c | 7 ++++--- src/feature/rend/rendcommon.c | 5 +++-- src/feature/rend/rendparse.c | 17 +++++++++++------ 6 files changed, 26 insertions(+), 14 deletions(-) diff --git a/src/feature/hs/hs_client.c b/src/feature/hs/hs_client.c index dfad216abb..a8a4aa776f 100644 --- a/src/feature/hs/hs_client.c +++ b/src/feature/hs/hs_client.c @@ -1517,7 +1517,10 @@ parse_auth_file_content(const char *client_key_str) auth = tor_malloc_zero(sizeof(hs_client_service_authorization_t)); if (base32_decode((char *) auth->enc_seckey.secret_key, sizeof(auth->enc_seckey.secret_key), - seckey_b32, strlen(seckey_b32)) < 0) { + seckey_b32, strlen(seckey_b32)) != + sizeof(auth->enc_seckey.secret_key)) { + log_warn(LD_REND, "Client authorization encoded base32 private key " + "can't be decoded: %s", seckey_b32); goto err; } strncpy(auth->onion_address, onion_address, HS_SERVICE_ADDR_LEN_BASE32); diff --git a/src/feature/hs/hs_common.c b/src/feature/hs/hs_common.c index 8dbd9485ea..97229ac11e 100644 --- a/src/feature/hs/hs_common.c +++ b/src/feature/hs/hs_common.c @@ -926,7 +926,8 @@ hs_parse_address(const char *address, ed25519_public_key_t *key_out, } /* Decode address so we can extract needed fields. */ - if (base32_decode(decoded, sizeof(decoded), address, strlen(address)) < 0) { + if (base32_decode(decoded, sizeof(decoded), address, strlen(address)) + != sizeof(decoded)) { log_warn(LD_REND, "Service address %s can't be decoded.", escaped_safe_str(address)); goto invalid; diff --git a/src/feature/hs/hs_service.c b/src/feature/hs/hs_service.c index 623a239d50..6f6cf01053 100644 --- a/src/feature/hs/hs_service.c +++ b/src/feature/hs/hs_service.c @@ -1179,7 +1179,8 @@ parse_authorized_client(const char *client_key_str) client = tor_malloc_zero(sizeof(hs_service_authorized_client_t)); if (base32_decode((char *) client->client_pk.public_key, sizeof(client->client_pk.public_key), - pubkey_b32, strlen(pubkey_b32)) < 0) { + pubkey_b32, strlen(pubkey_b32)) != + sizeof(client->client_pk.public_key)) { log_warn(LD_REND, "Client authorization public key cannot be decoded: %s", pubkey_b32); goto err; diff --git a/src/feature/rend/rendcache.c b/src/feature/rend/rendcache.c index ecd85e4a5a..699b5a4a48 100644 --- a/src/feature/rend/rendcache.c +++ b/src/feature/rend/rendcache.c @@ -593,10 +593,10 @@ rend_cache_lookup_v2_desc_as_dir(const char *desc_id, const char **desc) char desc_id_digest[DIGEST_LEN]; tor_assert(rend_cache_v2_dir); if (base32_decode(desc_id_digest, DIGEST_LEN, - desc_id, REND_DESC_ID_V2_LEN_BASE32) < 0) { + desc_id, REND_DESC_ID_V2_LEN_BASE32) != DIGEST_LEN) { log_fn(LOG_PROTOCOL_WARN, LD_REND, "Rejecting v2 rendezvous descriptor request -- descriptor ID " - "contains illegal characters: %s", + "has wrong length or illegal characters: %s", safe_str(desc_id)); return -1; } @@ -854,7 +854,8 @@ rend_cache_store_v2_desc_as_client(const char *desc, *entry = NULL; } if (base32_decode(want_desc_id, sizeof(want_desc_id), - desc_id_base32, strlen(desc_id_base32)) < 0) { + desc_id_base32, strlen(desc_id_base32)) != + sizeof(want_desc_id)) { log_warn(LD_BUG, "Couldn't decode base32 %s for descriptor id.", escaped_safe_str_client(desc_id_base32)); goto err; diff --git a/src/feature/rend/rendcommon.c b/src/feature/rend/rendcommon.c index 15e4534fca..80edf549f1 100644 --- a/src/feature/rend/rendcommon.c +++ b/src/feature/rend/rendcommon.c @@ -171,9 +171,10 @@ rend_compute_v2_desc_id(char *desc_id_out, const char *service_id, } /* Convert service ID to binary. */ if (base32_decode(service_id_binary, REND_SERVICE_ID_LEN, - service_id, REND_SERVICE_ID_LEN_BASE32) < 0) { + service_id, REND_SERVICE_ID_LEN_BASE32) != + REND_SERVICE_ID_LEN) { log_warn(LD_REND, "Could not compute v2 descriptor ID: " - "Illegal characters in service ID: %s", + "Illegal characters or wrong length for service ID: %s", safe_str_client(service_id)); return -1; } diff --git a/src/feature/rend/rendparse.c b/src/feature/rend/rendparse.c index e2378e340f..c79f861b51 100644 --- a/src/feature/rend/rendparse.c +++ b/src/feature/rend/rendparse.c @@ -143,8 +143,9 @@ rend_parse_v2_service_descriptor(rend_service_descriptor_t **parsed_out, goto err; } if (base32_decode(desc_id_out, DIGEST_LEN, - tok->args[0], REND_DESC_ID_V2_LEN_BASE32) < 0) { - log_warn(LD_REND, "Descriptor ID contains illegal characters: %s", + tok->args[0], REND_DESC_ID_V2_LEN_BASE32) != DIGEST_LEN) { + log_warn(LD_REND, + "Descriptor ID has wrong length or illegal characters: %s", tok->args[0]); goto err; } @@ -174,8 +175,10 @@ rend_parse_v2_service_descriptor(rend_service_descriptor_t **parsed_out, log_warn(LD_REND, "Invalid secret ID part: '%s'", tok->args[0]); goto err; } - if (base32_decode(secret_id_part, DIGEST_LEN, tok->args[0], 32) < 0) { - log_warn(LD_REND, "Secret ID part contains illegal characters: %s", + if (base32_decode(secret_id_part, DIGEST_LEN, tok->args[0], 32) != + DIGEST_LEN) { + log_warn(LD_REND, + "Secret ID part has wrong length or illegal characters: %s", tok->args[0]); goto err; } @@ -429,8 +432,10 @@ rend_parse_introduction_points(rend_service_descriptor_t *parsed, /* Parse identifier. */ tok = find_by_keyword(tokens, R_IPO_IDENTIFIER); if (base32_decode(info->identity_digest, DIGEST_LEN, - tok->args[0], REND_INTRO_POINT_ID_LEN_BASE32) < 0) { - log_warn(LD_REND, "Identity digest contains illegal characters: %s", + tok->args[0], REND_INTRO_POINT_ID_LEN_BASE32) != + DIGEST_LEN) { + log_warn(LD_REND, + "Identity digest has wrong length or illegal characters: %s", tok->args[0]); rend_intro_point_free(intro); goto err; From c08fc2e19e7695c5d3fd111fb8c1a46afc23b5e8 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 17 Jan 2019 16:10:02 -0500 Subject: [PATCH 0423/2557] Speed up the deterministic prng in test_prob_distr, by a lot. Using a single xof object and squeezing it repeatedly should make everything MUCH faster here. --- src/test/test_prob_distr.c | 47 +++++++++++++++++++++++++------------- 1 file changed, 31 insertions(+), 16 deletions(-) diff --git a/src/test/test_prob_distr.c b/src/test/test_prob_distr.c index aac9d23e46..114057feca 100644 --- a/src/test/test_prob_distr.c +++ b/src/test/test_prob_distr.c @@ -1115,13 +1115,30 @@ test_psi_dist_sample(const struct dist *dist) } /* This is the seed of the deterministic randomness */ -static uint32_t deterministic_rand_counter; +static uint8_t rng_seed[16]; +static crypto_xof_t *rng_xof = NULL; /** Initialize the seed of the deterministic randomness. */ static void init_deterministic_rand(void) { - deterministic_rand_counter = crypto_rand_u32(); + crypto_rand((char*)rng_seed, sizeof(rng_seed)); + crypto_xof_free(rng_xof); + rng_xof = crypto_xof_new(); + crypto_xof_add_bytes(rng_xof, rng_seed, sizeof(rng_seed)); +} + +static void +teardown_deterministic_rand(void) +{ + crypto_xof_free(rng_xof); +} + +static void +dump_seed(void) +{ + printf("\nSeed: %s\n", + hex_str((const char*)rng_seed, sizeof(rng_seed))); } /** Produce deterministic randomness for the stochastic tests using the global @@ -1134,15 +1151,8 @@ static void crypto_rand_deterministic(char *out, size_t n) { /* Use a XOF to squeeze bytes out of that silly counter */ - crypto_xof_t *xof = crypto_xof_new(); - tor_assert(xof); - crypto_xof_add_bytes(xof, (uint8_t*)&deterministic_rand_counter, - sizeof(deterministic_rand_counter)); - crypto_xof_squeeze_bytes(xof, (uint8_t*)out, n); - crypto_xof_free(xof); - - /* Increase counter for next run */ - deterministic_rand_counter++; + tor_assert(rng_xof); + crypto_xof_squeeze_bytes(rng_xof, (uint8_t*)out, n); } static void @@ -1287,8 +1297,9 @@ test_stochastic_genpareto(void *arg) done: if (tests_failed) { - printf("seed: %"PRIu32, deterministic_rand_counter); + dump_seed(); } + teardown_deterministic_rand(); UNMOCK(crypto_rand); } @@ -1316,8 +1327,9 @@ test_stochastic_geometric(void *arg) done: if (tests_failed) { - printf("seed: %"PRIu32, deterministic_rand_counter); + dump_seed(); } + teardown_deterministic_rand(); UNMOCK(crypto_rand); } @@ -1344,8 +1356,9 @@ test_stochastic_logistic(void *arg) done: if (tests_failed) { - printf("seed: %"PRIu32, deterministic_rand_counter); + dump_seed(); } + teardown_deterministic_rand(); UNMOCK(crypto_rand); } @@ -1372,8 +1385,9 @@ test_stochastic_log_logistic(void *arg) done: if (tests_failed) { - printf("seed: %"PRIu32, deterministic_rand_counter); + dump_seed(); } + teardown_deterministic_rand(); UNMOCK(crypto_rand); } @@ -1402,8 +1416,9 @@ test_stochastic_weibull(void *arg) done: if (tests_failed) { - printf("seed: %"PRIu32, deterministic_rand_counter); + dump_seed(); } + teardown_deterministic_rand(); UNMOCK(crypto_rand); } From 81f1b89efc94723f2c13b0b80d311364a2245d85 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 17 Jan 2019 16:24:08 -0500 Subject: [PATCH 0424/2557] Better failure message on stochastic test failure --- src/test/test_prob_distr.c | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/test/test_prob_distr.c b/src/test/test_prob_distr.c index 114057feca..42cc6d70f6 100644 --- a/src/test/test_prob_distr.c +++ b/src/test/test_prob_distr.c @@ -1137,7 +1137,11 @@ teardown_deterministic_rand(void) static void dump_seed(void) { - printf("\nSeed: %s\n", + printf("\n" + "NOTE: This is a stochastic test, and we expect it to fail from\n" + "time to time, with some low probability. If you see it fail more\n" + "than one trial in 100, though, please tell us.\n\n" + "Seed: %s\n", hex_str((const char*)rng_seed, sizeof(rng_seed))); } @@ -1190,7 +1194,7 @@ test_stochastic_uniform(void *arg) .a = -4e-324, .b = 4e-310, }; - bool ok = true; + bool ok = true, tests_failed = true; init_deterministic_rand(); MOCK(crypto_rand, crypto_rand_deterministic); @@ -1204,8 +1208,14 @@ test_stochastic_uniform(void *arg) tt_assert(ok); + tests_failed = false; + done: - ; + if (tests_failed) { + dump_seed(); + } + teardown_deterministic_rand(); + UNMOCK(crypto_rand); } static bool From 88818eacbdcbdd1fddb3452c3eddf8fba3f26e9c Mon Sep 17 00:00:00 2001 From: rl1987 Date: Fri, 18 Jan 2019 13:49:30 +0200 Subject: [PATCH 0425/2557] Cleanup shellcheck warnings in autogen.sh --- autogen.sh | 5 +++-- changes/ticket26069 | 2 ++ 2 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 changes/ticket26069 diff --git a/autogen.sh b/autogen.sh index 276dd4047c..63ef6d49ef 100755 --- a/autogen.sh +++ b/autogen.sh @@ -1,9 +1,9 @@ #!/bin/sh -if [ -x "`which autoreconf 2>/dev/null`" ] ; then +if command -v autoreconf; then opt="-i -f -W all,error" - for i in $@; do + for i in "$@"; do case "$i" in -v) opt="${opt} -v" @@ -11,6 +11,7 @@ if [ -x "`which autoreconf 2>/dev/null`" ] ; then esac done + # shellcheck disable=SC2086 exec autoreconf $opt fi diff --git a/changes/ticket26069 b/changes/ticket26069 new file mode 100644 index 0000000000..caed9be348 --- /dev/null +++ b/changes/ticket26069 @@ -0,0 +1,2 @@ + o Code simplification and refactoring (shell scripts): + - Cleanup autogen.sh to silence shellcheck warnings. Closes ticket 26069. From 67f275f8dacc53f10b34b431028cebe346d0369c Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 18 Jan 2019 10:08:57 -0500 Subject: [PATCH 0426/2557] Bump to 0.4.0.1-alpha-dev --- configure.ac | 2 +- contrib/win32build/tor-mingw.nsi.in | 2 +- src/win32/orconfig.h | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/configure.ac b/configure.ac index 6d1b58923d..75bb0c7bd9 100644 --- a/configure.ac +++ b/configure.ac @@ -4,7 +4,7 @@ dnl Copyright (c) 2007-2019, The Tor Project, Inc. dnl See LICENSE for licensing information AC_PREREQ([2.63]) -AC_INIT([tor],[0.4.0.1-alpha]) +AC_INIT([tor],[0.4.0.1-alpha-dev]) AC_CONFIG_SRCDIR([src/app/main/tor_main.c]) AC_CONFIG_MACRO_DIR([m4]) diff --git a/contrib/win32build/tor-mingw.nsi.in b/contrib/win32build/tor-mingw.nsi.in index bc8ab7de94..4f2f82d007 100644 --- a/contrib/win32build/tor-mingw.nsi.in +++ b/contrib/win32build/tor-mingw.nsi.in @@ -8,7 +8,7 @@ !include "LogicLib.nsh" !include "FileFunc.nsh" !insertmacro GetParameters -!define VERSION "0.4.0.1-alpha" +!define VERSION "0.4.0.1-alpha-dev" !define INSTALLER "tor-${VERSION}-win32.exe" !define WEBSITE "https://www.torproject.org/" !define LICENSE "LICENSE" diff --git a/src/win32/orconfig.h b/src/win32/orconfig.h index 51b78b0b33..c618c5b49d 100644 --- a/src/win32/orconfig.h +++ b/src/win32/orconfig.h @@ -218,7 +218,7 @@ #define USING_TWOS_COMPLEMENT /* Version number of package */ -#define VERSION "0.4.0.1-alpha" +#define VERSION "0.4.0.1-alpha-dev" From aaef95ada5d304117d5697a042d0f43b87329ecc Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 18 Jan 2019 10:15:02 -0500 Subject: [PATCH 0427/2557] format_changelog: treat "issue NNNN" as a track ticket --- scripts/maint/format_changelog.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/maint/format_changelog.py b/scripts/maint/format_changelog.py index 8dce4b6e51..08b2155fa3 100755 --- a/scripts/maint/format_changelog.py +++ b/scripts/maint/format_changelog.py @@ -401,7 +401,7 @@ class ChangeLog(object): self.dumpEndOfChangelog() # Let's turn bugs to html. -BUG_PAT = re.compile('(bug|ticket|feature)\s+(\d{4,5})', re.I) +BUG_PAT = re.compile('(bug|ticket|issue|feature)\s+(\d{4,5})', re.I) def bug_html(m): return "%s %s" % (m.group(1), m.group(2), m.group(2)) From 9d7b4f1ca8c8e096c56e963fa34eec480f22fa03 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 18 Jan 2019 10:22:06 -0500 Subject: [PATCH 0428/2557] Fix the year for 0.3.4.10 and 0.3.3.11 --- ChangeLog | 4 ++-- ReleaseNotes | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ChangeLog b/ChangeLog index 0be9ac5700..e6bcc806e2 100644 --- a/ChangeLog +++ b/ChangeLog @@ -314,7 +314,7 @@ Changes in version 0.4.0.1-alpha - 2019-01-18 issue 28008. -Changes in version 0.3.3.11 - 2018-01-07 +Changes in version 0.3.3.11 - 2019-01-07 Tor 0.3.3.11 backports numerous fixes from later versions of Tor. numerous fixes, including an important fix for anyone using OpenSSL 1.1.1. Anyone running an earlier version of Tor 0.3.3 should upgrade @@ -487,7 +487,7 @@ Changes in version 0.3.3.11 - 2018-01-07 bugfix on 0.3.0.1-alpha. -Changes in version 0.3.4.10 - 2018-01-07 +Changes in version 0.3.4.10 - 2019-01-07 Tor 0.3.4.9 is the second stable release in its series; it backports numerous fixes, including an important fix for relays, and for anyone using OpenSSL 1.1.1. Anyone running an earlier version of Tor 0.3.4 diff --git a/ReleaseNotes b/ReleaseNotes index 7c8ab01661..6c9aa3c294 100644 --- a/ReleaseNotes +++ b/ReleaseNotes @@ -2,7 +2,7 @@ This document summarizes new features and bugfixes in each stable release of Tor. If you want to see more detailed descriptions of the changes in each development snapshot, see the ChangeLog file. -Changes in version 0.3.3.11 - 2018-01-07 +Changes in version 0.3.3.11 - 2019-01-07 Tor 0.3.3.11 backports numerous fixes from later versions of Tor. numerous fixes, including an important fix for anyone using OpenSSL 1.1.1. Anyone running an earlier version of Tor 0.3.3 should upgrade @@ -175,7 +175,7 @@ Changes in version 0.3.3.11 - 2018-01-07 bugfix on 0.3.0.1-alpha. -Changes in version 0.3.4.10 - 2018-01-07 +Changes in version 0.3.4.10 - 2019-01-07 Tor 0.3.4.9 is the second stable release in its series; it backports numerous fixes, including an important fix for relays, and for anyone using OpenSSL 1.1.1. Anyone running an earlier version of Tor 0.3.4 From 2529b29a75e24dcc1a6c92ef8f19a5916028a63f Mon Sep 17 00:00:00 2001 From: rl1987 Date: Sat, 19 Jan 2019 17:39:48 +0200 Subject: [PATCH 0429/2557] Fix shellcheck warning SC2145 in torify script --- changes/ticket29070 | 2 ++ contrib/client-tools/torify | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 changes/ticket29070 diff --git a/changes/ticket29070 b/changes/ticket29070 new file mode 100644 index 0000000000..2716915359 --- /dev/null +++ b/changes/ticket29070 @@ -0,0 +1,2 @@ + o Code simplification and refactoring (shell scripts): + - Fix shellcheck warning in torify script. Resolves issue 29070. diff --git a/contrib/client-tools/torify b/contrib/client-tools/torify index 54acfed654..ac4c9b5c7f 100755 --- a/contrib/client-tools/torify +++ b/contrib/client-tools/torify @@ -53,7 +53,7 @@ pathfind() { if pathfind torsocks; then exec torsocks "$@" - echo "$0: Failed to exec torsocks $@" >&2 + echo "$0: Failed to exec torsocks $*" >&2 exit 1 else echo "$0: torsocks not found in your PATH. Perhaps it isn't installed? (tsocks is no longer supported, for security reasons.)" >&2 From 00fff96e4884e295a767532aa570abf8d27e75ee Mon Sep 17 00:00:00 2001 From: rl1987 Date: Tue, 22 Jan 2019 15:14:16 +0200 Subject: [PATCH 0430/2557] Fix shellcheck warning in test_rebind.sh --- changes/bug29063 | 2 ++ src/test/test_rebind.sh | 9 +++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) create mode 100644 changes/bug29063 diff --git a/changes/bug29063 b/changes/bug29063 new file mode 100644 index 0000000000..8cbcbebc6e --- /dev/null +++ b/changes/bug29063 @@ -0,0 +1,2 @@ + o Code simplification and refactoring (shell scripts): + - Fix issues shellcheck found in test_rebind.sh. Resolves issue 29063. diff --git a/src/test/test_rebind.sh b/src/test/test_rebind.sh index 498072de35..a8f07c7c1e 100755 --- a/src/test/test_rebind.sh +++ b/src/test/test_rebind.sh @@ -15,10 +15,15 @@ fi exitcode=0 tmpdir= -clean () { test -n "$tmpdir" && test -d "$tmpdir" && rm -rf "$tmpdir" || :; } +clean () { + if [ -n "$tmpdir" ] && [ -d "$tmpdir" ]; then + rm -rf "$tmpdir" + fi +} + trap clean EXIT HUP INT TERM -tmpdir="`mktemp -d -t tor_rebind_test.XXXXXX`" +tmpdir="$(mktemp -d -t tor_rebind_test.XXXXXX)" if [ -z "$tmpdir" ]; then echo >&2 mktemp failed exit 2 From 6243133a71e816ba9a462c9fa667c63729483627 Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Mon, 21 Jan 2019 12:36:33 +0200 Subject: [PATCH 0431/2557] Fix intermittent failures of test_circuitpadding_wronghop(). We fix it by disabling the scheduling of actual padding. Fixes #29122. --- changes/bug29122 | 3 +++ src/test/test_circuitpadding.c | 4 ++++ 2 files changed, 7 insertions(+) create mode 100644 changes/bug29122 diff --git a/changes/bug29122 b/changes/bug29122 new file mode 100644 index 0000000000..020052ff8f --- /dev/null +++ b/changes/bug29122 @@ -0,0 +1,3 @@ + o Minor bugfixes (unit tests): + - Fix intermittent failures on an adaptive padding unittest. Fixes bug + 29122; bugfix on 0.4.0.1-alpha diff --git a/src/test/test_circuitpadding.c b/src/test/test_circuitpadding.c index f4d003969e..61bd174e6c 100644 --- a/src/test/test_circuitpadding.c +++ b/src/test/test_circuitpadding.c @@ -1239,6 +1239,10 @@ test_circuitpadding_wronghop(void *arg) MOCK(circuitmux_attach_circuit, circuitmux_attach_circuit_mock); + /* Mock this function so that our cell counting tests don't get confused by + * padding that gets sent by scheduled timers. */ + MOCK(circpad_machine_schedule_padding,circpad_machine_schedule_padding_mock); + client_side = (circuit_t *)origin_circuit_new(); dummy_channel.cmux = circuitmux_alloc(); relay_side = (circuit_t *)new_fake_orcirc(&dummy_channel, From db3ee1d862272a36fb23adb208bfe84013e4b8f7 Mon Sep 17 00:00:00 2001 From: Peter Gerber Date: Tue, 22 Jan 2019 21:47:43 +0000 Subject: [PATCH 0432/2557] =?UTF-8?q?Allow=20getsockopt(=E2=80=A6,=20SOL?= =?UTF-8?q?=5FSOCKET,=20SO=5FACCEPTCONN,=20=E2=80=A6)=20in=20sandbox?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SO_ACCEPTCONN checks whether socket listening is enabled and is used ever since 9369152aae9527cc3764 has been merged. Closes ticket #29150 --- src/lib/sandbox/sandbox.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/lib/sandbox/sandbox.c b/src/lib/sandbox/sandbox.c index 1f0f5d858f..b652397f5a 100644 --- a/src/lib/sandbox/sandbox.c +++ b/src/lib/sandbox/sandbox.c @@ -832,6 +832,12 @@ sb_getsockopt(scmp_filter_ctx ctx, sandbox_cfg_t *filter) if (rc) return rc; + rc = seccomp_rule_add_2(ctx, SCMP_ACT_ALLOW, SCMP_SYS(getsockopt), + SCMP_CMP(1, SCMP_CMP_EQ, SOL_SOCKET), + SCMP_CMP(2, SCMP_CMP_EQ, SO_ACCEPTCONN)); + if (rc) + return rc; + #ifdef HAVE_SYSTEMD rc = seccomp_rule_add_2(ctx, SCMP_ACT_ALLOW, SCMP_SYS(getsockopt), SCMP_CMP(1, SCMP_CMP_EQ, SOL_SOCKET), From 99fffc6c2fbc00cdeb9edd07ebb580f55d0a3513 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Mon, 21 Jan 2019 13:01:52 +0200 Subject: [PATCH 0433/2557] Add a pre-commit hook that runs code and changelog entry formatting checks --- changes/feature28976 | 4 ++++ scripts/maint/pre-commit.git-hook | 23 +++++++++++++++++++++++ 2 files changed, 27 insertions(+) create mode 100644 changes/feature28976 create mode 100755 scripts/maint/pre-commit.git-hook diff --git a/changes/feature28976 b/changes/feature28976 new file mode 100644 index 0000000000..c7ebc207f7 --- /dev/null +++ b/changes/feature28976 @@ -0,0 +1,4 @@ + o Minor features (developer tooling): + - Provide a git pre-commit hook that disallows commiting if we have any + failures in our code and changelog formatting checks. It is now available + in scripts/maint/pre-commit.git-hook. Implements feature 28976. diff --git a/scripts/maint/pre-commit.git-hook b/scripts/maint/pre-commit.git-hook new file mode 100755 index 0000000000..0868559606 --- /dev/null +++ b/scripts/maint/pre-commit.git-hook @@ -0,0 +1,23 @@ +#!/bin/bash +# +# To install this script, copy it to .git/hooks/pre-commit in local copy of +# tor git repo and make sure it has permission to execute. +# +# This is pre-commit git hook script that prevents commiting your changeset if +# it fails our code formatting or changelog entry formatting checkers. + +workdir=$(git rev-parse --show-toplevel) + +cd "$workdir" || exit 1 + +python scripts/maint/lintChanges.py ./changes + +perl scripts/maint/checkSpace.pl -C \ +src/lib/*/*.[ch] \ +src/core/*/*.[ch] \ +src/feature/*/*.[ch] \ +src/app/*/*.[ch] \ +src/test/*.[ch] \ +src/test/*/*.[ch] \ +src/tools/*.[ch] + From 611e8dcda7e29617db0b78c6fc5a47f66e4d1842 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Thu, 24 Jan 2019 12:22:28 +0200 Subject: [PATCH 0434/2557] Also run checkInclude.py before every commit --- scripts/maint/pre-commit.git-hook | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/maint/pre-commit.git-hook b/scripts/maint/pre-commit.git-hook index 0868559606..724cd77321 100755 --- a/scripts/maint/pre-commit.git-hook +++ b/scripts/maint/pre-commit.git-hook @@ -21,3 +21,5 @@ src/test/*.[ch] \ src/test/*/*.[ch] \ src/tools/*.[ch] +python scripts/maint/checkIncludes.py + From 107e50a90f67ed26a7a24cc6135e5191601184fb Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 24 Jan 2019 08:05:42 -0500 Subject: [PATCH 0435/2557] lintChanges.py didn't take a directory argument in 0.2.9 --- scripts/maint/pre-commit.git-hook | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/maint/pre-commit.git-hook b/scripts/maint/pre-commit.git-hook index 724cd77321..a0d0a8bb6c 100755 --- a/scripts/maint/pre-commit.git-hook +++ b/scripts/maint/pre-commit.git-hook @@ -10,7 +10,7 @@ workdir=$(git rev-parse --show-toplevel) cd "$workdir" || exit 1 -python scripts/maint/lintChanges.py ./changes +python scripts/maint/lintChanges.py ./changes/* perl scripts/maint/checkSpace.pl -C \ src/lib/*/*.[ch] \ From dd22c84ccf07214d3d57642c2ba161be496f3607 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 24 Jan 2019 08:06:09 -0500 Subject: [PATCH 0436/2557] Only run the checkIncludes.py script if it exists (It was added in 0.3.5) --- scripts/maint/pre-commit.git-hook | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/maint/pre-commit.git-hook b/scripts/maint/pre-commit.git-hook index a0d0a8bb6c..b4c4ce2061 100755 --- a/scripts/maint/pre-commit.git-hook +++ b/scripts/maint/pre-commit.git-hook @@ -21,5 +21,6 @@ src/test/*.[ch] \ src/test/*/*.[ch] \ src/tools/*.[ch] -python scripts/maint/checkIncludes.py - +if test -e scripts/maint/checkIncludes.py; then + python scripts/maint/checkIncludes.py +fi From c985940de9ec743cbb3e9a3e32fd482bd859c9ed Mon Sep 17 00:00:00 2001 From: Neel Chauhan Date: Mon, 14 Jan 2019 14:09:53 -0500 Subject: [PATCH 0437/2557] Add version 3 onion service support to HSFETCH --- changes/ticket25417 | 4 ++++ src/feature/control/control.c | 26 ++++++++++++++++++++------ src/feature/hs/hs_client.c | 18 ++++++++++++++++++ src/feature/hs/hs_client.h | 4 ++++ src/feature/hs/hs_control.c | 14 ++++++++++++++ src/feature/hs/hs_control.h | 4 ++++ 6 files changed, 64 insertions(+), 6 deletions(-) create mode 100644 changes/ticket25417 diff --git a/changes/ticket25417 b/changes/ticket25417 new file mode 100644 index 0000000000..41f2acc988 --- /dev/null +++ b/changes/ticket25417 @@ -0,0 +1,4 @@ + o Minor features (controller): + - Add onion service version 3 support to HSFETCH. Previously, only + version 2 onion services were supported. Closes ticket 25417. + Patch by Neel Chauhan diff --git a/src/feature/control/control.c b/src/feature/control/control.c index 170037e060..d7299a4cf6 100644 --- a/src/feature/control/control.c +++ b/src/feature/control/control.c @@ -4426,6 +4426,8 @@ handle_control_hsfetch(control_connection_t *conn, uint32_t len, static const char *v2_str = "v2-"; const size_t v2_str_len = strlen(v2_str); rend_data_t *rend_query = NULL; + ed25519_public_key_t v3_pk; + uint32_t version; /* Make sure we have at least one argument, the HSAddress. */ args = getargs_helper(hsfetch_command, conn, body, 1, -1); @@ -4438,6 +4440,7 @@ handle_control_hsfetch(control_connection_t *conn, uint32_t len, /* Test if it's an HS address without the .onion part. */ if (rend_valid_v2_service_id(arg1)) { hsaddress = arg1; + version = HS_VERSION_TWO; } else if (strcmpstart(arg1, v2_str) == 0 && rend_valid_descriptor_id(arg1 + v2_str_len) && base32_decode(digest, sizeof(digest), arg1 + v2_str_len, @@ -4445,6 +4448,11 @@ handle_control_hsfetch(control_connection_t *conn, uint32_t len, /* We have a well formed version 2 descriptor ID. Keep the decoded value * of the id. */ desc_id = digest; + version = HS_VERSION_TWO; + } else if (hs_address_is_valid(arg1)) { + hsaddress = arg1; + version = HS_VERSION_THREE; + hs_parse_address(hsaddress, &v3_pk, NULL, NULL); } else { connection_printf_to_buf(conn, "513 Invalid argument \"%s\"\r\n", arg1); @@ -4481,11 +4489,13 @@ handle_control_hsfetch(control_connection_t *conn, uint32_t len, } } - rend_query = rend_data_client_create(hsaddress, desc_id, NULL, - REND_NO_AUTH); - if (rend_query == NULL) { - connection_printf_to_buf(conn, "551 Error creating the HS query\r\n"); - goto done; + if (version == HS_VERSION_TWO) { + rend_query = rend_data_client_create(hsaddress, desc_id, NULL, + REND_NO_AUTH); + if (rend_query == NULL) { + connection_printf_to_buf(conn, "551 Error creating the HS query\r\n"); + goto done; + } } /* Using a descriptor ID, we force the user to provide at least one @@ -4504,7 +4514,11 @@ handle_control_hsfetch(control_connection_t *conn, uint32_t len, /* Trigger the fetch using the built rend query and possibly a list of HS * directory to use. This function ignores the client cache thus this will * always send a fetch command. */ - rend_client_fetch_v2_desc(rend_query, hsdirs); + if (version == HS_VERSION_TWO) { + rend_client_fetch_v2_desc(rend_query, hsdirs); + } else if (version == HS_VERSION_THREE) { + hs_control_hsfetch_command(&v3_pk, hsdirs); + } done: SMARTLIST_FOREACH(args, char *, cp, tor_free(cp)); diff --git a/src/feature/hs/hs_client.c b/src/feature/hs/hs_client.c index 5fded92fe3..7dc856ae54 100644 --- a/src/feature/hs/hs_client.c +++ b/src/feature/hs/hs_client.c @@ -459,6 +459,24 @@ fetch_v3_desc, (const ed25519_public_key_t *onion_identity_pk)) return directory_launch_v3_desc_fetch(onion_identity_pk, hsdir_rs); } +/* With a given onion_identity_pk, fetch its descriptor. If + * hsdirs is specified, use the directory servers specified in the list. + * Else, use a random server. */ +void +hs_client_launch_v3_desc_fetch(const ed25519_public_key_t *onion_identity_pk, + const smartlist_t *hsdirs) +{ + tor_assert(onion_identity_pk); + + if (hsdirs != NULL) { + SMARTLIST_FOREACH_BEGIN(hsdirs, const routerstatus_t *, hsdir) { + directory_launch_v3_desc_fetch(onion_identity_pk, hsdir); + } SMARTLIST_FOREACH_END(hsdir); + } else { + fetch_v3_desc(onion_identity_pk); + } +} + /* Make sure that the given v3 origin circuit circ is a valid correct * introduction circuit. This will BUG() on any problems and hard assert if * the anonymity of the circuit is not ok. Return 0 on success else -1 where diff --git a/src/feature/hs/hs_client.h b/src/feature/hs/hs_client.h index f6fb167ea2..47c525242f 100644 --- a/src/feature/hs/hs_client.h +++ b/src/feature/hs/hs_client.h @@ -44,6 +44,10 @@ typedef struct hs_client_service_authorization_t { void hs_client_note_connection_attempt_succeeded( const edge_connection_t *conn); +void hs_client_launch_v3_desc_fetch( + const ed25519_public_key_t *onion_identity_pk, + const smartlist_t *hsdirs); + int hs_client_decode_descriptor( const char *desc_str, const ed25519_public_key_t *service_identity_pk, diff --git a/src/feature/hs/hs_control.c b/src/feature/hs/hs_control.c index a21788ecd7..df8c1958b5 100644 --- a/src/feature/hs/hs_control.c +++ b/src/feature/hs/hs_control.c @@ -10,6 +10,7 @@ #include "feature/control/control.h" #include "lib/crypt_ops/crypto_format.h" #include "lib/crypt_ops/crypto_util.h" +#include "feature/hs/hs_client.h" #include "feature/hs/hs_common.h" #include "feature/hs/hs_control.h" #include "feature/hs/hs_descriptor.h" @@ -259,3 +260,16 @@ hs_control_hspost_command(const char *body, const char *onion_address, smartlist_free(hsdirs); return ret; } + +/* With a given onion_identity_pk, fetch its descriptor, optionally + * using the list of directory servers given in hsdirs, or a random + * server if it is NULL. This function calls hs_client_launch_v3_desc_fetch(). + */ +void +hs_control_hsfetch_command(const ed25519_public_key_t *onion_identity_pk, + const smartlist_t *hsdirs) +{ + tor_assert(onion_identity_pk); + + hs_client_launch_v3_desc_fetch(onion_identity_pk, hsdirs); +} diff --git a/src/feature/hs/hs_control.h b/src/feature/hs/hs_control.h index 63e3fe13d6..2f5dcd2154 100644 --- a/src/feature/hs/hs_control.h +++ b/src/feature/hs/hs_control.h @@ -48,5 +48,9 @@ void hs_control_desc_event_content(const hs_ident_dir_conn_t *ident, int hs_control_hspost_command(const char *body, const char *onion_address, const smartlist_t *hsdirs_rs); +/* Command "HSFETCH [...]" */ +void hs_control_hsfetch_command(const ed25519_public_key_t *onion_identity_pk, + const smartlist_t *hsdirs); + #endif /* !defined(TOR_HS_CONTROL_H) */ From e19222a0daee40302218ab8efce359c154e575f0 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 24 Jan 2019 13:20:21 -0500 Subject: [PATCH 0438/2557] Use tt_u64_op() in test_circuitpadding.c to fix compilation warnings Fixes bug 29169. --- changes/bug29169 | 3 ++ src/test/test_circuitpadding.c | 69 +++++++++++++++++----------------- 2 files changed, 37 insertions(+), 35 deletions(-) create mode 100644 changes/bug29169 diff --git a/changes/bug29169 b/changes/bug29169 new file mode 100644 index 0000000000..41d4b76ef5 --- /dev/null +++ b/changes/bug29169 @@ -0,0 +1,3 @@ + o Minor bugfixes (compilation): + - Fix compilation warnings in test_circuitpadding.c. Fixes bug 29169; + bugfix on 0.4.0.1-alpha. diff --git a/src/test/test_circuitpadding.c b/src/test/test_circuitpadding.c index 61bd174e6c..12a07fa957 100644 --- a/src/test/test_circuitpadding.c +++ b/src/test/test_circuitpadding.c @@ -321,12 +321,12 @@ test_circuitpadding_rtt(void *arg) /* Test 1: Test measuring RTT */ circpad_cell_event_nonpadding_received((circuit_t*)relay_side); - tt_int_op(relay_side->padding_info[0]->last_received_time_usec, OP_NE, 0); + tt_u64_op(relay_side->padding_info[0]->last_received_time_usec, OP_NE, 0); timers_advance_and_run(20); circpad_cell_event_nonpadding_sent((circuit_t*)relay_side); - tt_int_op(relay_side->padding_info[0]->last_received_time_usec, OP_EQ, 0); + tt_u64_op(relay_side->padding_info[0]->last_received_time_usec, OP_EQ, 0); tt_int_op(relay_side->padding_info[0]->rtt_estimate_usec, OP_GE, 19000); tt_int_op(relay_side->padding_info[0]->rtt_estimate_usec, OP_LE, 30000); @@ -338,11 +338,11 @@ test_circuitpadding_rtt(void *arg) circpad_cell_event_nonpadding_received((circuit_t*)relay_side); circpad_cell_event_nonpadding_received((circuit_t*)relay_side); - tt_int_op(relay_side->padding_info[0]->last_received_time_usec, OP_NE, 0); + tt_u64_op(relay_side->padding_info[0]->last_received_time_usec, OP_NE, 0); timers_advance_and_run(20); circpad_cell_event_nonpadding_sent((circuit_t*)relay_side); circpad_cell_event_nonpadding_sent((circuit_t*)relay_side); - tt_int_op(relay_side->padding_info[0]->last_received_time_usec, OP_EQ, 0); + tt_u64_op(relay_side->padding_info[0]->last_received_time_usec, OP_EQ, 0); tt_int_op(relay_side->padding_info[0]->rtt_estimate_usec, OP_GE, 20000); tt_int_op(relay_side->padding_info[0]->rtt_estimate_usec, OP_LE, 21000); @@ -362,7 +362,7 @@ test_circuitpadding_rtt(void *arg) tt_int_op(relay_side->padding_info[0]->rtt_estimate_usec, OP_EQ, rtt_estimate); - tt_int_op(relay_side->padding_info[0]->last_received_time_usec, OP_EQ, 0); + tt_u64_op(relay_side->padding_info[0]->last_received_time_usec, OP_EQ, 0); tt_int_op(relay_side->padding_info[0]->stop_rtt_update, OP_EQ, 1); tt_int_op(circpad_histogram_bin_to_usec(relay_side->padding_info[0], 0), OP_EQ, @@ -372,11 +372,11 @@ test_circuitpadding_rtt(void *arg) /* Test 3: Make sure client side machine properly ignores RTT */ circpad_cell_event_nonpadding_received((circuit_t*)client_side); - tt_int_op(client_side->padding_info[0]->last_received_time_usec, OP_EQ, 0); + tt_u64_op(client_side->padding_info[0]->last_received_time_usec, OP_EQ, 0); timers_advance_and_run(20); circpad_cell_event_nonpadding_sent((circuit_t*)client_side); - tt_int_op(client_side->padding_info[0]->last_received_time_usec, OP_EQ, 0); + tt_u64_op(client_side->padding_info[0]->last_received_time_usec, OP_EQ, 0); tt_int_op(client_side->padding_info[0]->rtt_estimate_usec, OP_EQ, 0); tt_int_op(circpad_histogram_bin_to_usec(client_side->padding_info[0], 0), @@ -1861,7 +1861,7 @@ test_circuitpadding_circuitsetup_machine(void *arg) tt_int_op(relay_side->padding_info[0]->current_state, OP_EQ, CIRCPAD_STATE_BURST); - tt_int_op(client_side->padding_info[0]->padding_scheduled_at_usec, + tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec, OP_NE, 0); tt_int_op(relay_side->padding_info[0]->is_padding_timer_scheduled, OP_EQ, 0); @@ -1872,73 +1872,73 @@ test_circuitpadding_circuitsetup_machine(void *arg) tt_int_op(relay_side->padding_info[0]->current_state, OP_EQ, CIRCPAD_STATE_GAP); - tt_int_op(client_side->padding_info[0]->padding_scheduled_at_usec, + tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec, OP_EQ, 0); - tt_int_op(relay_side->padding_info[0]->padding_scheduled_at_usec, + tt_u64_op(relay_side->padding_info[0]->padding_scheduled_at_usec, OP_NE, 0); timers_advance_and_run(5000); tt_int_op(n_client_cells, OP_EQ, 2); tt_int_op(n_relay_cells, OP_EQ, 2); - tt_int_op(client_side->padding_info[0]->padding_scheduled_at_usec, + tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec, OP_NE, 0); - tt_int_op(relay_side->padding_info[0]->padding_scheduled_at_usec, + tt_u64_op(relay_side->padding_info[0]->padding_scheduled_at_usec, OP_EQ, 0); timers_advance_and_run(2000); tt_int_op(n_client_cells, OP_EQ, 3); tt_int_op(n_relay_cells, OP_EQ, 2); - tt_int_op(client_side->padding_info[0]->padding_scheduled_at_usec, + tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec, OP_EQ, 0); - tt_int_op(relay_side->padding_info[0]->padding_scheduled_at_usec, + tt_u64_op(relay_side->padding_info[0]->padding_scheduled_at_usec, OP_NE, 0); timers_advance_and_run(5000); tt_int_op(n_client_cells, OP_EQ, 3); tt_int_op(n_relay_cells, OP_EQ, 3); - tt_int_op(client_side->padding_info[0]->padding_scheduled_at_usec, + tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec, OP_NE, 0); - tt_int_op(relay_side->padding_info[0]->padding_scheduled_at_usec, + tt_u64_op(relay_side->padding_info[0]->padding_scheduled_at_usec, OP_EQ, 0); timers_advance_and_run(2000); tt_int_op(n_client_cells, OP_EQ, 4); tt_int_op(n_relay_cells, OP_EQ, 3); - tt_int_op(client_side->padding_info[0]->padding_scheduled_at_usec, + tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec, OP_EQ, 0); - tt_int_op(relay_side->padding_info[0]->padding_scheduled_at_usec, + tt_u64_op(relay_side->padding_info[0]->padding_scheduled_at_usec, OP_NE, 0); timers_advance_and_run(5000); tt_int_op(n_client_cells, OP_EQ, 4); tt_int_op(n_relay_cells, OP_EQ, 4); - tt_int_op(client_side->padding_info[0]->padding_scheduled_at_usec, + tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec, OP_NE, 0); - tt_int_op(relay_side->padding_info[0]->padding_scheduled_at_usec, + tt_u64_op(relay_side->padding_info[0]->padding_scheduled_at_usec, OP_EQ, 0); timers_advance_and_run(2000); tt_int_op(n_client_cells, OP_EQ, 5); tt_int_op(n_relay_cells, OP_EQ, 4); - tt_int_op(client_side->padding_info[0]->padding_scheduled_at_usec, + tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec, OP_EQ, 0); - tt_int_op(relay_side->padding_info[0]->padding_scheduled_at_usec, + tt_u64_op(relay_side->padding_info[0]->padding_scheduled_at_usec, OP_NE, 0); timers_advance_and_run(5000); tt_int_op(n_client_cells, OP_EQ, 5); tt_int_op(n_relay_cells, OP_EQ, 5); - tt_int_op(client_side->padding_info[0]->padding_scheduled_at_usec, + tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec, OP_NE, 0); - tt_int_op(relay_side->padding_info[0]->padding_scheduled_at_usec, + tt_u64_op(relay_side->padding_info[0]->padding_scheduled_at_usec, OP_EQ, 0); timers_advance_and_run(2000); tt_int_op(n_client_cells, OP_EQ, 6); tt_int_op(n_relay_cells, OP_EQ, 5); - tt_int_op(client_side->padding_info[0]->padding_scheduled_at_usec, + tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec, OP_EQ, 0); - tt_int_op(relay_side->padding_info[0]->padding_scheduled_at_usec, + tt_u64_op(relay_side->padding_info[0]->padding_scheduled_at_usec, OP_NE, 0); timers_advance_and_run(5000); tt_int_op(n_client_cells, OP_EQ, 6); @@ -1946,11 +1946,11 @@ test_circuitpadding_circuitsetup_machine(void *arg) tt_int_op(client_side->padding_info[0]->current_state, OP_EQ, CIRCPAD_STATE_END); - tt_int_op(client_side->padding_info[0]->padding_scheduled_at_usec, + tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec, OP_EQ, 0); tt_int_op(relay_side->padding_info[0]->current_state, OP_EQ, CIRCPAD_STATE_GAP); - tt_int_op(relay_side->padding_info[0]->padding_scheduled_at_usec, + tt_u64_op(relay_side->padding_info[0]->padding_scheduled_at_usec, OP_EQ, 0); /* Verify we can't schedule padding in END state */ @@ -1986,17 +1986,17 @@ test_circuitpadding_circuitsetup_machine(void *arg) tt_int_op(n_client_cells, OP_EQ, 8); tt_int_op(n_relay_cells, OP_EQ, 8); - tt_int_op(client_side->padding_info[0]->padding_scheduled_at_usec, + tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec, OP_NE, 0); - tt_int_op(relay_side->padding_info[0]->padding_scheduled_at_usec, + tt_u64_op(relay_side->padding_info[0]->padding_scheduled_at_usec, OP_NE, 0); /* Test timer cancel due to state rules */ circpad_cell_event_nonpadding_sent(client_side); - tt_int_op(client_side->padding_info[0]->padding_scheduled_at_usec, + tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec, OP_EQ, 0); circpad_cell_event_padding_received(client_side); - tt_int_op(client_side->padding_info[0]->padding_scheduled_at_usec, + tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec, OP_NE, 0); /* Simulate application traffic to cancel timer */ @@ -2030,9 +2030,9 @@ test_circuitpadding_circuitsetup_machine(void *arg) tt_int_op(relay_side->padding_info[0]->current_state, OP_EQ, CIRCPAD_STATE_GAP); - tt_int_op(client_side->padding_info[0]->padding_scheduled_at_usec, + tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec, OP_NE, 0); - tt_int_op(relay_side->padding_info[0]->padding_scheduled_at_usec, + tt_u64_op(relay_side->padding_info[0]->padding_scheduled_at_usec, OP_NE, 0); circuit_mark_for_close(client_side, END_CIRC_REASON_FLAG_REMOTE); free_fake_orcirc(relay_side); @@ -2357,4 +2357,3 @@ struct testcase_t circuitpadding_tests[] = { TEST_CIRCUITPADDING(circuitpadding_token_removal_exact, TT_FORK), END_OF_TESTCASES }; - From d9010c5b67a60e9f0b9051ba6597005e33e58266 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 24 Jan 2019 15:23:06 -0500 Subject: [PATCH 0439/2557] One more 32-bit clang warning --- src/test/test_process_slow.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/test_process_slow.c b/src/test/test_process_slow.c index d7cce59493..1322d7b833 100644 --- a/src/test/test_process_slow.c +++ b/src/test/test_process_slow.c @@ -240,7 +240,7 @@ test_callbacks(void *arg) tt_int_op(smartlist_len(process_data->stdout_data), OP_EQ, 12); tt_int_op(smartlist_len(process_data->stderr_data), OP_EQ, 3); tt_assert(process_data->did_exit); - tt_int_op(process_data->exit_code, OP_EQ, 0); + tt_u64_op(process_data->exit_code, OP_EQ, 0); /* Check stdout output. */ char argv0_expected[256]; From bbd893d6bd8e79ab6e303152c871027bc0380a38 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 29 Jan 2019 16:18:41 +0100 Subject: [PATCH 0440/2557] Write consensus files in binary mode This will help us out on windows now that we mmap files. Fixes part of ticket 28614. --- changes/ticket28614 | 4 ++++ src/feature/nodelist/networkstatus.c | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) create mode 100644 changes/ticket28614 diff --git a/changes/ticket28614 b/changes/ticket28614 new file mode 100644 index 0000000000..c022baf668 --- /dev/null +++ b/changes/ticket28614 @@ -0,0 +1,4 @@ + o Major bugfixes (windows, startup): + - When writing a consensus file to disk, always write in + "binary" mode so that we can safely map it into memory later. + Fixes part bug 28614; bugfix on 0.4.0.1-alpha. diff --git a/src/feature/nodelist/networkstatus.c b/src/feature/nodelist/networkstatus.c index d9659b67c0..64169f6a4b 100644 --- a/src/feature/nodelist/networkstatus.c +++ b/src/feature/nodelist/networkstatus.c @@ -1991,7 +1991,7 @@ networkstatus_set_current_consensus(const char *consensus, waiting->set_at = now; waiting->dl_failed = 0; if (!from_cache) { - write_bytes_to_file(unverified_fname, consensus, consensus_len, 0); + write_bytes_to_file(unverified_fname, consensus, consensus_len, 1); } if (dl_certs) authority_certs_fetch_missing(c, now, source_dir); @@ -2142,7 +2142,7 @@ networkstatus_set_current_consensus(const char *consensus, } if (!from_cache) { - write_bytes_to_file(consensus_fname, consensus, consensus_len, 0); + write_bytes_to_file(consensus_fname, consensus, consensus_len, 1); } warn_early_consensus(c, flavor, now); From cdda3dc48457af31a9d1b3df1222d449b736f3b1 Mon Sep 17 00:00:00 2001 From: teor Date: Wed, 30 Jan 2019 15:15:41 +0100 Subject: [PATCH 0441/2557] hs: Move get_lspecs_from_node to nodelist.c Also: * rename to node_get_link_specifier_smartlist * rewrite to return a smartlist * add link_specifier_smartlist_free Part of 23576. --- src/feature/hs/hs_circuit.c | 86 ++--------------------------- src/feature/nodelist/nodelist.c | 96 +++++++++++++++++++++++++++++++++ src/feature/nodelist/nodelist.h | 5 ++ 3 files changed, 105 insertions(+), 82 deletions(-) diff --git a/src/feature/hs/hs_circuit.c b/src/feature/hs/hs_circuit.c index e3873d2f18..253c24d643 100644 --- a/src/feature/hs/hs_circuit.c +++ b/src/feature/hs/hs_circuit.c @@ -565,81 +565,6 @@ retry_service_rendezvous_point(const origin_circuit_t *circ) return; } -/* Add all possible link specifiers in node to lspecs: - * - legacy ID is mandatory thus MUST be present in node; - * - include ed25519 link specifier if present in the node, and the node - * supports ed25519 link authentication, even if its link versions are not - * compatible with us; - * - include IPv4 link specifier, if the primary address is not IPv4, log a - * BUG() warning, and return an empty smartlist; - * - include IPv6 link specifier if present in the node. */ -static void -get_lspecs_from_node(const node_t *node, smartlist_t *lspecs) -{ - link_specifier_t *ls; - tor_addr_port_t ap; - - tor_assert(node); - tor_assert(lspecs); - - /* Get the relay's IPv4 address. */ - node_get_prim_orport(node, &ap); - - /* We expect the node's primary address to be a valid IPv4 address. - * This conforms to the protocol, which requires either an IPv4 or IPv6 - * address (or both). */ - if (BUG(!tor_addr_is_v4(&ap.addr)) || - BUG(!tor_addr_port_is_valid_ap(&ap, 0))) { - return; - } - - ls = link_specifier_new(); - link_specifier_set_ls_type(ls, LS_IPV4); - link_specifier_set_un_ipv4_addr(ls, tor_addr_to_ipv4h(&ap.addr)); - link_specifier_set_un_ipv4_port(ls, ap.port); - /* Four bytes IPv4 and two bytes port. */ - link_specifier_set_ls_len(ls, sizeof(ap.addr.addr.in_addr) + - sizeof(ap.port)); - smartlist_add(lspecs, ls); - - /* Legacy ID is mandatory and will always be present in node. */ - ls = link_specifier_new(); - link_specifier_set_ls_type(ls, LS_LEGACY_ID); - memcpy(link_specifier_getarray_un_legacy_id(ls), node->identity, - link_specifier_getlen_un_legacy_id(ls)); - link_specifier_set_ls_len(ls, link_specifier_getlen_un_legacy_id(ls)); - smartlist_add(lspecs, ls); - - /* ed25519 ID is only included if the node has it, and the node declares a - protocol version that supports ed25519 link authentication, even if that - link version is not compatible with us. (We are sending the ed25519 key - to another tor, which may support different link versions.) */ - if (!ed25519_public_key_is_zero(&node->ed25519_id) && - node_supports_ed25519_link_authentication(node, 0)) { - ls = link_specifier_new(); - link_specifier_set_ls_type(ls, LS_ED25519_ID); - memcpy(link_specifier_getarray_un_ed25519_id(ls), &node->ed25519_id, - link_specifier_getlen_un_ed25519_id(ls)); - link_specifier_set_ls_len(ls, link_specifier_getlen_un_ed25519_id(ls)); - smartlist_add(lspecs, ls); - } - - /* Check for IPv6. If so, include it as well. */ - if (node_has_ipv6_orport(node)) { - ls = link_specifier_new(); - node_get_pref_ipv6_orport(node, &ap); - link_specifier_set_ls_type(ls, LS_IPV6); - size_t addr_len = link_specifier_getlen_un_ipv6_addr(ls); - const uint8_t *in6_addr = tor_addr_to_in6_addr8(&ap.addr); - uint8_t *ipv6_array = link_specifier_getarray_un_ipv6_addr(ls); - memcpy(ipv6_array, in6_addr, addr_len); - link_specifier_set_un_ipv6_port(ls, ap.port); - /* Sixteen bytes IPv6 and two bytes port. */ - link_specifier_set_ls_len(ls, addr_len + sizeof(ap.port)); - smartlist_add(lspecs, ls); - } -} - /* Using the given descriptor intro point ip, the node of the * rendezvous point rp_node and the service's subcredential, populate the * already allocated intro1_data object with the needed key material and link @@ -662,10 +587,9 @@ setup_introduce1_data(const hs_desc_intro_point_t *ip, tor_assert(subcredential); tor_assert(intro1_data); - /* Build the link specifiers from the extend information of the rendezvous - * circuit that we've picked previously. */ - rp_lspecs = smartlist_new(); - get_lspecs_from_node(rp_node, rp_lspecs); + /* Build the link specifiers from the node at the end of the rendezvous + * circuit that we opened for this introduction. */ + rp_lspecs = node_get_link_specifier_smartlist(rp_node, 0); if (smartlist_len(rp_lspecs) == 0) { /* We can't rendezvous without link specifiers. */ smartlist_free(rp_lspecs); @@ -1044,9 +968,7 @@ hs_circ_handle_introduce2(const hs_service_t *service, ret = 0; done: - SMARTLIST_FOREACH(data.link_specifiers, link_specifier_t *, lspec, - link_specifier_free(lspec)); - smartlist_free(data.link_specifiers); + link_specifier_smartlist_free(data.link_specifiers); memwipe(&data, 0, sizeof(data)); return ret; } diff --git a/src/feature/nodelist/nodelist.c b/src/feature/nodelist/nodelist.c index 9a27701803..32ee2677bb 100644 --- a/src/feature/nodelist/nodelist.c +++ b/src/feature/nodelist/nodelist.c @@ -1189,6 +1189,102 @@ node_get_rsa_id_digest(const node_t *node) return (const uint8_t*)node->identity; } +/* Returns a new smartlist with all possible link specifiers from node: + * - legacy ID is mandatory thus MUST be present in node; + * - include ed25519 link specifier if present in the node, and the node + * supports ed25519 link authentication, and: + * - if direct_conn is true, its link versions are compatible with us, + * - if direct_conn is false, regardless of its link versions; + * - include IPv4 link specifier, if the primary address is not IPv4, log a + * BUG() warning, and return an empty smartlist; + * - include IPv6 link specifier if present in the node. + * + * If node is NULL, returns an empty smartlist. + * + * The smartlist must be freed using link_specifier_smartlist_free(). */ +smartlist_t * +node_get_link_specifier_smartlist(const node_t *node, bool direct_conn) +{ + link_specifier_t *ls; + tor_addr_port_t ap; + smartlist_t *lspecs = smartlist_new(); + + if (!node) + return lspecs; + + /* Get the relay's IPv4 address. */ + node_get_prim_orport(node, &ap); + + /* We expect the node's primary address to be a valid IPv4 address. + * This conforms to the protocol, which requires either an IPv4 or IPv6 + * address (or both). */ + if (BUG(!tor_addr_is_v4(&ap.addr)) || + BUG(!tor_addr_port_is_valid_ap(&ap, 0))) { + return lspecs; + } + + ls = link_specifier_new(); + link_specifier_set_ls_type(ls, LS_IPV4); + link_specifier_set_un_ipv4_addr(ls, tor_addr_to_ipv4h(&ap.addr)); + link_specifier_set_un_ipv4_port(ls, ap.port); + /* Four bytes IPv4 and two bytes port. */ + link_specifier_set_ls_len(ls, sizeof(ap.addr.addr.in_addr) + + sizeof(ap.port)); + smartlist_add(lspecs, ls); + + /* Legacy ID is mandatory and will always be present in node. */ + ls = link_specifier_new(); + link_specifier_set_ls_type(ls, LS_LEGACY_ID); + memcpy(link_specifier_getarray_un_legacy_id(ls), node->identity, + link_specifier_getlen_un_legacy_id(ls)); + link_specifier_set_ls_len(ls, link_specifier_getlen_un_legacy_id(ls)); + smartlist_add(lspecs, ls); + + /* ed25519 ID is only included if the node has it, and the node declares a + protocol version that supports ed25519 link authentication. + If direct_conn is true, we also require that the node's link version is + compatible with us. (Otherwise, we will be sending the ed25519 key + to another tor, which may support different link versions.) */ + if (!ed25519_public_key_is_zero(&node->ed25519_id) && + node_supports_ed25519_link_authentication(node, direct_conn)) { + ls = link_specifier_new(); + link_specifier_set_ls_type(ls, LS_ED25519_ID); + memcpy(link_specifier_getarray_un_ed25519_id(ls), &node->ed25519_id, + link_specifier_getlen_un_ed25519_id(ls)); + link_specifier_set_ls_len(ls, link_specifier_getlen_un_ed25519_id(ls)); + smartlist_add(lspecs, ls); + } + + /* Check for IPv6. If so, include it as well. */ + if (node_has_ipv6_orport(node)) { + ls = link_specifier_new(); + node_get_pref_ipv6_orport(node, &ap); + link_specifier_set_ls_type(ls, LS_IPV6); + size_t addr_len = link_specifier_getlen_un_ipv6_addr(ls); + const uint8_t *in6_addr = tor_addr_to_in6_addr8(&ap.addr); + uint8_t *ipv6_array = link_specifier_getarray_un_ipv6_addr(ls); + memcpy(ipv6_array, in6_addr, addr_len); + link_specifier_set_un_ipv6_port(ls, ap.port); + /* Sixteen bytes IPv6 and two bytes port. */ + link_specifier_set_ls_len(ls, addr_len + sizeof(ap.port)); + smartlist_add(lspecs, ls); + } + + return lspecs; +} + +/* Free a link specifier list. */ +void +link_specifier_smartlist_free_(smartlist_t *ls_list) +{ + if (!ls_list) + return; + + SMARTLIST_FOREACH(ls_list, link_specifier_t *, lspec, + link_specifier_free(lspec)); + smartlist_free(ls_list); +} + /** Return the nickname of node, or NULL if we can't find one. */ const char * node_get_nickname(const node_t *node) diff --git a/src/feature/nodelist/nodelist.h b/src/feature/nodelist/nodelist.h index 3420959618..a3d65347a8 100644 --- a/src/feature/nodelist/nodelist.h +++ b/src/feature/nodelist/nodelist.h @@ -77,6 +77,11 @@ int node_supports_v3_hsdir(const node_t *node); int node_supports_ed25519_hs_intro(const node_t *node); int node_supports_v3_rendezvous_point(const node_t *node); const uint8_t *node_get_rsa_id_digest(const node_t *node); +smartlist_t *node_get_link_specifier_smartlist(const node_t *node, + bool direct_conn); +void link_specifier_smartlist_free_(smartlist_t *ls_list); +#define link_specifier_smartlist_free(ls_list) \ + FREE_AND_NULL(smartlist_t, link_specifier_smartlist_free_, (ls_list)) int node_has_ipv6_addr(const node_t *node); int node_has_ipv6_orport(const node_t *node); From 58cbe517084d4ec8cd4ff3c297f841e561804cc3 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Wed, 30 Jan 2019 17:13:40 +0200 Subject: [PATCH 0442/2557] Use RFC5737-compliant example IP addresses in manpage when describing MapAddress --- changes/doc28623 | 3 +++ doc/tor.1.txt | 13 +++++++------ 2 files changed, 10 insertions(+), 6 deletions(-) create mode 100644 changes/doc28623 diff --git a/changes/doc28623 b/changes/doc28623 new file mode 100644 index 0000000000..3c3313abdd --- /dev/null +++ b/changes/doc28623 @@ -0,0 +1,3 @@ + o Documentation: + - In manpage entry describing MapAddress torrc setting, use example + IP addresses from ranges specified by RFC 5737. Resolves issue 28623. diff --git a/doc/tor.1.txt b/doc/tor.1.txt index 00d6929c19..eb16037430 100644 --- a/doc/tor.1.txt +++ b/doc/tor.1.txt @@ -1159,17 +1159,18 @@ The following options are useful only for clients (that is, if 1. When evaluating MapAddress expressions Tor stops when it hits the most recently added expression that matches the requested address. So if you - have the following in your torrc, www.torproject.org will map to 1.1.1.1: + have the following in your torrc, www.torproject.org will map to + 198.51.100.1: - MapAddress www.torproject.org 2.2.2.2 - MapAddress www.torproject.org 1.1.1.1 + MapAddress www.torproject.org 192.0.2.1 + MapAddress www.torproject.org 198.51.100.1 2. Tor evaluates the MapAddress configuration until it finds no matches. So if you have the following in your torrc, www.torproject.org will map to - 2.2.2.2: + 203.0.113.1: - MapAddress 1.1.1.1 2.2.2.2 - MapAddress www.torproject.org 1.1.1.1 + MapAddress 198.51.100.1 203.0.113.1 + MapAddress www.torproject.org 198.51.100.1 3. The following MapAddress expression is invalid (and will be ignored) because you cannot map from a specific address to a wildcard From 6170d3fcf12f4345ec13561d0f444930f2f12e84 Mon Sep 17 00:00:00 2001 From: teor Date: Tue, 31 Jul 2018 14:30:17 +1000 Subject: [PATCH 0443/2557] hs: Onion services put IPv6 addresses in service descriptors Rewrite service_intro_point_new() to take a node_t. Since node_get_link_specifier_smartlist() supports IPv6 link specifiers, this refactor adds IPv6 addresses to onion service descriptors. Part of 23576, implements 26992. --- changes/bug23576 | 7 +++ src/feature/hs/hs_common.c | 18 ++++++- src/feature/hs/hs_service.c | 96 +++++++---------------------------- src/feature/hs/hs_service.h | 5 +- src/test/test_hs_cell.c | 4 +- src/test/test_hs_intropoint.c | 4 +- src/test/test_hs_service.c | 2 +- 7 files changed, 48 insertions(+), 88 deletions(-) create mode 100644 changes/bug23576 diff --git a/changes/bug23576 b/changes/bug23576 new file mode 100644 index 0000000000..edcae02e5e --- /dev/null +++ b/changes/bug23576 @@ -0,0 +1,7 @@ + o Minor features (IPv6, v3 onion services): + - Make v3 onion services put IPv6 addresses in service + descriptors. Before this change, service descriptors only + contained IPv4 addressesd. Implements 26992. + o Code simplification and refactoring: + - Simplify v3 onion service link specifier handling code. + Fixes bug 23576; bugfix on 0.3.2.1-alpha. diff --git a/src/feature/hs/hs_common.c b/src/feature/hs/hs_common.c index ebe49f09a5..5d8f54230f 100644 --- a/src/feature/hs/hs_common.c +++ b/src/feature/hs/hs_common.c @@ -1697,6 +1697,12 @@ hs_get_extend_info_from_lspecs(const smartlist_t *lspecs, tor_assert(lspecs); + if (smartlist_len(lspecs) == 0) { + log_fn(LOG_PROTOCOL_WARN, LD_REND, "Empty link specifier list."); + /* Return NULL. */ + goto done; + } + SMARTLIST_FOREACH_BEGIN(lspecs, const link_specifier_t *, ls) { switch (link_specifier_get_ls_type(ls)) { case LS_IPV4: @@ -1730,6 +1736,12 @@ hs_get_extend_info_from_lspecs(const smartlist_t *lspecs, /* Legacy ID is mandatory, and we require IPv4. */ if (!have_v4 || !have_legacy_id) { + bool both = !have_v4 && !have_legacy_id; + log_fn(LOG_PROTOCOL_WARN, LD_REND, "Missing %s%s%s link specifier%s.", + !have_v4 ? "IPv4" : "", + both ? " and " : "", + !have_legacy_id ? "legacy ID" : "", + both ? "s" : ""); goto done; } @@ -1748,6 +1760,10 @@ hs_get_extend_info_from_lspecs(const smartlist_t *lspecs, * release. */ } else { /* If we can't reach IPv4, return NULL. */ + log_fn(LOG_PROTOCOL_WARN, LD_REND, + "Received an IPv4 link specifier, " + "but the address is not reachable: %s:%u", + fmt_addr(&addr_v4), port_v4); goto done; } @@ -1755,7 +1771,7 @@ hs_get_extend_info_from_lspecs(const smartlist_t *lspecs, validate: /* We'll validate now that the address we've picked isn't a private one. If - * it is, are we allowing to extend to private address? */ + * it is, are we allowed to extend to private addresses? */ if (!extend_info_addr_is_allowed(&addr_v4)) { log_fn(LOG_PROTOCOL_WARN, LD_REND, "Requested address is private and we are not allowed to extend to " diff --git a/src/feature/hs/hs_service.c b/src/feature/hs/hs_service.c index b94dd9a481..6e4da8856a 100644 --- a/src/feature/hs/hs_service.c +++ b/src/feature/hs/hs_service.c @@ -426,23 +426,16 @@ service_intro_point_free_void(void *obj) } /* Return a newly allocated service intro point and fully initialized from the - * given extend_info_t ei if non NULL. - * If is_legacy is true, we also generate the legacy key. - * If supports_ed25519_link_handshake_any is true, we add the relay's ed25519 - * key to the link specifiers. + * given node_t node, if non NULL. * - * If ei is NULL, returns a hs_service_intro_point_t with an empty link + * If node is NULL, returns a hs_service_intro_point_t with an empty link * specifier list and no onion key. (This is used for testing.) * On any other error, NULL is returned. * - * ei must be an extend_info_t containing an IPv4 address. (We will add supoort - * for IPv6 in a later release.) When calling extend_info_from_node(), pass - * 0 in for_direct_connection to make sure ei always has an IPv4 address. */ + * node must be an node_t with an IPv4 address. */ STATIC hs_service_intro_point_t * -service_intro_point_new(const extend_info_t *ei, unsigned int is_legacy, - unsigned int supports_ed25519_link_handshake_any) +service_intro_point_new(const node_t *node) { - hs_desc_link_specifier_t *ls; hs_service_intro_point_t *ip; ip = tor_malloc_zero(sizeof(*ip)); @@ -472,12 +465,17 @@ service_intro_point_new(const extend_info_t *ei, unsigned int is_legacy, ip->replay_cache = replaycache_new(0, 0); /* Initialize the base object. We don't need the certificate object. */ - ip->base.link_specifiers = smartlist_new(); + ip->base.link_specifiers = node_get_link_specifier_smartlist(node, 0); + + if (node == NULL) { + goto done; + } /* Generate the encryption key for this intro point. */ curve25519_keypair_generate(&ip->enc_key_kp, 0); - /* Figure out if this chosen node supports v3 or is legacy only. */ - if (is_legacy) { + /* Figure out if this chosen node supports v3 or is legacy only. + * NULL nodes are used in the unit tests. */ + if (!node_supports_ed25519_hs_intro(node)) { ip->base.is_only_legacy = 1; /* Legacy mode that is doesn't support v3+ with ed25519 auth key. */ ip->legacy_key = crypto_pk_new(); @@ -490,40 +488,9 @@ service_intro_point_new(const extend_info_t *ei, unsigned int is_legacy, } } - if (ei == NULL) { - goto done; - } - - /* We'll try to add all link specifiers. Legacy is mandatory. - * IPv4 or IPv6 is required, and we always send IPv4. */ - ls = hs_desc_link_specifier_new(ei, LS_IPV4); - /* It is impossible to have an extend info object without a v4. */ - if (BUG(!ls)) { - goto err; - } - smartlist_add(ip->base.link_specifiers, ls); - - ls = hs_desc_link_specifier_new(ei, LS_LEGACY_ID); - /* It is impossible to have an extend info object without an identity - * digest. */ - if (BUG(!ls)) { - goto err; - } - smartlist_add(ip->base.link_specifiers, ls); - - /* ed25519 identity key is optional for intro points. If the node supports - * ed25519 link authentication, we include it. */ - if (supports_ed25519_link_handshake_any) { - ls = hs_desc_link_specifier_new(ei, LS_ED25519_ID); - if (ls) { - smartlist_add(ip->base.link_specifiers, ls); - } - } - - /* IPv6 is not supported in this release. */ - - /* Finally, copy onion key from the extend_info_t object. */ - memcpy(&ip->onion_key, &ei->curve25519_onion_key, sizeof(ip->onion_key)); + /* Finally, copy onion key from the node. */ + memcpy(&ip->onion_key, node_get_curve25519_onion_key(node), + sizeof(ip->onion_key)); done: return ip; @@ -2106,7 +2073,6 @@ static hs_service_intro_point_t * pick_intro_point(unsigned int direct_conn, smartlist_t *exclude_nodes) { const node_t *node; - extend_info_t *info = NULL; hs_service_intro_point_t *ip = NULL; /* Normal 3-hop introduction point flags. */ router_crn_flags_t flags = CRN_NEED_UPTIME | CRN_NEED_DESC; @@ -2127,43 +2093,17 @@ pick_intro_point(unsigned int direct_conn, smartlist_t *exclude_nodes) * we don't want to use that node anymore. */ smartlist_add(exclude_nodes, (void *) node); - /* We do this to ease our life but also this call makes appropriate checks - * of the node object such as validating ntor support for instance. - * - * We must provide an extend_info for clients to connect over a 3-hop path, - * so we don't pass direct_conn here. */ - info = extend_info_from_node(node, 0); - if (BUG(info == NULL)) { - goto err; - } + /* Create our objects and populate them with the node information. */ + ip = service_intro_point_new(node); - /* Let's do a basic sanity check here so that we don't end up advertising the - * ed25519 identity key of relays that don't actually support the link - * protocol */ - if (!node_supports_ed25519_link_authentication(node, 0)) { - tor_assert_nonfatal(ed25519_public_key_is_zero(&info->ed_identity)); - } else { - /* Make sure we *do* have an ed key if we support the link authentication. - * Sending an empty key would result in a failure to extend. */ - tor_assert_nonfatal(!ed25519_public_key_is_zero(&info->ed_identity)); - } - - /* Create our objects and populate them with the node information. - * We don't care if the intro's link auth is compatible with us, because - * we are sending the ed25519 key to a remote client via the descriptor. */ - ip = service_intro_point_new(info, !node_supports_ed25519_hs_intro(node), - node_supports_ed25519_link_authentication(node, - 0)); if (ip == NULL) { goto err; } - log_info(LD_REND, "Picked intro point: %s", extend_info_describe(info)); - extend_info_free(info); + log_info(LD_REND, "Picked intro point: %s", node_describe(node)); return ip; err: service_intro_point_free(ip); - extend_info_free(info); return NULL; } diff --git a/src/feature/hs/hs_service.h b/src/feature/hs/hs_service.h index ec53f2f23b..8d7f773219 100644 --- a/src/feature/hs/hs_service.h +++ b/src/feature/hs/hs_service.h @@ -369,10 +369,7 @@ STATIC hs_service_t *find_service(hs_service_ht *map, STATIC void remove_service(hs_service_ht *map, hs_service_t *service); STATIC int register_service(hs_service_ht *map, hs_service_t *service); /* Service introduction point functions. */ -STATIC hs_service_intro_point_t *service_intro_point_new( - const extend_info_t *ei, - unsigned int is_legacy, - unsigned int supports_ed25519_link_handshake_any); +STATIC hs_service_intro_point_t *service_intro_point_new(const node_t *node); STATIC void service_intro_point_free_(hs_service_intro_point_t *ip); #define service_intro_point_free(ip) \ FREE_AND_NULL(hs_service_intro_point_t, \ diff --git a/src/test/test_hs_cell.c b/src/test/test_hs_cell.c index 0c93f593ce..6e00e8807e 100644 --- a/src/test/test_hs_cell.c +++ b/src/test/test_hs_cell.c @@ -39,7 +39,7 @@ test_gen_establish_intro_cell(void *arg) attempt to parse it. */ { /* We only need the auth key pair here. */ - hs_service_intro_point_t *ip = service_intro_point_new(NULL, 0, 0); + hs_service_intro_point_t *ip = service_intro_point_new(NULL); /* Auth key pair is generated in the constructor so we are all set for * using this IP object. */ ret = hs_cell_build_establish_intro(circ_nonce, ip, buf); @@ -107,7 +107,7 @@ test_gen_establish_intro_cell_bad(void *arg) ed25519_sign_prefixed() function and make it fail. */ cell = trn_cell_establish_intro_new(); tt_assert(cell); - ip = service_intro_point_new(NULL, 0, 0); + ip = service_intro_point_new(NULL); cell_len = hs_cell_build_establish_intro(circ_nonce, ip, NULL); service_intro_point_free(ip); expect_log_msg_containing("Unable to make signature for " diff --git a/src/test/test_hs_intropoint.c b/src/test/test_hs_intropoint.c index 660f21ffd8..b7163c5c13 100644 --- a/src/test/test_hs_intropoint.c +++ b/src/test/test_hs_intropoint.c @@ -50,7 +50,7 @@ new_establish_intro_cell(const char *circ_nonce, /* Auth key pair is generated in the constructor so we are all set for * using this IP object. */ - ip = service_intro_point_new(NULL, 0, 0); + ip = service_intro_point_new(NULL); tt_assert(ip); cell_len = hs_cell_build_establish_intro(circ_nonce, ip, buf); tt_i64_op(cell_len, OP_GT, 0); @@ -76,7 +76,7 @@ new_establish_intro_encoded_cell(const char *circ_nonce, uint8_t *cell_out) /* Auth key pair is generated in the constructor so we are all set for * using this IP object. */ - ip = service_intro_point_new(NULL, 0, 0); + ip = service_intro_point_new(NULL); tt_assert(ip); cell_len = hs_cell_build_establish_intro(circ_nonce, ip, cell_out); tt_i64_op(cell_len, OP_GT, 0); diff --git a/src/test/test_hs_service.c b/src/test/test_hs_service.c index 43bf894383..e8bdcd86e2 100644 --- a/src/test/test_hs_service.c +++ b/src/test/test_hs_service.c @@ -329,7 +329,7 @@ static hs_service_intro_point_t * helper_create_service_ip(void) { hs_desc_link_specifier_t *ls; - hs_service_intro_point_t *ip = service_intro_point_new(NULL, 0, 0); + hs_service_intro_point_t *ip = service_intro_point_new(NULL); tor_assert(ip); /* Add a first unused link specifier. */ ls = tor_malloc_zero(sizeof(*ls)); From cb1072790f17cd43787a41c04b36a24833fbf7ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20M=2E=20Guisado?= Date: Thu, 31 Jan 2019 13:27:42 +0100 Subject: [PATCH 0444/2557] Warn about missing ContactInfo when MyFamily set MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Operators should be warned when setting MyFamily in addition to missing ContactInfo Signed-off-by: José M. Guisado --- changes/ticket25110 | 4 ++++ src/app/config/config.c | 4 ++++ 2 files changed, 8 insertions(+) create mode 100644 changes/ticket25110 diff --git a/changes/ticket25110 b/changes/ticket25110 new file mode 100644 index 0000000000..298e33287f --- /dev/null +++ b/changes/ticket25110 @@ -0,0 +1,4 @@ + o Minor bugfixes (logging, configuration): + - Warn operators when MyFamily option is set but ContactInfo + is missing, as the latter should be set too. + Fixes bug 25110; bugfix on 0.3.3.1-alpha. diff --git a/src/app/config/config.c b/src/app/config/config.c index 952b9cd301..6b9162a3df 100644 --- a/src/app/config/config.c +++ b/src/app/config/config.c @@ -4179,6 +4179,10 @@ options_validate(or_options_t *old_options, or_options_t *options, "You should also make sure you aren't listing this bridge's " "fingerprint in any other MyFamily."); } + if (options->MyFamily_lines && !options->ContactInfo) { + log_warn(LD_CONFIG, "MyFamily is set but ContactInfo is not configured. " + "ContactInfo should always be set when MyFamily option is too."); + } if (normalize_nickname_list(&options->MyFamily, options->MyFamily_lines, "MyFamily", msg)) return -1; From 10455aeff228042ee769205e43788d0e81cde910 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Fri, 1 Feb 2019 20:53:40 +0200 Subject: [PATCH 0445/2557] Fix shellcheck warnings in test_rust.sh --- changes/ticket29064 | 2 ++ src/test/test_rust.sh | 6 +++--- 2 files changed, 5 insertions(+), 3 deletions(-) create mode 100644 changes/ticket29064 diff --git a/changes/ticket29064 b/changes/ticket29064 new file mode 100644 index 0000000000..616b8aa77e --- /dev/null +++ b/changes/ticket29064 @@ -0,0 +1,2 @@ + o Code simplification and refactoring (shell scripts): + - Fix shellcheck warning in test_rust.sh. Fixes issue 29064. diff --git a/src/test/test_rust.sh b/src/test/test_rust.sh index 00b3e88d37..da2bd32d21 100755 --- a/src/test/test_rust.sh +++ b/src/test/test_rust.sh @@ -16,10 +16,10 @@ for cargo_toml_dir in "${abs_top_srcdir:-../../..}"/src/rust/*; do if [ -e "${cargo_toml_dir}/Cargo.toml" ]; then cd "${abs_top_builddir:-../../..}/src/rust" && \ CARGO_TARGET_DIR="${abs_top_builddir:-../../..}/src/rust/target" \ - "${CARGO:-cargo}" test ${CARGO_ONLINE-"--frozen"} \ + "${CARGO:-cargo}" test "${CARGO_ONLINE-'--frozen'}" \ --features "test_linking_hack" \ - --target $rustc_host \ - ${EXTRA_CARGO_OPTIONS} \ + --target "$rustc_host" \ + "${EXTRA_CARGO_OPTIONS}" \ --manifest-path "${cargo_toml_dir}/Cargo.toml" || exitcode=1 fi done From a84dc1973bc459dd3bc39a13dc5b5af59d9d5f21 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Fri, 1 Feb 2019 21:15:10 +0200 Subject: [PATCH 0446/2557] Fix instances of SC2006 in test_keygen.sh --- src/test/test_keygen.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/test/test_keygen.sh b/src/test/test_keygen.sh index 455f9e7d42..d807bd419c 100755 --- a/src/test/test_keygen.sh +++ b/src/test/test_keygen.sh @@ -13,7 +13,7 @@ if [ $# -eq 0 ] || [ ! -f ${1} ] || [ ! -x ${1} ]; then fi fi -UNAME_OS=`uname -s | cut -d_ -f1` +UNAME_OS=$(uname -s | cut -d_ -f1) if test "$UNAME_OS" = 'CYGWIN' || \ test "$UNAME_OS" = 'MSYS' || \ test "$UNAME_OS" = 'MINGW'; then @@ -65,10 +65,10 @@ die() { echo "$1" >&2 ; exit 5; } check_dir() { [ -d "$1" ] || die "$1 did not exist"; } check_file() { [ -e "$1" ] || die "$1 did not exist"; } check_no_file() { [ -e "$1" ] && die "$1 was not supposed to exist" || true; } -check_files_eq() { cmp "$1" "$2" || die "$1 and $2 did not match: `dump $1` vs `dump $2`"; } +check_files_eq() { cmp "$1" "$2" || die "$1 and $2 did not match: $(dump $1) vs $(dump $2)"; } check_keys_eq() { check_files_eq "${SRC}/keys/${1}" "${ME}/keys/${1}"; } -DATA_DIR=`mktemp -d -t tor_keygen_tests.XXXXXX` +DATA_DIR=$(mktemp -d -t tor_keygen_tests.XXXXXX) if [ -z "$DATA_DIR" ]; then echo "Failure: mktemp invocation returned empty string" >&2 exit 3 @@ -80,7 +80,7 @@ fi trap "rm -rf '$DATA_DIR'" 0 # Use an absolute path for this or Tor will complain -DATA_DIR=`cd "${DATA_DIR}" && pwd` +DATA_DIR=$(cd "${DATA_DIR}" && pwd) touch "${DATA_DIR}/empty_torrc" @@ -143,7 +143,7 @@ ME="${DATA_DIR}/case2a" SRC="${DATA_DIR}/orig" mkdir -p "${ME}/keys" cp "${SRC}/keys/ed25519_master_id_public_key" "${ME}/keys/" -${TOR} --DataDirectory "${ME}" --list-fingerprint > "${ME}/stdout" && die "Somehow succeeded when missing secret key, certs: `cat ${ME}/stdout`" || true +${TOR} --DataDirectory "${ME}" --list-fingerprint > "${ME}/stdout" && die "Somehow succeeded when missing secret key, certs: $(cat ${ME}/stdout)" || true check_files_eq "${SRC}/keys/ed25519_master_id_public_key" "${ME}/keys/ed25519_master_id_public_key" grep "We needed to load a secret key.*but couldn't find it" "${ME}/stdout" >/dev/null || die "Tor didn't declare that it was missing a secret key" From 82813315edf56a980147bd33778f8d16403fb249 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Fri, 1 Feb 2019 21:24:06 +0200 Subject: [PATCH 0447/2557] Fix instances of SC2086 in test_keygen.sh --- src/test/test_keygen.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/test/test_keygen.sh b/src/test/test_keygen.sh index d807bd419c..cc38397836 100755 --- a/src/test/test_keygen.sh +++ b/src/test/test_keygen.sh @@ -6,7 +6,7 @@ umask 077 set -e -if [ $# -eq 0 ] || [ ! -f ${1} ] || [ ! -x ${1} ]; then +if [ $# -eq 0 ] || [ ! -f "${1}" ] || [ ! -x "${1}" ]; then if [ "$TESTING_TOR_BINARY" = "" ] ; then echo "Usage: ${0} PATH_TO_TOR [case-number]" exit 1 @@ -65,7 +65,7 @@ die() { echo "$1" >&2 ; exit 5; } check_dir() { [ -d "$1" ] || die "$1 did not exist"; } check_file() { [ -e "$1" ] || die "$1 did not exist"; } check_no_file() { [ -e "$1" ] && die "$1 was not supposed to exist" || true; } -check_files_eq() { cmp "$1" "$2" || die "$1 and $2 did not match: $(dump $1) vs $(dump $2)"; } +check_files_eq() { cmp "$1" "$2" || die "$1 and $2 did not match: $(dump "$1") vs $(dump "$2")"; } check_keys_eq() { check_files_eq "${SRC}/keys/${1}" "${ME}/keys/${1}"; } DATA_DIR=$(mktemp -d -t tor_keygen_tests.XXXXXX) @@ -143,7 +143,7 @@ ME="${DATA_DIR}/case2a" SRC="${DATA_DIR}/orig" mkdir -p "${ME}/keys" cp "${SRC}/keys/ed25519_master_id_public_key" "${ME}/keys/" -${TOR} --DataDirectory "${ME}" --list-fingerprint > "${ME}/stdout" && die "Somehow succeeded when missing secret key, certs: $(cat ${ME}/stdout)" || true +${TOR} --DataDirectory "${ME}" --list-fingerprint > "${ME}/stdout" && die "Somehow succeeded when missing secret key, certs: $(cat "${ME}/stdout")" || true check_files_eq "${SRC}/keys/ed25519_master_id_public_key" "${ME}/keys/ed25519_master_id_public_key" grep "We needed to load a secret key.*but couldn't find it" "${ME}/stdout" >/dev/null || die "Tor didn't declare that it was missing a secret key" From 3ca1d585619d56cb424c9916cd889d1ff9f5b857 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Fri, 1 Feb 2019 21:26:29 +0200 Subject: [PATCH 0448/2557] Fix SC2064 --- src/test/test_keygen.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/test_keygen.sh b/src/test/test_keygen.sh index cc38397836..849d159af5 100755 --- a/src/test/test_keygen.sh +++ b/src/test/test_keygen.sh @@ -77,7 +77,7 @@ if [ ! -d "$DATA_DIR" ]; then echo "Failure: mktemp invocation result doesn't point to directory" >&2 exit 3 fi -trap "rm -rf '$DATA_DIR'" 0 +trap 'rm -rf "$DATA_DIR"' 0 # Use an absolute path for this or Tor will complain DATA_DIR=$(cd "${DATA_DIR}" && pwd) From 3f5459cb955f14d8aeb77fabe7f9d5c5018f4f24 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Fri, 1 Feb 2019 21:41:14 +0200 Subject: [PATCH 0449/2557] Fix all instances of SC2015 in test_keygen.sh --- src/test/test_keygen.sh | 34 +++++++++++++++++++++++++--------- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/src/test/test_keygen.sh b/src/test/test_keygen.sh index 849d159af5..9fbf7dd578 100755 --- a/src/test/test_keygen.sh +++ b/src/test/test_keygen.sh @@ -64,7 +64,7 @@ dump() { xxd -p "$1" | tr -d '\n '; } die() { echo "$1" >&2 ; exit 5; } check_dir() { [ -d "$1" ] || die "$1 did not exist"; } check_file() { [ -e "$1" ] || die "$1 did not exist"; } -check_no_file() { [ -e "$1" ] && die "$1 was not supposed to exist" || true; } +check_no_file() { if [ -e "$1" ]; then die "$1 was not supposed to exist"; fi } check_files_eq() { cmp "$1" "$2" || die "$1 and $2 did not match: $(dump "$1") vs $(dump "$2")"; } check_keys_eq() { check_files_eq "${SRC}/keys/${1}" "${ME}/keys/${1}"; } @@ -143,7 +143,9 @@ ME="${DATA_DIR}/case2a" SRC="${DATA_DIR}/orig" mkdir -p "${ME}/keys" cp "${SRC}/keys/ed25519_master_id_public_key" "${ME}/keys/" -${TOR} --DataDirectory "${ME}" --list-fingerprint > "${ME}/stdout" && die "Somehow succeeded when missing secret key, certs: $(cat "${ME}/stdout")" || true +if ${TOR} --DataDirectory "${ME}" --list-fingerprint > "${ME}/stdout"; then + die "Somehow succeeded when missing secret key, certs: $(cat "${ME}/stdout")" +fi check_files_eq "${SRC}/keys/ed25519_master_id_public_key" "${ME}/keys/ed25519_master_id_public_key" grep "We needed to load a secret key.*but couldn't find it" "${ME}/stdout" >/dev/null || die "Tor didn't declare that it was missing a secret key" @@ -280,7 +282,9 @@ SRC="${DATA_DIR}/encrypted" mkdir -p "${ME}/keys" cp "${SRC}/keys/ed25519_master_id_secret_key_encrypted" "${ME}/keys/" cp "${SRC}/keys/ed25519_master_id_public_key" "${ME}/keys/" -${TOR} --DataDirectory "${ME}" --list-fingerprint > "${ME}/stdout" && die "Tor started with encrypted secret key and no certs" || true +if ${TOR} --DataDirectory "${ME}" --list-fingerprint > "${ME}/stdout"; then + die "Tor started with encrypted secret key and no certs" +fi check_no_file "${ME}/keys/ed25519_signing_cert" check_no_file "${ME}/keys/ed25519_signing_secret_key" @@ -369,7 +373,9 @@ mkdir -p "${ME}/keys" cp "${SRC}/keys/ed25519_master_id_public_key" "${ME}/keys/" cp "${OTHER}/keys/ed25519_master_id_secret_key" "${ME}/keys/" -${TOR} --DataDirectory "${ME}" --list-fingerprint >"${ME}/stdout" && die "Successfully started with mismatched keys!?" || true +if ${TOR} --DataDirectory "${ME}" --list-fingerprint >"${ME}/stdout"; then + die "Successfully started with mismatched keys!?" +fi grep "public_key does not match.*secret_key" "${ME}/stdout" >/dev/null || die "Tor didn't declare that there was a key mismatch" @@ -385,7 +391,9 @@ ME="${DATA_DIR}/case11a" mkdir -p "${ME}/keys" -${TOR} --DataDirectory "${ME}" --passphrase-fd 1 > "${ME}/stdout" && die "Successfully started with passphrase-fd but no keygen?" || true +if ${TOR} --DataDirectory "${ME}" --passphrase-fd 1 > "${ME}/stdout"; then + die "Successfully started with passphrase-fd but no keygen?" +fi grep "passphrase-fd specified without --keygen" "${ME}/stdout" >/dev/null || die "Tor didn't declare that there was a problem with the arguments." @@ -401,7 +409,9 @@ ME="${DATA_DIR}/case11b" mkdir -p "${ME}/keys" -${TOR} --DataDirectory "${ME}" --no-passphrase > "${ME}/stdout" && die "Successfully started with no-passphrase but no keygen?" || true +if ${TOR} --DataDirectory "${ME}" --no-passphrase > "${ME}/stdout"; then + die "Successfully started with no-passphrase but no keygen?" +fi grep "no-passphrase specified without --keygen" "${ME}/stdout" >/dev/null || die "Tor didn't declare that there was a problem with the arguments." @@ -417,7 +427,9 @@ ME="${DATA_DIR}/case11C" mkdir -p "${ME}/keys" -${TOR} --DataDirectory "${ME}" --newpass > "${ME}/stdout" && die "Successfully started with newpass but no keygen?" || true +if ${TOR} --DataDirectory "${ME}" --newpass > "${ME}/stdout"; then + die "Successfully started with newpass but no keygen?" +fi grep "newpass specified without --keygen" "${ME}/stdout" >/dev/null || die "Tor didn't declare that there was a problem with the arguments." @@ -455,7 +467,9 @@ ME="${DATA_DIR}/case11E" mkdir -p "${ME}/keys" -${TOR} --DataDirectory "${ME}" --keygen --passphrase-fd ewigeblumenkraft > "${ME}/stdout" && die "Successfully started with bogus passphrase-fd?" || true +if ${TOR} --DataDirectory "${ME}" --keygen --passphrase-fd ewigeblumenkraft > "${ME}/stdout"; then + die "Successfully started with bogus passphrase-fd?" +fi grep "Invalid --passphrase-fd value" "${ME}/stdout" >/dev/null || die "Tor didn't declare that there was a problem with the arguments." @@ -472,7 +486,9 @@ ME="${DATA_DIR}/case11F" mkdir -p "${ME}/keys" -${TOR} --DataDirectory "${ME}" --keygen --passphrase-fd 1 --no-passphrase > "${ME}/stdout" && die "Successfully started with bogus passphrase-fd combination?" || true +if ${TOR} --DataDirectory "${ME}" --keygen --passphrase-fd 1 --no-passphrase > "${ME}/stdout"; then + die "Successfully started with bogus passphrase-fd combination?" +fi grep "no-passphrase specified with --passphrase-fd" "${ME}/stdout" >/dev/null || die "Tor didn't declare that there was a problem with the arguments." From 0b245e418e91cbbc325dfdc399fc184b067e9471 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Fri, 1 Feb 2019 21:43:45 +0200 Subject: [PATCH 0450/2557] Add changes file --- changes/ticket29062 | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 changes/ticket29062 diff --git a/changes/ticket29062 b/changes/ticket29062 new file mode 100644 index 0000000000..de05c621f1 --- /dev/null +++ b/changes/ticket29062 @@ -0,0 +1,3 @@ + o Code simplification and refactoring (shell scripts): + - Cleanup test_keygen.sh to silence all shellcheck warnings. Closes + ticket 29062. From 7341d9acdc4699498a593d0ff848be5e7d7fe0c5 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Sat, 2 Feb 2019 16:32:37 +0200 Subject: [PATCH 0451/2557] Fix all instances of SC2166 in test-network.sh --- src/test/test-network.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/test/test-network.sh b/src/test/test-network.sh index b7a9f1b3c0..51b298ca84 100755 --- a/src/test/test-network.sh +++ b/src/test/test-network.sh @@ -5,7 +5,7 @@ # If we already know CHUTNEY_PATH, don't bother with argument parsing TEST_NETWORK="$CHUTNEY_PATH/tools/test-network.sh" # Call the chutney version of this script, if it exists, and we can find it -if [ -d "$CHUTNEY_PATH" -a -x "$TEST_NETWORK" ]; then +if [ -d "$CHUTNEY_PATH" ] && [ -x "$TEST_NETWORK" ]; then # we can't produce any output, because we might be --quiet # this preserves arguments with spaces correctly exec "$TEST_NETWORK" "$@" @@ -52,12 +52,12 @@ done # - if $PWD looks like a tor build directory, set it to $PWD, or # - unset $TOR_DIR, and let chutney fall back to finding tor binaries in $PATH if [ ! -d "$TOR_DIR" ]; then - if [ -d "$BUILDDIR/src/core/or" -a -d "$BUILDDIR/src/tools" ]; then + if [ -d "$BUILDDIR/src/core/or" ] && [ -d "$BUILDDIR/src/tools" ]; then # Choose the build directory # But only if it looks like one $ECHO "$myname: \$TOR_DIR not set, trying \$BUILDDIR" TOR_DIR="$BUILDDIR" - elif [ -d "$PWD/src/core/or" -a -d "$PWD/src/tools" ]; then + elif [ -d "$PWD/src/core/or" ] && [ -d "$PWD/src/tools" ]; then # Guess the tor directory is the current directory # But only if it looks like one $ECHO "$myname: \$TOR_DIR not set, trying \$PWD" @@ -73,12 +73,12 @@ fi # - if $PWD looks like a chutney directory, set it to $PWD, or # - set it based on $TOR_DIR, expecting chutney to be next to tor, or # - fail and tell the user how to clone the chutney repository -if [ ! -d "$CHUTNEY_PATH" -o ! -x "$CHUTNEY_PATH/chutney" ]; then +if [ ! -d "$CHUTNEY_PATH" ] || [ ! -x "$CHUTNEY_PATH/chutney" ]; then if [ -x "$PWD/chutney" ]; then $ECHO "$myname: \$CHUTNEY_PATH not valid, trying \$PWD" CHUTNEY_PATH="$PWD" - elif [ -d "$TOR_DIR" -a -d "$TOR_DIR/../chutney" -a \ - -x "$TOR_DIR/../chutney/chutney" ]; then + elif [ -d "$TOR_DIR" ] && [ -d "$TOR_DIR/../chutney" ] && \ + [ -x "$TOR_DIR/../chutney/chutney" ]; then $ECHO "$myname: \$CHUTNEY_PATH not valid, trying \$TOR_DIR/../chutney" CHUTNEY_PATH="$TOR_DIR/../chutney" else @@ -94,7 +94,7 @@ fi TEST_NETWORK="$CHUTNEY_PATH/tools/test-network.sh" # Call the chutney version of this script, if it exists, and we can find it -if [ -d "$CHUTNEY_PATH" -a -x "$TEST_NETWORK" ]; then +if [ -d "$CHUTNEY_PATH" ] && [ -x "$TEST_NETWORK" ]; then $ECHO "$myname: Calling newer chutney script $TEST_NETWORK" # this may fail if some arguments have spaces in them # if so, set CHUTNEY_PATH before calling test-network.sh, and spaces From f888b3e2ee23f6e0394c9c7589c10e320714deda Mon Sep 17 00:00:00 2001 From: rl1987 Date: Sat, 2 Feb 2019 16:46:30 +0200 Subject: [PATCH 0452/2557] Update test-network.sh to bash script to use array This lets us to save original script argument to array (POSIX shell does not support that). Fixes shellcheck warnings SC2124 and SC2086. --- src/test/test-network.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/test/test-network.sh b/src/test/test-network.sh index 51b298ca84..e382eec66e 100755 --- a/src/test/test-network.sh +++ b/src/test/test-network.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # This script calls the equivalent script in chutney/tools @@ -20,7 +20,7 @@ myname=$(basename $0) # Save the arguments before we destroy them # This might not preserve arguments with spaces in them -ORIGINAL_ARGS="$@" +ORIGINAL_ARGS=( "$@" ) # We need to find CHUTNEY_PATH, so that we can call the version of this script # in chutney/tools with the same arguments. We also need to respect --quiet. @@ -99,7 +99,7 @@ if [ -d "$CHUTNEY_PATH" ] && [ -x "$TEST_NETWORK" ]; then # this may fail if some arguments have spaces in them # if so, set CHUTNEY_PATH before calling test-network.sh, and spaces # will be handled correctly - exec "$TEST_NETWORK" $ORIGINAL_ARGS + exec "$TEST_NETWORK" "${ORIGINAL_ARGS[@]}" # $ORIGINAL_ARGS else $ECHO "$myname: Could not find tools/test-network.sh in CHUTNEY_PATH." $ECHO "$myname: Please update your chutney using 'git pull'." From d7e5086694e722573dc46fd45f1a5571e10d5590 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Sat, 2 Feb 2019 16:49:19 +0200 Subject: [PATCH 0453/2557] Fix one last SC2086 --- src/test/test-network.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/test-network.sh b/src/test/test-network.sh index e382eec66e..6c678849b6 100755 --- a/src/test/test-network.sh +++ b/src/test/test-network.sh @@ -16,7 +16,7 @@ fi # Do we output anything at all? ECHO="${ECHO:-echo}" # Output is prefixed with the name of the script -myname=$(basename $0) +myname=$(basename "$0") # Save the arguments before we destroy them # This might not preserve arguments with spaces in them From 583e20615cf0826a4b1774eead50c40d246f6962 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Sat, 2 Feb 2019 16:54:50 +0200 Subject: [PATCH 0454/2557] Add changes file --- changes/ticket29060 | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 changes/ticket29060 diff --git a/changes/ticket29060 b/changes/ticket29060 new file mode 100644 index 0000000000..380cc8eb11 --- /dev/null +++ b/changes/ticket29060 @@ -0,0 +1,2 @@ + o Code simplification and refactoring (shell scripts): + - Fix shellcheck warnings in test-network.sh. Resolves issue 29060. From 83b1ca9b07195cc2f2ad514043eaebfaf9e85641 Mon Sep 17 00:00:00 2001 From: Peter Gerber Date: Sat, 2 Feb 2019 23:09:44 +0000 Subject: [PATCH 0455/2557] Add release note for bug 29150 --- changes/bug29150 | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 changes/bug29150 diff --git a/changes/bug29150 b/changes/bug29150 new file mode 100644 index 0000000000..7696b90378 --- /dev/null +++ b/changes/bug29150 @@ -0,0 +1,3 @@ + o Minor bugfixes (linux seccomp sandbox): + - Fix startup crash when experimental sandbox support is enabled. + Fixes bug 29150; bugfix on 0.4.0.1-alpha. Patch by Peter Gerber. From 1fc8bbff9cefd243725c04890426a28204d7b0d2 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Wed, 6 Feb 2019 10:20:11 +0200 Subject: [PATCH 0456/2557] Fix two instances of SC2004 in nagios-check-tor-authority --- contrib/dirauth-tools/nagios-check-tor-authority-cert | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/dirauth-tools/nagios-check-tor-authority-cert b/contrib/dirauth-tools/nagios-check-tor-authority-cert index 46dc7284b7..7a60da16f0 100755 --- a/contrib/dirauth-tools/nagios-check-tor-authority-cert +++ b/contrib/dirauth-tools/nagios-check-tor-authority-cert @@ -74,10 +74,10 @@ now=$(date +%s) if [ "$now" -ge "$expiryunix" ]; then echo "CRITICAL: Certificate expired $expirydate (authority $identity)." exit 2 -elif [ "$(( $now + 7*24*60*60 ))" -ge "$expiryunix" ]; then +elif [ "$(( now + 7*24*60*60 ))" -ge "$expiryunix" ]; then echo "CRITICAL: Certificate expires $expirydate (authority $identity)." exit 2 -elif [ "$(( $now + 30*24*60*60 ))" -ge "$expiryunix" ]; then +elif [ "$(( now + 30*24*60*60 ))" -ge "$expiryunix" ]; then echo "WARNING: Certificate expires $expirydate (authority $identity)." exit 1 else From 2649239c620b57cc11c2499ed35fa571af996d18 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Wed, 6 Feb 2019 10:27:16 +0200 Subject: [PATCH 0457/2557] Use mktemp to fix SC2186; also, fix SC2006. --- contrib/dirauth-tools/nagios-check-tor-authority-cert | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/dirauth-tools/nagios-check-tor-authority-cert b/contrib/dirauth-tools/nagios-check-tor-authority-cert index 7a60da16f0..932a780618 100755 --- a/contrib/dirauth-tools/nagios-check-tor-authority-cert +++ b/contrib/dirauth-tools/nagios-check-tor-authority-cert @@ -49,7 +49,7 @@ DIRSERVERS="$DIRSERVERS 80.190.246.100:80" # gabelmoo DIRSERVERS="$DIRSERVERS 194.109.206.212:80" # dizum DIRSERVERS="$DIRSERVERS 213.73.91.31:80" # dannenberg -TMPFILE="`tempfile`" +TMPFILE=$(mktemp) trap 'rm -f "$TMPFILE"' 0 for dirserver in $DIRSERVERS; do From 7e415ec05a1bb1cf5cda7088f2c27e9202b3b3f1 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Wed, 6 Feb 2019 10:51:54 +0200 Subject: [PATCH 0458/2557] Fix SC2181 --- contrib/dirauth-tools/nagios-check-tor-authority-cert | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/dirauth-tools/nagios-check-tor-authority-cert b/contrib/dirauth-tools/nagios-check-tor-authority-cert index 932a780618..75ff479a53 100755 --- a/contrib/dirauth-tools/nagios-check-tor-authority-cert +++ b/contrib/dirauth-tools/nagios-check-tor-authority-cert @@ -53,8 +53,8 @@ TMPFILE=$(mktemp) trap 'rm -f "$TMPFILE"' 0 for dirserver in $DIRSERVERS; do - wget -q -O "$TMPFILE" "http://$dirserver/tor/keys/fp/$identity" - if [ "$?" = 0 ]; then + if wget -q -O "$TMPFILE" "http://$dirserver/tor/keys/fp/$identity" + then break else cat /dev/null > "$TMPFILE" From 4e3880607ad2f09711e18429afda1514a2cb7e28 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Wed, 6 Feb 2019 10:53:12 +0200 Subject: [PATCH 0459/2557] Add changes file --- changes/ticket29071 | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 changes/ticket29071 diff --git a/changes/ticket29071 b/changes/ticket29071 new file mode 100644 index 0000000000..0997a8d22f --- /dev/null +++ b/changes/ticket29071 @@ -0,0 +1,3 @@ + o Code simplification and refactoring (shell scripts): + - Fix shellcheck warnings in nagios-check-tor-authority-cert script. + Resolves issue 29071. From daff9e1ba11bdf1d6fe55eeeda45dd4bb2918874 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Sat, 2 Feb 2019 17:14:49 +0200 Subject: [PATCH 0460/2557] Fix shellcheck warning in fuzz_static_testcases.sh --- changes/ticket29059 | 3 +++ src/test/fuzz_static_testcases.sh | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 changes/ticket29059 diff --git a/changes/ticket29059 b/changes/ticket29059 new file mode 100644 index 0000000000..d47d0e2a3b --- /dev/null +++ b/changes/ticket29059 @@ -0,0 +1,3 @@ + o Code simplification and refactoring (shell scripts): + - Fix shellcheck warnings in fuzz_static_testcases.sh. Resolves ticket + 29059. diff --git a/src/test/fuzz_static_testcases.sh b/src/test/fuzz_static_testcases.sh index f7b3adffb1..b883352402 100755 --- a/src/test/fuzz_static_testcases.sh +++ b/src/test/fuzz_static_testcases.sh @@ -14,7 +14,7 @@ fi for fuzzer in "${builddir:-.}"/src/test/fuzz/fuzz-* ; do - f=`basename $fuzzer` + f=$(basename "$fuzzer") case="${f#fuzz-}" if [ -d "${TOR_FUZZ_CORPORA}/${case}" ]; then echo "Running tests for ${case}" From 8ca808f81d28bbae9e9059172a7b7479d20e9594 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 5 Feb 2019 16:58:56 -0500 Subject: [PATCH 0461/2557] Code for anonymous mappings via mmap() or CreateFileMapping(). Using an anonymous mmap() is a good way to get pages that we can set kernel-level flags on, like minherit() or madvise() or mlock(). We're going to use that so that we can make uninheritable locked pages to store PRNG data. --- configure.ac | 3 + src/lib/malloc/include.am | 6 +- src/lib/malloc/map_anon.c | 213 ++++++++++++++++++++++++++++++++++++++ src/lib/malloc/map_anon.h | 37 +++++++ src/test/test_util.c | 107 +++++++++++++++++++ 5 files changed, 364 insertions(+), 2 deletions(-) create mode 100644 src/lib/malloc/map_anon.c create mode 100644 src/lib/malloc/map_anon.h diff --git a/configure.ac b/configure.ac index 75bb0c7bd9..7de3c3a46e 100644 --- a/configure.ac +++ b/configure.ac @@ -605,8 +605,10 @@ AC_CHECK_FUNCS( llround \ localtime_r \ lround \ + madvise \ memmem \ memset_s \ + minherit \ mmap \ pipe \ pipe2 \ @@ -1450,6 +1452,7 @@ AC_CHECK_HEADERS([errno.h \ inttypes.h \ limits.h \ linux/types.h \ + mach/vm_inherit.h \ machine/limits.h \ malloc.h \ malloc/malloc.h \ diff --git a/src/lib/malloc/include.am b/src/lib/malloc/include.am index 502cc1c6b7..95d96168e1 100644 --- a/src/lib/malloc/include.am +++ b/src/lib/malloc/include.am @@ -6,7 +6,8 @@ noinst_LIBRARIES += src/lib/libtor-malloc-testing.a endif src_lib_libtor_malloc_a_SOURCES = \ - src/lib/malloc/malloc.c + src/lib/malloc/malloc.c \ + src/lib/malloc/map_anon.c if USE_OPENBSD_MALLOC src_lib_libtor_malloc_a_SOURCES += src/ext/OpenBSD_malloc_Linux.c @@ -18,4 +19,5 @@ src_lib_libtor_malloc_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_malloc_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) noinst_HEADERS += \ - src/lib/malloc/malloc.h + src/lib/malloc/malloc.h \ + src/lib/malloc/map_anon.h diff --git a/src/lib/malloc/map_anon.c b/src/lib/malloc/map_anon.c new file mode 100644 index 0000000000..2fc6e89ea2 --- /dev/null +++ b/src/lib/malloc/map_anon.c @@ -0,0 +1,213 @@ +/* Copyright (c) 2003-2004, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file map_anon.c + * \brief Manage anonymous mappings. + **/ + +#include "orconfig.h" +#include "lib/malloc/map_anon.h" +#include "lib/malloc/malloc.h" +#include "lib/err/torerr.h" + +#ifdef HAVE_SYS_MMAN_H +#include +#endif +#ifdef HAVE_SYS_TYPES_H +#include +#endif +#ifdef HAVE_MACH_VM_INHERIT_H +#include +#endif + +#ifdef _WIN32 +#include +#endif + +/** + * Macro to get the high bytes of a size_t, if there are high bytes. + * Windows needs this; other operating systems define a size_t that does + * what it should. + */ +#if SIZEOF_SIZE_T > 4 +#define HIGH_SIZE_T_BYTES(sz) ((sz) >> 32) +#else +#define HIGH_SIZE_T_BYTES(sz) (0) +#endif + +/* Here we define a MINHERIT macro that is minherit() or madvise(), depending + * on what we actually want. + * + * If there's a flag that sets pages to zero after fork, we define FLAG_ZERO + * to be that flag. If there's a flag unmaps pages after fork, we define + * FLAG_NOINHERIT to be that flag. + */ +#if defined(HAVE_MINHERIT) +#define MINHERIT minherit + +#ifdef INHERIT_ZERO +#define FLAG_ZERO INHERIT_ZERO +#endif +#ifdef INHERIT_NONE +#define FLAG_NOINHERIT INHERIT_NONE +#elif defined(VM_INHERIT_NONE) +#define FLAG_NOINHERIT VM_INHERIT_NONE +#endif + +#elif defined(HAVE_MADVISE) + +#define MINHERIT madvise + +#ifdef MADV_WIPEONFORK +#define FLAG_ZERO MADV_WIPEONFORK +#endif +#ifdef MADV_DONTFORK +#define FLAG_NOINHERIT MADV_DONTFORK +#endif + +#endif + +/** + * Helper: try to prevent the sz bytes at mem from being swapped + * to disk. Return 0 on success or if the facility is not available on this + * OS; return -1 on failure. + */ +static int +lock_mem(void *mem, size_t sz) +{ +#ifdef _WIN32 + return VirtualLock(mem, sz) ? 0 : -1; +#elif defined(HAVE_MLOCK) + return mlock(mem, sz); +#else + (void) mem; + (void) sz; + + return 0; +#endif +} + +/** + * Helper: try to prevent the sz bytes at mem from appearing in + * a core dump. Return 0 on success or if the facility is not available on + * this OS; return -1 on failure. + */ +static int +nodump_mem(void *mem, size_t sz) +{ +#if defined(MADV_DONTDUMP) + return madvise(mem, sz, MADV_DONTDUMP); +#else + (void) mem; + (void) sz; + return 0; +#endif +} + +/** + * Helper: try to prevent the sz bytes at mem from being + * accessible in child processes -- ideally by having them set to 0 after a + * fork, and if that doesn't work, by having them unmapped after a fork. + * Return 0 on success or if the facility is not available on this OS; return + * -1 on failure. + */ +static int +noinherit_mem(void *mem, size_t sz) +{ +#ifdef FLAG_ZERO + int r = MINHERIT(mem, sz, FLAG_ZERO); + if (r == 0) + return 0; +#endif +#ifdef FLAG_NOINHERIT + return MINHERIT(mem, sz, FLAG_NOINHERIT); +#else + (void)mem; + (void)sz; + return 0; +#endif +} + +/** + * Return a new anonymous memory mapping that holds sz bytes. + * + * Memory mappings are unlike the results from malloc() in that they are + * handled separately by the operating system, and as such can have different + * kernel-level flags set on them. + * + * The "flags" argument may be zero or more of ANONMAP_PRIVATE and + * ANONMAP_NOINHERIT. + * + * Memory returned from this function must be released with + * tor_munmap_anonymous(). + * + * [Note: OS people use the word "anonymous" here to mean that the memory + * isn't associated with any file. This has *nothing* to do with the kind of + * anonymity that Tor is trying to provide.] + */ +void * +tor_mmap_anonymous(size_t sz, unsigned flags) +{ + void *ptr; +#if defined(_WIN32) + HANDLE mapping = CreateFileMapping(INVALID_HANDLE_VALUE, + NULL, /*attributes*/ + PAGE_READWRITE, + HIGH_SIZE_T_BYTES(sz), + sz & 0xffffffff, + NULL /* name */); + raw_assert(mapping != NULL); + ptr = MapViewOfFile(mapping, FILE_MAP_WRITE, + 0, 0, /* Offset */ + 0 /* Extend to end of mapping */); + raw_assert(ptr); + CloseHandle(mapping); /* mapped view holds a reference */ +#elif defined(HAVE_SYS_MMAN_H) + ptr = mmap(NULL, sz, + PROT_READ|PROT_WRITE, + MAP_ANON|MAP_PRIVATE, + -1, 0); + raw_assert(ptr != MAP_FAILED); + raw_assert(ptr != NULL); +#else + ptr = tor_malloc_zero(sz); +#endif + + if (flags & ANONMAP_PRIVATE) { + int lock_result = lock_mem(ptr, sz); + raw_assert(lock_result == 0); + int nodump_result = nodump_mem(ptr, sz); + raw_assert(nodump_result == 0); + } + + if (flags & ANONMAP_NOINHERIT) { + int noinherit_result = noinherit_mem(ptr, sz); + raw_assert(noinherit_result == 0); + } + + return ptr; +} + +/** + * Release sz bytes of memory that were previously mapped at + * mapping by tor_mmap_anonymous(). + **/ +void +tor_munmap_anonymous(void *mapping, size_t sz) +{ + if (!mapping) + return; + +#if defined(_WIN32) + (void)sz; + UnmapViewOfFile(mapping); +#elif defined(HAVE_SYS_MMAN_H) + munmap(mapping, sz); +#else + (void)sz; + tor_free(mapping); +#endif +} diff --git a/src/lib/malloc/map_anon.h b/src/lib/malloc/map_anon.h new file mode 100644 index 0000000000..cc5797e4ec --- /dev/null +++ b/src/lib/malloc/map_anon.h @@ -0,0 +1,37 @@ +/* Copyright (c) 2003-2004, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file map_anon.h + * \brief Headers for map_anon.c + **/ + +#ifndef TOR_MAP_ANON_H +#define TOR_MAP_ANON_H + +#include "lib/malloc/malloc.h" +#include + +/** + * When this flag is specified, try to prevent the mapping from being + * swapped or dumped. + * + * In some operating systems, this flag is not implemented. + */ +#define ANONMAP_PRIVATE (1u<<0) +/** + * When this flag is specified, try to prevent the mapping from being + * inherited after a fork(). In some operating systems, trying to access it + * afterwards will cause its contents to be zero. In others, trying to access + * it afterwards will cause a crash. + * + * In some operating systems, this flag is not implemented at all. + */ +#define ANONMAP_NOINHERIT (1u<<1) + +void *tor_mmap_anonymous(size_t sz, unsigned flags); +void tor_munmap_anonymous(void *mapping, size_t sz); + +#endif /* !defined(TOR_MAP_ANON_H) */ diff --git a/src/test/test_util.c b/src/test/test_util.c index 4e95303f2e..ba78b55936 100644 --- a/src/test/test_util.c +++ b/src/test/test_util.c @@ -40,6 +40,7 @@ #include "lib/time/tvdiff.h" #include "lib/encoding/confline.h" #include "lib/net/socketpair.h" +#include "lib/malloc/map_anon.h" #ifdef HAVE_PWD_H #include @@ -59,6 +60,12 @@ #ifdef HAVE_UNISTD_H #include #endif +#ifdef HAVE_SYS_MMAN_H +#include +#endif +#ifdef HAVE_SYS_WAIT_H +#include +#endif #ifdef _WIN32 #include @@ -6093,6 +6100,104 @@ test_util_log_mallinfo(void *arg) tor_free(mem); } +static void +test_util_map_anon(void *arg) +{ + (void)arg; + char *ptr = NULL; + size_t sz = 16384; + + /* Basic checks. */ + ptr = tor_mmap_anonymous(sz, 0); + tt_ptr_op(ptr, OP_NE, 0); + ptr[sz-1] = 3; + tt_int_op(ptr[0], OP_EQ, 0); + tt_int_op(ptr[sz-2], OP_EQ, 0); + tt_int_op(ptr[sz-1], OP_EQ, 3); + + /* Try again, with a private (non-swappable) mapping. */ + tor_munmap_anonymous(ptr, sz); + ptr = tor_mmap_anonymous(sz, ANONMAP_PRIVATE); + tt_ptr_op(ptr, OP_NE, 0); + ptr[sz-1] = 10; + tt_int_op(ptr[0], OP_EQ, 0); + tt_int_op(ptr[sz/2], OP_EQ, 0); + tt_int_op(ptr[sz-1], OP_EQ, 10); + + /* Now let's test a drop-on-fork mapping. */ + tor_munmap_anonymous(ptr, sz); + ptr = tor_mmap_anonymous(sz, ANONMAP_NOINHERIT); + tt_ptr_op(ptr, OP_NE, 0); + ptr[sz-1] = 10; + tt_int_op(ptr[0], OP_EQ, 0); + tt_int_op(ptr[sz/2], OP_EQ, 0); + tt_int_op(ptr[sz-1], OP_EQ, 10); + + done: + tor_munmap_anonymous(ptr, sz); +} + +static void +test_util_map_anon_nofork(void *arg) +{ + (void)arg; +#if !defined(HAVE_MADVISE) && !defined(HAVE_MINHERIT) + /* The operating system doesn't support this. */ + tt_skip(); + done: + ; +#else + /* We have the right OS support. We're going to try marking the buffer as + * either zero-on-fork or as drop-on-fork, whichever is supported. Then we + * will fork and send a byte back to the parent process. This will either + * crash, or send zero. */ + + char *ptr = NULL; + size_t sz = 16384; + int pipefd[2] = {-1, -1}; + + tor_munmap_anonymous(ptr, sz); + ptr = tor_mmap_anonymous(sz, ANONMAP_NOINHERIT); + tt_ptr_op(ptr, OP_NE, 0); + memset(ptr, 0xd0, sz); + + tt_int_op(0, OP_EQ, pipe(pipefd)); + pid_t child = fork(); + if (child == 0) { + /* We're in the child. */ + close(pipefd[0]); + ssize_t r = write(pipefd[1], &ptr[sz-1], 1); /* This may crash. */ + close(pipefd[1]); + if (r < 0) + exit(1); + exit(0); + } + tt_int_op(child, OP_GT, 0); + /* In the parent. */ + close(pipefd[1]); + pipefd[1] = -1; + char buf[1]; + ssize_t r = read(pipefd[0], buf, 1); +#if defined(INHERIT_ZERO) || defined(MADV_WIPEONFORK) + tt_int_op((int)r, OP_EQ, 1); // child should send us a byte. + tt_int_op(buf[0], OP_EQ, 0); +#else + tt_int_op(r, OP_LE, 0); // child said nothing; it should have crashed. +#endif + int ws; + waitpid(child, &ws, 0); + + done: + tor_munmap_anonymous(ptr, sz); + if (pipefd[0] >= 0) { + close(pipefd[0]); + } + if (pipefd[1] >= 0) { + close(pipefd[1]); + } +#endif +} + #define UTIL_LEGACY(name) \ { #name, test_util_ ## name , 0, NULL, NULL } @@ -6230,5 +6335,7 @@ struct testcase_t util_tests[] = { UTIL_TEST(htonll, 0), UTIL_TEST(get_unquoted_path, 0), UTIL_TEST(log_mallinfo, 0), + UTIL_TEST(map_anon, 0), + UTIL_TEST(map_anon_nofork, 0), END_OF_TESTCASES }; From 21d184a184b2ac4e4b8a1ff9853cb50de31a41fe Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 6 Feb 2019 12:37:02 -0500 Subject: [PATCH 0462/2557] Remove extraneous #if/#endif wrapper in crypto_rand.c I don't know how this got here, but this kind of a wrapper only belongs in a header file. --- src/lib/crypt_ops/crypto_rand.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/lib/crypt_ops/crypto_rand.c b/src/lib/crypt_ops/crypto_rand.c index f45ff719d3..5a739a8f1b 100644 --- a/src/lib/crypt_ops/crypto_rand.c +++ b/src/lib/crypt_ops/crypto_rand.c @@ -11,7 +11,6 @@ * number generators, and working with randomness. **/ -#ifndef CRYPTO_RAND_PRIVATE #define CRYPTO_RAND_PRIVATE #include "lib/crypt_ops/crypto_rand.h" @@ -737,5 +736,3 @@ crypto_force_rand_ssleay(void) #endif return 0; } - -#endif /* !defined(CRYPTO_RAND_PRIVATE) */ From a49149fc13e8b81e30df17579f19408ae8f4200f Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 6 Feb 2019 12:40:01 -0500 Subject: [PATCH 0463/2557] Extract numeric CSPRNG functions into a new module. Some of the code for getting a random value within a range wants to be shared between crypto_rand() and the new crypto_fast_rng() code. --- src/lib/crypt_ops/crypto_rand.c | 108 ---------------------- src/lib/crypt_ops/crypto_rand_numeric.c | 117 ++++++++++++++++++++++++ src/lib/crypt_ops/include.am | 1 + 3 files changed, 118 insertions(+), 108 deletions(-) create mode 100644 src/lib/crypt_ops/crypto_rand_numeric.c diff --git a/src/lib/crypt_ops/crypto_rand.c b/src/lib/crypt_ops/crypto_rand.c index 5a739a8f1b..14c2434a84 100644 --- a/src/lib/crypt_ops/crypto_rand.c +++ b/src/lib/crypt_ops/crypto_rand.c @@ -538,114 +538,6 @@ crypto_rand_u32(void) return rand; } -/** - * Return a pseudorandom integer, chosen uniformly from the values - * between 0 and max-1 inclusive. max must be between 1 and - * INT_MAX+1, inclusive. - */ -int -crypto_rand_int(unsigned int max) -{ - unsigned int val; - unsigned int cutoff; - tor_assert(max <= ((unsigned int)INT_MAX)+1); - tor_assert(max > 0); /* don't div by 0 */ - - /* We ignore any values that are >= 'cutoff,' to avoid biasing the - * distribution with clipping at the upper end of unsigned int's - * range. - */ - cutoff = UINT_MAX - (UINT_MAX%max); - while (1) { - crypto_rand((char*)&val, sizeof(val)); - if (val < cutoff) - return val % max; - } -} - -/** - * Return a pseudorandom integer, chosen uniformly from the values i such - * that min <= i < max. - * - * min MUST be in range [0, max). - * max MUST be in range (min, INT_MAX]. - **/ -int -crypto_rand_int_range(unsigned int min, unsigned int max) -{ - tor_assert(min < max); - tor_assert(max <= INT_MAX); - - /* The overflow is avoided here because crypto_rand_int() returns a value - * between 0 and (max - min) inclusive. */ - return min + crypto_rand_int(max - min); -} - -/** - * As crypto_rand_int_range, but supports uint64_t. - **/ -uint64_t -crypto_rand_uint64_range(uint64_t min, uint64_t max) -{ - tor_assert(min < max); - return min + crypto_rand_uint64(max - min); -} - -/** - * As crypto_rand_int_range, but supports time_t. - **/ -time_t -crypto_rand_time_range(time_t min, time_t max) -{ - tor_assert(min < max); - return min + (time_t)crypto_rand_uint64(max - min); -} - -/** - * Return a pseudorandom 64-bit integer, chosen uniformly from the values - * between 0 and max-1 inclusive. - **/ -uint64_t -crypto_rand_uint64(uint64_t max) -{ - uint64_t val; - uint64_t cutoff; - tor_assert(max < UINT64_MAX); - tor_assert(max > 0); /* don't div by 0 */ - - /* We ignore any values that are >= 'cutoff,' to avoid biasing the - * distribution with clipping at the upper end of unsigned int's - * range. - */ - cutoff = UINT64_MAX - (UINT64_MAX%max); - while (1) { - crypto_rand((char*)&val, sizeof(val)); - if (val < cutoff) - return val % max; - } -} - -/** - * Return a pseudorandom double d, chosen uniformly from the range - * 0.0 <= d < 1.0. - **/ -double -crypto_rand_double(void) -{ - /* We just use an unsigned int here; we don't really care about getting - * more than 32 bits of resolution */ - unsigned int u; - crypto_rand((char*)&u, sizeof(u)); -#if SIZEOF_INT == 4 -#define UINT_MAX_AS_DOUBLE 4294967296.0 -#elif SIZEOF_INT == 8 -#define UINT_MAX_AS_DOUBLE 1.8446744073709552e+19 -#else -#error SIZEOF_INT is neither 4 nor 8 -#endif /* SIZEOF_INT == 4 || ... */ - return ((double)u) / UINT_MAX_AS_DOUBLE; -} - /** * Generate and return a new random hostname starting with prefix, * ending with suffix, and containing no fewer than diff --git a/src/lib/crypt_ops/crypto_rand_numeric.c b/src/lib/crypt_ops/crypto_rand_numeric.c new file mode 100644 index 0000000000..97fe3bd180 --- /dev/null +++ b/src/lib/crypt_ops/crypto_rand_numeric.c @@ -0,0 +1,117 @@ +/** + * \file crypto_rand_numeric.c + * + * \brief Functions for retrieving uniformly distributed numbers + * from our PRNGs. + **/ + +#include "lib/crypt_ops/crypto_rand.h" +#include "lib/log/util_bug.h" + +/** + * Return a pseudorandom integer, chosen uniformly from the values + * between 0 and max-1 inclusive. max must be between 1 and + * INT_MAX+1, inclusive. + */ +int +crypto_rand_int(unsigned int max) +{ + unsigned int val; + unsigned int cutoff; + tor_assert(max <= ((unsigned int)INT_MAX)+1); + tor_assert(max > 0); /* don't div by 0 */ + + /* We ignore any values that are >= 'cutoff,' to avoid biasing the + * distribution with clipping at the upper end of unsigned int's + * range. + */ + cutoff = UINT_MAX - (UINT_MAX%max); + while (1) { + crypto_rand((char*)&val, sizeof(val)); + if (val < cutoff) + return val % max; + } +} + +/** + * Return a pseudorandom integer, chosen uniformly from the values i such + * that min <= i < max. + * + * min MUST be in range [0, max). + * max MUST be in range (min, INT_MAX]. + **/ +int +crypto_rand_int_range(unsigned int min, unsigned int max) +{ + tor_assert(min < max); + tor_assert(max <= INT_MAX); + + /* The overflow is avoided here because crypto_rand_int() returns a value + * between 0 and (max - min) inclusive. */ + return min + crypto_rand_int(max - min); +} + +/** + * As crypto_rand_int_range, but supports uint64_t. + **/ +uint64_t +crypto_rand_uint64_range(uint64_t min, uint64_t max) +{ + tor_assert(min < max); + return min + crypto_rand_uint64(max - min); +} + +/** + * As crypto_rand_int_range, but supports time_t. + **/ +time_t +crypto_rand_time_range(time_t min, time_t max) +{ + tor_assert(min < max); + return min + (time_t)crypto_rand_uint64(max - min); +} + +/** + * Return a pseudorandom 64-bit integer, chosen uniformly from the values + * between 0 and max-1 inclusive. + **/ +uint64_t +crypto_rand_uint64(uint64_t max) +{ + uint64_t val; + uint64_t cutoff; + tor_assert(max < UINT64_MAX); + tor_assert(max > 0); /* don't div by 0 */ + + /* We ignore any values that are >= 'cutoff,' to avoid biasing the + * distribution with clipping at the upper end of unsigned int's + * range. + */ + cutoff = UINT64_MAX - (UINT64_MAX%max); + while (1) { + crypto_rand((char*)&val, sizeof(val)); + if (val < cutoff) + return val % max; + } +} + +/** + * Return a pseudorandom double d, chosen uniformly from the range + * 0.0 <= d < 1.0. + **/ +double +crypto_rand_double(void) +{ + /* We just use an unsigned int here; we don't really care about getting + * more than 32 bits of resolution */ + unsigned int u; + crypto_rand((char*)&u, sizeof(u)); +#if SIZEOF_INT == 4 +#define UINT_MAX_AS_DOUBLE 4294967296.0 +#elif SIZEOF_INT == 8 +#define UINT_MAX_AS_DOUBLE 1.8446744073709552e+19 +#else +#error SIZEOF_INT is neither 4 nor 8 +#endif /* SIZEOF_INT == 4 || ... */ + return ((double)u) / UINT_MAX_AS_DOUBLE; +} diff --git a/src/lib/crypt_ops/include.am b/src/lib/crypt_ops/include.am index d0ccc13bff..19cfee1355 100644 --- a/src/lib/crypt_ops/include.am +++ b/src/lib/crypt_ops/include.am @@ -17,6 +17,7 @@ src_lib_libtor_crypt_ops_a_SOURCES = \ src/lib/crypt_ops/crypto_ope.c \ src/lib/crypt_ops/crypto_pwbox.c \ src/lib/crypt_ops/crypto_rand.c \ + src/lib/crypt_ops/crypto_rand_numeric.c \ src/lib/crypt_ops/crypto_rsa.c \ src/lib/crypt_ops/crypto_s2k.c \ src/lib/crypt_ops/crypto_util.c \ From 8df6a65e6b86be70ab1d7f57b097913068ace72c Mon Sep 17 00:00:00 2001 From: rl1987 Date: Thu, 7 Feb 2019 17:22:06 +0200 Subject: [PATCH 0464/2557] Fix shellcheck warning in zero_length_keys.sh --- changes/ticket29068 | 2 ++ src/test/zero_length_keys.sh | 6 +++--- 2 files changed, 5 insertions(+), 3 deletions(-) create mode 100644 changes/ticket29068 diff --git a/changes/ticket29068 b/changes/ticket29068 new file mode 100644 index 0000000000..77ef304f1d --- /dev/null +++ b/changes/ticket29068 @@ -0,0 +1,2 @@ + o Code simplification and refactoring (shell scripts): + - Fix shellcheck warnings in zero_length_keys.sh. Resolves issue 29068. diff --git a/src/test/zero_length_keys.sh b/src/test/zero_length_keys.sh index 3c61f8d465..4069148e0b 100755 --- a/src/test/zero_length_keys.sh +++ b/src/test/zero_length_keys.sh @@ -19,7 +19,7 @@ # 3: a command failed - the test could not be completed # -if [ $# -eq 0 ] || [ ! -f ${1} ] || [ ! -x ${1} ]; then +if [ $# -eq 0 ] || [ ! -f "${1}" ] || [ ! -x "${1}" ]; then echo "Usage: ${0} PATH_TO_TOR [-z|-d|-e]" exit 1 elif [ $# -eq 1 ]; then @@ -31,7 +31,7 @@ else #[$# -gt 1 ]; then shift fi -DATA_DIR=`mktemp -d -t tor_zero_length_keys.XXXXXX` +DATA_DIR=$(mktemp -d -t tor_zero_length_keys.XXXXXX) if [ -z "$DATA_DIR" ]; then echo "Failure: mktemp invocation returned empty string" >&2 exit 3 @@ -40,7 +40,7 @@ if [ ! -d "$DATA_DIR" ]; then echo "Failure: mktemp invocation result doesn't point to directory" >&2 exit 3 fi -trap "rm -rf '$DATA_DIR'" 0 +trap 'rm -rf "$DATA_DIR"' 0 touch "$DATA_DIR"/empty_torrc From b53fee46223a425ed27f37944ac0d26569cead78 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Fri, 8 Feb 2019 16:51:49 +0200 Subject: [PATCH 0465/2557] Fix SC2086 warning in test_workqueue_*.sh shell scripts --- changes/ticket29067 | 3 +++ src/test/test_workqueue_cancel.sh | 2 +- src/test/test_workqueue_efd.sh | 2 +- src/test/test_workqueue_efd2.sh | 2 +- src/test/test_workqueue_pipe.sh | 2 +- src/test/test_workqueue_pipe2.sh | 2 +- src/test/test_workqueue_socketpair.sh | 2 +- 7 files changed, 9 insertions(+), 6 deletions(-) create mode 100644 changes/ticket29067 diff --git a/changes/ticket29067 b/changes/ticket29067 new file mode 100644 index 0000000000..a660648775 --- /dev/null +++ b/changes/ticket29067 @@ -0,0 +1,3 @@ + o Code simplification and refactoring (shell scripts): + - Fix test_workqueue_*.sh scripts to silence shellcheck SC2086 + warnings. Fixes issue 29067. diff --git a/src/test/test_workqueue_cancel.sh b/src/test/test_workqueue_cancel.sh index f7c663171e..e50b884f26 100755 --- a/src/test/test_workqueue_cancel.sh +++ b/src/test/test_workqueue_cancel.sh @@ -1,4 +1,4 @@ #!/bin/sh -${builddir:-.}/src/test/test_workqueue -C 1 +"${builddir:-.}/src/test/test_workqueue" -C 1 diff --git a/src/test/test_workqueue_efd.sh b/src/test/test_workqueue_efd.sh index 4d89396819..592841fc91 100755 --- a/src/test/test_workqueue_efd.sh +++ b/src/test/test_workqueue_efd.sh @@ -1,4 +1,4 @@ #!/bin/sh -${builddir:-.}/src/test/test_workqueue \ +"${builddir:-.}/src/test/test_workqueue" \ --no-eventfd2 --no-pipe2 --no-pipe --no-socketpair diff --git a/src/test/test_workqueue_efd2.sh b/src/test/test_workqueue_efd2.sh index 7cfff45ff3..4cf1b76cbe 100755 --- a/src/test/test_workqueue_efd2.sh +++ b/src/test/test_workqueue_efd2.sh @@ -1,4 +1,4 @@ #!/bin/sh -${builddir:-.}/src/test/test_workqueue \ +"${builddir:-.}/src/test/test_workqueue" \ --no-eventfd --no-pipe2 --no-pipe --no-socketpair diff --git a/src/test/test_workqueue_pipe.sh b/src/test/test_workqueue_pipe.sh index afcef87853..fc3ef34c6c 100755 --- a/src/test/test_workqueue_pipe.sh +++ b/src/test/test_workqueue_pipe.sh @@ -1,4 +1,4 @@ #!/bin/sh -${builddir:-.}/src/test/test_workqueue \ +"${builddir:-.}/src/test/test_workqueue" \ --no-eventfd2 --no-eventfd --no-pipe2 --no-socketpair diff --git a/src/test/test_workqueue_pipe2.sh b/src/test/test_workqueue_pipe2.sh index a20a1427e0..7f19ea880d 100755 --- a/src/test/test_workqueue_pipe2.sh +++ b/src/test/test_workqueue_pipe2.sh @@ -1,4 +1,4 @@ #!/bin/sh -${builddir:-.}/src/test/test_workqueue \ +"${builddir:-.}/src/test/test_workqueue" \ --no-eventfd2 --no-eventfd --no-pipe --no-socketpair diff --git a/src/test/test_workqueue_socketpair.sh b/src/test/test_workqueue_socketpair.sh index 76af79746d..1ee1776447 100755 --- a/src/test/test_workqueue_socketpair.sh +++ b/src/test/test_workqueue_socketpair.sh @@ -1,4 +1,4 @@ #!/bin/sh -${builddir:-.}/src/test/test_workqueue \ +"${builddir:-.}/src/test/test_workqueue" \ --no-eventfd2 --no-eventfd --no-pipe2 --no-pipe From f4c76661d1bb49a47829c6999a07c1de63b55cc9 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 8 Feb 2019 11:21:09 -0500 Subject: [PATCH 0466/2557] Add a script to check for coverage nondeterminism Closes ticket 29436. --- changes/ticket29436 | 4 +++ scripts/test/cov-test-determinism.sh | 48 ++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+) create mode 100644 changes/ticket29436 create mode 100755 scripts/test/cov-test-determinism.sh diff --git a/changes/ticket29436 b/changes/ticket29436 new file mode 100644 index 0000000000..025be619e5 --- /dev/null +++ b/changes/ticket29436 @@ -0,0 +1,4 @@ + o Minor features (testing): + - We now have a script, cov-test-determinism.sh, to identify places + where our unit test coverage has become nondeterministic. + Closes ticket 29436. diff --git a/scripts/test/cov-test-determinism.sh b/scripts/test/cov-test-determinism.sh new file mode 100755 index 0000000000..3b4f372e04 --- /dev/null +++ b/scripts/test/cov-test-determinism.sh @@ -0,0 +1,48 @@ +#!/bin/sh + +# To use this script, build Tor with coverage enabled, and then say: +# ./scripts/test/cov-test-determinism.sh run +# +# Let it run for a long time so it can run the tests over and over. It +# will put their coverage outputs in coverage-raw/coverage-*/. +# +# Then say: +# ./scripts/test/cov-test-determinism.sh check +# +# It will diff the other coverage outputs to the first one, and put their +# diffs in coverage-raw/diff-coverage-*. + +run=0 +check=0 + +if test "$1" = run; then + run=1 +elif test "$1" = check; then + check=1 +else + echo "First use 'run' with this script, then use 'check'." + exit 1 +fi + +if test "$run" = 1; then + while true; do + make reset-gcov + CD=coverage-raw/coverage-$(date +%s) + make -j5 check + mkdir -p "$CD" + ./scripts/test/coverage "$CD" + done +fi + +if test "$check" = 1; then + cd coverage-raw || exit 1 + + FIRST="$(find . -name "coverage-*" -type d | head -1)" + rm -f A + ln -sf "$FIRST" A + for dir in coverage-*; do + rm -f B + ln -sf "$dir" B + ../scripts/test/cov-diff A B > "diff-$dir" + done +fi From a036c0187a7938718a1056f45ca95ddbca744b49 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 8 Feb 2019 11:22:55 -0500 Subject: [PATCH 0467/2557] cov-diff: better handle the case where a file stops having coverage --- scripts/test/cov-diff | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/test/cov-diff b/scripts/test/cov-diff index f3ca856888..8751800966 100755 --- a/scripts/test/cov-diff +++ b/scripts/test/cov-diff @@ -16,6 +16,5 @@ for B in "$DIRB"/*; do fi perl -pe 's/^\s*\!*\d+(\*?):/ 1$1:/; s/^([^:]+:)[\d\s]+:/$1/; s/^ *-:(Runs|Programs):.*//;' "$B" > "$B.tmp" diff -u "$A.tmp" "$B.tmp" |perl -pe 's/^((?:\+\+\+|---)(?:.*tmp))\s+.*/$1/;' - rm "$A.tmp" "$B.tmp" + rm -f "$A.tmp" "$B.tmp" done - From 09a3c949f8388c83d4b62e281474687a26596a59 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Sat, 9 Feb 2019 16:06:32 +0200 Subject: [PATCH 0468/2557] Add connection_dir_buf_add() helper function --- src/core/mainloop/connection.c | 17 +++++++++++++++++ src/core/mainloop/connection.h | 2 ++ 2 files changed, 19 insertions(+) diff --git a/src/core/mainloop/connection.c b/src/core/mainloop/connection.c index 37f35c8b8d..b53552a53d 100644 --- a/src/core/mainloop/connection.c +++ b/src/core/mainloop/connection.c @@ -4341,6 +4341,23 @@ connection_write_to_buf_impl_,(const char *string, size_t len, connection_write_to_buf_commit(conn, written); } +/** + * Write a string (of size len to directory connection + * dir_conn. Apply compression if connection is configured to use + * it and finalize it if done is true. + */ +void +connection_dir_buf_add(const char *string, size_t len, + dir_connection_t *dir_conn, int done) +{ + if (dir_conn->compress_state != NULL) { + connection_buf_add_compress(string, len, dir_conn, done); + return; + } + + connection_buf_add(string, len, TO_CONN(dir_conn)); +} + void connection_buf_add_compress(const char *string, size_t len, dir_connection_t *conn, int done) diff --git a/src/core/mainloop/connection.h b/src/core/mainloop/connection.h index f4f0e839ae..de6473251d 100644 --- a/src/core/mainloop/connection.h +++ b/src/core/mainloop/connection.h @@ -226,6 +226,8 @@ MOCK_DECL(void, connection_write_to_buf_impl_, /* DOCDOC connection_write_to_buf */ static void connection_buf_add(const char *string, size_t len, connection_t *conn); +void connection_dir_buf_add(const char *string, size_t len, + dir_connection_t *dir_conn, int done); static inline void connection_buf_add(const char *string, size_t len, connection_t *conn) { From 4c102213326705079be60d3ce335b81c94b76499 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Sat, 9 Feb 2019 16:46:31 +0200 Subject: [PATCH 0469/2557] Use compress_dir_buf_add() function in a few places --- src/feature/dircache/dircache.c | 28 ++++++++++------------------ src/feature/dircache/dirserv.c | 20 +++++++------------- 2 files changed, 17 insertions(+), 31 deletions(-) diff --git a/src/feature/dircache/dircache.c b/src/feature/dircache/dircache.c index f6e57c5064..ee6e4f7a81 100644 --- a/src/feature/dircache/dircache.c +++ b/src/feature/dircache/dircache.c @@ -1068,13 +1068,11 @@ handle_get_status_vote(dir_connection_t *conn, const get_handler_args_t *args) if (compress_method != NO_METHOD) { conn->compress_state = tor_compress_new(1, compress_method, choose_compression_level(estimated_len)); - SMARTLIST_FOREACH(items, const char *, c, - connection_buf_add_compress(c, strlen(c), conn, 0)); - connection_buf_add_compress("", 0, conn, 1); - } else { - SMARTLIST_FOREACH(items, const char *, c, - connection_buf_add(c, strlen(c), TO_CONN(conn))); } + + SMARTLIST_FOREACH(items, const char *, c, + connection_dir_buf_add(c, strlen(c), conn, + c_sl_idx == c_sl_len - 1)); } else { SMARTLIST_FOREACH(dir_items, cached_dir_t *, d, connection_buf_add(compress_method != NO_METHOD ? @@ -1325,19 +1323,13 @@ handle_get_keys(dir_connection_t *conn, const get_handler_args_t *args) if (compress_method != NO_METHOD) { conn->compress_state = tor_compress_new(1, compress_method, choose_compression_level(len)); - SMARTLIST_FOREACH(certs, authority_cert_t *, c, - connection_buf_add_compress( - c->cache_info.signed_descriptor_body, - c->cache_info.signed_descriptor_len, - conn, 0)); - connection_buf_add_compress("", 0, conn, 1); - } else { - SMARTLIST_FOREACH(certs, authority_cert_t *, c, - connection_buf_add(c->cache_info.signed_descriptor_body, - c->cache_info.signed_descriptor_len, - TO_CONN(conn))); } - keys_done: + + SMARTLIST_FOREACH(certs, authority_cert_t *, c, + connection_dir_buf_add(c->cache_info.signed_descriptor_body, + c->cache_info.signed_descriptor_len, + conn, c_sl_idx == c_sl_len - 1)); + keys_done: smartlist_free(certs); goto done; } diff --git a/src/feature/dircache/dirserv.c b/src/feature/dircache/dirserv.c index 4be6836fe1..79400bf15f 100644 --- a/src/feature/dircache/dirserv.c +++ b/src/feature/dircache/dirserv.c @@ -583,11 +583,9 @@ spooled_resource_flush_some(spooled_resource_t *spooled, /* Absent objects count as "done". */ return SRFS_DONE; } - if (conn->compress_state) { - connection_buf_add_compress((const char*)body, bodylen, conn, 0); - } else { - connection_buf_add((const char*)body, bodylen, TO_CONN(conn)); - } + + connection_dir_buf_add((const char*)body, bodylen, conn, 0); + return SRFS_DONE; } else { cached_dir_t *cached = spooled->cached_dir_ref; @@ -622,14 +620,10 @@ spooled_resource_flush_some(spooled_resource_t *spooled, if (BUG(remaining < 0)) return SRFS_ERR; ssize_t bytes = (ssize_t) MIN(DIRSERV_CACHED_DIR_CHUNK_SIZE, remaining); - if (conn->compress_state) { - connection_buf_add_compress( - ptr + spooled->cached_dir_offset, - bytes, conn, 0); - } else { - connection_buf_add(ptr + spooled->cached_dir_offset, - bytes, TO_CONN(conn)); - } + + connection_dir_buf_add(ptr + spooled->cached_dir_offset, + bytes, conn, 0); + spooled->cached_dir_offset += bytes; if (spooled->cached_dir_offset >= (off_t)total_len) { return SRFS_DONE; From 8d04dc416bd6aa3fefc0914add8708083dc99d52 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Sat, 9 Feb 2019 16:56:54 +0200 Subject: [PATCH 0470/2557] Add changes file --- changes/ticket28816 | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 changes/ticket28816 diff --git a/changes/ticket28816 b/changes/ticket28816 new file mode 100644 index 0000000000..02878ccfdc --- /dev/null +++ b/changes/ticket28816 @@ -0,0 +1,4 @@ + o Code simplification and refactoring: + - Introduce a connection_dir_buf_add() helper function that checks for + compress_state of dir_connection_t and automatically writes a string to + directory connection with or without compression. Resolves issue 28816. From 72b978c3a58983d9b4d57f56db618e074ddae31e Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 30 Jan 2019 09:22:36 +0100 Subject: [PATCH 0471/2557] On windows, if we fail to load a consensus and it has a CRLF, retry. Fixes bug 28614; bugfix on 0.4.0.1-alpha when we started mmapping the consensus. --- changes/ticket28614 | 6 ++- src/feature/nodelist/networkstatus.c | 81 +++++++++++++++++----------- 2 files changed, 56 insertions(+), 31 deletions(-) diff --git a/changes/ticket28614 b/changes/ticket28614 index c022baf668..6c65ce49de 100644 --- a/changes/ticket28614 +++ b/changes/ticket28614 @@ -1,4 +1,8 @@ o Major bugfixes (windows, startup): - When writing a consensus file to disk, always write in "binary" mode so that we can safely map it into memory later. - Fixes part bug 28614; bugfix on 0.4.0.1-alpha. + Fixes part of bug 28614; bugfix on 0.4.0.1-alpha. + - When reading a consensus file from disk, detect whether it + was written in text mode, and re-read it in text mode if it + Fixes part of bug 28614; bugfix on 0.4.0.1-alpha. + diff --git a/src/feature/nodelist/networkstatus.c b/src/feature/nodelist/networkstatus.c index 64169f6a4b..2c34754621 100644 --- a/src/feature/nodelist/networkstatus.c +++ b/src/feature/nodelist/networkstatus.c @@ -178,6 +178,10 @@ static void update_consensus_bootstrap_multiple_downloads( static int networkstatus_check_required_protocols(const networkstatus_t *ns, int client_mode, char **warning_out); +static int reload_consensus_from_file(const char *fname, + const char *flavor, + unsigned flags, + const char *source_dir); /** Forget that we've warned about anything networkstatus-related, so we will * give fresh warnings if the same behavior happens again. */ @@ -269,27 +273,15 @@ router_reload_consensus_networkstatus(void) /* FFFF Suppress warnings if cached consensus is bad? */ for (flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) { const char *flavor = networkstatus_get_flavor_name(flav); - tor_mmap_t *m = networkstatus_map_cached_consensus_impl(flav, flavor, 0); - if (m) { - if (networkstatus_set_current_consensus(m->data, m->size, - flavor, flags, NULL) < -1) { - log_warn(LD_FS, "Couldn't load consensus %s networkstatus from cache", - flavor); - } - tor_munmap_file(m); - } + char *fname = networkstatus_get_cache_fname(flav, flavor, 0); + reload_consensus_from_file(fname, flavor, flags, NULL); + tor_free(fname); - m = networkstatus_map_cached_consensus_impl(flav, flavor, 1); - if (m) { - if (networkstatus_set_current_consensus(m->data, m->size, - flavor, - flags | NSSET_WAS_WAITING_FOR_CERTS, - NULL)) { - log_info(LD_FS, "Couldn't load unverified consensus %s networkstatus " - "from cache", flavor); - } - tor_munmap_file(m); - } + fname = networkstatus_get_cache_fname(flav, flavor, 1); + reload_consensus_from_file(fname, flavor, + flags | NSSET_WAS_WAITING_FOR_CERTS, + NULL); + tor_free(fname); } update_certificate_downloads(time(NULL)); @@ -1750,6 +1742,41 @@ networkstatus_set_current_consensus_from_ns(networkstatus_t *c, } #endif /* defined(TOR_UNIT_TESTS) */ +/** + * Helper: Read a the current consensus of type flavor from + * fname. Flags and return values are as for + * networkstatus_set_current_consensus(). + **/ +static int +reload_consensus_from_file(const char *fname, + const char *flavor, + unsigned flags, + const char *source_dir) +{ + tor_mmap_t *map = tor_mmap_file(fname); + if (!map) + return 0; + + int rv = networkstatus_set_current_consensus(map->data, map->size, + flavor, flags, source_dir); +#ifdef _WIN32 + if (rv < 0 && tor_memstr(map->data, map->size, "\r\n")) { + log_info(LD_GENERAL, "Found CRLF in consensus file %s; falling back to " + "read_file_to_string.", escaped(fname)); + char *content = read_file_to_str(fname, RFTS_IGNORE_MISSING, NULL); + rv = networkstatus_set_current_consensus(content, strlen(content), + flavor, flags, source_dir); + tor_free(content); + } +#endif + if (rv < -1) { + log_warn(LD_GENERAL, "Couldn't set consensus from cache file %s", + escaped(fname)); + } + tor_munmap_file(map); + return rv; +} + /** * Helper for handle_missing_protocol_warning: handles either the * client case (if is_client is set) or the server case otherwise. @@ -2178,16 +2205,10 @@ networkstatus_note_certs_arrived(const char *source_dir) if (!waiting->consensus) continue; if (networkstatus_check_consensus_signature(waiting->consensus, 0)>=0) { - tor_mmap_t *mapping = networkstatus_map_cached_consensus_impl( - i, flavor_name, 1); - if (mapping) { - networkstatus_set_current_consensus(mapping->data, - mapping->size, - flavor_name, - NSSET_WAS_WAITING_FOR_CERTS, - source_dir); - } - tor_munmap_file(mapping); + char *fname = networkstatus_get_cache_fname(i, flavor_name, 1); + reload_consensus_from_file(fname, flavor_name, + NSSET_WAS_WAITING_FOR_CERTS, source_dir); + tor_free(fname); } } } From fc3e90a7b64714f30c19572900523078e8e7c0d6 Mon Sep 17 00:00:00 2001 From: juga0 Date: Tue, 6 Nov 2018 14:49:20 +0000 Subject: [PATCH 0472/2557] bwauth: Add function to get the digest of a bw file --- src/app/config/config.c | 3 +- src/feature/dirauth/bwauth.c | 26 ++++++++++++--- src/feature/dirauth/bwauth.h | 4 +-- src/test/test_dir.c | 62 ++++++++++++++++++++++++++---------- 4 files changed, 72 insertions(+), 23 deletions(-) diff --git a/src/app/config/config.c b/src/app/config/config.c index 8e4794315f..7734c19a04 100644 --- a/src/app/config/config.c +++ b/src/app/config/config.c @@ -3554,7 +3554,8 @@ options_validate(or_options_t *old_options, or_options_t *options, "(Bridge/V3)AuthoritativeDir is set."); /* If we have a v3bandwidthsfile and it's broken, complain on startup */ if (options->V3BandwidthsFile && !old_options) { - dirserv_read_measured_bandwidths(options->V3BandwidthsFile, NULL, NULL); + dirserv_read_measured_bandwidths(options->V3BandwidthsFile, NULL, NULL, + NULL); } /* same for guardfraction file */ if (options->GuardfractionFile && !old_options) { diff --git a/src/feature/dirauth/bwauth.c b/src/feature/dirauth/bwauth.c index a31050ff9c..1929c14384 100644 --- a/src/feature/dirauth/bwauth.c +++ b/src/feature/dirauth/bwauth.c @@ -20,6 +20,7 @@ #include "feature/nodelist/routerinfo_st.h" #include "feature/nodelist/vote_routerstatus_st.h" +#include "lib/crypt_ops/crypto_format.h" #include "lib/encoding/keyval.h" /** Total number of routers with measured bandwidth; this is set by @@ -205,7 +206,8 @@ dirserv_get_credible_bandwidth_kb(const routerinfo_t *ri) int dirserv_read_measured_bandwidths(const char *from_file, smartlist_t *routerstatuses, - smartlist_t *bw_file_headers) + smartlist_t *bw_file_headers, + uint8_t *digest_out) { FILE *fp = tor_fopen_cloexec(from_file, "r"); int applied_lines = 0; @@ -219,6 +221,7 @@ dirserv_read_measured_bandwidths(const char *from_file, int rv = -1; char *line = NULL; size_t n = 0; + crypto_digest_t *digest = crypto_digest256_new(DIGEST_SHA256); /* Initialise line, so that we can't possibly run off the end. */ @@ -233,11 +236,14 @@ dirserv_read_measured_bandwidths(const char *from_file, log_warn(LD_DIRSERV, "Empty bandwidth file"); goto err; } + /* If the line could be gotten, add it to the digest */ + crypto_digest_add_bytes(digest, (const char *) line, strlen(line)); if (!strlen(line) || line[strlen(line)-1] != '\n') { log_warn(LD_DIRSERV, "Long or truncated time in bandwidth file: %s", escaped(line)); - goto err; + /* Continue adding lines to the digest. */ + goto continue_digest; } line[strlen(line)-1] = '\0'; @@ -245,14 +251,14 @@ dirserv_read_measured_bandwidths(const char *from_file, if (!ok) { log_warn(LD_DIRSERV, "Non-integer time in bandwidth file: %s", escaped(line)); - goto err; + goto continue_digest; } now = time(NULL); if ((now - file_time) > MAX_MEASUREMENT_AGE) { log_warn(LD_DIRSERV, "Bandwidth measurement file stale. Age: %u", (unsigned)(time(NULL) - file_time)); - goto err; + goto continue_digest; } /* If timestamp was correct and bw_file_headers is not NULL, @@ -267,6 +273,7 @@ dirserv_read_measured_bandwidths(const char *from_file, while (!feof(fp)) { measured_bw_line_t parsed_line; if (tor_getline(&line, &n, fp) >= 0) { + crypto_digest_add_bytes(digest, (const char *) line, strlen(line)); if (measured_bw_line_parse(&parsed_line, line, line_is_after_headers) != -1) { /* This condition will be true when the first complete valid bw line @@ -305,6 +312,14 @@ dirserv_read_measured_bandwidths(const char *from_file, "Applied %d measurements.", applied_lines); rv = 0; + continue_digest: + /* Continue parsing lines to return the digest of the Bandwidth File. */ + while (!feof(fp)) { + if (tor_getline(&line, &n, fp) >= 0) { + crypto_digest_add_bytes(digest, (const char *) line, strlen(line)); + } + } + err: if (line) { // we need to raw_free this buffer because we got it from tor_getdelim() @@ -312,6 +327,9 @@ dirserv_read_measured_bandwidths(const char *from_file, } if (fp) fclose(fp); + if (digest_out) + crypto_digest_get_digest(digest, (char *) digest_out, DIGEST256_LEN); + crypto_digest_free(digest); return rv; } diff --git a/src/feature/dirauth/bwauth.h b/src/feature/dirauth/bwauth.h index 4507728458..8b7acc4a1c 100644 --- a/src/feature/dirauth/bwauth.h +++ b/src/feature/dirauth/bwauth.h @@ -21,8 +21,8 @@ int dirserv_read_measured_bandwidths(const char *from_file, smartlist_t *routerstatuses, - smartlist_t *bw_file_headers); - + smartlist_t *bw_file_headers, + uint8_t *digest_out); int dirserv_query_measured_bw_cache_kb(const char *node_id, long *bw_out, time_t *as_of_out); diff --git a/src/test/test_dir.c b/src/test/test_dir.c index dab0b7444a..08b5cdddfc 100644 --- a/src/test/test_dir.c +++ b/src/test/test_dir.c @@ -1765,7 +1765,8 @@ test_dir_dirserv_read_measured_bandwidths(void *arg) write_str_to_file(fname, "", 0); setup_capture_of_logs(LOG_WARN); tt_int_op(-1, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL, - bw_file_headers)); + bw_file_headers, + NULL)); expect_log_msg("Empty bandwidth file\n"); teardown_capture_of_logs(); bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL); @@ -1781,7 +1782,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg) write_str_to_file(fname, content, 0); tor_free(content); tt_int_op(-1, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL, - bw_file_headers)); + bw_file_headers, + NULL)); + bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL); tt_str_op("", OP_EQ, bw_file_headers_str); SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c)); @@ -1792,7 +1795,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg) write_str_to_file(fname, header_lines_v100, 0); bw_file_headers = smartlist_new(); tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL, - bw_file_headers)); + bw_file_headers, + NULL)); + bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL); tt_str_op(bw_file_headers_str_v100, OP_EQ, bw_file_headers_str); SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c)); @@ -1805,7 +1810,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg) write_str_to_file(fname, content, 0); tor_free(content); tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL, - bw_file_headers)); + bw_file_headers, + NULL)); + bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL); tt_str_op(bw_file_headers_str_v100, OP_EQ, bw_file_headers_str); SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c)); @@ -1816,7 +1823,8 @@ test_dir_dirserv_read_measured_bandwidths(void *arg) tor_asprintf(&content, "%s%s", header_lines_v100, relay_lines_v100); write_str_to_file(fname, content, 0); tor_free(content); - tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL, NULL)); + tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL, NULL, + NULL)); /* Test bandwidth file including v1.1.0 bandwidth headers and * v1.0.0 relay lines. bw_file_headers will contain the v1.1.0 headers. */ @@ -1826,7 +1834,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg) write_str_to_file(fname, content, 0); tor_free(content); tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL, - bw_file_headers)); + bw_file_headers, + NULL)); + bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL); tt_str_op(bw_file_headers_str_v110, OP_EQ, bw_file_headers_str); SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c)); @@ -1842,7 +1852,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg) write_str_to_file(fname, content, 0); tor_free(content); tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL, - bw_file_headers)); + bw_file_headers, + NULL)); + bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL); tt_str_op(bw_file_headers_str_v100, OP_EQ, bw_file_headers_str); SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c)); @@ -1859,7 +1871,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg) write_str_to_file(fname, content, 0); tor_free(content); tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL, - bw_file_headers)); + bw_file_headers, + NULL)); + bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL); tt_str_op(bw_file_headers_str_v100, OP_EQ, bw_file_headers_str); SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c)); @@ -1870,7 +1884,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg) bw_file_headers = smartlist_new(); write_str_to_file(fname, header_lines_v110_no_terminator, 0); tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL, - bw_file_headers)); + bw_file_headers, + NULL)); + bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL); tt_str_op(bw_file_headers_str_v110, OP_EQ, bw_file_headers_str); SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c)); @@ -1881,7 +1897,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg) bw_file_headers = smartlist_new(); write_str_to_file(fname, header_lines_v110, 0); tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL, - bw_file_headers)); + bw_file_headers, + NULL)); + bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL); tt_str_op(bw_file_headers_str_v110, OP_EQ, bw_file_headers_str); SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c)); @@ -1896,7 +1914,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg) write_str_to_file(fname, content, 0); tor_free(content); tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL, - bw_file_headers)); + bw_file_headers, + NULL)); + bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL); tt_str_op(bw_file_headers_str_v110, OP_EQ, bw_file_headers_str); SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c)); @@ -1911,7 +1931,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg) write_str_to_file(fname, content, 0); tor_free(content); tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL, - bw_file_headers)); + bw_file_headers, + NULL)); + bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL); tt_str_op(bw_file_headers_str_v110, OP_EQ, bw_file_headers_str); SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c)); @@ -1927,7 +1949,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg) write_str_to_file(fname, content, 0); tor_free(content); tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL, - bw_file_headers)); + bw_file_headers, + NULL)); + bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL); tt_str_op(bw_file_headers_str_v110, OP_EQ, bw_file_headers_str); SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c)); @@ -1944,7 +1968,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg) write_str_to_file(fname, content, 0); tor_free(content); tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL, - bw_file_headers)); + bw_file_headers, + NULL)); + bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL); tt_str_op(bw_file_headers_str_bad, OP_EQ, bw_file_headers_str); SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c)); @@ -1962,7 +1988,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg) write_str_to_file(fname, content, 0); tor_free(content); tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL, - bw_file_headers)); + bw_file_headers, + NULL)); + tt_int_op(MAX_BW_FILE_HEADER_COUNT_IN_VOTE, OP_EQ, smartlist_len(bw_file_headers)); bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL); @@ -1983,7 +2011,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg) write_str_to_file(fname, content, 0); tor_free(content); tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL, - bw_file_headers)); + bw_file_headers, + NULL)); + tt_int_op(MAX_BW_FILE_HEADER_COUNT_IN_VOTE, OP_EQ, smartlist_len(bw_file_headers)); /* force bw_file_headers to be bigger than From 28490fa23edc11f0b22ef4365be97eb3fb88768d Mon Sep 17 00:00:00 2001 From: juga0 Date: Tue, 6 Nov 2018 14:56:05 +0000 Subject: [PATCH 0473/2557] test: Add test to get the digest of a bw file --- src/feature/dirauth/bwauth.c | 2 +- src/test/test_dir.c | 61 ++++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 1 deletion(-) diff --git a/src/feature/dirauth/bwauth.c b/src/feature/dirauth/bwauth.c index 1929c14384..7d7dea4dfa 100644 --- a/src/feature/dirauth/bwauth.c +++ b/src/feature/dirauth/bwauth.c @@ -254,7 +254,7 @@ dirserv_read_measured_bandwidths(const char *from_file, goto continue_digest; } - now = time(NULL); + now = approx_time(); if ((now - file_time) > MAX_MEASUREMENT_AGE) { log_warn(LD_DIRSERV, "Bandwidth measurement file stale. Age: %u", (unsigned)(time(NULL) - file_time)); diff --git a/src/test/test_dir.c b/src/test/test_dir.c index 08b5cdddfc..4132d42d12 100644 --- a/src/test/test_dir.c +++ b/src/test/test_dir.c @@ -91,6 +91,9 @@ #ifdef HAVE_SYS_STAT_H #include #endif +#ifdef HAVE_UNISTD_H +#include +#endif #define NS_MODULE dir @@ -2027,6 +2030,7 @@ test_dir_dirserv_read_measured_bandwidths(void *arg) tor_free(bw_file_headers_str); done: + unlink(fname); tor_free(fname); tor_free(header_lines_v100); tor_free(header_lines_v110_no_terminator); @@ -3860,6 +3864,62 @@ mock_get_options(void) return mock_options; } +/** + * Test dirauth_get_b64_digest_bw_file. + * This function should be near the other bwauth functions, but it needs + * mock_get_options, that is only defined here. + */ + +static void +test_dir_bwauth_bw_file_digest256(void *arg) +{ + (void)arg; + const char *content = + "1541171221\n" + "node_id=$68A483E05A2ABDCA6DA5A3EF8DB5177638A27F80 " + "master_key_ed25519=YaqV4vbvPYKucElk297eVdNArDz9HtIwUoIeo0+cVIpQ " + "bw=760 nick=Test time=2018-05-08T16:13:26\n"; + + char *fname = tor_strdup(get_fname("V3BandwidthsFile")); + /* Initialize to a wrong digest. */ + uint8_t digest[DIGEST256_LEN] = "01234567890123456789abcdefghijkl"; + + /* Digest of an empty string. Initialize to a wrong digest. */ + char digest_empty_str[DIGEST256_LEN] = "01234567890123456789abcdefghijkl"; + crypto_digest256(digest_empty_str, "", 0, DIGEST_SHA256); + + /* Digest of the content. Initialize to a wrong digest. */ + char digest_expected[DIGEST256_LEN] = "01234567890123456789abcdefghijkl"; + crypto_digest256(digest_expected, content, strlen(content), DIGEST_SHA256); + + /* When the bandwidth file can not be found. */ + tt_int_op(-1, OP_EQ, + dirserv_read_measured_bandwidths(fname, + NULL, NULL, digest)); + tt_mem_op(digest, OP_EQ, digest_empty_str, DIGEST256_LEN); + + /* When there is a timestamp but it is too old. */ + write_str_to_file(fname, content, 0); + tt_int_op(-1, OP_EQ, + dirserv_read_measured_bandwidths(fname, + NULL, NULL, digest)); + /* The digest will be correct. */ + tt_mem_op(digest, OP_EQ, digest_expected, DIGEST256_LEN); + + update_approx_time(1541171221); + + /* When there is a bandwidth file and it can be read. */ + tt_int_op(0, OP_EQ, + dirserv_read_measured_bandwidths(fname, + NULL, NULL, digest)); + tt_mem_op(digest, OP_EQ, digest_expected, DIGEST256_LEN); + + done: + unlink(fname); + tor_free(fname); + update_approx_time(time(NULL)); +} + static void reset_routerstatus(routerstatus_t *rs, const char *hex_identity_digest, @@ -6441,6 +6501,7 @@ struct testcase_t dir_tests[] = { DIR_LEGACY(measured_bw_kb_line_is_after_headers), DIR_LEGACY(measured_bw_kb_cache), DIR_LEGACY(dirserv_read_measured_bandwidths), + DIR(bwauth_bw_file_digest256, 0), DIR_LEGACY(param_voting), DIR(param_voting_lookup, 0), DIR_LEGACY(v3_networkstatus), From ec7da50ab4d30eac64e02e1a5d37a72c91845214 Mon Sep 17 00:00:00 2001 From: juga0 Date: Tue, 6 Nov 2018 14:57:18 +0000 Subject: [PATCH 0474/2557] dirvote: Add the bandwidth file digest in the vote --- src/feature/dirauth/dirvote.c | 40 +++++++++++++++++++++++-- src/feature/nodelist/networkstatus_st.h | 3 ++ 2 files changed, 40 insertions(+), 3 deletions(-) diff --git a/src/feature/dirauth/dirvote.c b/src/feature/dirauth/dirvote.c index 9587f36550..ba7b2f1de6 100644 --- a/src/feature/dirauth/dirvote.c +++ b/src/feature/dirauth/dirvote.c @@ -61,6 +61,9 @@ #include "lib/encoding/confline.h" #include "lib/crypt_ops/crypto_format.h" +/* Algorithm to use for the bandwidth file digest. */ +#define DIGEST_ALG_BW_FILE DIGEST_SHA256 + /** * \file dirvote.c * \brief Functions to compute directory consensus, and schedule voting. @@ -269,6 +272,7 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key, char *flag_thresholds = dirserv_get_flag_thresholds_line(); char *params; char *bw_headers_line = NULL; + char *bw_file_digest = NULL; authority_cert_t *cert = v3_ns->cert; char *methods = make_consensus_method_list(MIN_SUPPORTED_CONSENSUS_METHOD, @@ -308,6 +312,28 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key, tor_free(bw_file_headers); } + /* Create bandwidth-file-digest if applicable. + * v3_ns->b64_digest_bw_file will contain the digest when V3BandwidthsFile + * is configured and the bandwidth file could be read, even if it was not + * parseable. + */ + if (!tor_digest256_is_zero((const char *)v3_ns->bw_file_digest256)) { + /* Encode the digest. */ + char b64_digest_bw_file[BASE64_DIGEST256_LEN+1] = {0}; + if (digest256_to_base64(b64_digest_bw_file, + (const char *)v3_ns->bw_file_digest256)>0) { + /* "bandwidth-file-digest" 1*(SP algorithm "=" digest) NL */ + char *digest_algo_b64_digest_bw_file = NULL; + tor_asprintf(&digest_algo_b64_digest_bw_file, "%s=%s", + crypto_digest_algorithm_get_name(DIGEST_ALG_BW_FILE), + b64_digest_bw_file); + /* No need for tor_strdup(""), format_line_if_present does it. */ + bw_file_digest = format_line_if_present( + "bandwidth-file-digest", digest_algo_b64_digest_bw_file); + tor_free(digest_algo_b64_digest_bw_file); + } + } + smartlist_add_asprintf(chunks, "network-status-version 3\n" "vote-status %s\n" @@ -327,6 +353,7 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key, "contact %s\n" "%s" /* shared randomness information */ "%s" /* bandwidth file headers */ + "%s" /* bandwidth file */ , v3_ns->type == NS_TYPE_VOTE ? "vote" : "opinion", methods, @@ -345,7 +372,8 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key, shared_random_vote_str ? shared_random_vote_str : "", bw_headers_line ? - bw_headers_line : ""); + bw_headers_line : "", + bw_file_digest ? bw_file_digest: ""); tor_free(params); tor_free(flags); @@ -353,6 +381,7 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key, tor_free(methods); tor_free(shared_random_vote_str); tor_free(bw_headers_line); + tor_free(bw_file_digest); if (!tor_digest_is_zero(voter->legacy_id_digest)) { char fpbuf[HEX_DIGEST_LEN+1]; @@ -4425,6 +4454,7 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key, const int vote_on_reachability = running_long_enough_to_decide_unreachable(); smartlist_t *microdescriptors = NULL; smartlist_t *bw_file_headers = NULL; + uint8_t bw_file_digest256[DIGEST256_LEN] = {0}; tor_assert(private_key); tor_assert(cert); @@ -4462,7 +4492,8 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key, * set_routerstatus_from_routerinfo() see up-to-date bandwidth info. */ if (options->V3BandwidthsFile) { - dirserv_read_measured_bandwidths(options->V3BandwidthsFile, NULL, NULL); + dirserv_read_measured_bandwidths(options->V3BandwidthsFile, NULL, NULL, + NULL); } else { /* * No bandwidths file; clear the measured bandwidth cache in case we had @@ -4567,7 +4598,9 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key, /* Only set bw_file_headers when V3BandwidthsFile is configured */ bw_file_headers = smartlist_new(); dirserv_read_measured_bandwidths(options->V3BandwidthsFile, - routerstatuses, bw_file_headers); + routerstatuses, bw_file_headers, + bw_file_digest256); + } else { /* * No bandwidths file; clear the measured bandwidth cache in case we had @@ -4664,6 +4697,7 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key, smartlist_sort_strings(v3_out->net_params); } v3_out->bw_file_headers = bw_file_headers; + memcpy(v3_out->bw_file_digest256, bw_file_digest256, DIGEST256_LEN); voter = tor_malloc_zero(sizeof(networkstatus_voter_info_t)); voter->nickname = tor_strdup(options->Nickname); diff --git a/src/feature/nodelist/networkstatus_st.h b/src/feature/nodelist/networkstatus_st.h index 6160f12361..5c1eea3259 100644 --- a/src/feature/nodelist/networkstatus_st.h +++ b/src/feature/nodelist/networkstatus_st.h @@ -99,6 +99,9 @@ struct networkstatus_t { /** List of key=value strings from the headers of the bandwidth list file */ smartlist_t *bw_file_headers; + + /** A SHA256 digest of the bandwidth file used in a vote. */ + uint8_t bw_file_digest256[DIGEST256_LEN]; }; #endif From b61c3c6dfa072a9717c878675ed431c3f8f74d24 Mon Sep 17 00:00:00 2001 From: juga0 Date: Tue, 6 Nov 2018 14:58:15 +0000 Subject: [PATCH 0475/2557] changes: Add changes file for #26698 --- changes/ticket26698 | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 changes/ticket26698 diff --git a/changes/ticket26698 b/changes/ticket26698 new file mode 100644 index 0000000000..6b029a1b73 --- /dev/null +++ b/changes/ticket26698 @@ -0,0 +1,4 @@ + o Minor features (directory authority): + - When a directory authority is using a bandwidth file to obtain the + bandwidth values, include the digest of the file in the vote. + Closes ticket 26698. From 4f9061868b04724bf3eaecddf9b536c189bd34da Mon Sep 17 00:00:00 2001 From: rl1987 Date: Wed, 13 Feb 2019 14:51:42 +0200 Subject: [PATCH 0476/2557] Use env to find bash --- src/test/test-network.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/test-network.sh b/src/test/test-network.sh index 6c678849b6..4d56e83806 100755 --- a/src/test/test-network.sh +++ b/src/test/test-network.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # This script calls the equivalent script in chutney/tools From ad48aab0565e8498fd68428fb2cfaa46b39ffc01 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Wed, 13 Feb 2019 15:04:12 +0200 Subject: [PATCH 0477/2557] Let's not double-quote EXTRA_CARGO_OPTIONS after all --- src/test/test_rust.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/test/test_rust.sh b/src/test/test_rust.sh index da2bd32d21..804d2ada36 100755 --- a/src/test/test_rust.sh +++ b/src/test/test_rust.sh @@ -14,12 +14,13 @@ rustc_host=$(rustc -vV | grep host | sed 's/host: //') for cargo_toml_dir in "${abs_top_srcdir:-../../..}"/src/rust/*; do if [ -e "${cargo_toml_dir}/Cargo.toml" ]; then + # shellcheck disable=SC2086 cd "${abs_top_builddir:-../../..}/src/rust" && \ CARGO_TARGET_DIR="${abs_top_builddir:-../../..}/src/rust/target" \ "${CARGO:-cargo}" test "${CARGO_ONLINE-'--frozen'}" \ --features "test_linking_hack" \ --target "$rustc_host" \ - "${EXTRA_CARGO_OPTIONS}" \ + ${EXTRA_CARGO_OPTIONS} \ --manifest-path "${cargo_toml_dir}/Cargo.toml" || exitcode=1 fi done From f5a6d4c6ea78e73e8f27292fac6d4153e8e84aa0 Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Thu, 14 Feb 2019 12:07:10 +0200 Subject: [PATCH 0478/2557] Disable unstable circuit padding unittest. until #29298 is implemented. --- changes/bug29298 | 5 +++++ src/test/test_circuitpadding.c | 8 +++++--- 2 files changed, 10 insertions(+), 3 deletions(-) create mode 100644 changes/bug29298 diff --git a/changes/bug29298 b/changes/bug29298 new file mode 100644 index 0000000000..df12db77d7 --- /dev/null +++ b/changes/bug29298 @@ -0,0 +1,5 @@ + o Minor bugfixes (testing, circuit padding): + - Disabled unstable circuit padding unittest that was causing intermittent + test failures because of ill-defined small histogram. Such histograms + will be allowed again after 29298 is implemented. Fixes second case of + bug 29122; bugfix on 0.4.0.1-alpha. \ No newline at end of file diff --git a/src/test/test_circuitpadding.c b/src/test/test_circuitpadding.c index 12a07fa957..eee1edc50c 100644 --- a/src/test/test_circuitpadding.c +++ b/src/test/test_circuitpadding.c @@ -46,7 +46,6 @@ void test_circuitpadding_conditions(void *arg); void test_circuitpadding_serialize(void *arg); void test_circuitpadding_rtt(void *arg); void test_circuitpadding_tokens(void *arg); -void test_circuitpadding_circuitsetup_machine(void *arg); static void simulate_single_hop_extend(circuit_t *client, circuit_t *mid_relay, @@ -1809,6 +1808,8 @@ test_circuitpadding_conditions(void *arg) return; } +/** Disabled unstable test until #29298 is implemented (see #29122) */ +#if 0 void test_circuitpadding_circuitsetup_machine(void *arg) { @@ -2054,6 +2055,7 @@ test_circuitpadding_circuitsetup_machine(void *arg) return; } +#endif /** Helper function: Initializes a padding machine where every state uses the * uniform probability distribution. */ @@ -2340,11 +2342,11 @@ test_circuitpadding_global_rate_limiting(void *arg) { #name, test_##name, (flags), NULL, NULL } struct testcase_t circuitpadding_tests[] = { - //TEST_CIRCUITPADDING(circuitpadding_circuitsetup_machine, 0), TEST_CIRCUITPADDING(circuitpadding_tokens, TT_FORK), TEST_CIRCUITPADDING(circuitpadding_negotiation, TT_FORK), TEST_CIRCUITPADDING(circuitpadding_wronghop, TT_FORK), - TEST_CIRCUITPADDING(circuitpadding_circuitsetup_machine, TT_FORK), + /** Disabled unstable test until #29298 is implemented (see #29122) */ + // TEST_CIRCUITPADDING(circuitpadding_circuitsetup_machine, TT_FORK), TEST_CIRCUITPADDING(circuitpadding_conditions, TT_FORK), TEST_CIRCUITPADDING(circuitpadding_rtt, TT_FORK), TEST_CIRCUITPADDING(circuitpadding_sample_distribution, TT_FORK), From 622a9a8a364520f2f1d0294b8d4389f1b29ba376 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 6 Feb 2019 12:51:50 -0500 Subject: [PATCH 0479/2557] Extract the common body of our random-int functions into a macro This is the second part of refactoring the random-int-in-range code. --- src/lib/crypt_ops/crypto_rand_numeric.c | 69 +++++++++++++++---------- 1 file changed, 41 insertions(+), 28 deletions(-) diff --git a/src/lib/crypt_ops/crypto_rand_numeric.c b/src/lib/crypt_ops/crypto_rand_numeric.c index 97fe3bd180..d07657440e 100644 --- a/src/lib/crypt_ops/crypto_rand_numeric.c +++ b/src/lib/crypt_ops/crypto_rand_numeric.c @@ -8,6 +8,39 @@ #include "lib/crypt_ops/crypto_rand.h" #include "lib/log/util_bug.h" +/** + * Implementation macro: yields code that returns a uniform unbiased + * random number between 0 and limit. "type" is the type of the number to + * return; "maxval" is the largest possible value of "type"; and "fill_stmt" + * is a code snippet that fills an object named "val" with random bits. + **/ +#define IMPLEMENT_RAND_UNSIGNED(type, maxval, limit, fill_stmt) \ + do { \ + type val; \ + type cutoff; \ + tor_assert((limit) > 0); \ + \ + /* We ignore any values that are >= 'cutoff,' to avoid biasing */ \ + /* the distribution with clipping at the upper end of the type's */ \ + /* range. */ \ + cutoff = (maxval) - ((maxval)%(limit)); \ + while (1) { \ + fill_stmt; \ + if (val < cutoff) \ + return val % (limit); \ + } \ + } while (0) + +/** Helper: Return a pseudorandom integer chosen uniformly from the + * values between 0 and limit-1 inclusive. */ +static unsigned +crypto_rand_uint(unsigned limit) +{ + tor_assert(limit < UINT_MAX); + IMPLEMENT_RAND_UNSIGNED(unsigned, UINT_MAX, limit, + crypto_rand((char*)&val, sizeof(val))); +} + /** * Return a pseudorandom integer, chosen uniformly from the values * between 0 and max-1 inclusive. max must be between 1 and @@ -16,21 +49,13 @@ int crypto_rand_int(unsigned int max) { - unsigned int val; - unsigned int cutoff; - tor_assert(max <= ((unsigned int)INT_MAX)+1); - tor_assert(max > 0); /* don't div by 0 */ - - /* We ignore any values that are >= 'cutoff,' to avoid biasing the - * distribution with clipping at the upper end of unsigned int's - * range. + /* We can't use IMPLEMENT_RAND_UNSIGNED directly, since we're trying + * to return a signed type. Instead we make sure that the range is + * reasonable for a nonnegative int, use crypto_rand_uint(), and cast. */ - cutoff = UINT_MAX - (UINT_MAX%max); - while (1) { - crypto_rand((char*)&val, sizeof(val)); - if (val < cutoff) - return val % max; - } + tor_assert(max <= ((unsigned int)INT_MAX)+1); + + return (int)crypto_rand_uint(max); } /** @@ -78,21 +103,9 @@ crypto_rand_time_range(time_t min, time_t max) uint64_t crypto_rand_uint64(uint64_t max) { - uint64_t val; - uint64_t cutoff; tor_assert(max < UINT64_MAX); - tor_assert(max > 0); /* don't div by 0 */ - - /* We ignore any values that are >= 'cutoff,' to avoid biasing the - * distribution with clipping at the upper end of unsigned int's - * range. - */ - cutoff = UINT64_MAX - (UINT64_MAX%max); - while (1) { - crypto_rand((char*)&val, sizeof(val)); - if (val < cutoff) - return val % max; - } + IMPLEMENT_RAND_UNSIGNED(uint64_t, UINT64_MAX, max, + crypto_rand((char*)&val, sizeof(val))); } /** From 3d3578ab41c9be602fad5c4172a880668994c8c3 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 6 Feb 2019 12:28:05 -0500 Subject: [PATCH 0480/2557] Extract RNG tests into a new test module test_crypto.c is pretty big; it wouldn't hurt to split it up some more before I start adding stuff to the PRNG tests. --- src/test/include.am | 1 + src/test/test.c | 1 + src/test/test.h | 1 + src/test/test_crypto.c | 171 --------------------------------- src/test/test_crypto_rng.c | 187 +++++++++++++++++++++++++++++++++++++ 5 files changed, 190 insertions(+), 171 deletions(-) create mode 100644 src/test/test_crypto_rng.c diff --git a/src/test/include.am b/src/test/include.am index b276500fd5..1b8f7f5ad3 100644 --- a/src/test/include.am +++ b/src/test/include.am @@ -120,6 +120,7 @@ src_test_test_SOURCES += \ src/test/test_controller_events.c \ src/test/test_crypto.c \ src/test/test_crypto_ope.c \ + src/test/test_crypto_rng.c \ src/test/test_data.c \ src/test/test_dir.c \ src/test/test_dir_common.c \ diff --git a/src/test/test.c b/src/test/test.c index 82bdba676e..25e9da5591 100644 --- a/src/test/test.c +++ b/src/test/test.c @@ -866,6 +866,7 @@ struct testgroup_t testgroups[] = { { "crypto/openssl/", crypto_openssl_tests }, #endif { "crypto/pem/", pem_tests }, + { "crypto/rng/", crypto_rng_tests }, { "dir/", dir_tests }, { "dir/md/", microdesc_tests }, { "dir/voting/flags/", voting_flags_tests }, diff --git a/src/test/test.h b/src/test/test.h index 5f549e5421..2564432985 100644 --- a/src/test/test.h +++ b/src/test/test.h @@ -206,6 +206,7 @@ extern struct testcase_t controller_event_tests[]; extern struct testcase_t controller_tests[]; extern struct testcase_t crypto_ope_tests[]; extern struct testcase_t crypto_openssl_tests[]; +extern struct testcase_t crypto_rng_tests[]; extern struct testcase_t crypto_tests[]; extern struct testcase_t dir_handle_get_tests[]; extern struct testcase_t dir_tests[]; diff --git a/src/test/test_crypto.c b/src/test/test_crypto.c index fa79f4cc47..0b57448bcf 100644 --- a/src/test/test_crypto.c +++ b/src/test/test_crypto.c @@ -254,168 +254,6 @@ test_crypto_openssl_version(void *arg) ; } -/** Run unit tests for our random number generation function and its wrappers. - */ -static void -test_crypto_rng(void *arg) -{ - int i, j, allok; - char data1[100], data2[100]; - double d; - char *h=NULL; - - /* Try out RNG. */ - (void)arg; - tt_assert(! crypto_seed_rng()); - crypto_rand(data1, 100); - crypto_rand(data2, 100); - tt_mem_op(data1,OP_NE, data2,100); - allok = 1; - for (i = 0; i < 100; ++i) { - uint64_t big; - char *host; - j = crypto_rand_int(100); - if (j < 0 || j >= 100) - allok = 0; - big = crypto_rand_uint64(UINT64_C(1)<<40); - if (big >= (UINT64_C(1)<<40)) - allok = 0; - big = crypto_rand_uint64(UINT64_C(5)); - if (big >= 5) - allok = 0; - d = crypto_rand_double(); - tt_assert(d >= 0); - tt_assert(d < 1.0); - host = crypto_random_hostname(3,8,"www.",".onion"); - if (strcmpstart(host,"www.") || - strcmpend(host,".onion") || - strlen(host) < 13 || - strlen(host) > 18) - allok = 0; - tor_free(host); - } - - /* Make sure crypto_random_hostname clips its inputs properly. */ - h = crypto_random_hostname(20000, 9000, "www.", ".onion"); - tt_assert(! strcmpstart(h,"www.")); - tt_assert(! strcmpend(h,".onion")); - tt_int_op(63+4+6, OP_EQ, strlen(h)); - - tt_assert(allok); - done: - tor_free(h); -} - -static void -test_crypto_rng_range(void *arg) -{ - int got_smallest = 0, got_largest = 0; - int i; - - (void)arg; - for (i = 0; i < 1000; ++i) { - int x = crypto_rand_int_range(5,9); - tt_int_op(x, OP_GE, 5); - tt_int_op(x, OP_LT, 9); - if (x == 5) - got_smallest = 1; - if (x == 8) - got_largest = 1; - } - /* These fail with probability 1/10^603. */ - tt_assert(got_smallest); - tt_assert(got_largest); - - got_smallest = got_largest = 0; - const uint64_t ten_billion = 10 * ((uint64_t)1000000000000); - for (i = 0; i < 1000; ++i) { - uint64_t x = crypto_rand_uint64_range(ten_billion, ten_billion+10); - tt_u64_op(x, OP_GE, ten_billion); - tt_u64_op(x, OP_LT, ten_billion+10); - if (x == ten_billion) - got_smallest = 1; - if (x == ten_billion+9) - got_largest = 1; - } - - tt_assert(got_smallest); - tt_assert(got_largest); - - const time_t now = time(NULL); - for (i = 0; i < 2000; ++i) { - time_t x = crypto_rand_time_range(now, now+60); - tt_i64_op(x, OP_GE, now); - tt_i64_op(x, OP_LT, now+60); - if (x == now) - got_smallest = 1; - if (x == now+59) - got_largest = 1; - } - - tt_assert(got_smallest); - tt_assert(got_largest); - done: - ; -} - -static void -test_crypto_rng_strongest(void *arg) -{ - const char *how = arg; - int broken = 0; - - if (how == NULL) { - ; - } else if (!strcmp(how, "nosyscall")) { - break_strongest_rng_syscall = 1; - } else if (!strcmp(how, "nofallback")) { - break_strongest_rng_fallback = 1; - } else if (!strcmp(how, "broken")) { - broken = break_strongest_rng_syscall = break_strongest_rng_fallback = 1; - } - -#define N 128 - uint8_t combine_and[N]; - uint8_t combine_or[N]; - int i, j; - - memset(combine_and, 0xff, N); - memset(combine_or, 0, N); - - for (i = 0; i < 100; ++i) { /* 2^-100 chances just don't happen. */ - uint8_t output[N]; - memset(output, 0, N); - if (how == NULL) { - /* this one can't fail. */ - crypto_strongest_rand(output, sizeof(output)); - } else { - int r = crypto_strongest_rand_raw(output, sizeof(output)); - if (r == -1) { - if (broken) { - goto done; /* we're fine. */ - } - /* This function is allowed to break, but only if it always breaks. */ - tt_int_op(i, OP_EQ, 0); - tt_skip(); - } else { - tt_assert(! broken); - } - } - for (j = 0; j < N; ++j) { - combine_and[j] &= output[j]; - combine_or[j] |= output[j]; - } - } - - for (j = 0; j < N; ++j) { - tt_int_op(combine_and[j], OP_EQ, 0); - tt_int_op(combine_or[j], OP_EQ, 0xff); - } - done: - ; -#undef N -} - /** Run unit tests for our AES128 functionality */ static void test_crypto_aes128(void *arg) @@ -3140,15 +2978,6 @@ test_crypto_failure_modes(void *arg) struct testcase_t crypto_tests[] = { CRYPTO_LEGACY(formats), - CRYPTO_LEGACY(rng), - { "rng_range", test_crypto_rng_range, 0, NULL, NULL }, - { "rng_strongest", test_crypto_rng_strongest, TT_FORK, NULL, NULL }, - { "rng_strongest_nosyscall", test_crypto_rng_strongest, TT_FORK, - &passthrough_setup, (void*)"nosyscall" }, - { "rng_strongest_nofallback", test_crypto_rng_strongest, TT_FORK, - &passthrough_setup, (void*)"nofallback" }, - { "rng_strongest_broken", test_crypto_rng_strongest, TT_FORK, - &passthrough_setup, (void*)"broken" }, { "openssl_version", test_crypto_openssl_version, TT_FORK, NULL, NULL }, { "aes_AES", test_crypto_aes128, TT_FORK, &passthrough_setup, (void*)"aes" }, { "aes_EVP", test_crypto_aes128, TT_FORK, &passthrough_setup, (void*)"evp" }, diff --git a/src/test/test_crypto_rng.c b/src/test/test_crypto_rng.c new file mode 100644 index 0000000000..008753d971 --- /dev/null +++ b/src/test/test_crypto_rng.c @@ -0,0 +1,187 @@ +/* Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "orconfig.h" +#define CRYPTO_RAND_PRIVATE +#include "core/or/or.h" +#include "test/test.h" +#include "lib/crypt_ops/aes.h" +#include "lib/crypt_ops/crypto_format.h" +#include "lib/crypt_ops/crypto_rand.h" + +/** Run unit tests for our random number generation function and its wrappers. + */ +static void +test_crypto_rng(void *arg) +{ + int i, j, allok; + char data1[100], data2[100]; + double d; + char *h=NULL; + + /* Try out RNG. */ + (void)arg; + tt_assert(! crypto_seed_rng()); + crypto_rand(data1, 100); + crypto_rand(data2, 100); + tt_mem_op(data1,OP_NE, data2,100); + allok = 1; + for (i = 0; i < 100; ++i) { + uint64_t big; + char *host; + j = crypto_rand_int(100); + if (j < 0 || j >= 100) + allok = 0; + big = crypto_rand_uint64(UINT64_C(1)<<40); + if (big >= (UINT64_C(1)<<40)) + allok = 0; + big = crypto_rand_uint64(UINT64_C(5)); + if (big >= 5) + allok = 0; + d = crypto_rand_double(); + tt_assert(d >= 0); + tt_assert(d < 1.0); + host = crypto_random_hostname(3,8,"www.",".onion"); + if (strcmpstart(host,"www.") || + strcmpend(host,".onion") || + strlen(host) < 13 || + strlen(host) > 18) + allok = 0; + tor_free(host); + } + + /* Make sure crypto_random_hostname clips its inputs properly. */ + h = crypto_random_hostname(20000, 9000, "www.", ".onion"); + tt_assert(! strcmpstart(h,"www.")); + tt_assert(! strcmpend(h,".onion")); + tt_int_op(63+4+6, OP_EQ, strlen(h)); + + tt_assert(allok); + done: + tor_free(h); +} + +static void +test_crypto_rng_range(void *arg) +{ + int got_smallest = 0, got_largest = 0; + int i; + + (void)arg; + for (i = 0; i < 1000; ++i) { + int x = crypto_rand_int_range(5,9); + tt_int_op(x, OP_GE, 5); + tt_int_op(x, OP_LT, 9); + if (x == 5) + got_smallest = 1; + if (x == 8) + got_largest = 1; + } + /* These fail with probability 1/10^603. */ + tt_assert(got_smallest); + tt_assert(got_largest); + + got_smallest = got_largest = 0; + const uint64_t ten_billion = 10 * ((uint64_t)1000000000000); + for (i = 0; i < 1000; ++i) { + uint64_t x = crypto_rand_uint64_range(ten_billion, ten_billion+10); + tt_u64_op(x, OP_GE, ten_billion); + tt_u64_op(x, OP_LT, ten_billion+10); + if (x == ten_billion) + got_smallest = 1; + if (x == ten_billion+9) + got_largest = 1; + } + + tt_assert(got_smallest); + tt_assert(got_largest); + + const time_t now = time(NULL); + for (i = 0; i < 2000; ++i) { + time_t x = crypto_rand_time_range(now, now+60); + tt_i64_op(x, OP_GE, now); + tt_i64_op(x, OP_LT, now+60); + if (x == now) + got_smallest = 1; + if (x == now+59) + got_largest = 1; + } + + tt_assert(got_smallest); + tt_assert(got_largest); + done: + ; +} + +static void +test_crypto_rng_strongest(void *arg) +{ + const char *how = arg; + int broken = 0; + + if (how == NULL) { + ; + } else if (!strcmp(how, "nosyscall")) { + break_strongest_rng_syscall = 1; + } else if (!strcmp(how, "nofallback")) { + break_strongest_rng_fallback = 1; + } else if (!strcmp(how, "broken")) { + broken = break_strongest_rng_syscall = break_strongest_rng_fallback = 1; + } + +#define N 128 + uint8_t combine_and[N]; + uint8_t combine_or[N]; + int i, j; + + memset(combine_and, 0xff, N); + memset(combine_or, 0, N); + + for (i = 0; i < 100; ++i) { /* 2^-100 chances just don't happen. */ + uint8_t output[N]; + memset(output, 0, N); + if (how == NULL) { + /* this one can't fail. */ + crypto_strongest_rand(output, sizeof(output)); + } else { + int r = crypto_strongest_rand_raw(output, sizeof(output)); + if (r == -1) { + if (broken) { + goto done; /* we're fine. */ + } + /* This function is allowed to break, but only if it always breaks. */ + tt_int_op(i, OP_EQ, 0); + tt_skip(); + } else { + tt_assert(! broken); + } + } + for (j = 0; j < N; ++j) { + combine_and[j] &= output[j]; + combine_or[j] |= output[j]; + } + } + + for (j = 0; j < N; ++j) { + tt_int_op(combine_and[j], OP_EQ, 0); + tt_int_op(combine_or[j], OP_EQ, 0xff); + } + done: + ; +#undef N +} + +struct testcase_t crypto_rng_tests[] = { + { "rng", test_crypto_rng, 0, NULL, NULL }, + { "rng_range", test_crypto_rng_range, 0, NULL, NULL }, + { "rng_strongest", test_crypto_rng_strongest, TT_FORK, NULL, NULL }, + { "rng_strongest_nosyscall", test_crypto_rng_strongest, TT_FORK, + &passthrough_setup, (void*)"nosyscall" }, + { "rng_strongest_nofallback", test_crypto_rng_strongest, TT_FORK, + &passthrough_setup, (void*)"nofallback" }, + { "rng_strongest_broken", test_crypto_rng_strongest, TT_FORK, + &passthrough_setup, (void*)"broken" }, + END_OF_TESTCASES +}; From f3cbd6426cbb27b9ab4e5492a50a785cce77f805 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 5 Feb 2019 12:49:04 -0500 Subject: [PATCH 0481/2557] Implement a fast aes-ctr prng This module is currently implemented to use the same technique as libottery (later used by the bsds' arc4random replacement), using AES-CTR-256 as its underlying stream cipher. It's backtracking- resistant immediately after each call, and prediction-resistant after a while. Here's how it works: We generate psuedorandom bytes using AES-CTR-256. We generate BUFLEN bytes at a time. When we do this, we keep the first SEED_LEN bytes as the key and the IV for our next invocation of AES_CTR, and yield the remaining BUFLEN - SEED_LEN bytes to the user as they invoke the PRNG. As we yield bytes to the user, we clear them from the buffer. Every RESEED_AFTER times we refill the buffer, we mix in an additional SEED_LEN bytes from our strong PRNG into the seed. If the user ever asks for a huge number of bytes at once, we pull SEED_LEN bytes from the PRNG and use them with our stream cipher to fill the user's request. --- src/lib/crypt_ops/crypto_rand.h | 32 +++ src/lib/crypt_ops/crypto_rand_fast.c | 263 ++++++++++++++++++++++++ src/lib/crypt_ops/crypto_rand_numeric.c | 56 ++++- src/lib/crypt_ops/include.am | 1 + src/test/test_crypto_rng.c | 117 +++++++++++ 5 files changed, 459 insertions(+), 10 deletions(-) create mode 100644 src/lib/crypt_ops/crypto_rand_fast.c diff --git a/src/lib/crypt_ops/crypto_rand.h b/src/lib/crypt_ops/crypto_rand.h index cc2762842a..8a81a4acdc 100644 --- a/src/lib/crypt_ops/crypto_rand.h +++ b/src/lib/crypt_ops/crypto_rand.h @@ -16,6 +16,7 @@ #include "lib/cc/compat_compiler.h" #include "lib/cc/torint.h" #include "lib/testsupport/testsupport.h" +#include "lib/malloc/malloc.h" /* random numbers */ int crypto_seed_rng(void) ATTR_WUR; @@ -24,6 +25,7 @@ void crypto_rand_unmocked(char *to, size_t n); void crypto_strongest_rand(uint8_t *out, size_t out_len); MOCK_DECL(void,crypto_strongest_rand_,(uint8_t *out, size_t out_len)); int crypto_rand_int(unsigned int max); +unsigned crypto_rand_uint(unsigned limit); int crypto_rand_int_range(unsigned int min, unsigned int max); uint64_t crypto_rand_uint64_range(uint64_t min, uint64_t max); time_t crypto_rand_time_range(time_t min, time_t max); @@ -41,6 +43,36 @@ void *smartlist_choose(const struct smartlist_t *sl); void smartlist_shuffle(struct smartlist_t *sl); int crypto_force_rand_ssleay(void); +/** + * A fast PRNG, for use when the PRNG provided by our crypto library isn't + * fast enough. This one _should_ be cryptographically strong, but + * has seen less auditing than the PRNGs in OpenSSL and NSS. Use with + * caution. + * + * Note that this object is NOT thread-safe. If you need a thread-safe + * prng, use crypto_rand(), or wrap this in a mutex. + **/ +typedef struct crypto_fast_rng_t crypto_fast_rng_t; +/** + * Number of bytes used to seed a crypto_rand_fast_t. + **/ +crypto_fast_rng_t *crypto_fast_rng_new(void); +#define CRYPTO_FAST_RNG_SEED_LEN 48 +crypto_fast_rng_t *crypto_fast_rng_new_from_seed(const uint8_t *seed); +void crypto_fast_rng_getbytes(crypto_fast_rng_t *rng, uint8_t *out, size_t n); +void crypto_fast_rng_free_(crypto_fast_rng_t *); +#define crypto_fast_rng_free(c) \ + FREE_AND_NULL(crypto_fast_rng_t, crypto_fast_rng_free_, (c)) + +unsigned crypto_fast_rng_get_uint(crypto_fast_rng_t *rng, unsigned limit); +uint64_t crypto_fast_rng_get_uint64(crypto_fast_rng_t *rng, uint64_t limit); +double crypto_fast_rng_get_double(crypto_fast_rng_t *rng); + +#if defined(TOR_UNIT_TESTS) +/* Used for white-box testing */ +size_t crypto_fast_rng_get_bytes_used_per_stream(void); +#endif + #ifdef CRYPTO_RAND_PRIVATE STATIC int crypto_strongest_rand_raw(uint8_t *out, size_t out_len); diff --git a/src/lib/crypt_ops/crypto_rand_fast.c b/src/lib/crypt_ops/crypto_rand_fast.c new file mode 100644 index 0000000000..34e763bf51 --- /dev/null +++ b/src/lib/crypt_ops/crypto_rand_fast.c @@ -0,0 +1,263 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file crypto_rand_fast.c + * + * \brief A fast strong PRNG for use when our underlying cryptographic + * library's PRNG isn't fast enough. + **/ + +/* This library is currently implemented to use the same implementation + * technique as libottery, using AES-CTR-256 as our underlying stream cipher. + * It's backtracking-resistant immediately, and prediction-resistant after + * a while. + * + * Here's how it works: + * + * We generate pseudorandom bytes using AES-CTR-256. We generate BUFLEN bytes + * at a time. When we do this, we keep the first SEED_LEN bytes as the key + * and the IV for our next invocation of AES_CTR, and yield the remaining + * BUFLEN - SEED_LEN bytes to the user as they invoke the PRNG. As we yield + * bytes to the user, we clear them from the buffer. + * + * After we have refilled the buffer RESEED_AFTER times, we mix in an + * additional SEED_LEN bytes from our strong PRNG into the seed. + * + * If the user ever asks for a huge number of bytes at once, we pull SEED_LEN + * bytes from the PRNG and use them with our stream cipher to fill the user's + * request. + */ + +#define CRYPTO_RAND_FAST_PRIVATE + +#include "lib/crypt_ops/crypto_rand.h" +#include "lib/crypt_ops/crypto_cipher.h" +#include "lib/crypt_ops/crypto_digest.h" +#include "lib/crypt_ops/crypto_util.h" +#include "lib/intmath/cmp.h" +#include "lib/cc/ctassert.h" +#include "lib/malloc/map_anon.h" + +#include "lib/log/util_bug.h" + +#include + +/* Alias for CRYPTO_FAST_RNG_SEED_LEN to make our code shorter. + */ +#define SEED_LEN (CRYPTO_FAST_RNG_SEED_LEN) + +/* The amount of space that we mmap for a crypto_fast_rng_t. + */ +#define MAPLEN 4096 + +/* The number of random bytes that we can yield to the user after each + * time we fill a crypto_fast_rng_t's buffer. + */ +#define BUFLEN (MAPLEN - 2*sizeof(uint16_t) - SEED_LEN) + +/* The number of buffer refills after which we should fetch more + * entropy from crypto_strongest_rand(). + */ +#define RESEED_AFTER 16 + +/* The length of the stream cipher key we will use for the PRNG, in bytes. + */ +#define KEY_LEN (CRYPTO_FAST_RNG_SEED_LEN - CIPHER_IV_LEN) +/* The length of the stream cipher key we will use for the PRNG, in bits. + */ +#define KEY_BITS (KEY_LEN * 8) + +/* Make sure that we have a key length we can actually use with AES. */ +CTASSERT(KEY_BITS == 128 || KEY_BITS == 192 || KEY_BITS == 256); + +struct crypto_fast_rng_t { + /** How many more fills does this buffer have before we should mix + * in the output of crypto_rand()? */ + uint16_t n_till_reseed; + /** How many bytes are remaining in cbuf.bytes? */ + uint16_t bytes_left; + struct cbuf { + /** The seed (key and IV) that we will use the next time that we refill + * cbuf. */ + uint8_t seed[SEED_LEN]; + /** + * Bytes that we are yielding to the user. The next byte to be + * yielded is at bytes[BUFLEN-bytes_left]; all other bytes in this + * array are set to zero. + */ + uint8_t bytes[BUFLEN]; + } buf; +}; + +/* alignof(uint8_t) should be 1, so there shouldn't be any padding in cbuf. + */ +CTASSERT(sizeof(struct cbuf) == BUFLEN+SEED_LEN); +/* We're trying to fit all of the RNG state into a nice mmapable chunk. + */ +CTASSERT(sizeof(crypto_fast_rng_t) <= MAPLEN); + +/** + * Initialize and return a new fast PRNG, using a strong random seed. + * + * Note that this object is NOT thread-safe. If you need a thread-safe + * prng, use crypto_rand(), or wrap this in a mutex. + **/ +crypto_fast_rng_t * +crypto_fast_rng_new(void) +{ + uint8_t seed[SEED_LEN]; + crypto_strongest_rand(seed, sizeof(seed)); + crypto_fast_rng_t *result = crypto_fast_rng_new_from_seed(seed); + memwipe(seed, 0, sizeof(seed)); + return result; +} + +/** + * Initialize and return a new fast PRNG, using a seed value specified + * in seed. This value must be CRYPTO_FAST_RNG_SEED_LEN bytes + * long. + * + * Note that this object is NOT thread-safe. If you need a thread-safe + * prng, use crypto_rand(), or wrap this in a mutex. + **/ +crypto_fast_rng_t * +crypto_fast_rng_new_from_seed(const uint8_t *seed) +{ + /* We try to allocate this object as securely as we can, to avoid + * having it get dumped, swapped, or shared after fork. + */ + crypto_fast_rng_t *result = tor_mmap_anonymous(sizeof(*result), + ANONMAP_PRIVATE | ANONMAP_NOINHERIT); + + memcpy(result->buf.seed, seed, SEED_LEN); + /* Causes an immediate refill once the user asks for data. */ + result->bytes_left = 0; + result->n_till_reseed = RESEED_AFTER; + return result; +} + +/** + * Helper: create a crypto_cipher_t object from SEED_LEN bytes of + * input. The first KEY_LEN bytes are used as the stream cipher's key, + * and the remaining CIPHER_IV_LEN bytes are used as its IV. + **/ +static inline crypto_cipher_t * +cipher_from_seed(const uint8_t *seed) +{ + return crypto_cipher_new_with_iv_and_bits(seed, seed+KEY_LEN, KEY_BITS); +} + +/** + * Helper: refill the seed bytes and output buffer of rng, using + * the input seed bytes as input (key and IV) for the stream cipher. + * + * If the n_till_reseed counter has reached zero, mix more random bytes into + * the seed before refilling the buffer. + **/ +static void +crypto_fast_rng_refill(crypto_fast_rng_t *rng) +{ + if (rng->n_till_reseed-- == 0) { + /* It's time to reseed the RNG. We'll do this by using our XOF to mix the + * old value for the seed with some additional bytes from + * crypto_strongest_rand(). */ + crypto_xof_t *xof = crypto_xof_new(); + crypto_xof_add_bytes(xof, rng->buf.seed, SEED_LEN); + { + uint8_t seedbuf[SEED_LEN]; + crypto_strongest_rand(seedbuf, SEED_LEN); + crypto_xof_add_bytes(xof, seedbuf, SEED_LEN); + memwipe(seedbuf, 0, SEED_LEN); + } + crypto_xof_squeeze_bytes(xof, rng->buf.seed, SEED_LEN); + crypto_xof_free(xof); + + rng->n_till_reseed = RESEED_AFTER; + } + /* Now fill rng->buf with output from our stream cipher, initialized from + * that seed value. */ + crypto_cipher_t *c = cipher_from_seed(rng->buf.seed); + memset(&rng->buf, 0, sizeof(rng->buf)); + crypto_cipher_crypt_inplace(c, (char*)&rng->buf, sizeof(rng->buf)); + crypto_cipher_free(c); + + rng->bytes_left = sizeof(rng->buf.bytes); +} + +/** + * Release all storage held by rng. + **/ +void +crypto_fast_rng_free_(crypto_fast_rng_t *rng) +{ + if (!rng) + return; + memwipe(rng, 0, sizeof(*rng)); + tor_munmap_anonymous(rng, sizeof(*rng)); +} + +/** + * Helper: extract bytes from the PRNG, refilling it as necessary. Does not + * optimize the case when the user has asked for a huge output. + **/ +static void +crypto_fast_rng_getbytes_impl(crypto_fast_rng_t *rng, uint8_t *out, + const size_t n) +{ + size_t bytes_to_yield = n; + + while (bytes_to_yield) { + if (rng->bytes_left == 0) + crypto_fast_rng_refill(rng); + + const size_t to_copy = MIN(rng->bytes_left, bytes_to_yield); + + tor_assert(sizeof(rng->buf.bytes) >= rng->bytes_left); + uint8_t *copy_from = rng->buf.bytes + + (sizeof(rng->buf.bytes) - rng->bytes_left); + memcpy(out, copy_from, to_copy); + memset(copy_from, 0, to_copy); + + out += to_copy; + bytes_to_yield -= to_copy; + rng->bytes_left -= to_copy; + } +} + +/** + * Extract n bytes from rng into the buffer at out. + **/ +void +crypto_fast_rng_getbytes(crypto_fast_rng_t *rng, uint8_t *out, size_t n) +{ + if (PREDICT_UNLIKELY(n > BUFLEN)) { + /* The user has asked for a lot of output; generate it from a stream + * cipher seeded by the PRNG rather than by pulling it out of the PRNG + * directly. + */ + uint8_t seed[SEED_LEN]; + crypto_fast_rng_getbytes_impl(rng, seed, SEED_LEN); + crypto_cipher_t *c = cipher_from_seed(seed); + memset(out, 0, n); + crypto_cipher_crypt_inplace(c, (char*)out, n); + crypto_cipher_free(c); + memwipe(seed, 0, sizeof(seed)); + return; + } + + crypto_fast_rng_getbytes_impl(rng, out, n); +} + +#if defined(TOR_UNIT_TESTS) +/** for white-box testing: return the number of bytes that are returned from + * the user for each invocation of the stream cipher in this RNG. */ +size_t +crypto_fast_rng_get_bytes_used_per_stream(void) +{ + return BUFLEN; +} +#endif diff --git a/src/lib/crypt_ops/crypto_rand_numeric.c b/src/lib/crypt_ops/crypto_rand_numeric.c index d07657440e..d02c5cdcfa 100644 --- a/src/lib/crypt_ops/crypto_rand_numeric.c +++ b/src/lib/crypt_ops/crypto_rand_numeric.c @@ -31,9 +31,11 @@ } \ } while (0) -/** Helper: Return a pseudorandom integer chosen uniformly from the - * values between 0 and limit-1 inclusive. */ -static unsigned +/** + * Return a pseudorandom integer chosen uniformly from the values between 0 + * and limit-1 inclusive. limit must be strictly between 0 and + * UINT_MAX. */ +unsigned crypto_rand_uint(unsigned limit) { tor_assert(limit < UINT_MAX); @@ -108,6 +110,14 @@ crypto_rand_uint64(uint64_t max) crypto_rand((char*)&val, sizeof(val))); } +#if SIZEOF_INT == 4 +#define UINT_MAX_AS_DOUBLE 4294967296.0 +#elif SIZEOF_INT == 8 +#define UINT_MAX_AS_DOUBLE 1.8446744073709552e+19 +#else +#error SIZEOF_INT is neither 4 nor 8 +#endif /* SIZEOF_INT == 4 || ... */ + /** * Return a pseudorandom double d, chosen uniformly from the range * 0.0 <= d < 1.0. @@ -119,12 +129,38 @@ crypto_rand_double(void) * more than 32 bits of resolution */ unsigned int u; crypto_rand((char*)&u, sizeof(u)); -#if SIZEOF_INT == 4 -#define UINT_MAX_AS_DOUBLE 4294967296.0 -#elif SIZEOF_INT == 8 -#define UINT_MAX_AS_DOUBLE 1.8446744073709552e+19 -#else -#error SIZEOF_INT is neither 4 nor 8 -#endif /* SIZEOF_INT == 4 || ... */ + return ((double)u) / UINT_MAX_AS_DOUBLE; +} + +/** + * As crypto_rand_uint, but extract the result from a crypto_fast_rng_t + */ +unsigned +crypto_fast_rng_get_uint(crypto_fast_rng_t *rng, unsigned limit) +{ + tor_assert(limit < UINT_MAX); + IMPLEMENT_RAND_UNSIGNED(unsigned, UINT_MAX, limit, + crypto_fast_rng_getbytes(rng, (void*)&val, sizeof(val))); +} + +/** + * As crypto_rand_uint64, but extract the result from a crypto_fast_rng_t. + */ +uint64_t +crypto_fast_rng_get_uint64(crypto_fast_rng_t *rng, uint64_t limit) +{ + tor_assert(limit < UINT64_MAX); + IMPLEMENT_RAND_UNSIGNED(uint64_t, UINT64_MAX, limit, + crypto_fast_rng_getbytes(rng, (void*)&val, sizeof(val))); +} + +/** + * As crypto_rand_, but extract the result from a crypto_fast_rng_t. + */ +double +crypto_fast_rng_get_double(crypto_fast_rng_t *rng) +{ + unsigned int u; + crypto_fast_rng_getbytes(rng, (void*)&u, sizeof(u)); return ((double)u) / UINT_MAX_AS_DOUBLE; } diff --git a/src/lib/crypt_ops/include.am b/src/lib/crypt_ops/include.am index 19cfee1355..4730440143 100644 --- a/src/lib/crypt_ops/include.am +++ b/src/lib/crypt_ops/include.am @@ -17,6 +17,7 @@ src_lib_libtor_crypt_ops_a_SOURCES = \ src/lib/crypt_ops/crypto_ope.c \ src/lib/crypt_ops/crypto_pwbox.c \ src/lib/crypt_ops/crypto_rand.c \ + src/lib/crypt_ops/crypto_rand_fast.c \ src/lib/crypt_ops/crypto_rand_numeric.c \ src/lib/crypt_ops/crypto_rsa.c \ src/lib/crypt_ops/crypto_s2k.c \ diff --git a/src/test/test_crypto_rng.c b/src/test/test_crypto_rng.c index 008753d971..0513d2a807 100644 --- a/src/test/test_crypto_rng.c +++ b/src/test/test_crypto_rng.c @@ -173,6 +173,121 @@ test_crypto_rng_strongest(void *arg) #undef N } +static void +test_crypto_rng_fast(void *arg) +{ + (void)arg; + crypto_fast_rng_t *rng = crypto_fast_rng_new(); + tt_assert(rng); + + /* Rudimentary black-block test to make sure that our prng outputs + * have all bits sometimes on and all bits sometimes off. */ + uint64_t m1 = 0, m2 = ~(uint64_t)0; + const int N = 128; + + for (int i=0; i < N; ++i) { + uint64_t v; + crypto_fast_rng_getbytes(rng, (void*)&v, sizeof(v)); + m1 |= v; + m2 &= v; + } + + tt_u64_op(m1, OP_EQ, ~(uint64_t)0); + tt_u64_op(m2, OP_EQ, 0); + + /* Check range functions. */ + int counts[5]; + memset(counts, 0, sizeof(counts)); + for (int i=0; i < N; ++i) { + unsigned u = crypto_fast_rng_get_uint(rng, 5); + tt_int_op(u, OP_GE, 0); + tt_int_op(u, OP_LT, 5); + counts[u]++; + + uint64_t u64 = crypto_fast_rng_get_uint64(rng, UINT64_C(1)<<40); + tt_u64_op(u64, OP_GE, 0); + tt_u64_op(u64, OP_LT, UINT64_C(1)<<40); + } + + /* All values should have come up once. */ + for (int i=0; i<5; ++i) { + tt_int_op(counts[i], OP_GT, 0); + } + + done: + crypto_fast_rng_free(rng); +} + +static void +test_crypto_rng_fast_whitebox(void *arg) +{ + (void)arg; + const size_t buflen = crypto_fast_rng_get_bytes_used_per_stream(); + char *buf = tor_malloc_zero(buflen); + char *buf2 = tor_malloc_zero(buflen); + + crypto_cipher_t *cipher = NULL; + uint8_t seed[CRYPTO_FAST_RNG_SEED_LEN]; + memset(seed, 0, sizeof(seed)); + + /* Start with a prng with zero key and zero IV. */ + crypto_fast_rng_t *rng = crypto_fast_rng_new_from_seed(seed); + tt_assert(rng); + + /* We'll use a stream cipher to keep in sync */ + cipher = crypto_cipher_new_with_iv_and_bits(seed, seed+32, 256); + + /* The first 48 bytes are used for the next seed -- let's make sure we have + * them. + */ + memset(seed, 0, sizeof(seed)); + crypto_cipher_crypt_inplace(cipher, (char*)seed, sizeof(seed)); + + /* if we get 128 bytes, they should match the bytes from the aes256-counter + * stream, starting at position 48. + */ + crypto_fast_rng_getbytes(rng, (uint8_t*)buf, 128); + memset(buf2, 0, 128); + crypto_cipher_crypt_inplace(cipher, buf2, 128); + tt_mem_op(buf, OP_EQ, buf2, 128); + + /* Try that again, with an odd number of bytes. */ + crypto_fast_rng_getbytes(rng, (uint8_t*)buf, 199); + memset(buf2, 0, 199); + crypto_cipher_crypt_inplace(cipher, buf2, 199); + tt_mem_op(buf, OP_EQ, buf2, 199); + + /* Make sure that refilling works as expected: skip all but the last 5 bytes + * of this steam. */ + size_t skip = buflen - (199+128) - 5; + crypto_fast_rng_getbytes(rng, (uint8_t*)buf, skip); + crypto_cipher_crypt_inplace(cipher, buf2, skip); + + /* Now get the next 128 bytes. The first 5 will come from this stream, and + * the next 5 will come from the stream keyed by the new value of 'seed'. */ + crypto_fast_rng_getbytes(rng, (uint8_t*)buf, 128); + memset(buf2, 0, 128); + crypto_cipher_crypt_inplace(cipher, buf2, 5); + crypto_cipher_free(cipher); + cipher = crypto_cipher_new_with_iv_and_bits(seed, seed+32, 256); + memset(seed, 0, sizeof(seed)); + crypto_cipher_crypt_inplace(cipher, (char*)seed, sizeof(seed)); + crypto_cipher_crypt_inplace(cipher, buf2+5, 128-5); + tt_mem_op(buf, OP_EQ, buf2, 128); + + /* And check the next 7 bytes to make sure we didn't discard anything. */ + crypto_fast_rng_getbytes(rng, (uint8_t*)buf, 7); + memset(buf2, 0, 7); + crypto_cipher_crypt_inplace(cipher, buf2, 7); + tt_mem_op(buf, OP_EQ, buf2, 7); + + done: + crypto_fast_rng_free(rng); + crypto_cipher_free(cipher); + tor_free(buf); + tor_free(buf2); +} + struct testcase_t crypto_rng_tests[] = { { "rng", test_crypto_rng, 0, NULL, NULL }, { "rng_range", test_crypto_rng_range, 0, NULL, NULL }, @@ -183,5 +298,7 @@ struct testcase_t crypto_rng_tests[] = { &passthrough_setup, (void*)"nofallback" }, { "rng_strongest_broken", test_crypto_rng_strongest, TT_FORK, &passthrough_setup, (void*)"broken" }, + { "fast", test_crypto_rng_fast, 0, NULL, NULL }, + { "fast_whitebox", test_crypto_rng_fast_whitebox, 0, NULL, NULL }, END_OF_TESTCASES }; From 490e187056d87ddd87981e0ea1b624b61bcbb666 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Sat, 2 Feb 2019 16:01:28 +0100 Subject: [PATCH 0482/2557] Add a benchmark for our several PRNGs. --- src/test/bench.c | 62 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/src/test/bench.c b/src/test/bench.c index 0713eb6719..65fa617cbd 100644 --- a/src/test/bench.c +++ b/src/test/bench.c @@ -14,6 +14,8 @@ #include "core/crypto/onion_tap.h" #include "core/crypto/relay_crypto.h" +#include "lib/intmath/weakrng.h" + #ifdef ENABLE_OPENSSL #include #include @@ -335,6 +337,65 @@ bench_ed25519(void) } } +static void +bench_rand_len(int len) +{ + const int N = 100000; + int i; + char *buf = tor_malloc(len); + uint64_t start,end; + + start = perftime(); + for (i = 0; i < N; ++i) { + crypto_rand(buf, len); + } + end = perftime(); + printf("crypto_rand(%d): %f nsec.\n", len, NANOCOUNT(start,end,N)); + + crypto_fast_rng_t *fr = crypto_fast_rng_new(); + start = perftime(); + for (i = 0; i < N; ++i) { + crypto_fast_rng_getbytes(fr,(uint8_t*)buf,len); + } + end = perftime(); + printf("crypto_fast_rng_getbytes(%d): %f nsec.\n", len, + NANOCOUNT(start,end,N)); + crypto_fast_rng_free(fr); + + if (len <= 32) { + start = perftime(); + for (i = 0; i < N; ++i) { + crypto_strongest_rand((uint8_t*)buf, len); + } + end = perftime(); + printf("crypto_strongest_rand(%d): %f nsec.\n", len, + NANOCOUNT(start,end,N)); + } + + if (len == 4) { + tor_weak_rng_t weak; + tor_init_weak_random(&weak, 1337); + + start = perftime(); + uint32_t t=0; + for (i = 0; i < N; ++i) { + t += tor_weak_random(&weak); + } + end = perftime(); + printf("weak_rand(4): %f nsec.\n", NANOCOUNT(start,end,N)); + } + + tor_free(buf); +} + +static void +bench_rand(void) +{ + bench_rand_len(4); + bench_rand_len(16); + bench_rand_len(128); +} + static void bench_cell_aes(void) { @@ -695,6 +756,7 @@ static struct benchmark_t benchmarks[] = { ENT(onion_TAP), ENT(onion_ntor), ENT(ed25519), + ENT(rand), ENT(cell_aes), ENT(cell_ops), From acbde10fce5d688d70b5a4bfb3a736da838bb4cc Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 6 Feb 2019 09:36:12 -0500 Subject: [PATCH 0483/2557] Add a test-rng program so we can pipe to dieharder. --- .gitignore | 2 ++ src/test/include.am | 11 +++++++-- src/test/test_rng.c | 59 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 70 insertions(+), 2 deletions(-) create mode 100644 src/test/test_rng.c diff --git a/.gitignore b/.gitignore index 6a49285b8a..a40cda02dd 100644 --- a/.gitignore +++ b/.gitignore @@ -248,6 +248,7 @@ uptime-*.json /src/test/test-memwipe /src/test/test-ntor-cl /src/test/test-hs-ntor-cl +/src/test/test-rng /src/test/test-switch-id /src/test/test-timers /src/test/test_workqueue @@ -258,6 +259,7 @@ uptime-*.json /src/test/test-ntor-cl.exe /src/test/test-hs-ntor-cl.exe /src/test/test-memwipe.exe +/src/test/test-rng.exe /src/test/test-switch-id.exe /src/test/test-timers.exe /src/test/test_workqueue.exe diff --git a/src/test/include.am b/src/test/include.am index 1b8f7f5ad3..8622fcbaa1 100644 --- a/src/test/include.am +++ b/src/test/include.am @@ -68,7 +68,8 @@ noinst_PROGRAMS+= \ src/test/test-process \ src/test/test_workqueue \ src/test/test-switch-id \ - src/test/test-timers + src/test/test-timers \ + src/test/test-rng endif src_test_AM_CPPFLAGS = -DSHARE_DATADIR="\"$(datadir)\"" \ @@ -258,7 +259,13 @@ src_test_test_LDADD = \ src_test_test_slow_CPPFLAGS = $(src_test_test_CPPFLAGS) src_test_test_slow_CFLAGS = $(src_test_test_CFLAGS) src_test_test_slow_LDADD = $(src_test_test_LDADD) -src_test_test_slow_LDFLAGS = $(src_test_test_LDFLAGS) +src_test_test_slow_LDFLAGS =@TOR_LDFLAGS_openssl@ + +src_test_test_rng_CPPFLAGS = $(src_test_test_CPPFLAGS) +src_test_test_rng_CFLAGS = $(src_test_test_CFLAGS) +src_test_test_rng_SOURCES = src/test/test_rng.c +src_test_test_rng_LDFLAGS = $(src_test_test_LDFLAGS) +src_test_test_rng_LDADD = $(src_test_test_LDADD) src_test_test_memwipe_CPPFLAGS = $(src_test_test_CPPFLAGS) # Don't use bugtrap cflags here: memwipe tests require memory violations. diff --git a/src/test/test_rng.c b/src/test/test_rng.c new file mode 100644 index 0000000000..c749de112a --- /dev/null +++ b/src/test/test_rng.c @@ -0,0 +1,59 @@ +/* Copyright (c) 2016-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/* + * Example usage: + * + * ./src/test/test-rng --emit | dieharder -g 200 -a + * + * Remember, dieharder can tell you that your RNG is completely broken, but if + * your RNG is not _completely_ broken, dieharder cannot tell you whether your + * RNG is actually secure. + */ + +#include "orconfig.h" + +#ifdef HAVE_UNISTD_H +#include +#endif +#include +#include +#include + +#include "lib/crypt_ops/crypto_rand.h" + +int +main(int argc, char **argv) +{ + uint8_t buf[0x123]; + + if (argc != 2 || strcmp(argv[1], "--emit")) { + fprintf(stderr, "If you want me to fill stdout with a bunch of random " + "bytes, you need to say --emit.\n"); + return 1; + } + + if (crypto_seed_rng() < 0) { + fprintf(stderr, "Can't seed RNG.\n"); + return 1; + } + +#if 0 + while (1) { + crypto_rand(buf, sizeof(buf)); + if (write(1 /*stdout*/, buf, sizeof(buf)) != sizeof(buf)) { + fprintf(stderr, "write() failed: %s\n", strerror(errno)); + return 1; + } + } +#endif + + crypto_fast_rng_t *rng = crypto_fast_rng_new(); + while (1) { + crypto_fast_rng_getbytes(rng, buf, sizeof(buf)); + if (write(1 /*stdout*/, buf, sizeof(buf)) != sizeof(buf)) { + fprintf(stderr, "write() failed: %s\n", strerror(errno)); + return 1; + } + } +} From 3f28b98220a49730b8c58c45e42598ad2bada47b Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 7 Feb 2019 08:18:08 -0500 Subject: [PATCH 0484/2557] Add test for crypto_fast_rng_get_double(). --- src/test/test_crypto_rng.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/test/test_crypto_rng.c b/src/test/test_crypto_rng.c index 0513d2a807..075c876b8a 100644 --- a/src/test/test_crypto_rng.c +++ b/src/test/test_crypto_rng.c @@ -207,6 +207,10 @@ test_crypto_rng_fast(void *arg) uint64_t u64 = crypto_fast_rng_get_uint64(rng, UINT64_C(1)<<40); tt_u64_op(u64, OP_GE, 0); tt_u64_op(u64, OP_LT, UINT64_C(1)<<40); + + double d = crypto_fast_rng_get_double(rng); + tt_assert(d >= 0.0); + tt_assert(d < 1.0); } /* All values should have come up once. */ From 6a29aa7b8c86151701df4b881aded4fc152ea116 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 7 Feb 2019 08:18:46 -0500 Subject: [PATCH 0485/2557] Add whitebox test for the long-output optimization of fast_rng --- src/test/test_crypto_rng.c | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/src/test/test_crypto_rng.c b/src/test/test_crypto_rng.c index 075c876b8a..23b0c66514 100644 --- a/src/test/test_crypto_rng.c +++ b/src/test/test_crypto_rng.c @@ -229,8 +229,9 @@ test_crypto_rng_fast_whitebox(void *arg) const size_t buflen = crypto_fast_rng_get_bytes_used_per_stream(); char *buf = tor_malloc_zero(buflen); char *buf2 = tor_malloc_zero(buflen); + char *buf3 = NULL, *buf4 = NULL; - crypto_cipher_t *cipher = NULL; + crypto_cipher_t *cipher = NULL, *cipher2 = NULL; uint8_t seed[CRYPTO_FAST_RNG_SEED_LEN]; memset(seed, 0, sizeof(seed)); @@ -285,11 +286,26 @@ test_crypto_rng_fast_whitebox(void *arg) crypto_cipher_crypt_inplace(cipher, buf2, 7); tt_mem_op(buf, OP_EQ, buf2, 7); + /* Now try the optimization for long outputs. */ + buf3 = tor_malloc(65536); + crypto_fast_rng_getbytes(rng, (uint8_t*)buf3, 65536); + + buf4 = tor_malloc_zero(65536); + uint8_t seed2[CRYPTO_FAST_RNG_SEED_LEN]; + memset(seed2, 0, sizeof(seed2)); + crypto_cipher_crypt_inplace(cipher, (char*)seed2, sizeof(seed2)); + cipher2 = crypto_cipher_new_with_iv_and_bits(seed2, seed2+32, 256); + crypto_cipher_crypt_inplace(cipher2, buf4, 65536); + tt_mem_op(buf3, OP_EQ, buf4, 65536); + done: crypto_fast_rng_free(rng); crypto_cipher_free(cipher); + crypto_cipher_free(cipher2); tor_free(buf); tor_free(buf2); + tor_free(buf3); + tor_free(buf4); } struct testcase_t crypto_rng_tests[] = { From b054a6c6b9cdda5c35b905f98ff092f15d8b749d Mon Sep 17 00:00:00 2001 From: Matt Traudt Date: Thu, 14 Feb 2019 16:18:58 -0500 Subject: [PATCH 0486/2557] kist: When readding chans, check correct chan's sched_heap_idx Closes #29508 Signed-off-by: David Goulet --- changes/bug29508 | 3 +++ src/core/or/scheduler_kist.c | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 changes/bug29508 diff --git a/changes/bug29508 b/changes/bug29508 new file mode 100644 index 0000000000..ee728bbbc9 --- /dev/null +++ b/changes/bug29508 @@ -0,0 +1,3 @@ + o Minor bugfixes (scheduler): + - When readding channels to the pending list, check the correct channel's + sched_heap_idx. Fixes bug 29508; bugfix on 0.3.2.10 diff --git a/src/core/or/scheduler_kist.c b/src/core/or/scheduler_kist.c index 34e5672074..01be751ad2 100644 --- a/src/core/or/scheduler_kist.c +++ b/src/core/or/scheduler_kist.c @@ -724,7 +724,7 @@ kist_scheduler_run(void) SMARTLIST_FOREACH_BEGIN(to_readd, channel_t *, readd_chan) { scheduler_set_channel_state(readd_chan, SCHED_CHAN_PENDING); if (!smartlist_contains(cp, readd_chan)) { - if (!SCHED_BUG(chan->sched_heap_idx != -1, chan)) { + if (!SCHED_BUG(readd_chan->sched_heap_idx != -1, readd_chan)) { /* XXXX Note that the check above is in theory redundant with * the smartlist_contains check. But let's make sure we're * not messing anything up, and leave them both for now. */ From f07c6ae57cd2ac11d438232b94a15f99b96e8989 Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Wed, 13 Feb 2019 16:58:15 +0200 Subject: [PATCH 0487/2557] Add histogram fields in header file that allow specifying edges. --- src/core/or/circuitpadding.h | 57 +++++++++++++++++++++++++++++------- 1 file changed, 46 insertions(+), 11 deletions(-) diff --git a/src/core/or/circuitpadding.h b/src/core/or/circuitpadding.h index 92fd4fc2d5..57bd379816 100644 --- a/src/core/or/circuitpadding.h +++ b/src/core/or/circuitpadding.h @@ -228,13 +228,15 @@ typedef uint16_t circpad_statenum_t; #define CIRCPAD_STATENUM_MAX (UINT16_MAX) /** A histogram is used to sample padding delays given a machine state. This - * constant defines the maximum histogram width (i.e. the max number of bins) + * constant defines the maximum histogram width (i.e. the max number of bins). * - * Each histogram bin is twice as large as the previous. Two exceptions: The - * first bin has zero width (which means that minimum delay is applied to the - * next padding cell), and the last bin (infinity bin) has infinite width - * (which means that the next padding cell will be delayed infinitely). */ -#define CIRCPAD_MAX_HISTOGRAM_LEN (sizeof(circpad_delay_t)*8 + 1) + * The current limit is arbitrary and could be raised if there is a need, + * however too many bins will be hard to serialize in the future. + * + * Memory concerns are not so great here since the corresponding histogram and + * histogram_edges arrays are global and not per-circuit. + */ +#define CIRCPAD_MAX_HISTOGRAM_LEN (100) /** * A state of a padding state machine. The information here are immutable and @@ -248,15 +250,48 @@ typedef uint16_t circpad_statenum_t; * or the consensus. */ typedef struct circpad_state_t { - /** If a histogram is used for this state, this specifies the number of bins - * of this histogram. Histograms must have at least 2 bins. + /** + * If a histogram is used for this state, this specifies the number of bins + * of this histogram. Histograms must have at least 2 bins. * - * If a delay probability distribution is used for this state, this is set - * to 0. */ + * In particular, the following histogram: + * + * Tokens + * + + * 10 | +----+ + * 9 | | | +---------+ + * 8 | | | | | + * 7 | | | +-----+ | + * 6 +----+ Bin+-----+ | +---------------+ + * 5 | | #1 | | | | | + * | Bin| | Bin | Bin | Bin #4 | Bin #5 | + * | #0 | | #2 | #3 | | (infinity bin)| + * | | | | | | | + * | | | | | | | + * 0 +----+----+-----+-----+---------+---------------+ + * 0 100 200 350 500 1000 ∞ microseconds + * + * would be specified the following way: + * histogram_len = 6; + * histogram[] = { 6, 10, 6, 7, 9, 6 } + * histogram_edges[] = { 0, 100, 200, 350, 500, 1000 } + * + * If a delay probability distribution is used for this state, this is set + * to 0. */ circpad_hist_index_t histogram_len; /** The histogram itself: an array of uint16s of tokens, whose - * widths are exponentially spaced, in microseconds */ + * widths are exponentially spaced, in microseconds. + * + * This array must have histogram_len elements. */ circpad_hist_token_t histogram[CIRCPAD_MAX_HISTOGRAM_LEN]; + /* The histogram bin edges in usec. + * + * Each element of this array specifies the left edge of the corresponding + * bin. The rightmost edge is always infinity and is not specified in this + * array. + * + * This array must have histogram_len+1 elements. */ + circpad_delay_t histogram_edges[CIRCPAD_MAX_HISTOGRAM_LEN+1]; /** Total number of tokens in this histogram. This is a constant and is *not* * decremented every time we spend a token. It's used for initializing and * refilling the histogram. */ From 98af25e0137f283569aee3537a0ef19cda403426 Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Wed, 13 Feb 2019 16:17:01 +0200 Subject: [PATCH 0488/2557] Remove start_usec/range_usec and make equivalent fields for distributions. --- src/core/or/circuitpadding.c | 18 +++++++----------- src/core/or/circuitpadding.h | 25 +++++++------------------ src/test/test_circuitpadding.c | 24 ++++++++++++------------ 3 files changed, 26 insertions(+), 41 deletions(-) diff --git a/src/core/or/circuitpadding.c b/src/core/or/circuitpadding.c index 0dadc52139..a9c5796caa 100644 --- a/src/core/or/circuitpadding.c +++ b/src/core/or/circuitpadding.c @@ -399,17 +399,17 @@ circpad_choose_state_length(circpad_machine_state_t *mi) */ static circpad_delay_t circpad_distribution_sample_iat_delay(const circpad_state_t *state, - circpad_delay_t start_usec) + circpad_delay_t min_delay) { double val = circpad_distribution_sample(state->iat_dist); /* These comparisons are safe, because the output is in the range * [0, 2**32), and double has a precision of 53 bits. */ val = MAX(0, val); - val = MIN(val, state->range_usec); + val = MIN(val, state->dist_max_usec); - /* This addition is exact: val is at most 2**32-1, start_usec + /* This addition is exact: val is at most 2**32-1, min_delay * is at most 2**32-1, and doubles have a precision of 53 bits. */ - val += start_usec; + val += min_delay; /* Clamp the distribution at infinite delay val */ return (circpad_delay_t)MIN(tor_llround(val), CIRCPAD_DELAY_INFINITE); @@ -429,7 +429,6 @@ circpad_machine_sample_delay(circpad_machine_state_t *mi) const circpad_hist_token_t *histogram = NULL; circpad_hist_index_t curr_bin = 0; circpad_delay_t bin_start, bin_end; - circpad_delay_t start_usec; /* These three must all be larger than circpad_hist_token_t, because * we sum several circpad_hist_token_t values across the histogram */ uint64_t curr_weight = 0; @@ -438,14 +437,11 @@ circpad_machine_sample_delay(circpad_machine_state_t *mi) tor_assert(state); - if (state->use_rtt_estimate) - start_usec = mi->rtt_estimate_usec+state->start_usec; - else - start_usec = state->start_usec; - if (state->iat_dist.type != CIRCPAD_DIST_NONE) { /* Sample from a fixed IAT distribution and return */ - return circpad_distribution_sample_iat_delay(state, start_usec); + circpad_delay_t min_iat_delay = state->use_rtt_estimate ? + mi->rtt_estimate_usec + state->dist_min_usec : state->dist_min_usec; + return circpad_distribution_sample_iat_delay(state, min_iat_delay); } else if (state->token_removal != CIRCPAD_TOKEN_REMOVAL_NONE) { /* We have a mutable histogram. Do basic sanity check and apply: */ if (BUG(!mi->histogram) || diff --git a/src/core/or/circuitpadding.h b/src/core/or/circuitpadding.h index 57bd379816..918a7d0ddd 100644 --- a/src/core/or/circuitpadding.h +++ b/src/core/or/circuitpadding.h @@ -297,24 +297,6 @@ typedef struct circpad_state_t { * refilling the histogram. */ uint32_t histogram_total_tokens; - /** Minimum padding delay of this state in microseconds. - * - * If histograms are used, this is the left (and right) bound of the first - * bin (since it has zero width). - * - * If a delay probability distribution is used, this represents the minimum - * delay we can sample from the distribution. - */ - circpad_delay_t start_usec; - - /** If histograms are used, this is the width of the whole histogram in - * microseconds, and it's used to calculate individual bin width. - * - * If a delay probability distribution is used, this is used as the max - * delay we can sample from the distribution. - */ - circpad_delay_t range_usec; - /** * Represents a delay probability distribution (aka IAT distribution). It's a * parametrized way of encoding inter-packet delay information in @@ -327,6 +309,13 @@ typedef struct circpad_state_t { * results of sampling from this distribution (range_sec is used as a max). */ circpad_distribution_t iat_dist; + /* If a delay probability distribution is used, this represents the minimum + * delay we can sample from the distribution. */ + circpad_delay_t dist_min_usec; + /* If a delay probability distribution is used, this is used as the max + * delay we can sample from the distribution. + */ + circpad_delay_t dist_max_usec; /** * The length dist is a parameterized way of encoding how long this diff --git a/src/test/test_circuitpadding.c b/src/test/test_circuitpadding.c index 12a07fa957..63d8f8e099 100644 --- a/src/test/test_circuitpadding.c +++ b/src/test/test_circuitpadding.c @@ -2067,48 +2067,48 @@ helper_circpad_circ_distribution_machine_setup(int min, int max) zero_st->iat_dist.type = CIRCPAD_DIST_UNIFORM; zero_st->iat_dist.param1 = min; zero_st->iat_dist.param2 = max; - zero_st->start_usec = min; - zero_st->range_usec = max; + zero_st->dist_min_usec = min; + zero_st->dist_max_usec = max; circpad_state_t *first_st = &circ_client_machine.states[1]; first_st->next_state[CIRCPAD_EVENT_NONPADDING_RECV] = 2; first_st->iat_dist.type = CIRCPAD_DIST_LOGISTIC; first_st->iat_dist.param1 = min; first_st->iat_dist.param2 = max; - first_st->start_usec = min; - first_st->range_usec = max; + first_st->dist_min_usec = min; + first_st->dist_max_usec = max; circpad_state_t *second_st = &circ_client_machine.states[2]; second_st->next_state[CIRCPAD_EVENT_NONPADDING_RECV] = 3; second_st->iat_dist.type = CIRCPAD_DIST_LOG_LOGISTIC; second_st->iat_dist.param1 = min; second_st->iat_dist.param2 = max; - second_st->start_usec = min; - second_st->range_usec = max; + second_st->dist_min_usec = min; + second_st->dist_max_usec = max; circpad_state_t *third_st = &circ_client_machine.states[3]; third_st->next_state[CIRCPAD_EVENT_NONPADDING_RECV] = 4; third_st->iat_dist.type = CIRCPAD_DIST_GEOMETRIC; third_st->iat_dist.param1 = min; third_st->iat_dist.param2 = max; - third_st->start_usec = min; - third_st->range_usec = max; + third_st->dist_min_usec = min; + third_st->dist_max_usec = max; circpad_state_t *fourth_st = &circ_client_machine.states[4]; fourth_st->next_state[CIRCPAD_EVENT_NONPADDING_RECV] = 5; fourth_st->iat_dist.type = CIRCPAD_DIST_WEIBULL; fourth_st->iat_dist.param1 = min; fourth_st->iat_dist.param2 = max; - fourth_st->start_usec = min; - fourth_st->range_usec = max; + fourth_st->dist_min_usec = min; + fourth_st->dist_max_usec = max; circpad_state_t *fifth_st = &circ_client_machine.states[5]; fifth_st->next_state[CIRCPAD_EVENT_NONPADDING_RECV] = 6; fifth_st->iat_dist.type = CIRCPAD_DIST_PARETO; fifth_st->iat_dist.param1 = min; fifth_st->iat_dist.param2 = max; - fifth_st->start_usec = min; - fifth_st->range_usec = max; + fifth_st->dist_min_usec = min; + fifth_st->dist_max_usec = max; } /** Simple test that the padding delays sampled from a uniform distribution From 80abe4170d590ca3b5bcf136457f16d0d6e3692f Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Fri, 15 Feb 2019 17:16:18 +0200 Subject: [PATCH 0489/2557] Update all the histogram functions to use the new design. --- changes/bug29298 | 6 ++ src/core/or/circuitpadding.c | 113 ++++++++++++++--------------------- src/core/or/circuitpadding.h | 4 ++ 3 files changed, 55 insertions(+), 68 deletions(-) create mode 100644 changes/bug29298 diff --git a/changes/bug29298 b/changes/bug29298 new file mode 100644 index 0000000000..6e447b62dd --- /dev/null +++ b/changes/bug29298 @@ -0,0 +1,6 @@ + o Minor features (circuit padding): + - Allow the padding machine designer to pick the edges of their histogram + instead of trying to compute them automatically using an exponential + formula. Resolves some undefined behavior in the case of small histograms + and allows greater flexibility on machine design. Closes ticket 29298; + bugfix on 0.4.0.1-alpha. \ No newline at end of file diff --git a/src/core/or/circuitpadding.c b/src/core/or/circuitpadding.c index a9c5796caa..383e59d1e6 100644 --- a/src/core/or/circuitpadding.c +++ b/src/core/or/circuitpadding.c @@ -222,25 +222,15 @@ circpad_machine_current_state(const circpad_machine_state_t *mi) } /** - * Calculate the lower bound of a histogram bin. The upper bound - * is obtained by calling this function with bin+1, and subtracting 1. - * - * The 0th bin has a special value -- it only represents start_usec. - * This is so we can specify a probability on 0-delay values. - * - * After bin 0, bins are exponentially spaced, so that each subsequent - * bin is twice as large as the previous. This is done so that higher - * time resolution is given to lower time values. - * - * The infinity bin is a the last bin in the array (histogram_len-1). - * It has a usec value of CIRCPAD_DELAY_INFINITE (UINT32_MAX). + * Get the lower bound of a histogram bin. The upper bound is obtained by + * calling this function with bin+1, and subtracting 1. */ STATIC circpad_delay_t circpad_histogram_bin_to_usec(const circpad_machine_state_t *mi, circpad_hist_index_t bin) { const circpad_state_t *state = circpad_machine_current_state(mi); - circpad_delay_t start_usec; + circpad_delay_t rtt_add_usec = 0; /* Our state should have been checked to be non-null by the caller * (circpad_machine_remove_token()) */ @@ -248,27 +238,29 @@ circpad_histogram_bin_to_usec(const circpad_machine_state_t *mi, return CIRCPAD_DELAY_INFINITE; } - if (state->use_rtt_estimate) - start_usec = mi->rtt_estimate_usec+state->start_usec; - else - start_usec = state->start_usec; - - if (bin >= CIRCPAD_INFINITY_BIN(state)) + /* The infinity bin has an upper bound of infinity, so make sure we return + * that if they ask for it. */ + if (bin > CIRCPAD_INFINITY_BIN(mi)) { return CIRCPAD_DELAY_INFINITE; + } - if (bin == 0) - return start_usec; + /* If we are using an RTT estimate, consider it as well. */ + if (state->use_rtt_estimate) { + rtt_add_usec = mi->rtt_estimate_usec; + } - if (bin == 1) - return start_usec+1; + return state->histogram_edges[bin] + rtt_add_usec; +} - /* The bin widths double every index, so that we can have more resolution - * for lower time values in the histogram. */ - const circpad_time_t bin_width_exponent = - 1 << (CIRCPAD_INFINITY_BIN(state) - bin); - return (circpad_delay_t)MIN(start_usec + - state->range_usec/bin_width_exponent, - CIRCPAD_DELAY_INFINITE); +/** + * Like circpad_histogram_bin_to_usec() but return the upper bound of bin. + * (The upper bound is included in the bin.) + */ +STATIC circpad_delay_t +histogram_get_bin_upper_bound(const circpad_machine_state_t *mi, + circpad_hist_index_t bin) +{ + return circpad_histogram_bin_to_usec(mi, bin+1) - 1; } /** Return the midpoint of the histogram bin bin_index. */ @@ -287,19 +279,17 @@ circpad_get_histogram_bin_midpoint(const circpad_machine_state_t *mi, * Return the bin that contains the usec argument. * "Contains" is defined as us in [lower, upper). * - * This function will never return the infinity bin (histogram_len-1), - * in order to simplify the rest of the code. - * - * This means that technically the last bin (histogram_len-2) - * has range [start_usec+range_usec, CIRCPAD_DELAY_INFINITE]. + * This function will never return the infinity bin (histogram_len-1), in order + * to simplify the rest of the code, so if a usec is provided that falls above + * the highest non-infinity bin, that bin index will be returned. */ STATIC circpad_hist_index_t circpad_histogram_usec_to_bin(const circpad_machine_state_t *mi, circpad_delay_t usec) { const circpad_state_t *state = circpad_machine_current_state(mi); - circpad_delay_t start_usec; - int32_t bin; /* Larger than return type to properly clamp overflow */ + circpad_delay_t rtt_add_usec = 0; + int32_t bin; /* Our state should have been checked to be non-null by the caller * (circpad_machine_remove_token()) */ @@ -307,34 +297,25 @@ circpad_histogram_usec_to_bin(const circpad_machine_state_t *mi, return 0; } - if (state->use_rtt_estimate) - start_usec = mi->rtt_estimate_usec+state->start_usec; - else - start_usec = state->start_usec; + /* If we are using an RTT estimate, consider it as well. */ + if (state->use_rtt_estimate) { + rtt_add_usec = mi->rtt_estimate_usec; + } - /* The first bin (#0) has zero width and starts (and ends) at start_usec. */ - if (usec <= start_usec) - return 0; + /* Walk through the bins and check the upper bound of each bin, if 'usec' is + * less-or-equal to that, return that bin. If rtt_estimate is enabled then + * add that to the upper bound of each bin. + * + * We don't want to return the infinity bin here, so don't go there. */ + for (bin = 0 ; bin < CIRCPAD_INFINITY_BIN(state) ; bin++) { + if (usec <= histogram_get_bin_upper_bound(mi, bin) + rtt_add_usec) { + return bin; + } + } - if (usec == start_usec+1) - return 1; - - const circpad_time_t histogram_range_usec = state->range_usec; - /* We need to find the bin corresponding to our position in the range. - * Since bins are exponentially spaced in powers of two, we need to - * take the log2 of our position in histogram_range_usec. However, - * since tor_log2() returns the floor(log2(u64)), we have to adjust - * it to behave like ceil(log2(u64)). This is verified in our tests - * to properly invert the operation done in - * circpad_histogram_bin_to_usec(). */ - bin = CIRCPAD_INFINITY_BIN(state) - - tor_log2(2*histogram_range_usec/(usec-start_usec+1)); - - /* Clamp the return value to account for timevals before the start - * of bin 0, or after the last bin. Don't return the infinity bin - * index. */ - bin = MIN(MAX(bin, 1), CIRCPAD_INFINITY_BIN(state)-1); - return bin; + /* We don't want to return the infinity bin here, so if we still didn't find + * the right bin, return the highest non-infinity bin */ + return CIRCPAD_INFINITY_BIN(state)-1; } /** @@ -506,10 +487,6 @@ circpad_machine_sample_delay(circpad_machine_state_t *mi) * function below samples from [bin_start, bin_end) */ bin_end = circpad_histogram_bin_to_usec(mi, curr_bin+1); - /* Truncate the high bin in case it's the infinity bin: - * Don't actually schedule an "infinite"-1 delay */ - bin_end = MIN(bin_end, start_usec+state->range_usec); - // Sample uniformly between histogram[i] to histogram[i+1]-1, // but no need to sample if they are the same timeval (aka bin 0 or bin 1). if (bin_end <= bin_start+1) @@ -621,7 +598,7 @@ circpad_machine_first_higher_index(const circpad_machine_state_t *mi, /* Don't remove from the infinity bin */ for (; bin < CIRCPAD_INFINITY_BIN(mi); bin++) { if (mi->histogram[bin] && - circpad_histogram_bin_to_usec(mi, bin+1) > target_bin_usec) { + histogram_get_bin_upper_bound(mi, bin) >= target_bin_usec) { return bin; } } diff --git a/src/core/or/circuitpadding.h b/src/core/or/circuitpadding.h index 918a7d0ddd..b9e903b9f0 100644 --- a/src/core/or/circuitpadding.h +++ b/src/core/or/circuitpadding.h @@ -710,6 +710,10 @@ circpad_send_command_to_hop,(struct origin_circuit_t *circ, uint8_t hopnum, uint8_t relay_command, const uint8_t *payload, ssize_t payload_len)); +STATIC circpad_delay_t +histogram_get_bin_upper_bound(const circpad_machine_state_t *mi, + circpad_hist_index_t bin); + #ifdef TOR_UNIT_TESTS extern smartlist_t *origin_padding_machines; extern smartlist_t *relay_padding_machines; From 3093d8afbeb9f1a1901936d27cb788c03917dae0 Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Fri, 15 Feb 2019 17:27:09 +0200 Subject: [PATCH 0490/2557] Fix tests to use the new design. - All histogram tests were using start_usec/range_usec. We now manually specify the edges. - Also add a test for histogram_get_bin_upper_bound(). --- src/core/or/circuitpadding.c | 23 +++++-- src/test/test_circuitpadding.c | 107 +++++++++++++++++++++------------ 2 files changed, 87 insertions(+), 43 deletions(-) diff --git a/src/core/or/circuitpadding.c b/src/core/or/circuitpadding.c index 383e59d1e6..ceb9181f94 100644 --- a/src/core/or/circuitpadding.c +++ b/src/core/or/circuitpadding.c @@ -2086,13 +2086,15 @@ circpad_circ_client_machine_init(void) // FIXME: Tune this histogram circ_client_machine->states[CIRCPAD_STATE_BURST].histogram_len = 2; - circ_client_machine->states[CIRCPAD_STATE_BURST].start_usec = 500; - circ_client_machine->states[CIRCPAD_STATE_BURST].range_usec = 1000000; + circ_client_machine->states[CIRCPAD_STATE_BURST].histogram_edges[0]= 500; + circ_client_machine->states[CIRCPAD_STATE_BURST].histogram_edges[1]= 1000000; + /* We have 5 tokens in the histogram, which means that all circuits will look * like they have 7 hops (since we start this machine after the second hop, * and tokens are decremented for any valid hops, and fake extends are * used after that -- 2+5==7). */ circ_client_machine->states[CIRCPAD_STATE_BURST].histogram[0] = 5; + circ_client_machine->states[CIRCPAD_STATE_BURST].histogram_total_tokens = 5; circ_client_machine->machine_num = smartlist_len(origin_padding_machines); @@ -2143,8 +2145,9 @@ circpad_circ_responder_machine_init(void) circ_responder_machine->states[CIRCPAD_STATE_BURST].use_rtt_estimate = 1; /* The histogram is 2 bins: an empty one, and infinity */ circ_responder_machine->states[CIRCPAD_STATE_BURST].histogram_len = 2; - circ_responder_machine->states[CIRCPAD_STATE_BURST].start_usec = 5000; - circ_responder_machine->states[CIRCPAD_STATE_BURST].range_usec = 1000000; + circ_responder_machine->states[CIRCPAD_STATE_BURST].histogram_edges[0]= 500; + circ_responder_machine->states[CIRCPAD_STATE_BURST].histogram_edges[1] = + 1000000; /* During burst state we wait forever for padding to arrive. We are waiting for a padding cell from the client to come in, so that we @@ -2175,8 +2178,15 @@ circpad_circ_responder_machine_init(void) before you send a padding response */ circ_responder_machine->states[CIRCPAD_STATE_GAP].use_rtt_estimate = 1; circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram_len = 6; - circ_responder_machine->states[CIRCPAD_STATE_GAP].start_usec = 5000; - circ_responder_machine->states[CIRCPAD_STATE_GAP].range_usec = 1000000; + /* Specify histogram bins */ + circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram_edges[0]= 500; + circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram_edges[1]= 1000; + circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram_edges[2]= 2000; + circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram_edges[3]= 4000; + circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram_edges[4]= 8000; + circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram_edges[5]= 16000; + circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram_edges[6]=1000000; + /* Specify histogram tokens */ circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram[0] = 0; circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram[1] = 1; circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram[2] = 2; @@ -2184,6 +2194,7 @@ circpad_circ_responder_machine_init(void) circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram[4] = 1; /* Total number of tokens */ circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram_total_tokens = 6; + circ_responder_machine->states[CIRCPAD_STATE_GAP].token_removal = CIRCPAD_TOKEN_REMOVAL_CLOSEST_USEC; diff --git a/src/test/test_circuitpadding.c b/src/test/test_circuitpadding.c index 63d8f8e099..65bf529e42 100644 --- a/src/test/test_circuitpadding.c +++ b/src/test/test_circuitpadding.c @@ -334,7 +334,7 @@ test_circuitpadding_rtt(void *arg) OP_EQ, relay_side->padding_info[0]->rtt_estimate_usec+ circpad_machine_current_state( - relay_side->padding_info[0])->start_usec); + relay_side->padding_info[0])->histogram_edges[0]); circpad_cell_event_nonpadding_received((circuit_t*)relay_side); circpad_cell_event_nonpadding_received((circuit_t*)relay_side); @@ -350,7 +350,7 @@ test_circuitpadding_rtt(void *arg) OP_EQ, relay_side->padding_info[0]->rtt_estimate_usec+ circpad_machine_current_state( - relay_side->padding_info[0])->start_usec); + relay_side->padding_info[0])->histogram_edges[0]); /* Test 2: Termination of RTT measurement (from the previous test) */ tt_int_op(relay_side->padding_info[0]->stop_rtt_update, OP_EQ, 1); @@ -368,7 +368,7 @@ test_circuitpadding_rtt(void *arg) OP_EQ, relay_side->padding_info[0]->rtt_estimate_usec+ circpad_machine_current_state( - relay_side->padding_info[0])->start_usec); + relay_side->padding_info[0])->histogram_edges[0]); /* Test 3: Make sure client side machine properly ignores RTT */ circpad_cell_event_nonpadding_received((circuit_t*)client_side); @@ -384,7 +384,7 @@ test_circuitpadding_rtt(void *arg) tt_int_op(circpad_histogram_bin_to_usec(client_side->padding_info[0], 0), OP_EQ, circpad_machine_current_state( - client_side->padding_info[0])->start_usec); + client_side->padding_info[0])->histogram_edges[0]); done: free_fake_orcirc(relay_side); circuitmux_detach_all_circuits(dummy_channel.cmux, NULL); @@ -415,19 +415,25 @@ helper_create_basic_machine(void) circ_client_machine.states[CIRCPAD_STATE_BURST]. next_state[CIRCPAD_EVENT_NONPADDING_SENT] = CIRCPAD_STATE_CANCEL; - // FIXME: Is this what we want? circ_client_machine.states[CIRCPAD_STATE_BURST].token_removal = CIRCPAD_TOKEN_REMOVAL_HIGHER; - // FIXME: Tune this histogram circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_len = 5; - circ_client_machine.states[CIRCPAD_STATE_BURST].start_usec = 500; - circ_client_machine.states[CIRCPAD_STATE_BURST].range_usec = 1000000; + + circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_edges[0] = 500; + circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_edges[1] = 2500; + circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_edges[2] = 5000; + circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_edges[3] = 10000; + circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_edges[4] = 20000; + circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_edges[5] = 400000; + circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_edges[6] = 1000000; + circ_client_machine.states[CIRCPAD_STATE_BURST].histogram[0] = 1; circ_client_machine.states[CIRCPAD_STATE_BURST].histogram[1] = 0; circ_client_machine.states[CIRCPAD_STATE_BURST].histogram[2] = 2; circ_client_machine.states[CIRCPAD_STATE_BURST].histogram[3] = 2; circ_client_machine.states[CIRCPAD_STATE_BURST].histogram[4] = 2; + circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_total_tokens = 7; circ_client_machine.states[CIRCPAD_STATE_BURST].use_rtt_estimate = 1; @@ -459,15 +465,25 @@ helper_create_machine_with_big_histogram(circpad_removal_t removal_strategy) burst_state->token_removal = CIRCPAD_TOKEN_REMOVAL_HIGHER; burst_state->histogram_len = BIG_HISTOGRAM_LEN; - burst_state->start_usec = 0; - burst_state->range_usec = 1000; int n_tokens = 0; - for (int i = 0; i < BIG_HISTOGRAM_LEN ; i++) { + int i; + for (i = 0; i < BIG_HISTOGRAM_LEN ; i++) { burst_state->histogram[i] = tokens_per_bin; n_tokens += tokens_per_bin; } + burst_state->histogram_edges[0] = 0; + burst_state->histogram_edges[1] = 1; + burst_state->histogram_edges[2] = 7; + burst_state->histogram_edges[3] = 15; + burst_state->histogram_edges[4] = 31; + burst_state->histogram_edges[5] = 62; + burst_state->histogram_edges[6] = 125; + burst_state->histogram_edges[7] = 250; + burst_state->histogram_edges[8] = 500; + burst_state->histogram_edges[9] = 1000; + burst_state->histogram_total_tokens = n_tokens; burst_state->length_dist.type = CIRCPAD_DIST_UNIFORM; burst_state->length_dist.param1 = n_tokens; @@ -528,12 +544,20 @@ test_circuitpadding_token_removal_higher(void *arg) /* Test left boundaries of each histogram bin: */ const circpad_delay_t bin_left_bounds[] = - {0, 1, 7, 15, 31, 62, 125, 250, 500, CIRCPAD_DELAY_INFINITE}; - for (int i = 0; i < BIG_HISTOGRAM_LEN ; i++) { + {0, 1, 7, 15, 31, 62, 125, 250, 500, 1000, CIRCPAD_DELAY_INFINITE}; + for (int i = 0; i <= BIG_HISTOGRAM_LEN ; i++) { tt_uint_op(bin_left_bounds[i], OP_EQ, circpad_histogram_bin_to_usec(mi, i)); } + /* Test right boundaries of each histogram bin: */ + const circpad_delay_t bin_right_bounds[] = + {0, 6, 14, 30, 61, 124, 249, 499, 999, CIRCPAD_DELAY_INFINITE-1}; + for (int i = 0; i < BIG_HISTOGRAM_LEN ; i++) { + tt_uint_op(bin_right_bounds[i], OP_EQ, + histogram_get_bin_upper_bound(mi, i)); + } + /* Check that all bins have two tokens right now */ for (int i = 0; i < BIG_HISTOGRAM_LEN ; i++) { tt_int_op(mi->histogram[i], OP_EQ, 2); @@ -577,8 +601,8 @@ test_circuitpadding_token_removal_higher(void *arg) /* Test below the lowest bin, for coverage */ tt_int_op(client_side->padding_info[0]->current_state, OP_EQ, CIRCPAD_STATE_BURST); - circ_client_machine.states[CIRCPAD_STATE_BURST].start_usec = 100; - mi->padding_scheduled_at_usec = current_time - 1; + circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_edges[0] = 100; + mi->padding_scheduled_at_usec = current_time; circpad_machine_remove_token(mi); tt_int_op(mi->histogram[0], OP_EQ, 1); @@ -625,8 +649,8 @@ test_circuitpadding_token_removal_lower(void *arg) /* Test left boundaries of each histogram bin: */ const circpad_delay_t bin_left_bounds[] = - {0, 1, 7, 15, 31, 62, 125, 250, 500, CIRCPAD_DELAY_INFINITE}; - for (int i = 0; i < BIG_HISTOGRAM_LEN ; i++) { + {0, 1, 7, 15, 31, 62, 125, 250, 500, 1000, CIRCPAD_DELAY_INFINITE}; + for (int i = 0; i <= BIG_HISTOGRAM_LEN ; i++) { tt_uint_op(bin_left_bounds[i], OP_EQ, circpad_histogram_bin_to_usec(mi, i)); } @@ -674,7 +698,8 @@ test_circuitpadding_token_removal_lower(void *arg) /* Test above the highest bin, for coverage */ tt_int_op(client_side->padding_info[0]->current_state, OP_EQ, CIRCPAD_STATE_BURST); - circ_client_machine.states[CIRCPAD_STATE_BURST].start_usec = 100; + circ_client_machine.states[CIRCPAD_STATE_BURST]. + histogram_edges[BIG_HISTOGRAM_LEN-2] = 100; mi->padding_scheduled_at_usec = current_time - 29202; circpad_machine_remove_token(mi); tt_int_op(mi->histogram[BIG_HISTOGRAM_LEN-2], OP_EQ, 1); @@ -722,8 +747,8 @@ test_circuitpadding_closest_token_removal(void *arg) /* Test left boundaries of each histogram bin: */ const circpad_delay_t bin_left_bounds[] = - {0, 1, 7, 15, 31, 62, 125, 250, 500, CIRCPAD_DELAY_INFINITE}; - for (int i = 0; i < BIG_HISTOGRAM_LEN ; i++) { + {0, 1, 7, 15, 31, 62, 125, 250, 500, 1000, CIRCPAD_DELAY_INFINITE}; + for (int i = 0; i <= BIG_HISTOGRAM_LEN ; i++) { tt_uint_op(bin_left_bounds[i], OP_EQ, circpad_histogram_bin_to_usec(mi, i)); } @@ -770,7 +795,9 @@ test_circuitpadding_closest_token_removal(void *arg) /* Test below the lowest bin, for coverage */ tt_int_op(client_side->padding_info[0]->current_state, OP_EQ, CIRCPAD_STATE_BURST); - circ_client_machine.states[CIRCPAD_STATE_BURST].start_usec = 100; + circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_edges[0] = 100; + circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_edges[1] = 100; + circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_edges[2] = 120; mi->padding_scheduled_at_usec = current_time - 102; mi->histogram[0] = 0; circpad_machine_remove_token(mi); @@ -779,7 +806,6 @@ test_circuitpadding_closest_token_removal(void *arg) /* Test above the highest bin, for coverage */ tt_int_op(client_side->padding_info[0]->current_state, OP_EQ, CIRCPAD_STATE_BURST); - circ_client_machine.states[CIRCPAD_STATE_BURST].start_usec = 100; mi->padding_scheduled_at_usec = current_time - 29202; circpad_machine_remove_token(mi); tt_int_op(mi->histogram[BIG_HISTOGRAM_LEN-2], OP_EQ, 1); @@ -827,8 +853,8 @@ test_circuitpadding_closest_token_removal_usec(void *arg) /* Test left boundaries of each histogram bin: */ const circpad_delay_t bin_left_bounds[] = - {0, 1, 7, 15, 31, 62, 125, 250, 500, CIRCPAD_DELAY_INFINITE}; - for (int i = 0; i < BIG_HISTOGRAM_LEN ; i++) { + {0, 1, 7, 15, 31, 62, 125, 250, 500, 1000, CIRCPAD_DELAY_INFINITE}; + for (int i = 0; i <= BIG_HISTOGRAM_LEN ; i++) { tt_uint_op(bin_left_bounds[i], OP_EQ, circpad_histogram_bin_to_usec(mi, i)); } @@ -878,7 +904,9 @@ test_circuitpadding_closest_token_removal_usec(void *arg) /* Test below the lowest bin, for coverage */ tt_int_op(client_side->padding_info[0]->current_state, OP_EQ, CIRCPAD_STATE_BURST); - circ_client_machine.states[CIRCPAD_STATE_BURST].start_usec = 100; + circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_edges[0] = 100; + circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_edges[1] = 100; + circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_edges[2] = 120; mi->padding_scheduled_at_usec = current_time - 102; mi->histogram[0] = 0; circpad_machine_remove_token(mi); @@ -887,7 +915,8 @@ test_circuitpadding_closest_token_removal_usec(void *arg) /* Test above the highest bin, for coverage */ tt_int_op(client_side->padding_info[0]->current_state, OP_EQ, CIRCPAD_STATE_BURST); - circ_client_machine.states[CIRCPAD_STATE_BURST].start_usec = 100; + circ_client_machine.states[CIRCPAD_STATE_BURST]. + histogram_edges[BIG_HISTOGRAM_LEN-2] = 100; mi->padding_scheduled_at_usec = current_time - 29202; circpad_machine_remove_token(mi); tt_int_op(mi->histogram[BIG_HISTOGRAM_LEN-2], OP_EQ, 1); @@ -1044,7 +1073,7 @@ test_circuitpadding_tokens(void *arg) // Test 1: converting usec->bin->usec->bin // Bin 0+1 have different semantics. - for (circpad_delay_t i = 0; i <= state->start_usec+1; i++) { + for (circpad_delay_t i = 0; i <= state->histogram_edges[0]; i++) { int bin = circpad_histogram_usec_to_bin(client_side->padding_info[0], i); circpad_delay_t usec = @@ -1054,8 +1083,9 @@ test_circuitpadding_tokens(void *arg) tt_int_op(bin, OP_EQ, bin2); tt_int_op(i, OP_LE, usec); } - for (circpad_delay_t i = state->start_usec+1; - i <= state->start_usec + state->range_usec; i++) { + for (circpad_delay_t i = state->histogram_edges[0]+1; + i <= state->histogram_edges[0] + + state->histogram_edges[state->histogram_len-2]; i++) { int bin = circpad_histogram_usec_to_bin(client_side->padding_info[0], i); circpad_delay_t usec = @@ -1107,8 +1137,7 @@ test_circuitpadding_tokens(void *arg) /* 2.c. Bin 0 */ { tt_int_op(mi->histogram[0], OP_EQ, 1); - circpad_machine_remove_higher_token(mi, - state->start_usec/2); + circpad_machine_remove_higher_token(mi, state->histogram_edges[0]/2); tt_int_op(mi->histogram[0], OP_EQ, 0); } @@ -1127,8 +1156,7 @@ test_circuitpadding_tokens(void *arg) /* 3.a. Bin 0 */ { tt_int_op(mi->histogram[0], OP_EQ, 1); - circpad_machine_remove_higher_token(mi, - state->start_usec/2); + circpad_machine_remove_higher_token(mi, state->histogram_edges[0]/2); tt_int_op(mi->histogram[0], OP_EQ, 0); } @@ -1616,15 +1644,20 @@ helper_create_conditional_machine(void) ret->states[CIRCPAD_STATE_BURST]. next_state[CIRCPAD_EVENT_LENGTH_COUNT] = CIRCPAD_STATE_END; + /* Use EXACT removal strategy, otherwise setup_tokens() does not work */ ret->states[CIRCPAD_STATE_BURST].token_removal = - CIRCPAD_TOKEN_REMOVAL_NONE; + CIRCPAD_TOKEN_REMOVAL_EXACT; ret->states[CIRCPAD_STATE_BURST].histogram_len = 3; - ret->states[CIRCPAD_STATE_BURST].start_usec = 0; - ret->states[CIRCPAD_STATE_BURST].range_usec = 1000000; + + ret->states[CIRCPAD_STATE_BURST].histogram_edges[0] = 0; + ret->states[CIRCPAD_STATE_BURST].histogram_edges[1] = 0; + ret->states[CIRCPAD_STATE_BURST].histogram_edges[2] = 1000000; + ret->states[CIRCPAD_STATE_BURST].histogram[0] = 6; ret->states[CIRCPAD_STATE_BURST].histogram[1] = 0; - ret->states[CIRCPAD_STATE_BURST].histogram[1] = 0; + ret->states[CIRCPAD_STATE_BURST].histogram[2] = 0; + ret->states[CIRCPAD_STATE_BURST].histogram_total_tokens = 6; ret->states[CIRCPAD_STATE_BURST].use_rtt_estimate = 0; ret->states[CIRCPAD_STATE_BURST].length_includes_nonpadding = 1; From 6fdd34acd69e6e54562855acffa23df0cec2aee2 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Fri, 15 Feb 2019 19:18:24 +0200 Subject: [PATCH 0491/2557] Remove check-tor script --- changes/ticket29072 | 2 ++ contrib/or-tools/check-tor | 41 -------------------------------------- 2 files changed, 2 insertions(+), 41 deletions(-) create mode 100644 changes/ticket29072 delete mode 100755 contrib/or-tools/check-tor diff --git a/changes/ticket29072 b/changes/ticket29072 new file mode 100644 index 0000000000..3526330f30 --- /dev/null +++ b/changes/ticket29072 @@ -0,0 +1,2 @@ + o Removed features: + - Remove check-tor script from repository. Resolves issue 29072. diff --git a/contrib/or-tools/check-tor b/contrib/or-tools/check-tor deleted file mode 100755 index e981a35fcc..0000000000 --- a/contrib/or-tools/check-tor +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/sh - -## Originally written by Peter Palfrader. - -## This script lets you quickly check if a given router (by nickname) -## will let you do a TLS handshake, or will let you download a directory. - -## Usage: check-tor nickname - -#set -x - -router="$1" -dirserver="http://belegost.seul.org:80/tor/" - -lines=$( wget -q $dirserver --proxy=off -O - | grep -A5 '^router '"$router"' ' ) -line=$( echo "$lines" | head -n1 ) - -if [ -z "$line" ]; then - echo "Not found" >&2 - exit 1 -fi - -echo "$lines" -echo - -ipor=$( echo "$line" | awk '{printf "%s:%s", $3, $4}' ) - -op=$( echo "$line" | awk '{printf $6}' ) -ipop=$( echo "$line" | awk '{printf "%s:%s", $3, $6}' ) - -echo -echo ">>" openssl s_client -connect "$ipor" -timeout 5 openssl s_client -connect "$ipor" < /dev/null -if [ "$op" != "0" ]; then - echo - echo ">>" wget --proxy=off -O - http://$ipop/tor/ - timeout 5 wget --proxy=off -O - http://$ipop/tor/ | head -n3 -fi - -echo -echo -n "$router "; echo "$lines" | grep 'fingerprint' | sed -e 's/^opt //' -e 's/^fingerprint //'; From 384c5c618835a82d1d5455faff52e3f819683f43 Mon Sep 17 00:00:00 2001 From: Neel Chauhan Date: Sat, 16 Feb 2019 16:03:17 -0500 Subject: [PATCH 0492/2557] Make test-slow compile with libevent --- src/test/include.am | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/include.am b/src/test/include.am index 8622fcbaa1..9bfc626b28 100644 --- a/src/test/include.am +++ b/src/test/include.am @@ -259,7 +259,7 @@ src_test_test_LDADD = \ src_test_test_slow_CPPFLAGS = $(src_test_test_CPPFLAGS) src_test_test_slow_CFLAGS = $(src_test_test_CFLAGS) src_test_test_slow_LDADD = $(src_test_test_LDADD) -src_test_test_slow_LDFLAGS =@TOR_LDFLAGS_openssl@ +src_test_test_slow_LDFLAGS = @TOR_LDFLAGS_openssl@ @TOR_LDFLAGS_libevent@ src_test_test_rng_CPPFLAGS = $(src_test_test_CPPFLAGS) src_test_test_rng_CFLAGS = $(src_test_test_CFLAGS) From 5dcd44cbe2f03f545c15510de1954cd90cea4521 Mon Sep 17 00:00:00 2001 From: Roger Dingledine Date: Sun, 17 Feb 2019 16:55:55 -0500 Subject: [PATCH 0493/2557] fix some of the typos in Maintaining.md --- doc/HACKING/Maintaining.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/doc/HACKING/Maintaining.md b/doc/HACKING/Maintaining.md index 22d62b5471..4d5a7f6b76 100644 --- a/doc/HACKING/Maintaining.md +++ b/doc/HACKING/Maintaining.md @@ -4,7 +4,7 @@ This document details the duties and processes on maintaining the Tor code base. The first section describes who is the current Tor maintainer and what are the -responsabilities. Tor has one main single maintainer but does have many +responsibilities. Tor has one main single maintainer but does have many committers and subsystem maintainers. The second third section describes how the **alpha and master** branches are @@ -21,14 +21,14 @@ This document does not cover how Tor is released, please see The current maintainer is Nick Mathewson . The maintainer takes final decisions in terms of engineering, architecture and -protocol design. Releasing Tor falls under their responsability. +protocol design. Releasing Tor falls under their responsibility. ## Alpha and Master Branches -The Tor repository always has at all time a **master** branch which contains +The Tor repository always has at all times a **master** branch which contains the upstream ongoing development. -It may also contains a branch for a released feature freezed version which is +It may also contain a branch for a released feature freezed version which is called the **alpha** branch. The git tag and version number is always postfixed with `-alpha[-dev]`. For example: `tor-0.3.5.0-alpha-dev` or `tor-0.3.5.3-alpha`. @@ -39,7 +39,7 @@ code base but only commit (in most cases) into the subsystem they maintain. Upstream merges are restricted to the alpha and master branches. Subsystem maintainers should never push a patch into a stable branch which is the -responsability of the [stable branch maintainer](#stable-branches). +responsibility of the [stable branch maintainer](#stable-branches). ### Who @@ -68,7 +68,7 @@ maintain the following subsystems: These are the tasks of a subsystem maintainer: -1. Regurlarly go over `merge_ready` tickets relevant to the related subsystem +1. Regularly go over `merge_ready` tickets relevant to the related subsystem and for the current alpha or development (master branch) Milestone. 2. A subsystem maintainer is expected to contribute to any design changes @@ -86,7 +86,7 @@ These are few important items to follow when merging code upstream: **at least** one person that is not the original coder. Example A: If Alice writes a patch then Bob, a Tor network team member, - reviews it and flags it `merge_ready`. Then, the maintainter is required + reviews it and flags it `merge_ready`. Then, the maintainer is required to look at the patch and makes a decision. Example B: If the maintainer writes a patch then Bob, a Tor network @@ -100,7 +100,7 @@ These are few important items to follow when merging code upstream: 3. Trivial patches such as comment change, documentation, syntax issues or typos can be merged without a ticket or reviewers. -4. Tor uses the "merge forward" method that is if a patch applies to the +4. Tor uses the "merge forward" method, that is, if a patch applies to the alpha branch, it has to be merged there first and then merged forward into master. From 94f7e53d0416d80bacc5e2cf86e97ab07cfb09f2 Mon Sep 17 00:00:00 2001 From: Roger Dingledine Date: Sun, 17 Feb 2019 16:56:13 -0500 Subject: [PATCH 0494/2557] fix a bootstrapping string typo introduced in 85542ee5 next step is to fix it in torspec too --- src/feature/control/control_bootstrap.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/feature/control/control_bootstrap.c b/src/feature/control/control_bootstrap.c index a20a94ba89..8153d7595a 100644 --- a/src/feature/control/control_bootstrap.c +++ b/src/feature/control/control_bootstrap.c @@ -83,7 +83,7 @@ static const struct { { BOOTSTRAP_STATUS_AP_HANDSHAKE, "ap_handshake", "Finishing handshake with a relay to build circuits" }, { BOOTSTRAP_STATUS_AP_HANDSHAKE_DONE, "ap_handshake_done", - "Handshake fininshed with a relay to build circuits" }, + "Handshake finished with a relay to build circuits" }, /* Creating AP circuits */ From a798bd40fb108f83bcd3dea5c8fa8a60dbbb9fe2 Mon Sep 17 00:00:00 2001 From: teor Date: Tue, 8 Jan 2019 18:40:03 +1000 Subject: [PATCH 0495/2557] stats: Stop reporting statistics when ExtraInfoStatistics is 0 When ExtraInfoStatistics is 0, stop including bandwidth usage statistics, GeoIPFile hashes, ServerTransportPlugin lines, and bridge statistics by country in extra-info documents. Fixes bug 29018; bugfix on 0.2.4.1-alpha (and earlier versions). --- changes/bug29018 | 5 ++++ doc/tor.1.txt | 5 +++- src/feature/relay/router.c | 49 +++++++++++++++++++------------------- 3 files changed, 33 insertions(+), 26 deletions(-) create mode 100644 changes/bug29018 diff --git a/changes/bug29018 b/changes/bug29018 new file mode 100644 index 0000000000..b006ae36a7 --- /dev/null +++ b/changes/bug29018 @@ -0,0 +1,5 @@ + o Minor bugfixes (stats): + - When ExtraInfoStatistics is 0, stop including bandwidth usage statistics, + GeoIPFile hashes, ServerTransportPlugin lines, and bridge statistics + by country in extra-info documents. Fixes bug 29018; + bugfix on 0.2.4.1-alpha. diff --git a/doc/tor.1.txt b/doc/tor.1.txt index 52f5bfa0c3..d94e0dcac2 100644 --- a/doc/tor.1.txt +++ b/doc/tor.1.txt @@ -2313,7 +2313,8 @@ is non-zero): When this option is enabled and BridgeRelay is also enabled, and we have GeoIP data, Tor keeps a per-country count of how many client addresses have contacted it so that it can help the bridge authority guess - which countries have blocked access to it. (Default: 1) + which countries have blocked access to it. If ExtraInfoStatistics is + enabled, it will be published as part of extra-info document. (Default: 1) [[ServerDNSRandomizeCase]] **ServerDNSRandomizeCase** **0**|**1**:: When this option is set, Tor sets the case of each character randomly in @@ -2395,6 +2396,8 @@ is non-zero): [[ExtraInfoStatistics]] **ExtraInfoStatistics** **0**|**1**:: When this option is enabled, Tor includes previously gathered statistics in its extra-info documents that it uploads to the directory authorities. + Disabling this option also disables bandwidth usage statistics, GeoIPFile + hashes, and ServerTransportPlugin lists in the extra-info file. (Default: 1) [[ExtendAllowPrivateAddresses]] **ExtendAllowPrivateAddresses** **0**|**1**:: diff --git a/src/feature/relay/router.c b/src/feature/relay/router.c index cdd032f78d..93bc8c96cb 100644 --- a/src/feature/relay/router.c +++ b/src/feature/relay/router.c @@ -2942,7 +2942,6 @@ extrainfo_dump_to_string(char **s_out, extrainfo_t *extrainfo, char identity[HEX_DIGEST_LEN+1]; char published[ISO_TIME_LEN+1]; char digest[DIGEST_LEN]; - char *bandwidth_usage; int result; static int write_stats_to_extrainfo = 1; char sig[DIROBJ_MAX_SIG_LEN+1]; @@ -2957,7 +2956,6 @@ extrainfo_dump_to_string(char **s_out, extrainfo_t *extrainfo, base16_encode(identity, sizeof(identity), extrainfo->cache_info.identity_digest, DIGEST_LEN); format_iso_time(published, extrainfo->cache_info.published_on); - bandwidth_usage = rep_hist_get_bandwidth_lines(); if (emit_ed_sigs) { if (!extrainfo->cache_info.signing_key_cert->signing_key_included || !ed25519_pubkey_eq(&extrainfo->cache_info.signing_key_cert->signed_key, @@ -2983,21 +2981,25 @@ extrainfo_dump_to_string(char **s_out, extrainfo_t *extrainfo, ed_cert_line = tor_strdup(""); } - tor_asprintf(&pre, "extra-info %s %s\n%spublished %s\n%s", + tor_asprintf(&pre, "extra-info %s %s\n%spublished %s\n", extrainfo->nickname, identity, ed_cert_line, - published, bandwidth_usage); + published); smartlist_add(chunks, pre); - if (geoip_is_loaded(AF_INET)) - smartlist_add_asprintf(chunks, "geoip-db-digest %s\n", - geoip_db_digest(AF_INET)); - if (geoip_is_loaded(AF_INET6)) - smartlist_add_asprintf(chunks, "geoip6-db-digest %s\n", - geoip_db_digest(AF_INET6)); - if (options->ExtraInfoStatistics && write_stats_to_extrainfo) { log_info(LD_GENERAL, "Adding stats to extra-info descriptor."); + /* Bandwidth usage stats don't have their own option */ + { + contents = rep_hist_get_bandwidth_lines(); + smartlist_add(chunks, contents); + } + if (geoip_is_loaded(AF_INET)) + smartlist_add_asprintf(chunks, "geoip-db-digest %s\n", + geoip_db_digest(AF_INET)); + if (geoip_is_loaded(AF_INET6)) + smartlist_add_asprintf(chunks, "geoip6-db-digest %s\n", + geoip_db_digest(AF_INET6)); if (options->DirReqStatistics && load_stats_file("stats"PATH_SEPARATOR"dirreq-stats", "dirreq-stats-end", now, &contents) > 0) { @@ -3033,19 +3035,17 @@ extrainfo_dump_to_string(char **s_out, extrainfo_t *extrainfo, if (contents) smartlist_add(chunks, contents); } - } - - /* Add information about the pluggable transports we support. */ - if (options->ServerTransportPlugin) { - char *pluggable_transports = pt_get_extra_info_descriptor_string(); - if (pluggable_transports) - smartlist_add(chunks, pluggable_transports); - } - - if (should_record_bridge_info(options) && write_stats_to_extrainfo) { - const char *bridge_stats = geoip_get_bridge_stats_extrainfo(now); - if (bridge_stats) { - smartlist_add_strdup(chunks, bridge_stats); + /* Add information about the pluggable transports we support. */ + if (options->ServerTransportPlugin) { + char *pluggable_transports = pt_get_extra_info_descriptor_string(); + if (pluggable_transports) + smartlist_add(chunks, pluggable_transports); + } + if (should_record_bridge_info(options)) { + const char *bridge_stats = geoip_get_bridge_stats_extrainfo(now); + if (bridge_stats) { + smartlist_add_strdup(chunks, bridge_stats); + } } } @@ -3139,7 +3139,6 @@ extrainfo_dump_to_string(char **s_out, extrainfo_t *extrainfo, tor_free(s_dup); tor_free(ed_cert_line); extrainfo_free(ei_tmp); - tor_free(bandwidth_usage); return result; } From 6c5a506cdb887a655d8a4654fba5f6ea466aaeae Mon Sep 17 00:00:00 2001 From: teor Date: Thu, 10 Jan 2019 17:11:26 +1000 Subject: [PATCH 0496/2557] router: split router_build_fresh_descriptor() into static functions Split the body of router_build_fresh_descriptor() into static functions, by inserting function prologues and epilogues between existing sections. Write a new body for router_build_fresh_descriptor() that calls the new static functions. Initial refactor with no changes to the body of the old router_build_fresh_descriptor(), except for the split. Preparation for testing 29017 and 20918. --- src/feature/relay/router.c | 133 ++++++++++++++++++++++++++++++++++--- 1 file changed, 124 insertions(+), 9 deletions(-) diff --git a/src/feature/relay/router.c b/src/feature/relay/router.c index 93bc8c96cb..e57e9fb8d9 100644 --- a/src/feature/relay/router.c +++ b/src/feature/relay/router.c @@ -1941,18 +1941,15 @@ get_my_declared_family(const or_options_t *options) return result; } -/** Build a fresh routerinfo, signed server descriptor, and extra-info document - * for this OR. Set r to the generated routerinfo, e to the generated - * extra-info document. Return 0 on success, -1 on temporary error. Failure to - * generate an extra-info document is not an error and is indicated by setting - * e to NULL. Caller is responsible for freeing generated documents if 0 is - * returned. +/** Build a fresh routerinfo for this OR, without any of the fields that depend + * on the corresponding extrainfo. Set r to the generated routerinfo. + * Return 0 on success, -1 on temporary error. Caller is responsible for + * freeing the generated routerinfo if 0 is returned. */ -int -router_build_fresh_descriptor(routerinfo_t **r, extrainfo_t **e) +static int +router_build_fresh_routerinfo(routerinfo_t **r) { routerinfo_t *ri; - extrainfo_t *ei; uint32_t addr; char platform[256]; int hibernating = we_are_hibernating(); @@ -2057,6 +2054,21 @@ router_build_fresh_descriptor(routerinfo_t **r, extrainfo_t **e) ri->declared_family = get_my_declared_family(options); + *r = ri; + return 0; +} + +/** Build an extrainfo for this OR, based on the routerinfo ri. Set e to the + * generated extrainfo. Return 0 on success, -1 on temporary error. Failure to + * generate an extrainfo is not an error and is indicated by setting e to + * NULL. Caller is responsible for freeing the generated extrainfo if 0 is + * returned. + */ +static int +router_build_fresh_extrainfo(const routerinfo_t *ri, extrainfo_t **e) +{ + extrainfo_t *ei; + /* Now generate the extrainfo. */ ei = tor_malloc_zero(sizeof(extrainfo_t)); ei->cache_info.is_extrainfo = 1; @@ -2067,6 +2079,18 @@ router_build_fresh_descriptor(routerinfo_t **r, extrainfo_t **e) memcpy(ei->cache_info.identity_digest, ri->cache_info.identity_digest, DIGEST_LEN); + *e = ei; + return 0; +} + +/** Create a signed descriptor for ei, and add it to ei->cache_info. + * Return ei on success, free ei and return NULL on temporary error. + * Caller is responsible for freeing the returned extrainfo + * (if it is not NULL), including any extra fields set in ei->cache_info. + */ +static extrainfo_t * +router_update_extrainfo_descriptor_body(extrainfo_t *ei) +{ if (extrainfo_dump_to_string(&ei->cache_info.signed_descriptor_body, ei, get_server_identity_key(), get_master_signing_keypair()) < 0) { @@ -2085,6 +2109,15 @@ router_build_fresh_descriptor(routerinfo_t **r, extrainfo_t **e) DIGEST_SHA256); } + return ei; +} + +/** If ei is not NULL, set the fields in ri that depend on ei. + */ +static void +router_update_routerinfo_from_extrainfo(routerinfo_t *ri, + const extrainfo_t *ei) +{ /* Now finish the router descriptor. */ if (ei) { memcpy(ri->cache_info.extra_info_digest, @@ -2097,6 +2130,17 @@ router_build_fresh_descriptor(routerinfo_t **r, extrainfo_t **e) /* ri was allocated with tor_malloc_zero, so there is no need to * zero ri->cache_info.extra_info_digest here. */ } +} + +/** Create a signed descriptor for ri, and add it to ri->cache_info. + * Return 0 on success, free ri and ei and return -1 on temporary error. + * TODO: freeing ri and ei, but leaving dangling pointers, is a bad interface. + * Caller is responsible for freeing the generated ri and ei if 0 is returned, + * including any extra fields set in ri->cache_info. + */ +static int +router_update_routerinfo_descriptor_body(routerinfo_t *ri, extrainfo_t *ei) +{ if (! (ri->cache_info.signed_descriptor_body = router_dump_router_to_string(ri, get_server_identity_key(), get_onion_key(), @@ -2110,8 +2154,27 @@ router_build_fresh_descriptor(routerinfo_t **r, extrainfo_t **e) ri->cache_info.signed_descriptor_len = strlen(ri->cache_info.signed_descriptor_body); + return 0; +} + +/** Set the purpose field in ri. + */ +static void +router_update_routerinfo_purpose(routerinfo_t *ri) +{ + const or_options_t *options = get_options(); + ri->purpose = options->BridgeRelay ? ROUTER_PURPOSE_BRIDGE : ROUTER_PURPOSE_GENERAL; +} + +/** Set the cache_info.send_unencrypted fields in ri and ei. + */ +static void +router_update_info_send_unencrypted(routerinfo_t *ri, extrainfo_t *ei) +{ + const or_options_t *options = get_options(); + if (options->BridgeRelay) { /* Bridges shouldn't be able to send their descriptors unencrypted, anyway, since they don't have a DirPort, and always connect to the @@ -2125,10 +2188,62 @@ router_build_fresh_descriptor(routerinfo_t **r, extrainfo_t **e) if (ei) ei->cache_info.send_unencrypted = 1; } +} +/** Set signed_descriptor_digest in ri->cache_info. + */ +static void +router_update_routerinfo_digest(routerinfo_t *ri) +{ router_get_router_hash(ri->cache_info.signed_descriptor_body, strlen(ri->cache_info.signed_descriptor_body), ri->cache_info.signed_descriptor_digest); +} + +/** Build a fresh routerinfo, signed server descriptor, and extra-info document + * for this OR. Set r to the generated routerinfo, e to the generated + * extra-info document. Return 0 on success, -1 on temporary error. Failure to + * generate an extra-info document is not an error and is indicated by setting + * e to NULL. Caller is responsible for freeing generated documents if 0 is + * returned. + */ +int +router_build_fresh_descriptor(routerinfo_t **r, extrainfo_t **e) +{ + int result = -1; + routerinfo_t *ri = NULL; + extrainfo_t *ei = NULL; + + /* TODO: return ri */ + result = router_build_fresh_routerinfo(&ri); + if (result < 0) + return result; + + /* TODO: return ei */ + result = router_build_fresh_extrainfo(ri, &ei); + if (result < 0) + return result; + + /* TODO: this function frees ei on failure, instead, goto err */ + ei = router_update_extrainfo_descriptor_body(ei); + + /* TODO: don't rely on tor_malloc_zero */ + router_update_routerinfo_from_extrainfo(ri, ei); + + /* TODO: this function frees ri and ei on failure, instead, goto err */ + result = router_update_routerinfo_descriptor_body(ri, ei); + if (result < 0) + return result; + + /* TODO: fold into router_build_fresh_routerinfo() */ + router_update_routerinfo_purpose(ri); + + /* TODO: fold into router_update_extrainfo_descriptor_body() and + * router_update_routerinfo_descriptor_body() ? */ + router_update_info_send_unencrypted(ri, ei); + + /* TODO: fold into router_update_routerinfo_descriptor_body() */ + router_update_routerinfo_digest(ri); if (ei) { tor_assert(! From f19b64dce90c082b0e19f059b94c2d42b015a956 Mon Sep 17 00:00:00 2001 From: teor Date: Thu, 10 Jan 2019 19:47:24 +1000 Subject: [PATCH 0497/2557] router: refactor router_build_fresh_descriptor() static function interfaces Tidy the arguments and return values of these functions, and clean up their memory management. Preparation for testing 29017 and 20918. --- src/feature/relay/router.c | 169 ++++++++++++++++++++++++------------- src/feature/relay/router.h | 1 + 2 files changed, 112 insertions(+), 58 deletions(-) diff --git a/src/feature/relay/router.c b/src/feature/relay/router.c index e57e9fb8d9..d9242448c9 100644 --- a/src/feature/relay/router.c +++ b/src/feature/relay/router.c @@ -152,6 +152,8 @@ routerinfo_err_to_string(int err) return "Cannot generate descriptor"; case TOR_ROUTERINFO_ERROR_DESC_REBUILDING: return "Descriptor still rebuilding - not ready yet"; + case TOR_ROUTERINFO_ERROR_INTERNAL_BUG: + return "Internal bug, see logs for details"; } log_warn(LD_BUG, "unknown routerinfo error %d - shouldn't happen", err); @@ -1941,23 +1943,33 @@ get_my_declared_family(const or_options_t *options) return result; } -/** Build a fresh routerinfo for this OR, without any of the fields that depend - * on the corresponding extrainfo. Set r to the generated routerinfo. - * Return 0 on success, -1 on temporary error. Caller is responsible for - * freeing the generated routerinfo if 0 is returned. +/** Allocate a fresh, unsigned routerinfo for this OR, without any of the + * fields that depend on the corresponding extrainfo. + * + * On success, set ri_out to the new routerinfo, and return 0. + * Caller is responsible for freeing the generated routerinfo. + * + * Returns a negative value and sets ri_out to NULL on temporary error. */ static int -router_build_fresh_routerinfo(routerinfo_t **r) +router_build_fresh_routerinfo(routerinfo_t **ri_out) { - routerinfo_t *ri; + routerinfo_t *ri = NULL; uint32_t addr; char platform[256]; int hibernating = we_are_hibernating(); const or_options_t *options = get_options(); + int result = TOR_ROUTERINFO_ERROR_INTERNAL_BUG; + + if (BUG(!ri_out)) { + result = TOR_ROUTERINFO_ERROR_INTERNAL_BUG; + goto err; + } if (router_pick_published_address(options, &addr, 0) < 0) { log_warn(LD_CONFIG, "Don't know my address while generating descriptor"); - return TOR_ROUTERINFO_ERROR_NO_EXT_ADDR; + result = TOR_ROUTERINFO_ERROR_NO_EXT_ADDR; + goto err; } /* Log a message if the address in the descriptor doesn't match the ORPort @@ -2014,8 +2026,8 @@ router_build_fresh_routerinfo(routerinfo_t **r) ri->identity_pkey = crypto_pk_dup_key(get_server_identity_key()); if (BUG(crypto_pk_get_digest(ri->identity_pkey, ri->cache_info.identity_digest) < 0)) { - routerinfo_free(ri); - return TOR_ROUTERINFO_ERROR_DIGEST_FAILED; + result = TOR_ROUTERINFO_ERROR_DIGEST_FAILED; + goto err; } ri->cache_info.signing_key_cert = tor_cert_dup(get_master_signing_key_cert()); @@ -2054,20 +2066,26 @@ router_build_fresh_routerinfo(routerinfo_t **r) ri->declared_family = get_my_declared_family(options); - *r = ri; + goto done; + + err: + routerinfo_free(ri); + *ri_out = NULL; + return result; + + done: + *ri_out = ri; return 0; } -/** Build an extrainfo for this OR, based on the routerinfo ri. Set e to the - * generated extrainfo. Return 0 on success, -1 on temporary error. Failure to - * generate an extrainfo is not an error and is indicated by setting e to - * NULL. Caller is responsible for freeing the generated extrainfo if 0 is - * returned. +/** Allocate and return an extrainfo for this OR, based on the routerinfo ri. + * + * Caller is responsible for freeing the generated extrainfo. */ -static int -router_build_fresh_extrainfo(const routerinfo_t *ri, extrainfo_t **e) +static extrainfo_t * +router_build_fresh_extrainfo(const routerinfo_t *ri) { - extrainfo_t *ei; + extrainfo_t *ei = NULL; /* Now generate the extrainfo. */ ei = tor_malloc_zero(sizeof(extrainfo_t)); @@ -2079,24 +2097,23 @@ router_build_fresh_extrainfo(const routerinfo_t *ri, extrainfo_t **e) memcpy(ei->cache_info.identity_digest, ri->cache_info.identity_digest, DIGEST_LEN); - *e = ei; - return 0; + + return ei; } /** Create a signed descriptor for ei, and add it to ei->cache_info. - * Return ei on success, free ei and return NULL on temporary error. - * Caller is responsible for freeing the returned extrainfo - * (if it is not NULL), including any extra fields set in ei->cache_info. + * + * Return 0 on success, -1 on temporary error. + * On error, ei->cache_info is not modified. */ -static extrainfo_t * +static int router_update_extrainfo_descriptor_body(extrainfo_t *ei) { if (extrainfo_dump_to_string(&ei->cache_info.signed_descriptor_body, ei, get_server_identity_key(), get_master_signing_keypair()) < 0) { log_warn(LD_BUG, "Couldn't generate extra-info descriptor."); - extrainfo_free(ei); - ei = NULL; + return -1; } else { ei->cache_info.signed_descriptor_len = strlen(ei->cache_info.signed_descriptor_body); @@ -2107,12 +2124,11 @@ router_update_extrainfo_descriptor_body(extrainfo_t *ei) ei->cache_info.signed_descriptor_body, ei->cache_info.signed_descriptor_len, DIGEST_SHA256); + return 0; } - - return ei; } -/** If ei is not NULL, set the fields in ri that depend on ei. +/** Set the fields in ri that depend on ei. */ static void router_update_routerinfo_from_extrainfo(routerinfo_t *ri, @@ -2133,24 +2149,26 @@ router_update_routerinfo_from_extrainfo(routerinfo_t *ri, } /** Create a signed descriptor for ri, and add it to ri->cache_info. - * Return 0 on success, free ri and ei and return -1 on temporary error. - * TODO: freeing ri and ei, but leaving dangling pointers, is a bad interface. - * Caller is responsible for freeing the generated ri and ei if 0 is returned, - * including any extra fields set in ri->cache_info. + * + * Return 0 on success, and a negative value on temporary error. + * If ri is NULL, logs a BUG() warning and returns a negative value. + * On error, ri->cache_info is not modified. */ static int -router_update_routerinfo_descriptor_body(routerinfo_t *ri, extrainfo_t *ei) +router_update_routerinfo_descriptor_body(routerinfo_t *ri) { + if (BUG(!ri)) + return TOR_ROUTERINFO_ERROR_INTERNAL_BUG; + if (! (ri->cache_info.signed_descriptor_body = router_dump_router_to_string(ri, get_server_identity_key(), get_onion_key(), get_current_curve25519_keypair(), get_master_signing_keypair())) ) { log_warn(LD_BUG, "Couldn't generate router descriptor."); - routerinfo_free(ri); - extrainfo_free(ei); return TOR_ROUTERINFO_ERROR_CANNOT_GENERATE; } + ri->cache_info.signed_descriptor_len = strlen(ri->cache_info.signed_descriptor_body); @@ -2200,40 +2218,63 @@ router_update_routerinfo_digest(routerinfo_t *ri) ri->cache_info.signed_descriptor_digest); } -/** Build a fresh routerinfo, signed server descriptor, and extra-info document - * for this OR. Set r to the generated routerinfo, e to the generated - * extra-info document. Return 0 on success, -1 on temporary error. Failure to - * generate an extra-info document is not an error and is indicated by setting - * e to NULL. Caller is responsible for freeing generated documents if 0 is - * returned. +/** Build a fresh routerinfo, signed server descriptor, and signed extra-info + * document for this OR. + * + * Set r to the generated routerinfo, e to the generated extra-info document. + * Failure to generate an extra-info document is not an error and is indicated + * by setting e to NULL. + * Return 0 on success, and a negative value on temporary error. + * Caller is responsible for freeing generated documents on success. */ int router_build_fresh_descriptor(routerinfo_t **r, extrainfo_t **e) { - int result = -1; + int result = TOR_ROUTERINFO_ERROR_INTERNAL_BUG; routerinfo_t *ri = NULL; extrainfo_t *ei = NULL; - /* TODO: return ri */ + if (BUG(!r)) + goto err; + + if (BUG(!e)) + goto err; + result = router_build_fresh_routerinfo(&ri); - if (result < 0) - return result; + if (result < 0) { + goto err; + } + /* If ri is NULL, then result should be negative. So this check should be + * unreachable. */ + if (BUG(!ri)) { + result = TOR_ROUTERINFO_ERROR_INTERNAL_BUG; + goto err; + } - /* TODO: return ei */ - result = router_build_fresh_extrainfo(ri, &ei); - if (result < 0) - return result; + ei = router_build_fresh_extrainfo(ri); + /* Failing to create an ei is not an error, but at this stage, + * router_build_fresh_extrainfo() should not fail. */ + if (BUG(!ei)) + goto skip_ei; - /* TODO: this function frees ei on failure, instead, goto err */ - ei = router_update_extrainfo_descriptor_body(ei); + result = router_update_extrainfo_descriptor_body(ei); + if (result < 0) + goto skip_ei; /* TODO: don't rely on tor_malloc_zero */ router_update_routerinfo_from_extrainfo(ri, ei); - /* TODO: this function frees ri and ei on failure, instead, goto err */ - result = router_update_routerinfo_descriptor_body(ri, ei); + /* TODO: disentangle these GOTOs, or split into another function. */ + goto ei_ok; + + skip_ei: + extrainfo_free(ei); + ei = NULL; + + ei_ok: + result = router_update_routerinfo_descriptor_body(ri); if (result < 0) - return result; + goto err; /* TODO: fold into router_build_fresh_routerinfo() */ router_update_routerinfo_purpose(ri); @@ -2246,11 +2287,23 @@ router_build_fresh_descriptor(routerinfo_t **r, extrainfo_t **e) router_update_routerinfo_digest(ri); if (ei) { - tor_assert(! - routerinfo_incompatible_with_extrainfo(ri->identity_pkey, ei, - &ri->cache_info, NULL)); + if (BUG(routerinfo_incompatible_with_extrainfo(ri->identity_pkey, ei, + &ri->cache_info, NULL))) { + result = TOR_ROUTERINFO_ERROR_INTERNAL_BUG; + goto err; + } } + goto done; + + err: + routerinfo_free(ri); + extrainfo_free(ei); + *r = NULL; + *e = NULL; + return result; + + done: *r = ri; *e = ei; return 0; diff --git a/src/feature/relay/router.h b/src/feature/relay/router.h index 60bc857ceb..46364206ef 100644 --- a/src/feature/relay/router.h +++ b/src/feature/relay/router.h @@ -23,6 +23,7 @@ struct ed25519_keypair_t; #define TOR_ROUTERINFO_ERROR_DIGEST_FAILED (-4) #define TOR_ROUTERINFO_ERROR_CANNOT_GENERATE (-5) #define TOR_ROUTERINFO_ERROR_DESC_REBUILDING (-6) +#define TOR_ROUTERINFO_ERROR_INTERNAL_BUG (-7) crypto_pk_t *get_onion_key(void); time_t get_onion_key_set_at(void); From a65c101973f0b0dc7380470edff4f590b58c39d3 Mon Sep 17 00:00:00 2001 From: teor Date: Thu, 10 Jan 2019 19:49:46 +1000 Subject: [PATCH 0498/2557] router: check for NULL in router_build_fresh_descriptor() static functions Make sure that these static functions aren't passed NULL. If they are, log a BUG() warning, and return an error. Preparation for testing 29017 and 20918. --- src/feature/relay/router.c | 35 +++++++++++++++++++++++------------ 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/src/feature/relay/router.c b/src/feature/relay/router.c index d9242448c9..9aa4d56a4c 100644 --- a/src/feature/relay/router.c +++ b/src/feature/relay/router.c @@ -2080,6 +2080,7 @@ router_build_fresh_routerinfo(routerinfo_t **ri_out) /** Allocate and return an extrainfo for this OR, based on the routerinfo ri. * + * If ri is NULL, logs a BUG() warning and returns NULL. * Caller is responsible for freeing the generated extrainfo. */ static extrainfo_t * @@ -2087,6 +2088,9 @@ router_build_fresh_extrainfo(const routerinfo_t *ri) { extrainfo_t *ei = NULL; + if (BUG(!ri)) + return NULL; + /* Now generate the extrainfo. */ ei = tor_malloc_zero(sizeof(extrainfo_t)); ei->cache_info.is_extrainfo = 1; @@ -2104,11 +2108,15 @@ router_build_fresh_extrainfo(const routerinfo_t *ri) /** Create a signed descriptor for ei, and add it to ei->cache_info. * * Return 0 on success, -1 on temporary error. + * If ei is NULL, logs a BUG() warning and returns -1. * On error, ei->cache_info is not modified. */ static int router_update_extrainfo_descriptor_body(extrainfo_t *ei) { + if (BUG(!ei)) + return -1; + if (extrainfo_dump_to_string(&ei->cache_info.signed_descriptor_body, ei, get_server_identity_key(), get_master_signing_keypair()) < 0) { @@ -2129,23 +2137,27 @@ router_update_extrainfo_descriptor_body(extrainfo_t *ei) } /** Set the fields in ri that depend on ei. + * + * If ei is NULL, logs a BUG() warning and zeroes the relevant fields. */ static void router_update_routerinfo_from_extrainfo(routerinfo_t *ri, const extrainfo_t *ei) { - /* Now finish the router descriptor. */ - if (ei) { - memcpy(ri->cache_info.extra_info_digest, - ei->cache_info.signed_descriptor_digest, - DIGEST_LEN); - memcpy(ri->cache_info.extra_info_digest256, - ei->digest256, - DIGEST256_LEN); - } else { - /* ri was allocated with tor_malloc_zero, so there is no need to - * zero ri->cache_info.extra_info_digest here. */ + if (BUG(!ei)) { + /* Just to be safe, zero ri->cache_info.extra_info_digest* here. */ + memset(ri->cache_info.extra_info_digest, 0, DIGEST_LEN); + memset(ri->cache_info.extra_info_digest256, 0, DIGEST256_LEN); + return; } + + /* Now finish the router descriptor. */ + memcpy(ri->cache_info.extra_info_digest, + ei->cache_info.signed_descriptor_digest, + DIGEST_LEN); + memcpy(ri->cache_info.extra_info_digest256, + ei->digest256, + DIGEST256_LEN); } /** Create a signed descriptor for ri, and add it to ri->cache_info. @@ -2261,7 +2273,6 @@ router_build_fresh_descriptor(routerinfo_t **r, extrainfo_t **e) if (result < 0) goto skip_ei; - /* TODO: don't rely on tor_malloc_zero */ router_update_routerinfo_from_extrainfo(ri, ei); /* TODO: disentangle these GOTOs, or split into another function. */ From af0a43be2cf3e96ee9ac8e1f92c11aa5d5f6290f Mon Sep 17 00:00:00 2001 From: teor Date: Thu, 10 Jan 2019 20:01:28 +1000 Subject: [PATCH 0499/2557] router: eliminate tiny router_build_fresh_descriptor() static functions Remove some tiny static functions called by router_build_fresh_descriptor(), and move their code into more relevant functions. Then, give router_update_{router,extra}info_descriptor_body identical layouts. Preparation for testing 29017 and 20918. --- src/feature/relay/router.c | 58 +++++++++++++------------------------- 1 file changed, 20 insertions(+), 38 deletions(-) diff --git a/src/feature/relay/router.c b/src/feature/relay/router.c index 9aa4d56a4c..9a5e8b74ac 100644 --- a/src/feature/relay/router.c +++ b/src/feature/relay/router.c @@ -2066,6 +2066,9 @@ router_build_fresh_routerinfo(routerinfo_t **ri_out) ri->declared_family = get_my_declared_family(options); + ri->purpose = + options->BridgeRelay ? ROUTER_PURPOSE_BRIDGE : ROUTER_PURPOSE_GENERAL; + goto done; err: @@ -2122,18 +2125,20 @@ router_update_extrainfo_descriptor_body(extrainfo_t *ei) get_master_signing_keypair()) < 0) { log_warn(LD_BUG, "Couldn't generate extra-info descriptor."); return -1; - } else { - ei->cache_info.signed_descriptor_len = - strlen(ei->cache_info.signed_descriptor_body); - router_get_extrainfo_hash(ei->cache_info.signed_descriptor_body, - ei->cache_info.signed_descriptor_len, - ei->cache_info.signed_descriptor_digest); - crypto_digest256((char*) ei->digest256, - ei->cache_info.signed_descriptor_body, - ei->cache_info.signed_descriptor_len, - DIGEST_SHA256); - return 0; } + + ei->cache_info.signed_descriptor_len = + strlen(ei->cache_info.signed_descriptor_body); + + router_get_extrainfo_hash(ei->cache_info.signed_descriptor_body, + ei->cache_info.signed_descriptor_len, + ei->cache_info.signed_descriptor_digest); + crypto_digest256((char*) ei->digest256, + ei->cache_info.signed_descriptor_body, + ei->cache_info.signed_descriptor_len, + DIGEST_SHA256); + + return 0; } /** Set the fields in ri that depend on ei. @@ -2184,20 +2189,13 @@ router_update_routerinfo_descriptor_body(routerinfo_t *ri) ri->cache_info.signed_descriptor_len = strlen(ri->cache_info.signed_descriptor_body); + router_get_router_hash(ri->cache_info.signed_descriptor_body, + strlen(ri->cache_info.signed_descriptor_body), + ri->cache_info.signed_descriptor_digest); + return 0; } -/** Set the purpose field in ri. - */ -static void -router_update_routerinfo_purpose(routerinfo_t *ri) -{ - const or_options_t *options = get_options(); - - ri->purpose = - options->BridgeRelay ? ROUTER_PURPOSE_BRIDGE : ROUTER_PURPOSE_GENERAL; -} - /** Set the cache_info.send_unencrypted fields in ri and ei. */ static void @@ -2220,16 +2218,6 @@ router_update_info_send_unencrypted(routerinfo_t *ri, extrainfo_t *ei) } } -/** Set signed_descriptor_digest in ri->cache_info. - */ -static void -router_update_routerinfo_digest(routerinfo_t *ri) -{ - router_get_router_hash(ri->cache_info.signed_descriptor_body, - strlen(ri->cache_info.signed_descriptor_body), - ri->cache_info.signed_descriptor_digest); -} - /** Build a fresh routerinfo, signed server descriptor, and signed extra-info * document for this OR. * @@ -2287,16 +2275,10 @@ router_build_fresh_descriptor(routerinfo_t **r, extrainfo_t **e) if (result < 0) goto err; - /* TODO: fold into router_build_fresh_routerinfo() */ - router_update_routerinfo_purpose(ri); - /* TODO: fold into router_update_extrainfo_descriptor_body() and * router_update_routerinfo_descriptor_body() ? */ router_update_info_send_unencrypted(ri, ei); - /* TODO: fold into router_update_routerinfo_descriptor_body() */ - router_update_routerinfo_digest(ri); - if (ei) { if (BUG(routerinfo_incompatible_with_extrainfo(ri->identity_pkey, ei, &ri->cache_info, NULL))) { From 9cab988696b6f86f9b3b744c5694bee7d2a6cf70 Mon Sep 17 00:00:00 2001 From: teor Date: Thu, 10 Jan 2019 20:19:02 +1000 Subject: [PATCH 0500/2557] router: eliminate router_update_info_send_unencrypted() Remove router_update_info_send_unencrypted(), and move its code into the relevant functions. Then, re-use an options pointer. Preparation for testing 29017 and 20918. --- src/feature/relay/router.c | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/src/feature/relay/router.c b/src/feature/relay/router.c index 9a5e8b74ac..03af9aacd4 100644 --- a/src/feature/relay/router.c +++ b/src/feature/relay/router.c @@ -2066,8 +2066,17 @@ router_build_fresh_routerinfo(routerinfo_t **ri_out) ri->declared_family = get_my_declared_family(options); - ri->purpose = - options->BridgeRelay ? ROUTER_PURPOSE_BRIDGE : ROUTER_PURPOSE_GENERAL; + if (options->BridgeRelay) { + ri->purpose = ROUTER_PURPOSE_BRIDGE; + /* Bridges shouldn't be able to send their descriptors unencrypted, + anyway, since they don't have a DirPort, and always connect to the + bridge authority anonymously. But just in case they somehow think of + sending them on an unencrypted connection, don't allow them to try. */ + ri->cache_info.send_unencrypted = 0; + } else { + ri->purpose = ROUTER_PURPOSE_GENERAL; + ri->cache_info.send_unencrypted = 1; + } goto done; @@ -2090,6 +2099,7 @@ static extrainfo_t * router_build_fresh_extrainfo(const routerinfo_t *ri) { extrainfo_t *ei = NULL; + const or_options_t *options = get_options(); if (BUG(!ri)) return NULL; @@ -2097,7 +2107,7 @@ router_build_fresh_extrainfo(const routerinfo_t *ri) /* Now generate the extrainfo. */ ei = tor_malloc_zero(sizeof(extrainfo_t)); ei->cache_info.is_extrainfo = 1; - strlcpy(ei->nickname, get_options()->Nickname, sizeof(ei->nickname)); + strlcpy(ei->nickname, options->Nickname, sizeof(ei->nickname)); ei->cache_info.published_on = ri->cache_info.published_on; ei->cache_info.signing_key_cert = tor_cert_dup(get_master_signing_key_cert()); @@ -2105,6 +2115,13 @@ router_build_fresh_extrainfo(const routerinfo_t *ri) memcpy(ei->cache_info.identity_digest, ri->cache_info.identity_digest, DIGEST_LEN); + if (options->BridgeRelay) { + /* See note in router_build_fresh_routerinfo(). */ + ei->cache_info.send_unencrypted = 0; + } else { + ei->cache_info.send_unencrypted = 1; + } + return ei; } @@ -2275,10 +2292,6 @@ router_build_fresh_descriptor(routerinfo_t **r, extrainfo_t **e) if (result < 0) goto err; - /* TODO: fold into router_update_extrainfo_descriptor_body() and - * router_update_routerinfo_descriptor_body() ? */ - router_update_info_send_unencrypted(ri, ei); - if (ei) { if (BUG(routerinfo_incompatible_with_extrainfo(ri->identity_pkey, ei, &ri->cache_info, NULL))) { From a1f8558628881216917814989d293a028f9e48d7 Mon Sep 17 00:00:00 2001 From: teor Date: Thu, 10 Jan 2019 20:39:10 +1000 Subject: [PATCH 0501/2557] router: Move extrainfo signing into its own function This refactoring improves the structure of router_build_fresh_descriptor(). Preparation for testing 29017 and 20918. --- src/feature/relay/router.c | 57 ++++++++++++++++++++++++++------------ 1 file changed, 39 insertions(+), 18 deletions(-) diff --git a/src/feature/relay/router.c b/src/feature/relay/router.c index 03af9aacd4..0e872663f8 100644 --- a/src/feature/relay/router.c +++ b/src/feature/relay/router.c @@ -2158,6 +2158,40 @@ router_update_extrainfo_descriptor_body(extrainfo_t *ei) return 0; } +/** Allocate and return a fresh, signed extrainfo for this OR, based on the + * routerinfo ri. + * + * If ri is NULL, logs a BUG() warning and returns NULL. + * Caller is responsible for freeing the generated extrainfo. + */ +static extrainfo_t * +router_build_signed_extrainfo(const routerinfo_t *ri) +{ + int result = -1; + extrainfo_t *ei = NULL; + + if (BUG(!ri)) + return NULL; + + ei = router_build_fresh_extrainfo(ri); + /* router_build_fresh_extrainfo() should not fail. */ + if (BUG(!ei)) + goto err; + + result = router_update_extrainfo_descriptor_body(ei); + if (result < 0) + goto err; + + goto done; + + err: + extrainfo_free(ei); + return NULL; + + done: + return ei; +} + /** Set the fields in ri that depend on ei. * * If ei is NULL, logs a BUG() warning and zeroes the relevant fields. @@ -2268,26 +2302,13 @@ router_build_fresh_descriptor(routerinfo_t **r, extrainfo_t **e) goto err; } - ei = router_build_fresh_extrainfo(ri); - /* Failing to create an ei is not an error, but at this stage, - * router_build_fresh_extrainfo() should not fail. */ - if (BUG(!ei)) - goto skip_ei; + ei = router_build_signed_extrainfo(ri); - result = router_update_extrainfo_descriptor_body(ei); - if (result < 0) - goto skip_ei; + /* Failing to create an ei is not an error. */ + if (ei) { + router_update_routerinfo_from_extrainfo(ri, ei); + } - router_update_routerinfo_from_extrainfo(ri, ei); - - /* TODO: disentangle these GOTOs, or split into another function. */ - goto ei_ok; - - skip_ei: - extrainfo_free(ei); - ei = NULL; - - ei_ok: result = router_update_routerinfo_descriptor_body(ri); if (result < 0) goto err; From a9f852a0f65a24fc8c0e33caf82d1e3c845c23cf Mon Sep 17 00:00:00 2001 From: teor Date: Thu, 10 Jan 2019 20:47:37 +1000 Subject: [PATCH 0502/2557] router: Document the additional config and state used to dump descriptors Also, explicitly state when routerinfos and extra-infos are signed. And tidy up some other comments. Preparation for testing 29017 and 20918. --- src/feature/relay/router.c | 80 ++++++++++++++++++-------------------- src/feature/relay/router.h | 2 +- 2 files changed, 39 insertions(+), 43 deletions(-) diff --git a/src/feature/relay/router.c b/src/feature/relay/router.c index 0e872663f8..7e2161ef31 100644 --- a/src/feature/relay/router.c +++ b/src/feature/relay/router.c @@ -1952,7 +1952,7 @@ get_my_declared_family(const or_options_t *options) * Returns a negative value and sets ri_out to NULL on temporary error. */ static int -router_build_fresh_routerinfo(routerinfo_t **ri_out) +router_build_fresh_unsigned_routerinfo(routerinfo_t **ri_out) { routerinfo_t *ri = NULL; uint32_t addr; @@ -2090,13 +2090,17 @@ router_build_fresh_routerinfo(routerinfo_t **ri_out) return 0; } -/** Allocate and return an extrainfo for this OR, based on the routerinfo ri. +/** Allocate and return a fresh, unsigned extrainfo for this OR, based on the + * routerinfo ri. + * + * Uses options->Nickname to set the nickname, and options->BridgeRelay to set + * ei->cache_info.send_unencrypted. * * If ri is NULL, logs a BUG() warning and returns NULL. * Caller is responsible for freeing the generated extrainfo. */ static extrainfo_t * -router_build_fresh_extrainfo(const routerinfo_t *ri) +router_build_fresh_unsigned_extrainfo(const routerinfo_t *ri) { extrainfo_t *ei = NULL; const or_options_t *options = get_options(); @@ -2125,14 +2129,17 @@ router_build_fresh_extrainfo(const routerinfo_t *ri) return ei; } -/** Create a signed descriptor for ei, and add it to ei->cache_info. +/** Dump the extrainfo descriptor body for ei, sign it, and add the body and + * signature to ei->cache_info. Note that the extrainfo body is determined by + * ei, and some additional config and statistics state: see + * extrainfo_dump_to_string() for details. * * Return 0 on success, -1 on temporary error. * If ei is NULL, logs a BUG() warning and returns -1. * On error, ei->cache_info is not modified. */ static int -router_update_extrainfo_descriptor_body(extrainfo_t *ei) +router_dump_and_sign_extrainfo_descriptor_body(extrainfo_t *ei) { if (BUG(!ei)) return -1; @@ -2165,7 +2172,7 @@ router_update_extrainfo_descriptor_body(extrainfo_t *ei) * Caller is responsible for freeing the generated extrainfo. */ static extrainfo_t * -router_build_signed_extrainfo(const routerinfo_t *ri) +router_build_fresh_signed_extrainfo(const routerinfo_t *ri) { int result = -1; extrainfo_t *ei = NULL; @@ -2173,12 +2180,12 @@ router_build_signed_extrainfo(const routerinfo_t *ri) if (BUG(!ri)) return NULL; - ei = router_build_fresh_extrainfo(ri); - /* router_build_fresh_extrainfo() should not fail. */ + ei = router_build_fresh_unsigned_extrainfo(ri); + /* router_build_fresh_unsigned_extrainfo() should not fail. */ if (BUG(!ei)) goto err; - result = router_update_extrainfo_descriptor_body(ei); + result = router_dump_and_sign_extrainfo_descriptor_body(ei); if (result < 0) goto err; @@ -2201,7 +2208,7 @@ router_update_routerinfo_from_extrainfo(routerinfo_t *ri, const extrainfo_t *ei) { if (BUG(!ei)) { - /* Just to be safe, zero ri->cache_info.extra_info_digest* here. */ + /* Just to be safe, zero ri->cache_info.extra_info_digest here. */ memset(ri->cache_info.extra_info_digest, 0, DIGEST_LEN); memset(ri->cache_info.extra_info_digest256, 0, DIGEST256_LEN); return; @@ -2216,14 +2223,16 @@ router_update_routerinfo_from_extrainfo(routerinfo_t *ri, DIGEST256_LEN); } -/** Create a signed descriptor for ri, and add it to ri->cache_info. +/** Dump the descriptor body for ri, sign it, and add the body and signature to + * ri->cache_info. Note that the descriptor body is determined by ri, and some + * additional config and state: see router_dump_router_to_string() for details. * * Return 0 on success, and a negative value on temporary error. * If ri is NULL, logs a BUG() warning and returns a negative value. * On error, ri->cache_info is not modified. */ static int -router_update_routerinfo_descriptor_body(routerinfo_t *ri) +router_dump_and_sign_routerinfo_descriptor_body(routerinfo_t *ri) { if (BUG(!ri)) return TOR_ROUTERINFO_ERROR_INTERNAL_BUG; @@ -2247,32 +2256,10 @@ router_update_routerinfo_descriptor_body(routerinfo_t *ri) return 0; } -/** Set the cache_info.send_unencrypted fields in ri and ei. - */ -static void -router_update_info_send_unencrypted(routerinfo_t *ri, extrainfo_t *ei) -{ - const or_options_t *options = get_options(); - - if (options->BridgeRelay) { - /* Bridges shouldn't be able to send their descriptors unencrypted, - anyway, since they don't have a DirPort, and always connect to the - bridge authority anonymously. But just in case they somehow think of - sending them on an unencrypted connection, don't allow them to try. */ - ri->cache_info.send_unencrypted = 0; - if (ei) - ei->cache_info.send_unencrypted = 0; - } else { - ri->cache_info.send_unencrypted = 1; - if (ei) - ei->cache_info.send_unencrypted = 1; - } -} - -/** Build a fresh routerinfo, signed server descriptor, and signed extra-info +/** Build a fresh routerinfo, signed server descriptor, and signed extrainfo * document for this OR. * - * Set r to the generated routerinfo, e to the generated extra-info document. + * Set r to the generated routerinfo, e to the generated extrainfo document. * Failure to generate an extra-info document is not an error and is indicated * by setting e to NULL. * Return 0 on success, and a negative value on temporary error. @@ -2291,7 +2278,7 @@ router_build_fresh_descriptor(routerinfo_t **r, extrainfo_t **e) if (BUG(!e)) goto err; - result = router_build_fresh_routerinfo(&ri); + result = router_build_fresh_unsigned_routerinfo(&ri); if (result < 0) { goto err; } @@ -2302,14 +2289,14 @@ router_build_fresh_descriptor(routerinfo_t **r, extrainfo_t **e) goto err; } - ei = router_build_signed_extrainfo(ri); + ei = router_build_fresh_signed_extrainfo(ri); /* Failing to create an ei is not an error. */ if (ei) { router_update_routerinfo_from_extrainfo(ri, ei); } - result = router_update_routerinfo_descriptor_body(ri); + result = router_dump_and_sign_routerinfo_descriptor_body(ri); if (result < 0) goto err; @@ -2673,6 +2660,10 @@ get_platform_str(char *platform, size_t len) /** OR only: Given a routerinfo for this router, and an identity key to sign * with, encode the routerinfo as a signed server descriptor and return a new * string encoding the result, or NULL on failure. + * + * In addition to the fields in router, this function calls + * onion_key_lifetime(), get_options(), and we_are_hibernating(), and uses the + * results to populate some fields in the descriptor. */ char * router_dump_router_to_string(routerinfo_t *router, @@ -3125,9 +3116,14 @@ load_stats_file(const char *filename, const char *end_line, time_t now, return r; } -/** Write the contents of extrainfo and aggregated statistics to - * *s_out, signing them with ident_key. Return 0 on - * success, negative on failure. */ +/** Write the contents of extrainfo, to * *s_out, signing them + * with ident_key. + * + * If ExtraInfoStatistics is 1, also write aggregated statistics and related + * configuration data before signing. Most statistics also have an option that + * enables or disables that particular statistic. + * + * Return 0 on success, negative on failure. */ int extrainfo_dump_to_string(char **s_out, extrainfo_t *extrainfo, crypto_pk_t *ident_key, diff --git a/src/feature/relay/router.h b/src/feature/relay/router.h index 46364206ef..497d8d243e 100644 --- a/src/feature/relay/router.h +++ b/src/feature/relay/router.h @@ -115,7 +115,7 @@ void router_reset_reachability(void); void router_free_all(void); #ifdef ROUTER_PRIVATE -/* Used only by router.c and test.c */ +/* Used only by router.c and the unit tests */ STATIC void get_platform_str(char *platform, size_t len); STATIC int router_write_fingerprint(int hashed); STATIC smartlist_t *get_my_declared_family(const or_options_t *options); From 7c9450fb073c8b5fb38dab826de7f0356c4828e2 Mon Sep 17 00:00:00 2001 From: teor Date: Thu, 10 Jan 2019 17:12:15 +1000 Subject: [PATCH 0503/2557] test_router: Add comment to explain mocking Add comment in test_router_dump_router_to_string_no_bridge_distribution_method to explain the effect of a mocked function. --- src/test/test_router.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/test/test_router.c b/src/test/test_router.c index ea0ee3e84c..5477ab51e9 100644 --- a/src/test/test_router.c +++ b/src/test/test_router.c @@ -100,6 +100,9 @@ test_router_dump_router_to_string_no_bridge_distribution_method(void *arg) router = (routerinfo_t*)router_get_my_routerinfo(); tt_ptr_op(router, !=, NULL); + /* The real router_get_my_routerinfo() looks up onion_curve25519_pkey using + * get_current_curve25519_keypair(), but we don't initialise static data in + * this test. */ router->onion_curve25519_pkey = &ntor_keypair.pubkey; /* Generate our server descriptor and ensure that the substring From 53b49d1a35d2e7abf1cc7aff15553c23dde0f352 Mon Sep 17 00:00:00 2001 From: teor Date: Mon, 18 Feb 2019 15:24:26 +1000 Subject: [PATCH 0504/2557] test_dir: Unit tests for RSA-only router and extrainfo descriptor creation Tests 29017 and 29018. --- src/feature/nodelist/torcert.c | 4 +- src/feature/nodelist/torcert.h | 2 +- src/feature/relay/router.c | 18 ++--- src/feature/relay/router.h | 12 ++- src/test/test_dir.c | 134 ++++++++++++++++++++++++++++++--- 5 files changed, 146 insertions(+), 24 deletions(-) diff --git a/src/feature/nodelist/torcert.c b/src/feature/nodelist/torcert.c index b0197e9f13..56f1a8ac9f 100644 --- a/src/feature/nodelist/torcert.c +++ b/src/feature/nodelist/torcert.c @@ -290,8 +290,8 @@ tor_cert_describe_signature_status(const tor_cert_t *cert) } /** Return a new copy of cert */ -tor_cert_t * -tor_cert_dup(const tor_cert_t *cert) +MOCK_IMPL(tor_cert_t *, +tor_cert_dup,(const tor_cert_t *cert)) { tor_cert_t *newcert = tor_memdup(cert, sizeof(tor_cert_t)); if (cert->encoded) diff --git a/src/feature/nodelist/torcert.h b/src/feature/nodelist/torcert.h index 492275b514..03d5bdca93 100644 --- a/src/feature/nodelist/torcert.h +++ b/src/feature/nodelist/torcert.h @@ -71,7 +71,7 @@ int tor_cert_checksig(tor_cert_t *cert, const ed25519_public_key_t *pubkey, time_t now); const char *tor_cert_describe_signature_status(const tor_cert_t *cert); -tor_cert_t *tor_cert_dup(const tor_cert_t *cert); +MOCK_DECL(tor_cert_t *,tor_cert_dup,(const tor_cert_t *cert)); int tor_cert_eq(const tor_cert_t *cert1, const tor_cert_t *cert2); int tor_cert_opt_eq(const tor_cert_t *cert1, const tor_cert_t *cert2); diff --git a/src/feature/relay/router.c b/src/feature/relay/router.c index 7e2161ef31..18c8375299 100644 --- a/src/feature/relay/router.c +++ b/src/feature/relay/router.c @@ -196,8 +196,8 @@ set_onion_key(crypto_pk_t *k) /** Return the current onion key. Requires that the onion key has been * loaded or generated. */ -crypto_pk_t * -get_onion_key(void) +MOCK_IMPL(crypto_pk_t *, +get_onion_key,(void)) { tor_assert(onionkey); return onionkey; @@ -376,8 +376,8 @@ assert_identity_keys_ok(void) /** Returns the current server identity key; requires that the key has * been set, and that we are running as a Tor server. */ -crypto_pk_t * -get_server_identity_key(void) +MOCK_IMPL(crypto_pk_t *, +get_server_identity_key,(void)) { tor_assert(server_identitykey); tor_assert(server_mode(get_options())); @@ -1951,8 +1951,8 @@ get_my_declared_family(const or_options_t *options) * * Returns a negative value and sets ri_out to NULL on temporary error. */ -static int -router_build_fresh_unsigned_routerinfo(routerinfo_t **ri_out) +MOCK_IMPL(STATIC int, +router_build_fresh_unsigned_routerinfo,(routerinfo_t **ri_out)) { routerinfo_t *ri = NULL; uint32_t addr; @@ -2171,7 +2171,7 @@ router_dump_and_sign_extrainfo_descriptor_body(extrainfo_t *ei) * If ri is NULL, logs a BUG() warning and returns NULL. * Caller is responsible for freeing the generated extrainfo. */ -static extrainfo_t * +STATIC extrainfo_t * router_build_fresh_signed_extrainfo(const routerinfo_t *ri) { int result = -1; @@ -2203,7 +2203,7 @@ router_build_fresh_signed_extrainfo(const routerinfo_t *ri) * * If ei is NULL, logs a BUG() warning and zeroes the relevant fields. */ -static void +STATIC void router_update_routerinfo_from_extrainfo(routerinfo_t *ri, const extrainfo_t *ei) { @@ -2231,7 +2231,7 @@ router_update_routerinfo_from_extrainfo(routerinfo_t *ri, * If ri is NULL, logs a BUG() warning and returns a negative value. * On error, ri->cache_info is not modified. */ -static int +STATIC int router_dump_and_sign_routerinfo_descriptor_body(routerinfo_t *ri) { if (BUG(!ri)) diff --git a/src/feature/relay/router.h b/src/feature/relay/router.h index 497d8d243e..d4ad52c9d9 100644 --- a/src/feature/relay/router.h +++ b/src/feature/relay/router.h @@ -25,10 +25,10 @@ struct ed25519_keypair_t; #define TOR_ROUTERINFO_ERROR_DESC_REBUILDING (-6) #define TOR_ROUTERINFO_ERROR_INTERNAL_BUG (-7) -crypto_pk_t *get_onion_key(void); +MOCK_DECL(crypto_pk_t *,get_onion_key,(void)); time_t get_onion_key_set_at(void); void set_server_identity_key(crypto_pk_t *k); -crypto_pk_t *get_server_identity_key(void); +MOCK_DECL(crypto_pk_t *,get_server_identity_key,(void)); int server_identity_key_is_set(void); void set_client_identity_key(crypto_pk_t *k); crypto_pk_t *get_tlsclient_identity_key(void); @@ -124,6 +124,14 @@ STATIC smartlist_t *get_my_declared_family(const or_options_t *options); extern time_t desc_clean_since; extern const char *desc_dirty_reason; void set_server_identity_key_digest_testing(const uint8_t *digest); + +MOCK_DECL(STATIC int, + router_build_fresh_unsigned_routerinfo,(routerinfo_t **ri_out)); +STATIC extrainfo_t *router_build_fresh_signed_extrainfo( + const routerinfo_t *ri); +STATIC void router_update_routerinfo_from_extrainfo(routerinfo_t *ri, + const extrainfo_t *ei); +STATIC int router_dump_and_sign_routerinfo_descriptor_body(routerinfo_t *ri); #endif #endif diff --git a/src/test/test_dir.c b/src/test/test_dir.c index 4132d42d12..57adee414c 100644 --- a/src/test/test_dir.c +++ b/src/test/test_dir.c @@ -171,6 +171,31 @@ mock_get_configured_ports(void) return mocked_configured_ports; } +static tor_cert_t * +mock_tor_cert_dup_null(const tor_cert_t *cert) +{ + (void)cert; + return NULL; +} + +static crypto_pk_t *mocked_server_identitykey = NULL; + +/* Returns mocked_server_identitykey with no checks. */ +static crypto_pk_t * +mock_get_server_identity_key(void) +{ + return mocked_server_identitykey; +} + +static crypto_pk_t *mocked_onionkey = NULL; + +/* Returns mocked_onionkey with no checks. */ +static crypto_pk_t * +mock_get_onion_key(void) +{ + return mocked_onionkey; +} + /** Run unit tests for router descriptor generation logic. */ static void test_dir_formats(void *arg) @@ -182,8 +207,11 @@ test_dir_formats(void *arg) char *pk1_str = NULL, *pk2_str = NULL, *cp; size_t pk1_str_len, pk2_str_len; routerinfo_t *r1=NULL, *r2=NULL; + extrainfo_t *e1 = NULL, *e2 = NULL; crypto_pk_t *pk1 = NULL, *pk2 = NULL; + routerinfo_t *r2_out = NULL; routerinfo_t *rp1 = NULL, *rp2 = NULL; + extrainfo_t *ep1 = NULL, *ep2 = NULL; addr_policy_t *ex1, *ex2; routerlist_t *dir1 = NULL, *dir2 = NULL; uint8_t *rsa_cc = NULL; @@ -192,6 +220,8 @@ test_dir_formats(void *arg) time_t now = time(NULL); port_cfg_t orport, dirport; char cert_buf[256]; + int rv = -1; + const char *msg = NULL; (void)arg; pk1 = pk_generate(0); @@ -202,6 +232,8 @@ test_dir_formats(void *arg) hibernate_set_state_for_testing_(HIBERNATE_STATE_LIVE); get_platform_str(platform, sizeof(platform)); + + /* r1 is a minimal, RSA-only descriptor */ r1 = tor_malloc_zero(sizeof(routerinfo_t)); r1->addr = 0xc0a80001u; /* 192.168.0.1 */ r1->cache_info.published_on = 0; @@ -224,6 +256,7 @@ test_dir_formats(void *arg) r1->nickname = tor_strdup("Magri"); r1->platform = tor_strdup(platform); + /* r2 is a RSA + ed25519 descriptor, with an exit policy */ ex1 = tor_malloc_zero(sizeof(addr_policy_t)); ex2 = tor_malloc_zero(sizeof(addr_policy_t)); ex1->policy_type = ADDR_POLICY_ACCEPT; @@ -352,8 +385,86 @@ test_dir_formats(void *arg) crypto_pk_free(onion_pkey); tt_int_op(crypto_pk_cmp_keys(rp1->identity_pkey, pk2), OP_EQ, 0); tt_assert(rp1->supports_tunnelled_dir_requests); - //tt_assert(rp1->exit_policy == NULL); + tt_assert(rp1->policy_is_reject_star); + tor_free(buf); + routerinfo_free(rp1); + + /* Test extrainfo creation. + * We avoid calling router_build_fresh_unsigned_routerinfo(), because it's + * too complex. Instead, we re-use the manually-created routerinfos. + */ + + /* router_build_fresh_signed_extrainfo() requires options->Nickname */ + tor_free(options->Nickname); + options->Nickname = tor_strdup(r1->nickname); + /* router_build_fresh_signed_extrainfo() passes the result of + * get_master_signing_key_cert() directly to tor_cert_dup(), which fails on + * NULL. But we want a NULL ei->cache_info.signing_key_cert to test the + * non-ed key path. + */ + MOCK(tor_cert_dup, mock_tor_cert_dup_null); + /* router_build_fresh_signed_extrainfo() requires get_server_identity_key(). + * Use the same one as the call to router_dump_router_to_string() above. + */ + mocked_server_identitykey = pk2; + MOCK(get_server_identity_key, mock_get_server_identity_key); + /* router_dump_and_sign_routerinfo_descriptor_body() requires + * get_onion_key(). Use the same one as r1. + */ + mocked_onionkey = pk1; + MOCK(get_onion_key, mock_get_onion_key); + + /* Test some of the low-level static functions. */ + e1 = router_build_fresh_signed_extrainfo(r1); + tt_assert(e1); + router_update_routerinfo_from_extrainfo(r1, e1); + rv = router_dump_and_sign_routerinfo_descriptor_body(r1); + tt_assert(rv == 0); + msg = ""; + rv = routerinfo_incompatible_with_extrainfo(r1->identity_pkey, e1, + &r1->cache_info, &msg); + tt_str_op(msg, OP_EQ, ""); + tt_assert(rv == 0); + + /* Now cleanup */ + tor_free(options->Nickname); + UNMOCK(tor_cert_dup); + mocked_server_identitykey = NULL; + UNMOCK(get_server_identity_key); + mocked_onionkey = NULL; + UNMOCK(get_onion_key); + + /* Test that the signed ri is parseable */ + tt_assert(r1->cache_info.signed_descriptor_body); + cp = r1->cache_info.signed_descriptor_body; + rp1 = router_parse_entry_from_string((const char*)cp,NULL,1,0,NULL,NULL); + tt_assert(rp1); + tt_int_op(rp1->addr,OP_EQ, r1->addr); + tt_int_op(rp1->or_port,OP_EQ, r1->or_port); + tt_int_op(rp1->dir_port,OP_EQ, r1->dir_port); + tt_int_op(rp1->bandwidthrate,OP_EQ, r1->bandwidthrate); + tt_int_op(rp1->bandwidthburst,OP_EQ, r1->bandwidthburst); + tt_int_op(rp1->bandwidthcapacity,OP_EQ, r1->bandwidthcapacity); + onion_pkey = router_get_rsa_onion_pkey(rp1->onion_pkey, + rp1->onion_pkey_len); + tt_int_op(crypto_pk_cmp_keys(onion_pkey, pk1), OP_EQ, 0); + crypto_pk_free(onion_pkey); + tt_int_op(crypto_pk_cmp_keys(rp1->identity_pkey, pk2), OP_EQ, 0); + tt_assert(rp1->supports_tunnelled_dir_requests); + tt_assert(rp1->policy_is_reject_star); + + routerinfo_free(rp1); + + /* Test that the signed ei is parseable */ + tt_assert(e1->cache_info.signed_descriptor_body); + cp = e1->cache_info.signed_descriptor_body; + ep1 = extrainfo_parse_entry_from_string((const char*)cp,NULL,1,NULL,NULL); + tt_assert(ep1); + tt_str_op(ep1->nickname, OP_EQ, r1->nickname); + /* In future tests, we could check the actual extrainfo statistics. */ + + extrainfo_free(ep1); strlcpy(buf2, "router Fred 10.3.2.1 9005 0 0\n" @@ -503,20 +614,23 @@ test_dir_formats(void *arg) dirserv_free_fingerprint_list(); done: - if (r1) - routerinfo_free(r1); - if (r2) - routerinfo_free(r2); - if (rp2) - routerinfo_free(rp2); + routerinfo_free(r1); + routerinfo_free(r2); + routerinfo_free(r2_out); + routerinfo_free(rp1); + routerinfo_free(rp2); + + extrainfo_free(e1); + extrainfo_free(e2); + extrainfo_free(ep1); + extrainfo_free(ep2); tor_free(rsa_cc); tor_free(buf); tor_free(pk1_str); tor_free(pk2_str); - if (pk1) crypto_pk_free(pk1); - if (pk2) crypto_pk_free(pk2); - if (rp1) routerinfo_free(rp1); + crypto_pk_free(pk1); + crypto_pk_free(pk2); tor_free(dir1); /* XXXX And more !*/ tor_free(dir2); /* And more !*/ } From 7a2c8dadedcb3b17195111fee7aa91695d6bf6bb Mon Sep 17 00:00:00 2001 From: teor Date: Mon, 18 Feb 2019 17:03:33 +1000 Subject: [PATCH 0505/2557] test_dir: Split test_dir_formats into separate rsa and rsa_ed25519 tests --- src/test/test_dir.c | 186 +++++++++++++++++++++++++++++--------------- 1 file changed, 123 insertions(+), 63 deletions(-) diff --git a/src/test/test_dir.c b/src/test/test_dir.c index 57adee414c..785d114f77 100644 --- a/src/test/test_dir.c +++ b/src/test/test_dir.c @@ -196,9 +196,12 @@ mock_get_onion_key(void) return mocked_onionkey; } -/** Run unit tests for router descriptor generation logic. */ +/** Run unit tests for router descriptor generation logic for a RSA-only + * router. Tor versions without ed25519 (0.2.6 and earlier) are no longer + * officially supported, but the authorities still accept their descriptors. + */ static void -test_dir_formats(void *arg) +test_dir_formats_rsa(void *arg) { char *buf = NULL; char buf2[8192]; @@ -206,18 +209,14 @@ test_dir_formats(void *arg) char fingerprint[FINGERPRINT_LEN+1]; char *pk1_str = NULL, *pk2_str = NULL, *cp; size_t pk1_str_len, pk2_str_len; - routerinfo_t *r1=NULL, *r2=NULL; - extrainfo_t *e1 = NULL, *e2 = NULL; + routerinfo_t *r1 = NULL; + extrainfo_t *e1 = NULL; crypto_pk_t *pk1 = NULL, *pk2 = NULL; - routerinfo_t *r2_out = NULL; - routerinfo_t *rp1 = NULL, *rp2 = NULL; - extrainfo_t *ep1 = NULL, *ep2 = NULL; - addr_policy_t *ex1, *ex2; + routerinfo_t *rp1 = NULL; + extrainfo_t *ep1 = NULL; routerlist_t *dir1 = NULL, *dir2 = NULL; uint8_t *rsa_cc = NULL; or_options_t *options = get_options_mutable(); - const addr_policy_t *p; - time_t now = time(NULL); port_cfg_t orport, dirport; char cert_buf[256]; int rv = -1; @@ -256,57 +255,15 @@ test_dir_formats(void *arg) r1->nickname = tor_strdup("Magri"); r1->platform = tor_strdup(platform); - /* r2 is a RSA + ed25519 descriptor, with an exit policy */ - ex1 = tor_malloc_zero(sizeof(addr_policy_t)); - ex2 = tor_malloc_zero(sizeof(addr_policy_t)); - ex1->policy_type = ADDR_POLICY_ACCEPT; - tor_addr_from_ipv4h(&ex1->addr, 0); - ex1->maskbits = 0; - ex1->prt_min = ex1->prt_max = 80; - ex2->policy_type = ADDR_POLICY_REJECT; - tor_addr_from_ipv4h(&ex2->addr, 18<<24); - ex2->maskbits = 8; - ex2->prt_min = ex2->prt_max = 24; - r2 = tor_malloc_zero(sizeof(routerinfo_t)); - r2->addr = 0x0a030201u; /* 10.3.2.1 */ - ed25519_keypair_t kp1, kp2; - ed25519_secret_key_from_seed(&kp1.seckey, - (const uint8_t*)"YYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYY"); - ed25519_public_key_generate(&kp1.pubkey, &kp1.seckey); - ed25519_secret_key_from_seed(&kp2.seckey, - (const uint8_t*)"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"); - ed25519_public_key_generate(&kp2.pubkey, &kp2.seckey); - r2->cache_info.signing_key_cert = tor_cert_create(&kp1, - CERT_TYPE_ID_SIGNING, - &kp2.pubkey, - now, 86400, - CERT_FLAG_INCLUDE_SIGNING_KEY); - r2->platform = tor_strdup(platform); - r2->cache_info.published_on = 5; - r2->or_port = 9005; - r2->dir_port = 0; - r2->supports_tunnelled_dir_requests = 1; - router_set_rsa_onion_pkey(pk2, &r2->onion_pkey, &r2->onion_pkey_len); - curve25519_keypair_t r2_onion_keypair; - curve25519_keypair_generate(&r2_onion_keypair, 0); - r2->onion_curve25519_pkey = tor_memdup(&r2_onion_keypair.pubkey, - sizeof(curve25519_public_key_t)); - r2->identity_pkey = crypto_pk_dup_key(pk1); - r2->bandwidthrate = r2->bandwidthburst = r2->bandwidthcapacity = 3000; - r2->exit_policy = smartlist_new(); - smartlist_add(r2->exit_policy, ex1); - smartlist_add(r2->exit_policy, ex2); - r2->nickname = tor_strdup("Fred"); - tt_assert(!crypto_pk_write_public_key_to_string(pk1, &pk1_str, - &pk1_str_len)); + &pk1_str_len)); tt_assert(!crypto_pk_write_public_key_to_string(pk2 , &pk2_str, - &pk2_str_len)); + &pk2_str_len)); /* XXXX+++ router_dump_to_string should really take this from ri.*/ options->ContactInfo = tor_strdup("Magri White " ""); - /* Skip reachability checks for DirPort and tunnelled-dir-server */ + /* Skip reachability checks for DirPort, ORPort, and tunnelled-dir-server */ options->AssumeReachable = 1; /* Fake just enough of an ORPort and DirPort to get by */ @@ -466,6 +423,108 @@ test_dir_formats(void *arg) extrainfo_free(ep1); + done: + dirserv_free_fingerprint_list(); + + routerinfo_free(r1); + routerinfo_free(rp1); + + extrainfo_free(e1); + extrainfo_free(ep1); + + tor_free(rsa_cc); + tor_free(buf); + tor_free(pk1_str); + tor_free(pk2_str); + crypto_pk_free(pk1); + crypto_pk_free(pk2); + tor_free(dir1); /* XXXX And more !*/ + tor_free(dir2); /* And more !*/ +} + +/** Run unit tests for router descriptor generation logic for a RSA + ed25519 + * router. + */ +static void +test_dir_formats_rsa_ed25519(void *arg) +{ + char *buf = NULL; + char buf2[8192]; + char platform[256]; + char fingerprint[FINGERPRINT_LEN+1]; + char *pk1_str = NULL, *pk2_str = NULL, *cp; + size_t pk1_str_len, pk2_str_len; + routerinfo_t *r2 = NULL; + extrainfo_t *e2 = NULL; + crypto_pk_t *pk1 = NULL, *pk2 = NULL; + routerinfo_t *r2_out = NULL; + routerinfo_t *rp2 = NULL; + extrainfo_t *ep2 = NULL; + addr_policy_t *ex1, *ex2; + routerlist_t *dir1 = NULL, *dir2 = NULL; + uint8_t *rsa_cc = NULL; + or_options_t *options = get_options_mutable(); + const addr_policy_t *p; + time_t now = time(NULL); + port_cfg_t orport; + char cert_buf[256]; + + (void)arg; + pk1 = pk_generate(0); + pk2 = pk_generate(1); + + tt_assert(pk1 && pk2); + + hibernate_set_state_for_testing_(HIBERNATE_STATE_LIVE); + + get_platform_str(platform, sizeof(platform)); + /* r2 is a RSA + ed25519 descriptor, with an exit policy */ + ex1 = tor_malloc_zero(sizeof(addr_policy_t)); + ex2 = tor_malloc_zero(sizeof(addr_policy_t)); + ex1->policy_type = ADDR_POLICY_ACCEPT; + tor_addr_from_ipv4h(&ex1->addr, 0); + ex1->maskbits = 0; + ex1->prt_min = ex1->prt_max = 80; + ex2->policy_type = ADDR_POLICY_REJECT; + tor_addr_from_ipv4h(&ex2->addr, 18<<24); + ex2->maskbits = 8; + ex2->prt_min = ex2->prt_max = 24; + r2 = tor_malloc_zero(sizeof(routerinfo_t)); + r2->addr = 0x0a030201u; /* 10.3.2.1 */ + ed25519_keypair_t kp1, kp2; + ed25519_secret_key_from_seed(&kp1.seckey, + (const uint8_t*)"YYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYY"); + ed25519_public_key_generate(&kp1.pubkey, &kp1.seckey); + ed25519_secret_key_from_seed(&kp2.seckey, + (const uint8_t*)"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"); + ed25519_public_key_generate(&kp2.pubkey, &kp2.seckey); + r2->cache_info.signing_key_cert = tor_cert_create(&kp1, + CERT_TYPE_ID_SIGNING, + &kp2.pubkey, + now, 86400, + CERT_FLAG_INCLUDE_SIGNING_KEY); + r2->platform = tor_strdup(platform); + r2->cache_info.published_on = 5; + r2->or_port = 9005; + r2->dir_port = 0; + r2->supports_tunnelled_dir_requests = 1; + router_set_rsa_onion_pkey(pk2, &r2->onion_pkey, &r2->onion_pkey_len); + curve25519_keypair_t r2_onion_keypair; + curve25519_keypair_generate(&r2_onion_keypair, 0); + r2->onion_curve25519_pkey = tor_memdup(&r2_onion_keypair.pubkey, + sizeof(curve25519_public_key_t)); + r2->identity_pkey = crypto_pk_dup_key(pk1); + r2->bandwidthrate = r2->bandwidthburst = r2->bandwidthcapacity = 3000; + r2->exit_policy = smartlist_new(); + smartlist_add(r2->exit_policy, ex1); + smartlist_add(r2->exit_policy, ex2); + r2->nickname = tor_strdup("Fred"); + + tt_assert(!crypto_pk_write_public_key_to_string(pk1, &pk1_str, + &pk1_str_len)); + tt_assert(!crypto_pk_write_public_key_to_string(pk2 , &pk2_str, + &pk2_str_len)); + strlcpy(buf2, "router Fred 10.3.2.1 9005 0 0\n" "identity-ed25519\n" @@ -540,6 +599,9 @@ test_dir_formats(void *arg) strlcat(buf2, "tunnelled-dir-server\n", sizeof(buf2)); strlcat(buf2, "router-sig-ed25519 ", sizeof(buf2)); + /* Skip reachability checks for ORPort and tunnelled-dir-server */ + options->AssumeReachable = 1; + /* Fake just enough of an ORPort to get by */ MOCK(get_configured_ports, mock_get_configured_ports); mocked_configured_ports = smartlist_new(); @@ -577,8 +639,8 @@ test_dir_formats(void *arg) tt_mem_op(rp2->onion_curve25519_pkey->public_key,OP_EQ, r2->onion_curve25519_pkey->public_key, CURVE25519_PUBKEY_LEN); - onion_pkey = router_get_rsa_onion_pkey(rp2->onion_pkey, - rp2->onion_pkey_len); + crypto_pk_t *onion_pkey = router_get_rsa_onion_pkey(rp2->onion_pkey, + rp2->onion_pkey_len); tt_int_op(crypto_pk_cmp_keys(onion_pkey, pk2), OP_EQ, 0); crypto_pk_free(onion_pkey); tt_int_op(crypto_pk_cmp_keys(rp2->identity_pkey, pk1), OP_EQ, 0); @@ -611,18 +673,15 @@ test_dir_formats(void *arg) } #endif /* 0 */ - dirserv_free_fingerprint_list(); done: - routerinfo_free(r1); + dirserv_free_fingerprint_list(); + routerinfo_free(r2); routerinfo_free(r2_out); - routerinfo_free(rp1); routerinfo_free(rp2); - extrainfo_free(e1); extrainfo_free(e2); - extrainfo_free(ep1); extrainfo_free(ep2); tor_free(rsa_cc); @@ -6601,7 +6660,8 @@ test_dir_format_versions_list(void *arg) struct testcase_t dir_tests[] = { DIR_LEGACY(nicknames), - DIR_LEGACY(formats), + DIR_LEGACY(formats_rsa), + DIR_LEGACY(formats_rsa_ed25519), DIR(routerinfo_parsing, 0), DIR(extrainfo_parsing, 0), DIR(parse_router_list, TT_FORK), From 8e5df40018acd0bd80626073b16b6cc070129109 Mon Sep 17 00:00:00 2001 From: teor Date: Mon, 18 Feb 2019 17:37:47 +1000 Subject: [PATCH 0506/2557] test_dir: Test rsa + ed25519 extrainfo creation and parsing Also fix a missing mock in rsa-only parsing. --- src/feature/relay/router.c | 5 +- src/feature/relay/router.h | 2 + src/feature/relay/routerkeys.c | 8 +- src/feature/relay/routerkeys.h | 4 +- src/test/test_dir.c | 196 ++++++++++++++++++++++++++++++++- 5 files changed, 206 insertions(+), 9 deletions(-) diff --git a/src/feature/relay/router.c b/src/feature/relay/router.c index 18c8375299..cbd5f2f91e 100644 --- a/src/feature/relay/router.c +++ b/src/feature/relay/router.c @@ -271,11 +271,12 @@ expire_old_onion_keys(void) /** Return the current secret onion key for the ntor handshake. Must only * be called from the main thread. */ -static const curve25519_keypair_t * -get_current_curve25519_keypair(void) +MOCK_IMPL(STATIC const struct curve25519_keypair_t *, +get_current_curve25519_keypair,(void)) { return &curve25519_onion_key; } + /** Return a map from KEYID (the key itself) to keypairs for use in the ntor * handshake. Must only be called from the main thread. */ di_digest256_map_t * diff --git a/src/feature/relay/router.h b/src/feature/relay/router.h index d4ad52c9d9..f17e6a9438 100644 --- a/src/feature/relay/router.h +++ b/src/feature/relay/router.h @@ -124,6 +124,8 @@ STATIC smartlist_t *get_my_declared_family(const or_options_t *options); extern time_t desc_clean_since; extern const char *desc_dirty_reason; void set_server_identity_key_digest_testing(const uint8_t *digest); +MOCK_DECL(STATIC const struct curve25519_keypair_t *, + get_current_curve25519_keypair,(void)); MOCK_DECL(STATIC int, router_build_fresh_unsigned_routerinfo,(routerinfo_t **ri_out)); diff --git a/src/feature/relay/routerkeys.c b/src/feature/relay/routerkeys.c index 876f908d41..bdd7a82b58 100644 --- a/src/feature/relay/routerkeys.c +++ b/src/feature/relay/routerkeys.c @@ -631,14 +631,14 @@ get_master_identity_keypair(void) } #endif /* defined(TOR_UNIT_TESTS) */ -const ed25519_keypair_t * -get_master_signing_keypair(void) +MOCK_IMPL(const ed25519_keypair_t *, +get_master_signing_keypair,(void)) { return master_signing_key; } -const struct tor_cert_st * -get_master_signing_key_cert(void) +MOCK_IMPL(const struct tor_cert_st *, +get_master_signing_key_cert,(void)) { return signing_key_cert; } diff --git a/src/feature/relay/routerkeys.h b/src/feature/relay/routerkeys.h index 0badd34191..cde07b52c3 100644 --- a/src/feature/relay/routerkeys.h +++ b/src/feature/relay/routerkeys.h @@ -7,8 +7,8 @@ #include "lib/crypt_ops/crypto_ed25519.h" const ed25519_public_key_t *get_master_identity_key(void); -const ed25519_keypair_t *get_master_signing_keypair(void); -const struct tor_cert_st *get_master_signing_key_cert(void); +MOCK_DECL(const ed25519_keypair_t *, get_master_signing_keypair,(void)); +MOCK_DECL(const struct tor_cert_st *, get_master_signing_key_cert,(void)); const ed25519_keypair_t *get_current_auth_keypair(void); const struct tor_cert_st *get_current_link_cert_cert(void); diff --git a/src/test/test_dir.c b/src/test/test_dir.c index 785d114f77..ad8ab87e9f 100644 --- a/src/test/test_dir.c +++ b/src/test/test_dir.c @@ -196,6 +196,45 @@ mock_get_onion_key(void) return mocked_onionkey; } +static routerinfo_t *mocked_routerinfo = NULL; + +/* Returns 0 and sets ri_out to mocked_routerinfo. + * ri_out must not be NULL. There are no other checks. */ +static int +mock_router_build_fresh_unsigned_routerinfo(routerinfo_t **ri_out) +{ + tor_assert(ri_out); + *ri_out = mocked_routerinfo; + return 0; +} + +static ed25519_keypair_t *mocked_master_signing_key = NULL; + +/* Returns mocked_master_signing_key with no checks. */ +static const ed25519_keypair_t * +mock_get_master_signing_keypair(void) +{ + return mocked_master_signing_key; +} + +static struct tor_cert_st *mocked_signing_key_cert = NULL; + +/* Returns mocked_signing_key_cert with no checks. */ +static const struct tor_cert_st * +mock_get_master_signing_key_cert(void) +{ + return mocked_signing_key_cert; +} + +static curve25519_keypair_t *mocked_curve25519_onion_key = NULL; + +/* Returns mocked_curve25519_onion_key with no checks. */ +static const curve25519_keypair_t * +mock_get_current_curve25519_keypair(void) +{ + return mocked_curve25519_onion_key; +} + /** Run unit tests for router descriptor generation logic for a RSA-only * router. Tor versions without ed25519 (0.2.6 and earlier) are no longer * officially supported, but the authorities still accept their descriptors. @@ -372,6 +411,22 @@ test_dir_formats_rsa(void *arg) mocked_onionkey = pk1; MOCK(get_onion_key, mock_get_onion_key); + /* Fake just enough of an ORPort and DirPort to get by */ + MOCK(get_configured_ports, mock_get_configured_ports); + mocked_configured_ports = smartlist_new(); + + memset(&orport, 0, sizeof(orport)); + orport.type = CONN_TYPE_OR_LISTENER; + orport.addr.family = AF_INET; + orport.port = 9000; + smartlist_add(mocked_configured_ports, &orport); + + memset(&dirport, 0, sizeof(dirport)); + dirport.type = CONN_TYPE_DIR_LISTENER; + dirport.addr.family = AF_INET; + dirport.port = 9003; + smartlist_add(mocked_configured_ports, &dirport); + /* Test some of the low-level static functions. */ e1 = router_build_fresh_signed_extrainfo(r1); tt_assert(e1); @@ -392,6 +447,10 @@ test_dir_formats_rsa(void *arg) mocked_onionkey = NULL; UNMOCK(get_onion_key); + UNMOCK(get_configured_ports); + smartlist_free(mocked_configured_ports); + mocked_configured_ports = NULL; + /* Test that the signed ri is parseable */ tt_assert(r1->cache_info.signed_descriptor_body); cp = r1->cache_info.signed_descriptor_body; @@ -468,6 +527,7 @@ test_dir_formats_rsa_ed25519(void *arg) time_t now = time(NULL); port_cfg_t orport; char cert_buf[256]; + int rv = -1; (void)arg; pk1 = pk_generate(0); @@ -478,7 +538,10 @@ test_dir_formats_rsa_ed25519(void *arg) hibernate_set_state_for_testing_(HIBERNATE_STATE_LIVE); get_platform_str(platform, sizeof(platform)); - /* r2 is a RSA + ed25519 descriptor, with an exit policy */ + + /* We can't use init_mock_ed_keys() here, because the keys are seeded */ + + /* r2 is a RSA + ed25519 descriptor, with an exit policy */ ex1 = tor_malloc_zero(sizeof(addr_policy_t)); ex2 = tor_malloc_zero(sizeof(addr_policy_t)); ex1->policy_type = ADDR_POLICY_ACCEPT; @@ -662,6 +725,137 @@ test_dir_formats_rsa_ed25519(void *arg) tt_int_op(p->prt_min,OP_EQ, 24); tt_int_op(p->prt_max,OP_EQ, 24); + routerinfo_free(rp2); + + /* Test extrainfo creation. */ + + /* router_build_fresh_descriptor() requires + * router_build_fresh_unsigned_routerinfo(), but the implementation is + * too complex. Instead, we re-use r2. + */ + mocked_routerinfo = r2; + MOCK(router_build_fresh_unsigned_routerinfo, + mock_router_build_fresh_unsigned_routerinfo); + + /* router_build_fresh_signed_extrainfo() requires options->Nickname */ + tor_free(options->Nickname); + options->Nickname = tor_strdup(r2->nickname); + /* router_build_fresh_signed_extrainfo() requires get_server_identity_key(). + * Use the same one as the call to router_dump_router_to_string() above. + * For the second router, the keys are swapped. + */ + mocked_server_identitykey = pk1; + MOCK(get_server_identity_key, mock_get_server_identity_key); + /* router_dump_and_sign_routerinfo_descriptor_body() requires + * get_onion_key(). Use the same one as r1. + */ + mocked_onionkey = pk2; + MOCK(get_onion_key, mock_get_onion_key); + + /* r2 uses ed25519, so we need to mock the ed key functions */ + mocked_master_signing_key = &kp2; + MOCK(get_master_signing_keypair, mock_get_master_signing_keypair); + + mocked_signing_key_cert = r2->cache_info.signing_key_cert; + MOCK(get_master_signing_key_cert, mock_get_master_signing_key_cert); + + mocked_curve25519_onion_key = &r2_onion_keypair; + MOCK(get_current_curve25519_keypair, mock_get_current_curve25519_keypair); + + /* Fake just enough of an ORPort to get by */ + MOCK(get_configured_ports, mock_get_configured_ports); + mocked_configured_ports = smartlist_new(); + + memset(&orport, 0, sizeof(orport)); + orport.type = CONN_TYPE_OR_LISTENER; + orport.addr.family = AF_INET; + orport.port = 9005; + smartlist_add(mocked_configured_ports, &orport); + + /* Test the high-level interface. */ + rv = router_build_fresh_descriptor(&r2_out, &e2); + if (rv < 0) { + /* router_build_fresh_descriptor() frees r2 on failure. */ + r2 = NULL; + /* Get rid of an alias to rp2 */ + r2_out = NULL; + } + tt_assert(rv == 0); + tt_assert(r2_out); + tt_assert(e2); + /* Guaranteed by mock_router_build_fresh_unsigned_routerinfo() */ + tt_ptr_op(r2_out, OP_EQ, r2); + /* Get rid of an alias to r2 */ + r2_out = NULL; + + /* Now cleanup */ + mocked_routerinfo = NULL; + UNMOCK(router_build_fresh_unsigned_routerinfo); + tor_free(options->Nickname); + mocked_server_identitykey = NULL; + UNMOCK(get_server_identity_key); + mocked_onionkey = NULL; + UNMOCK(get_onion_key); + mocked_master_signing_key = NULL; + UNMOCK(get_master_signing_keypair); + mocked_signing_key_cert = NULL; + UNMOCK(get_master_signing_key_cert); + mocked_curve25519_onion_key = NULL; + UNMOCK(get_current_curve25519_keypair); + + UNMOCK(get_configured_ports); + smartlist_free(mocked_configured_ports); + mocked_configured_ports = NULL; + + /* Test that the signed ri is parseable */ + tt_assert(r2->cache_info.signed_descriptor_body); + cp = r2->cache_info.signed_descriptor_body; + rp2 = router_parse_entry_from_string((const char*)cp,NULL,1,0,NULL,NULL); + tt_assert(rp2); + tt_int_op(rp2->addr,OP_EQ, r2->addr); + tt_int_op(rp2->or_port,OP_EQ, r2->or_port); + tt_int_op(rp2->dir_port,OP_EQ, r2->dir_port); + tt_int_op(rp2->bandwidthrate,OP_EQ, r2->bandwidthrate); + tt_int_op(rp2->bandwidthburst,OP_EQ, r2->bandwidthburst); + tt_int_op(rp2->bandwidthcapacity,OP_EQ, r2->bandwidthcapacity); + tt_mem_op(rp2->onion_curve25519_pkey->public_key,OP_EQ, + r2->onion_curve25519_pkey->public_key, + CURVE25519_PUBKEY_LEN); + onion_pkey = router_get_rsa_onion_pkey(rp2->onion_pkey, + rp2->onion_pkey_len); + tt_int_op(crypto_pk_cmp_keys(onion_pkey, pk2), OP_EQ, 0); + crypto_pk_free(onion_pkey); + tt_int_op(crypto_pk_cmp_keys(rp2->identity_pkey, pk1), OP_EQ, 0); + tt_assert(rp2->supports_tunnelled_dir_requests); + + tt_int_op(smartlist_len(rp2->exit_policy),OP_EQ, 2); + + p = smartlist_get(rp2->exit_policy, 0); + tt_int_op(p->policy_type,OP_EQ, ADDR_POLICY_ACCEPT); + tt_assert(tor_addr_is_null(&p->addr)); + tt_int_op(p->maskbits,OP_EQ, 0); + tt_int_op(p->prt_min,OP_EQ, 80); + tt_int_op(p->prt_max,OP_EQ, 80); + + p = smartlist_get(rp2->exit_policy, 1); + tt_int_op(p->policy_type,OP_EQ, ADDR_POLICY_REJECT); + tt_assert(tor_addr_eq(&p->addr, &ex2->addr)); + tt_int_op(p->maskbits,OP_EQ, 8); + tt_int_op(p->prt_min,OP_EQ, 24); + tt_int_op(p->prt_max,OP_EQ, 24); + + routerinfo_free(rp2); + + /* Test that the signed ei is parseable */ + tt_assert(e2->cache_info.signed_descriptor_body); + cp = e2->cache_info.signed_descriptor_body; + ep2 = extrainfo_parse_entry_from_string((const char*)cp,NULL,1,NULL,NULL); + tt_assert(ep2); + tt_str_op(ep2->nickname, OP_EQ, r2->nickname); + /* In future tests, we could check the actual extrainfo statistics. */ + + extrainfo_free(ep2); + #if 0 /* Okay, now for the directories. */ { From 38fc52a50e12fbcf1b1451e5a3a3016d1ced9759 Mon Sep 17 00:00:00 2001 From: teor Date: Tue, 19 Feb 2019 11:19:44 +1000 Subject: [PATCH 0507/2557] test_dir: Refactor common code out of the dir_format unit tests Also: * delete some obsolete code that was #if 0 * improve cleanup on failure * make the dir_format tests more consistent with each other * construct the descriptors using smartlist chunks This refactor is incomplete, because removing the remaining duplicate code would be time-consuming. Part of 29017 and 29018. --- src/test/test_dir.c | 1009 +++++++++++++++++++++++++++---------------- 1 file changed, 637 insertions(+), 372 deletions(-) diff --git a/src/test/test_dir.c b/src/test/test_dir.c index ad8ab87e9f..cc0ef07fbf 100644 --- a/src/test/test_dir.c +++ b/src/test/test_dir.c @@ -162,6 +162,255 @@ test_dir_nicknames(void *arg) ; } +/* Allocate and return a new routerinfo, with the fields set from the + * arguments to this function. + * + * Also sets: + * - random RSA identity and onion keys, + * - the platform field using get_platform_str(), and + * - supports_tunnelled_dir_requests to 1. + * + * If rsa_onion_keypair_out is not NULL, it is set to the onion keypair. + * The caller must free this keypair. + */ +static routerinfo_t * +basic_routerinfo_new(const char *nickname, uint32_t ipv4_addr, + uint16_t or_port, uint16_t dir_port, + uint32_t bandwidthrate, uint32_t bandwidthburst, + uint32_t bandwidthcapacity, + time_t published_on, + crypto_pk_t **rsa_onion_keypair_out) +{ + char platform[256]; + + tor_assert(nickname); + + crypto_pk_t *pk1 = NULL, *pk2 = NULL; + /* These keys are random: idx is ignored. */ + pk1 = pk_generate(0); + pk2 = pk_generate(1); + + tor_assert(pk1); + tor_assert(pk2); + + get_platform_str(platform, sizeof(platform)); + + routerinfo_t *r1 = tor_malloc_zero(sizeof(routerinfo_t)); + + r1->nickname = tor_strdup(nickname); + r1->platform = tor_strdup(platform); + + r1->addr = ipv4_addr; + r1->or_port = or_port; + r1->dir_port = dir_port; + r1->supports_tunnelled_dir_requests = 1; + + router_set_rsa_onion_pkey(pk1, &r1->onion_pkey, &r1->onion_pkey_len); + r1->identity_pkey = pk2; + + r1->bandwidthrate = bandwidthrate; + r1->bandwidthburst = bandwidthburst; + r1->bandwidthcapacity = bandwidthcapacity; + + r1->cache_info.published_on = published_on; + + if (rsa_onion_keypair_out) { + *rsa_onion_keypair_out = pk1; + } else { + crypto_pk_free(pk1); + } + + return r1; +} + +/* Allocate and return a new string containing a "router" line for r1. */ +static char * +get_new_router_line(const routerinfo_t *r1) +{ + char *line = NULL; + + tor_assert(r1); + + tor_asprintf(&line, + "router %s %s %d 0 %d\n", + r1->nickname, fmt_addr32(r1->addr), + r1->or_port, r1->dir_port); + tor_assert(line); + + return line; +} + +/* Allocate and return a new string containing a "platform" line for the + * current Tor version and OS. */ +static char * +get_new_platform_line(void) +{ + char *line = NULL; + + tor_asprintf(&line, + "platform Tor %s on %s\n", + VERSION, get_uname()); + tor_assert(line); + + return line; +} + +/* Allocate and return a new string containing a "published" line for r1. + * r1->cache_info.published_on must be between 0 and 59 seconds. */ +static char * +get_new_published_line(const routerinfo_t *r1) +{ + char *line = NULL; + + tor_assert(r1); + + tor_assert(r1->cache_info.published_on >= 0); + tor_assert(r1->cache_info.published_on <= 59); + + tor_asprintf(&line, + "published 1970-01-01 00:00:%02u\n", + (unsigned)r1->cache_info.published_on); + tor_assert(line); + + return line; +} + +/* Allocate and return a new string containing a "fingerprint" line for r1. */ +static char * +get_new_fingerprint_line(const routerinfo_t *r1) +{ + char *line = NULL; + char fingerprint[FINGERPRINT_LEN+1]; + + tor_assert(r1); + + tor_assert(!crypto_pk_get_fingerprint(r1->identity_pkey, fingerprint, 1)); + tor_assert(strlen(fingerprint) > 0); + + tor_asprintf(&line, + "fingerprint %s\n", + fingerprint); + tor_assert(line); + + return line; +} + +/* Allocate and return a new string containing an "uptime" line with uptime t. + * + * You should pass a hard-coded value to this function, because even if we made + * it reflect uptime, that still wouldn't make it right, because the two + * descriptors might be made on different seconds. + */ +static char * +get_new_uptime_line(time_t t) +{ + char *line = NULL; + + tor_asprintf(&line, + "uptime %u\n", + (unsigned)t); + tor_assert(line); + + return line; +} + +/* Allocate and return a new string containing an "bandwidth" line for r1. + */ +static char * +get_new_bandwidth_line(const routerinfo_t *r1) +{ + char *line = NULL; + + tor_assert(r1); + + tor_asprintf(&line, + "bandwidth %u %u %u\n", + r1->bandwidthrate, + r1->bandwidthburst, + r1->bandwidthcapacity); + tor_assert(line); + + return line; +} + +/* Allocate and return a new string containing a key_name block for the + * RSA key pk1. + */ +static char * +get_new_rsa_key_block(const char *key_name, crypto_pk_t *pk1) +{ + char *block = NULL; + char *pk1_str = NULL; + size_t pk1_str_len = 0; + + tor_assert(key_name); + tor_assert(pk1); + + tor_assert(!crypto_pk_write_public_key_to_string(pk1, &pk1_str, + &pk1_str_len)); + tor_assert(pk1_str); + tor_assert(pk1_str_len); + + tor_asprintf(&block, + "%s\n%s", + key_name, + pk1_str); + tor_assert(block); + + return block; +} + +/* Allocate and return a new string containing an "onion-key" block for the + * router r1. + */ +static char * +get_new_onion_key_block(const routerinfo_t *r1) +{ + char *block = NULL; + tor_assert(r1); + crypto_pk_t *pk_tmp = router_get_rsa_onion_pkey(r1->onion_pkey, + r1->onion_pkey_len); + block = get_new_rsa_key_block("onion-key", pk_tmp); + crypto_pk_free(pk_tmp); + return block; +} + +/* Allocate and return a new string containing an "signing-key" block for the + * router r1. + */ +static char * +get_new_signing_key_block(const routerinfo_t *r1) +{ + tor_assert(r1); + return get_new_rsa_key_block("signing-key", r1->identity_pkey); +} + +/* Allocate and return a new string containing an "ntor-onion-key" line for + * the curve25519 public key ntor_onion_pubkey. + */ +static char * +get_new_ntor_onion_key_line(const curve25519_public_key_t *ntor_onion_pubkey) +{ + char *line = NULL; + char cert_buf[256]; + int rv = 0; + + tor_assert(ntor_onion_pubkey); + + rv = base64_encode(cert_buf, sizeof(cert_buf), + (const char*)ntor_onion_pubkey->public_key, 32, + BASE64_ENCODE_MULTILINE); + tor_assert(rv > 0); + tor_assert(strlen(cert_buf) > 0); + + tor_asprintf(&line, + "ntor-onion-key %s", + cert_buf); + tor_assert(line); + + return line; +} + static smartlist_t *mocked_configured_ports = NULL; /** Returns mocked_configured_ports */ @@ -235,6 +484,137 @@ mock_get_current_curve25519_keypair(void) return mocked_curve25519_onion_key; } +/* Unmock get_configured_ports() and free mocked_configured_ports. */ +static void +cleanup_mock_configured_ports(void) +{ + UNMOCK(get_configured_ports); + + if (mocked_configured_ports) { + SMARTLIST_FOREACH(mocked_configured_ports, port_cfg_t *, p, tor_free(p)); + smartlist_free(mocked_configured_ports); + } +} + +/* Mock get_configured_ports() with a list containing or_port and dir_port. + * If a port is 0, don't set it. + * Only sets the minimal data required for the tests to pass. */ +static void +setup_mock_configured_ports(uint16_t or_port, uint16_t dir_port) +{ + cleanup_mock_configured_ports(); + + /* Fake just enough of an ORPort and DirPort to get by */ + MOCK(get_configured_ports, mock_get_configured_ports); + mocked_configured_ports = smartlist_new(); + + if (or_port) { + port_cfg_t *or_port_cfg = tor_malloc_zero(sizeof(*or_port_cfg)); + or_port_cfg->type = CONN_TYPE_OR_LISTENER; + or_port_cfg->addr.family = AF_INET; + or_port_cfg->port = or_port; + smartlist_add(mocked_configured_ports, or_port_cfg); + } + + if (dir_port) { + port_cfg_t *dir_port_cfg = tor_malloc_zero(sizeof(*dir_port_cfg)); + dir_port_cfg->type = CONN_TYPE_DIR_LISTENER; + dir_port_cfg->addr.family = AF_INET; + dir_port_cfg->port = dir_port; + smartlist_add(mocked_configured_ports, dir_port_cfg); + } +} + +/* Clean up the data structures and unmock the functions needed for generating + * a fresh descriptor. */ +static void +cleanup_mocks_for_fresh_descriptor(void) +{ + tor_free(get_options_mutable()->Nickname); + + mocked_server_identitykey = NULL; + UNMOCK(get_server_identity_key); + + crypto_pk_free(mocked_onionkey); + UNMOCK(get_onion_key); +} + +/* Mock the data structures and functions needed for generating a fresh + * descriptor. + * + * Sets options->Nickname from r1->nickname. + * Mocks get_server_identity_key() with r1->identity_pkey. + * + * If rsa_onion_keypair is not NULL, it is used to mock get_onion_key(). + * Otherwise, the public key in r1->onion_pkey is used to mock get_onion_key(). + */ +static void +setup_mocks_for_fresh_descriptor(const routerinfo_t *r1, + crypto_pk_t *rsa_onion_keypair) +{ + cleanup_mocks_for_fresh_descriptor(); + + tor_assert(r1); + + /* router_build_fresh_signed_extrainfo() requires options->Nickname */ + get_options_mutable()->Nickname = tor_strdup(r1->nickname); + + /* router_build_fresh_signed_extrainfo() requires get_server_identity_key(). + * Use the same one as the call to router_dump_router_to_string() above. + */ + mocked_server_identitykey = r1->identity_pkey; + MOCK(get_server_identity_key, mock_get_server_identity_key); + + /* router_dump_and_sign_routerinfo_descriptor_body() requires + * get_onion_key(). Use the same one as r1. + */ + if (rsa_onion_keypair) { + mocked_onionkey = crypto_pk_dup_key(rsa_onion_keypair); + } else { + mocked_onionkey = router_get_rsa_onion_pkey(r1->onion_pkey, + r1->onion_pkey_len); + } + MOCK(get_onion_key, mock_get_onion_key); +} + +/* Check that routerinfos r1 and rp1 are consistent. + * Only performs some basic checks. + */ +#define CHECK_ROUTERINFO_CONSISTENCY(r1, rp1) \ +STMT_BEGIN \ + tt_assert(r1); \ + tt_assert(rp1); \ +\ + tt_int_op(rp1->addr,OP_EQ, r1->addr); \ + tt_int_op(rp1->or_port,OP_EQ, r1->or_port); \ + tt_int_op(rp1->dir_port,OP_EQ, r1->dir_port); \ + tt_int_op(rp1->bandwidthrate,OP_EQ, r1->bandwidthrate); \ + tt_int_op(rp1->bandwidthburst,OP_EQ, r1->bandwidthburst); \ + tt_int_op(rp1->bandwidthcapacity,OP_EQ, r1->bandwidthcapacity); \ + crypto_pk_t *rp1_onion_pkey = router_get_rsa_onion_pkey(rp1->onion_pkey, \ + rp1->onion_pkey_len); \ + crypto_pk_t *r1_onion_pkey = router_get_rsa_onion_pkey(r1->onion_pkey, \ + r1->onion_pkey_len); \ + tt_int_op(crypto_pk_cmp_keys(rp1_onion_pkey, r1_onion_pkey), OP_EQ, 0); \ + crypto_pk_free(rp1_onion_pkey); \ + crypto_pk_free(r1_onion_pkey); \ + tt_int_op(crypto_pk_cmp_keys(rp1->identity_pkey, r1->identity_pkey), \ + OP_EQ, 0); \ + tt_int_op(rp1->supports_tunnelled_dir_requests, OP_EQ, \ + r1->supports_tunnelled_dir_requests); \ +STMT_END + +/* Check that routerinfo r1 and extrainfo e1 are consistent. + * Only performs some basic checks. + */ +#define CHECK_EXTRAINFO_CONSISTENCY(r1, e1) \ +STMT_BEGIN \ + tt_assert(r1); \ + tt_assert(e1); \ +\ + tt_str_op(e1->nickname, OP_EQ, r1->nickname); \ +STMT_END + /** Run unit tests for router descriptor generation logic for a RSA-only * router. Tor versions without ed25519 (0.2.6 and earlier) are no longer * officially supported, but the authorities still accept their descriptors. @@ -243,144 +623,112 @@ static void test_dir_formats_rsa(void *arg) { char *buf = NULL; - char buf2[8192]; - char platform[256]; - char fingerprint[FINGERPRINT_LEN+1]; - char *pk1_str = NULL, *pk2_str = NULL, *cp; - size_t pk1_str_len, pk2_str_len; + char *buf2 = NULL; + char *cp = NULL; + + uint8_t *rsa_cc = NULL; + routerinfo_t *r1 = NULL; extrainfo_t *e1 = NULL; - crypto_pk_t *pk1 = NULL, *pk2 = NULL; routerinfo_t *rp1 = NULL; extrainfo_t *ep1 = NULL; - routerlist_t *dir1 = NULL, *dir2 = NULL; - uint8_t *rsa_cc = NULL; - or_options_t *options = get_options_mutable(); - port_cfg_t orport, dirport; - char cert_buf[256]; - int rv = -1; + + smartlist_t *chunks = NULL; const char *msg = NULL; + int rv = -1; + + or_options_t *options = get_options_mutable(); (void)arg; - pk1 = pk_generate(0); - pk2 = pk_generate(1); - - tt_assert(pk1 && pk2); hibernate_set_state_for_testing_(HIBERNATE_STATE_LIVE); - get_platform_str(platform, sizeof(platform)); + /* r1 is a minimal, RSA-only descriptor, with DirPort and IPv6 */ + r1 = basic_routerinfo_new("Magri", 0xc0a80001u /* 192.168.0.1 */, + 9000, 9003, + 1000, 5000, 10000, + 0, + NULL); - /* r1 is a minimal, RSA-only descriptor */ - r1 = tor_malloc_zero(sizeof(routerinfo_t)); - r1->addr = 0xc0a80001u; /* 192.168.0.1 */ - r1->cache_info.published_on = 0; - r1->or_port = 9000; - r1->dir_port = 9003; - r1->supports_tunnelled_dir_requests = 1; - tor_addr_parse(&r1->ipv6_addr, "1:2:3:4::"); - r1->ipv6_orport = 9999; - router_set_rsa_onion_pkey(pk1, &r1->onion_pkey, &r1->onion_pkey_len); - /* Fake just enough of an ntor key to get by */ + /* Fake just enough of an ntor key to get by */ curve25519_keypair_t r1_onion_keypair; curve25519_keypair_generate(&r1_onion_keypair, 0); r1->onion_curve25519_pkey = tor_memdup(&r1_onion_keypair.pubkey, sizeof(curve25519_public_key_t)); - r1->identity_pkey = crypto_pk_dup_key(pk2); - r1->bandwidthrate = 1000; - r1->bandwidthburst = 5000; - r1->bandwidthcapacity = 10000; + + /* Now add IPv6 */ + tor_addr_parse(&r1->ipv6_addr, "1:2:3:4::"); + r1->ipv6_orport = 9999; + r1->exit_policy = NULL; - r1->nickname = tor_strdup("Magri"); - r1->platform = tor_strdup(platform); - tt_assert(!crypto_pk_write_public_key_to_string(pk1, &pk1_str, - &pk1_str_len)); - tt_assert(!crypto_pk_write_public_key_to_string(pk2 , &pk2_str, - &pk2_str_len)); - - /* XXXX+++ router_dump_to_string should really take this from ri.*/ + /* XXXX+++ router_dump_to_string should really take this from ri. */ options->ContactInfo = tor_strdup("Magri White " ""); /* Skip reachability checks for DirPort, ORPort, and tunnelled-dir-server */ options->AssumeReachable = 1; - /* Fake just enough of an ORPort and DirPort to get by */ - MOCK(get_configured_ports, mock_get_configured_ports); - mocked_configured_ports = smartlist_new(); + setup_mock_configured_ports(r1->or_port, r1->dir_port); - memset(&orport, 0, sizeof(orport)); - orport.type = CONN_TYPE_OR_LISTENER; - orport.addr.family = AF_INET; - orport.port = 9000; - smartlist_add(mocked_configured_ports, &orport); - - memset(&dirport, 0, sizeof(dirport)); - dirport.type = CONN_TYPE_DIR_LISTENER; - dirport.addr.family = AF_INET; - dirport.port = 9003; - smartlist_add(mocked_configured_ports, &dirport); - - buf = router_dump_router_to_string(r1, pk2, NULL, NULL, NULL); - - UNMOCK(get_configured_ports); - smartlist_free(mocked_configured_ports); - mocked_configured_ports = NULL; - - tor_free(options->ContactInfo); + buf = router_dump_router_to_string(r1, r1->identity_pkey, NULL, NULL, NULL); tt_assert(buf); - strlcpy(buf2, "router Magri 192.168.0.1 9000 0 9003\n" - "or-address [1:2:3:4::]:9999\n" - "platform Tor "VERSION" on ", sizeof(buf2)); - strlcat(buf2, get_uname(), sizeof(buf2)); - strlcat(buf2, "\n" - "published 1970-01-01 00:00:00\n" - "fingerprint ", sizeof(buf2)); - tt_assert(!crypto_pk_get_fingerprint(pk2, fingerprint, 1)); - strlcat(buf2, fingerprint, sizeof(buf2)); - strlcat(buf2, "\nuptime 0\n" - /* XXX the "0" above is hard-coded, but even if we made it reflect - * uptime, that still wouldn't make it right, because the two - * descriptors might be made on different seconds... hm. */ - "bandwidth 1000 5000 10000\n" - "onion-key\n", sizeof(buf2)); - strlcat(buf2, pk1_str, sizeof(buf2)); - strlcat(buf2, "signing-key\n", sizeof(buf2)); - strlcat(buf2, pk2_str, sizeof(buf2)); - strlcat(buf2, "hidden-service-dir\n", sizeof(buf2)); - strlcat(buf2, "contact Magri White \n", - sizeof(buf2)); - strlcat(buf2, "ntor-onion-key ", sizeof(buf2)); - base64_encode(cert_buf, sizeof(cert_buf), - (const char*)r1_onion_keypair.pubkey.public_key, 32, - BASE64_ENCODE_MULTILINE); - strlcat(buf2, cert_buf, sizeof(buf2)); - strlcat(buf2, "reject *:*\n", sizeof(buf2)); - strlcat(buf2, "tunnelled-dir-server\nrouter-signature\n", sizeof(buf2)); + tor_free(options->ContactInfo); + cleanup_mock_configured_ports(); + + /* Synthesise a router descriptor, without the signature */ + chunks = smartlist_new(); + + smartlist_add(chunks, get_new_router_line(r1)); + smartlist_add_strdup(chunks, "or-address [1:2:3:4::]:9999\n"); + + smartlist_add(chunks, get_new_platform_line()); + smartlist_add(chunks, get_new_published_line(r1)); + smartlist_add(chunks, get_new_fingerprint_line(r1)); + + smartlist_add(chunks, get_new_uptime_line(0)); + smartlist_add(chunks, get_new_bandwidth_line(r1)); + + smartlist_add(chunks, get_new_onion_key_block(r1)); + smartlist_add(chunks, get_new_signing_key_block(r1)); + + smartlist_add_strdup(chunks, "hidden-service-dir\n"); + + smartlist_add_strdup(chunks, "contact Magri White " + "\n"); + + smartlist_add(chunks, get_new_ntor_onion_key_line(&r1_onion_keypair.pubkey)); + smartlist_add_strdup(chunks, "reject *:*\n"); + smartlist_add_strdup(chunks, "tunnelled-dir-server\n"); + + smartlist_add_strdup(chunks, "router-signature\n"); + + size_t len_out = 0; + buf2 = smartlist_join_strings(chunks, "", 0, &len_out); + SMARTLIST_FOREACH(chunks, char *, s, tor_free(s)); + smartlist_free(chunks); + + tt_assert(len_out > 0); + buf[strlen(buf2)] = '\0'; /* Don't compare the sig; it's never the same * twice */ tt_str_op(buf,OP_EQ, buf2); tor_free(buf); - buf = router_dump_router_to_string(r1, pk2, NULL, NULL, NULL); + setup_mock_configured_ports(r1->or_port, r1->dir_port); + + buf = router_dump_router_to_string(r1, r1->identity_pkey, NULL, NULL, NULL); tt_assert(buf); + + cleanup_mock_configured_ports(); + + /* Now, try to parse buf */ cp = buf; rp1 = router_parse_entry_from_string((const char*)cp,NULL,1,0,NULL,NULL); - tt_assert(rp1); - tt_int_op(rp1->addr,OP_EQ, r1->addr); - tt_int_op(rp1->or_port,OP_EQ, r1->or_port); - tt_int_op(rp1->dir_port,OP_EQ, r1->dir_port); - tt_int_op(rp1->bandwidthrate,OP_EQ, r1->bandwidthrate); - tt_int_op(rp1->bandwidthburst,OP_EQ, r1->bandwidthburst); - tt_int_op(rp1->bandwidthcapacity,OP_EQ, r1->bandwidthcapacity); - crypto_pk_t *onion_pkey = router_get_rsa_onion_pkey(rp1->onion_pkey, - rp1->onion_pkey_len); - tt_int_op(crypto_pk_cmp_keys(onion_pkey, pk1), OP_EQ, 0); - crypto_pk_free(onion_pkey); - tt_int_op(crypto_pk_cmp_keys(rp1->identity_pkey, pk2), OP_EQ, 0); - tt_assert(rp1->supports_tunnelled_dir_requests); + + CHECK_ROUTERINFO_CONSISTENCY(r1, rp1); + tt_assert(rp1->policy_is_reject_star); tor_free(buf); @@ -391,41 +739,18 @@ test_dir_formats_rsa(void *arg) * too complex. Instead, we re-use the manually-created routerinfos. */ - /* router_build_fresh_signed_extrainfo() requires options->Nickname */ - tor_free(options->Nickname); - options->Nickname = tor_strdup(r1->nickname); + /* Set up standard mocks and data */ + setup_mocks_for_fresh_descriptor(r1, NULL); + /* router_build_fresh_signed_extrainfo() passes the result of * get_master_signing_key_cert() directly to tor_cert_dup(), which fails on * NULL. But we want a NULL ei->cache_info.signing_key_cert to test the * non-ed key path. */ MOCK(tor_cert_dup, mock_tor_cert_dup_null); - /* router_build_fresh_signed_extrainfo() requires get_server_identity_key(). - * Use the same one as the call to router_dump_router_to_string() above. - */ - mocked_server_identitykey = pk2; - MOCK(get_server_identity_key, mock_get_server_identity_key); - /* router_dump_and_sign_routerinfo_descriptor_body() requires - * get_onion_key(). Use the same one as r1. - */ - mocked_onionkey = pk1; - MOCK(get_onion_key, mock_get_onion_key); /* Fake just enough of an ORPort and DirPort to get by */ - MOCK(get_configured_ports, mock_get_configured_ports); - mocked_configured_ports = smartlist_new(); - - memset(&orport, 0, sizeof(orport)); - orport.type = CONN_TYPE_OR_LISTENER; - orport.addr.family = AF_INET; - orport.port = 9000; - smartlist_add(mocked_configured_ports, &orport); - - memset(&dirport, 0, sizeof(dirport)); - dirport.type = CONN_TYPE_DIR_LISTENER; - dirport.addr.family = AF_INET; - dirport.port = 9003; - smartlist_add(mocked_configured_ports, &dirport); + setup_mock_configured_ports(r1->or_port, r1->dir_port); /* Test some of the low-level static functions. */ e1 = router_build_fresh_signed_extrainfo(r1); @@ -436,38 +761,26 @@ test_dir_formats_rsa(void *arg) msg = ""; rv = routerinfo_incompatible_with_extrainfo(r1->identity_pkey, e1, &r1->cache_info, &msg); + /* If they are incompatible, fail and show the msg string */ tt_str_op(msg, OP_EQ, ""); tt_assert(rv == 0); /* Now cleanup */ - tor_free(options->Nickname); - UNMOCK(tor_cert_dup); - mocked_server_identitykey = NULL; - UNMOCK(get_server_identity_key); - mocked_onionkey = NULL; - UNMOCK(get_onion_key); + cleanup_mocks_for_fresh_descriptor(); - UNMOCK(get_configured_ports); - smartlist_free(mocked_configured_ports); - mocked_configured_ports = NULL; + UNMOCK(tor_cert_dup); + + cleanup_mock_configured_ports(); + + CHECK_EXTRAINFO_CONSISTENCY(r1, e1); /* Test that the signed ri is parseable */ tt_assert(r1->cache_info.signed_descriptor_body); cp = r1->cache_info.signed_descriptor_body; rp1 = router_parse_entry_from_string((const char*)cp,NULL,1,0,NULL,NULL); - tt_assert(rp1); - tt_int_op(rp1->addr,OP_EQ, r1->addr); - tt_int_op(rp1->or_port,OP_EQ, r1->or_port); - tt_int_op(rp1->dir_port,OP_EQ, r1->dir_port); - tt_int_op(rp1->bandwidthrate,OP_EQ, r1->bandwidthrate); - tt_int_op(rp1->bandwidthburst,OP_EQ, r1->bandwidthburst); - tt_int_op(rp1->bandwidthcapacity,OP_EQ, r1->bandwidthcapacity); - onion_pkey = router_get_rsa_onion_pkey(rp1->onion_pkey, - rp1->onion_pkey_len); - tt_int_op(crypto_pk_cmp_keys(onion_pkey, pk1), OP_EQ, 0); - crypto_pk_free(onion_pkey); - tt_int_op(crypto_pk_cmp_keys(rp1->identity_pkey, pk2), OP_EQ, 0); - tt_assert(rp1->supports_tunnelled_dir_requests); + + CHECK_ROUTERINFO_CONSISTENCY(r1, rp1); + tt_assert(rp1->policy_is_reject_star); routerinfo_free(rp1); @@ -476,8 +789,9 @@ test_dir_formats_rsa(void *arg) tt_assert(e1->cache_info.signed_descriptor_body); cp = e1->cache_info.signed_descriptor_body; ep1 = extrainfo_parse_entry_from_string((const char*)cp,NULL,1,NULL,NULL); - tt_assert(ep1); - tt_str_op(ep1->nickname, OP_EQ, r1->nickname); + + CHECK_EXTRAINFO_CONSISTENCY(r1, ep1); + /* In future tests, we could check the actual extrainfo statistics. */ extrainfo_free(ep1); @@ -485,6 +799,17 @@ test_dir_formats_rsa(void *arg) done: dirserv_free_fingerprint_list(); + tor_free(options->ContactInfo); + tor_free(options->Nickname); + + cleanup_mock_configured_ports(); + cleanup_mocks_for_fresh_descriptor(); + + if (chunks) { + SMARTLIST_FOREACH(chunks, char *, s, tor_free(s)); + smartlist_free(chunks); + } + routerinfo_free(r1); routerinfo_free(rp1); @@ -492,15 +817,31 @@ test_dir_formats_rsa(void *arg) extrainfo_free(ep1); tor_free(rsa_cc); + tor_free(buf); - tor_free(pk1_str); - tor_free(pk2_str); - crypto_pk_free(pk1); - crypto_pk_free(pk2); - tor_free(dir1); /* XXXX And more !*/ - tor_free(dir2); /* And more !*/ + tor_free(buf2); } +/* Check that the exit policy in rp2 is as expected. */ +#define CHECK_PARSED_EXIT_POLICY(rp2) \ +STMT_BEGIN \ + tt_int_op(smartlist_len(rp2->exit_policy),OP_EQ, 2); \ + \ + p = smartlist_get(rp2->exit_policy, 0); \ + tt_int_op(p->policy_type,OP_EQ, ADDR_POLICY_ACCEPT); \ + tt_assert(tor_addr_is_null(&p->addr)); \ + tt_int_op(p->maskbits,OP_EQ, 0); \ + tt_int_op(p->prt_min,OP_EQ, 80); \ + tt_int_op(p->prt_max,OP_EQ, 80); \ + \ + p = smartlist_get(rp2->exit_policy, 1); \ + tt_int_op(p->policy_type,OP_EQ, ADDR_POLICY_REJECT); \ + tt_assert(tor_addr_eq(&p->addr, &ex2->addr)); \ + tt_int_op(p->maskbits,OP_EQ, 8); \ + tt_int_op(p->prt_min,OP_EQ, 24); \ + tt_int_op(p->prt_max,OP_EQ, 24); \ +STMT_END + /** Run unit tests for router descriptor generation logic for a RSA + ed25519 * router. */ @@ -508,52 +849,47 @@ static void test_dir_formats_rsa_ed25519(void *arg) { char *buf = NULL; - char buf2[8192]; - char platform[256]; - char fingerprint[FINGERPRINT_LEN+1]; - char *pk1_str = NULL, *pk2_str = NULL, *cp; - size_t pk1_str_len, pk2_str_len; + char *buf2 = NULL; + char *cp = NULL; + + crypto_pk_t *r2_onion_pkey = NULL; + char cert_buf[256]; + uint8_t *rsa_cc = NULL; + time_t now = time(NULL); + routerinfo_t *r2 = NULL; extrainfo_t *e2 = NULL; - crypto_pk_t *pk1 = NULL, *pk2 = NULL; routerinfo_t *r2_out = NULL; routerinfo_t *rp2 = NULL; extrainfo_t *ep2 = NULL; addr_policy_t *ex1, *ex2; - routerlist_t *dir1 = NULL, *dir2 = NULL; - uint8_t *rsa_cc = NULL; - or_options_t *options = get_options_mutable(); const addr_policy_t *p; - time_t now = time(NULL); - port_cfg_t orport; - char cert_buf[256]; + + smartlist_t *chunks = NULL; int rv = -1; - (void)arg; - pk1 = pk_generate(0); - pk2 = pk_generate(1); + or_options_t *options = get_options_mutable(); - tt_assert(pk1 && pk2); + (void)arg; hibernate_set_state_for_testing_(HIBERNATE_STATE_LIVE); - get_platform_str(platform, sizeof(platform)); + /* r2 is a RSA + ed25519 descriptor, with an exit policy, but no DirPort or + * IPv6 */ + r2 = basic_routerinfo_new("Fred", 0x0a030201u /* 10.3.2.1 */, + 9005, 0, + 3000, 3000, 3000, + 5, + &r2_onion_pkey); - /* We can't use init_mock_ed_keys() here, because the keys are seeded */ + /* Fake just enough of an ntor key to get by */ + curve25519_keypair_t r2_onion_keypair; + curve25519_keypair_generate(&r2_onion_keypair, 0); + r2->onion_curve25519_pkey = tor_memdup(&r2_onion_keypair.pubkey, + sizeof(curve25519_public_key_t)); - /* r2 is a RSA + ed25519 descriptor, with an exit policy */ - ex1 = tor_malloc_zero(sizeof(addr_policy_t)); - ex2 = tor_malloc_zero(sizeof(addr_policy_t)); - ex1->policy_type = ADDR_POLICY_ACCEPT; - tor_addr_from_ipv4h(&ex1->addr, 0); - ex1->maskbits = 0; - ex1->prt_min = ex1->prt_max = 80; - ex2->policy_type = ADDR_POLICY_REJECT; - tor_addr_from_ipv4h(&ex2->addr, 18<<24); - ex2->maskbits = 8; - ex2->prt_min = ex2->prt_max = 24; - r2 = tor_malloc_zero(sizeof(routerinfo_t)); - r2->addr = 0x0a030201u; /* 10.3.2.1 */ + /* Now add relay ed25519 keys + * We can't use init_mock_ed_keys() here, because the keys are seeded */ ed25519_keypair_t kp1, kp2; ed25519_secret_key_from_seed(&kp1.seckey, (const uint8_t*)"YYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYY"); @@ -566,72 +902,83 @@ test_dir_formats_rsa_ed25519(void *arg) &kp2.pubkey, now, 86400, CERT_FLAG_INCLUDE_SIGNING_KEY); - r2->platform = tor_strdup(platform); - r2->cache_info.published_on = 5; - r2->or_port = 9005; - r2->dir_port = 0; - r2->supports_tunnelled_dir_requests = 1; - router_set_rsa_onion_pkey(pk2, &r2->onion_pkey, &r2->onion_pkey_len); - curve25519_keypair_t r2_onion_keypair; - curve25519_keypair_generate(&r2_onion_keypair, 0); - r2->onion_curve25519_pkey = tor_memdup(&r2_onion_keypair.pubkey, - sizeof(curve25519_public_key_t)); - r2->identity_pkey = crypto_pk_dup_key(pk1); - r2->bandwidthrate = r2->bandwidthburst = r2->bandwidthcapacity = 3000; + + /* Now add an exit policy */ + ex1 = tor_malloc_zero(sizeof(addr_policy_t)); + ex2 = tor_malloc_zero(sizeof(addr_policy_t)); + ex1->policy_type = ADDR_POLICY_ACCEPT; + tor_addr_from_ipv4h(&ex1->addr, 0); + ex1->maskbits = 0; + ex1->prt_min = ex1->prt_max = 80; + ex2->policy_type = ADDR_POLICY_REJECT; + tor_addr_from_ipv4h(&ex2->addr, 18<<24); + ex2->maskbits = 8; + ex2->prt_min = ex2->prt_max = 24; + r2->exit_policy = smartlist_new(); smartlist_add(r2->exit_policy, ex1); smartlist_add(r2->exit_policy, ex2); - r2->nickname = tor_strdup("Fred"); - tt_assert(!crypto_pk_write_public_key_to_string(pk1, &pk1_str, - &pk1_str_len)); - tt_assert(!crypto_pk_write_public_key_to_string(pk2 , &pk2_str, - &pk2_str_len)); + /* Skip reachability checks for ORPort and tunnelled-dir-server */ + options->AssumeReachable = 1; - strlcpy(buf2, - "router Fred 10.3.2.1 9005 0 0\n" - "identity-ed25519\n" - "-----BEGIN ED25519 CERT-----\n", sizeof(buf2)); + /* Fake just enough of an ORPort to get by */ + setup_mock_configured_ports(r2->or_port, 0); + + buf = router_dump_router_to_string(r2, + r2->identity_pkey, r2_onion_pkey, + &r2_onion_keypair, &kp2); + tt_assert(buf); + + cleanup_mock_configured_ports(); + + chunks = smartlist_new(); + + /* Synthesise a router descriptor, without the signatures */ + smartlist_add(chunks, get_new_router_line(r2)); + + smartlist_add_strdup(chunks, + "identity-ed25519\n" + "-----BEGIN ED25519 CERT-----\n"); base64_encode(cert_buf, sizeof(cert_buf), (const char*)r2->cache_info.signing_key_cert->encoded, r2->cache_info.signing_key_cert->encoded_len, BASE64_ENCODE_MULTILINE); - strlcat(buf2, cert_buf, sizeof(buf2)); - strlcat(buf2, "-----END ED25519 CERT-----\n", sizeof(buf2)); - strlcat(buf2, "master-key-ed25519 ", sizeof(buf2)); + smartlist_add_strdup(chunks, cert_buf); + smartlist_add_strdup(chunks, "-----END ED25519 CERT-----\n"); + + smartlist_add_strdup(chunks, "master-key-ed25519 "); { char k[ED25519_BASE64_LEN+1]; tt_int_op(ed25519_public_to_base64(k, &r2->cache_info.signing_key_cert->signing_key), OP_GE, 0); - strlcat(buf2, k, sizeof(buf2)); - strlcat(buf2, "\n", sizeof(buf2)); + smartlist_add_strdup(chunks, k); + smartlist_add_strdup(chunks, "\n"); } - strlcat(buf2, "platform Tor "VERSION" on ", sizeof(buf2)); - strlcat(buf2, get_uname(), sizeof(buf2)); - strlcat(buf2, "\n" - "published 1970-01-01 00:00:05\n" - "fingerprint ", sizeof(buf2)); - tt_assert(!crypto_pk_get_fingerprint(pk1, fingerprint, 1)); - strlcat(buf2, fingerprint, sizeof(buf2)); - strlcat(buf2, "\nuptime 0\n" - "bandwidth 3000 3000 3000\n", sizeof(buf2)); - strlcat(buf2, "onion-key\n", sizeof(buf2)); - strlcat(buf2, pk2_str, sizeof(buf2)); - strlcat(buf2, "signing-key\n", sizeof(buf2)); - strlcat(buf2, pk1_str, sizeof(buf2)); + + smartlist_add(chunks, get_new_platform_line()); + smartlist_add(chunks, get_new_published_line(r2)); + smartlist_add(chunks, get_new_fingerprint_line(r2)); + + smartlist_add(chunks, get_new_uptime_line(0)); + smartlist_add(chunks, get_new_bandwidth_line(r2)); + + smartlist_add(chunks, get_new_onion_key_block(r2)); + smartlist_add(chunks, get_new_signing_key_block(r2)); + int rsa_cc_len; - rsa_cc = make_tap_onion_key_crosscert(pk2, + rsa_cc = make_tap_onion_key_crosscert(r2_onion_pkey, &kp1.pubkey, - pk1, + r2->identity_pkey, &rsa_cc_len); tt_assert(rsa_cc); base64_encode(cert_buf, sizeof(cert_buf), (char*)rsa_cc, rsa_cc_len, BASE64_ENCODE_MULTILINE); - strlcat(buf2, "onion-key-crosscert\n" - "-----BEGIN CROSSCERT-----\n", sizeof(buf2)); - strlcat(buf2, cert_buf, sizeof(buf2)); - strlcat(buf2, "-----END CROSSCERT-----\n", sizeof(buf2)); + smartlist_add_strdup(chunks, "onion-key-crosscert\n" + "-----BEGIN CROSSCERT-----\n"); + smartlist_add_strdup(chunks, cert_buf); + smartlist_add_strdup(chunks, "-----END CROSSCERT-----\n"); int ntor_cc_sign; { tor_cert_t *ntor_cc = NULL; @@ -646,89 +993,60 @@ test_dir_formats_rsa_ed25519(void *arg) BASE64_ENCODE_MULTILINE); tor_cert_free(ntor_cc); } - tor_snprintf(buf2+strlen(buf2), sizeof(buf2)-strlen(buf2), + smartlist_add_asprintf(chunks, "ntor-onion-key-crosscert %d\n" "-----BEGIN ED25519 CERT-----\n" "%s" "-----END ED25519 CERT-----\n", ntor_cc_sign, cert_buf); - strlcat(buf2, "hidden-service-dir\n", sizeof(buf2)); - strlcat(buf2, "ntor-onion-key ", sizeof(buf2)); - base64_encode(cert_buf, sizeof(cert_buf), - (const char*)r2_onion_keypair.pubkey.public_key, 32, - BASE64_ENCODE_MULTILINE); - strlcat(buf2, cert_buf, sizeof(buf2)); - strlcat(buf2, "accept *:80\nreject 18.0.0.0/8:24\n", sizeof(buf2)); - strlcat(buf2, "tunnelled-dir-server\n", sizeof(buf2)); - strlcat(buf2, "router-sig-ed25519 ", sizeof(buf2)); + smartlist_add_strdup(chunks, "hidden-service-dir\n"); - /* Skip reachability checks for ORPort and tunnelled-dir-server */ - options->AssumeReachable = 1; + smartlist_add(chunks, get_new_ntor_onion_key_line(&r2_onion_keypair.pubkey)); + smartlist_add_strdup(chunks, "accept *:80\nreject 18.0.0.0/8:24\n"); + smartlist_add_strdup(chunks, "tunnelled-dir-server\n"); - /* Fake just enough of an ORPort to get by */ - MOCK(get_configured_ports, mock_get_configured_ports); - mocked_configured_ports = smartlist_new(); + smartlist_add_strdup(chunks, "router-sig-ed25519 "); - memset(&orport, 0, sizeof(orport)); - orport.type = CONN_TYPE_OR_LISTENER; - orport.addr.family = AF_INET; - orport.port = 9005; - smartlist_add(mocked_configured_ports, &orport); + size_t len_out = 0; + buf2 = smartlist_join_strings(chunks, "", 0, &len_out); + SMARTLIST_FOREACH(chunks, char *, s, tor_free(s)); + smartlist_free(chunks); - buf = router_dump_router_to_string(r2, pk1, pk2, &r2_onion_keypair, &kp2); - tt_assert(buf); - buf[strlen(buf2)] = '\0'; /* Don't compare the sig; it's never the same + tt_assert(len_out > 0); + + buf[strlen(buf2)] = '\0'; /* Don't compare either sig; they're never the same * twice */ tt_str_op(buf, OP_EQ, buf2); tor_free(buf); - buf = router_dump_router_to_string(r2, pk1, NULL, NULL, NULL); + setup_mock_configured_ports(r2->or_port, 0); - UNMOCK(get_configured_ports); - smartlist_free(mocked_configured_ports); - mocked_configured_ports = NULL; + buf = router_dump_router_to_string(r2, r2->identity_pkey, NULL, NULL, NULL); + tt_assert(buf); - /* Reset for later */ + cleanup_mock_configured_ports(); + + /* Now, try to parse buf */ cp = buf; rp2 = router_parse_entry_from_string((const char*)cp,NULL,1,0,NULL,NULL); - tt_assert(rp2); - tt_int_op(rp2->addr,OP_EQ, r2->addr); - tt_int_op(rp2->or_port,OP_EQ, r2->or_port); - tt_int_op(rp2->dir_port,OP_EQ, r2->dir_port); - tt_int_op(rp2->bandwidthrate,OP_EQ, r2->bandwidthrate); - tt_int_op(rp2->bandwidthburst,OP_EQ, r2->bandwidthburst); - tt_int_op(rp2->bandwidthcapacity,OP_EQ, r2->bandwidthcapacity); + + CHECK_ROUTERINFO_CONSISTENCY(r2, rp2); + tt_mem_op(rp2->onion_curve25519_pkey->public_key,OP_EQ, r2->onion_curve25519_pkey->public_key, CURVE25519_PUBKEY_LEN); - crypto_pk_t *onion_pkey = router_get_rsa_onion_pkey(rp2->onion_pkey, - rp2->onion_pkey_len); - tt_int_op(crypto_pk_cmp_keys(onion_pkey, pk2), OP_EQ, 0); - crypto_pk_free(onion_pkey); - tt_int_op(crypto_pk_cmp_keys(rp2->identity_pkey, pk1), OP_EQ, 0); - tt_assert(rp2->supports_tunnelled_dir_requests); - tt_int_op(smartlist_len(rp2->exit_policy),OP_EQ, 2); - - p = smartlist_get(rp2->exit_policy, 0); - tt_int_op(p->policy_type,OP_EQ, ADDR_POLICY_ACCEPT); - tt_assert(tor_addr_is_null(&p->addr)); - tt_int_op(p->maskbits,OP_EQ, 0); - tt_int_op(p->prt_min,OP_EQ, 80); - tt_int_op(p->prt_max,OP_EQ, 80); - - p = smartlist_get(rp2->exit_policy, 1); - tt_int_op(p->policy_type,OP_EQ, ADDR_POLICY_REJECT); - tt_assert(tor_addr_eq(&p->addr, &ex2->addr)); - tt_int_op(p->maskbits,OP_EQ, 8); - tt_int_op(p->prt_min,OP_EQ, 24); - tt_int_op(p->prt_max,OP_EQ, 24); + CHECK_PARSED_EXIT_POLICY(rp2); + tor_free(buf); routerinfo_free(rp2); /* Test extrainfo creation. */ + /* Set up standard mocks and data */ + setup_mocks_for_fresh_descriptor(r2, r2_onion_pkey); + /* router_build_fresh_descriptor() requires * router_build_fresh_unsigned_routerinfo(), but the implementation is * too complex. Instead, we re-use r2. @@ -737,21 +1055,6 @@ test_dir_formats_rsa_ed25519(void *arg) MOCK(router_build_fresh_unsigned_routerinfo, mock_router_build_fresh_unsigned_routerinfo); - /* router_build_fresh_signed_extrainfo() requires options->Nickname */ - tor_free(options->Nickname); - options->Nickname = tor_strdup(r2->nickname); - /* router_build_fresh_signed_extrainfo() requires get_server_identity_key(). - * Use the same one as the call to router_dump_router_to_string() above. - * For the second router, the keys are swapped. - */ - mocked_server_identitykey = pk1; - MOCK(get_server_identity_key, mock_get_server_identity_key); - /* router_dump_and_sign_routerinfo_descriptor_body() requires - * get_onion_key(). Use the same one as r1. - */ - mocked_onionkey = pk2; - MOCK(get_onion_key, mock_get_onion_key); - /* r2 uses ed25519, so we need to mock the ed key functions */ mocked_master_signing_key = &kp2; MOCK(get_master_signing_keypair, mock_get_master_signing_keypair); @@ -763,14 +1066,7 @@ test_dir_formats_rsa_ed25519(void *arg) MOCK(get_current_curve25519_keypair, mock_get_current_curve25519_keypair); /* Fake just enough of an ORPort to get by */ - MOCK(get_configured_ports, mock_get_configured_ports); - mocked_configured_ports = smartlist_new(); - - memset(&orport, 0, sizeof(orport)); - orport.type = CONN_TYPE_OR_LISTENER; - orport.addr.family = AF_INET; - orport.port = 9005; - smartlist_add(mocked_configured_ports, &orport); + setup_mock_configured_ports(r2->or_port, 0); /* Test the high-level interface. */ rv = router_build_fresh_descriptor(&r2_out, &e2); @@ -789,13 +1085,10 @@ test_dir_formats_rsa_ed25519(void *arg) r2_out = NULL; /* Now cleanup */ + cleanup_mocks_for_fresh_descriptor(); + mocked_routerinfo = NULL; UNMOCK(router_build_fresh_unsigned_routerinfo); - tor_free(options->Nickname); - mocked_server_identitykey = NULL; - UNMOCK(get_server_identity_key); - mocked_onionkey = NULL; - UNMOCK(get_onion_key); mocked_master_signing_key = NULL; UNMOCK(get_master_signing_keypair); mocked_signing_key_cert = NULL; @@ -803,46 +1096,22 @@ test_dir_formats_rsa_ed25519(void *arg) mocked_curve25519_onion_key = NULL; UNMOCK(get_current_curve25519_keypair); - UNMOCK(get_configured_ports); - smartlist_free(mocked_configured_ports); - mocked_configured_ports = NULL; + cleanup_mock_configured_ports(); + + CHECK_EXTRAINFO_CONSISTENCY(r2, e2); /* Test that the signed ri is parseable */ tt_assert(r2->cache_info.signed_descriptor_body); cp = r2->cache_info.signed_descriptor_body; rp2 = router_parse_entry_from_string((const char*)cp,NULL,1,0,NULL,NULL); - tt_assert(rp2); - tt_int_op(rp2->addr,OP_EQ, r2->addr); - tt_int_op(rp2->or_port,OP_EQ, r2->or_port); - tt_int_op(rp2->dir_port,OP_EQ, r2->dir_port); - tt_int_op(rp2->bandwidthrate,OP_EQ, r2->bandwidthrate); - tt_int_op(rp2->bandwidthburst,OP_EQ, r2->bandwidthburst); - tt_int_op(rp2->bandwidthcapacity,OP_EQ, r2->bandwidthcapacity); + + CHECK_ROUTERINFO_CONSISTENCY(r2, rp2); + tt_mem_op(rp2->onion_curve25519_pkey->public_key,OP_EQ, r2->onion_curve25519_pkey->public_key, CURVE25519_PUBKEY_LEN); - onion_pkey = router_get_rsa_onion_pkey(rp2->onion_pkey, - rp2->onion_pkey_len); - tt_int_op(crypto_pk_cmp_keys(onion_pkey, pk2), OP_EQ, 0); - crypto_pk_free(onion_pkey); - tt_int_op(crypto_pk_cmp_keys(rp2->identity_pkey, pk1), OP_EQ, 0); - tt_assert(rp2->supports_tunnelled_dir_requests); - tt_int_op(smartlist_len(rp2->exit_policy),OP_EQ, 2); - - p = smartlist_get(rp2->exit_policy, 0); - tt_int_op(p->policy_type,OP_EQ, ADDR_POLICY_ACCEPT); - tt_assert(tor_addr_is_null(&p->addr)); - tt_int_op(p->maskbits,OP_EQ, 0); - tt_int_op(p->prt_min,OP_EQ, 80); - tt_int_op(p->prt_max,OP_EQ, 80); - - p = smartlist_get(rp2->exit_policy, 1); - tt_int_op(p->policy_type,OP_EQ, ADDR_POLICY_REJECT); - tt_assert(tor_addr_eq(&p->addr, &ex2->addr)); - tt_int_op(p->maskbits,OP_EQ, 8); - tt_int_op(p->prt_min,OP_EQ, 24); - tt_int_op(p->prt_max,OP_EQ, 24); + CHECK_PARSED_EXIT_POLICY(rp2); routerinfo_free(rp2); @@ -850,27 +1119,26 @@ test_dir_formats_rsa_ed25519(void *arg) tt_assert(e2->cache_info.signed_descriptor_body); cp = e2->cache_info.signed_descriptor_body; ep2 = extrainfo_parse_entry_from_string((const char*)cp,NULL,1,NULL,NULL); - tt_assert(ep2); - tt_str_op(ep2->nickname, OP_EQ, r2->nickname); + + CHECK_EXTRAINFO_CONSISTENCY(r2, ep2); + /* In future tests, we could check the actual extrainfo statistics. */ extrainfo_free(ep2); -#if 0 - /* Okay, now for the directories. */ - { - fingerprint_list = smartlist_new(); - crypto_pk_get_fingerprint(pk2, buf, 1); - add_fingerprint_to_dir(buf, fingerprint_list, 0); - crypto_pk_get_fingerprint(pk1, buf, 1); - add_fingerprint_to_dir(buf, fingerprint_list, 0); - } - -#endif /* 0 */ - done: dirserv_free_fingerprint_list(); + tor_free(options->Nickname); + + cleanup_mock_configured_ports(); + cleanup_mocks_for_fresh_descriptor(); + + if (chunks) { + SMARTLIST_FOREACH(chunks, char *, s, tor_free(s)); + smartlist_free(chunks); + } + routerinfo_free(r2); routerinfo_free(r2_out); routerinfo_free(rp2); @@ -879,13 +1147,10 @@ test_dir_formats_rsa_ed25519(void *arg) extrainfo_free(ep2); tor_free(rsa_cc); + crypto_pk_free(r2_onion_pkey); + tor_free(buf); - tor_free(pk1_str); - tor_free(pk2_str); - crypto_pk_free(pk1); - crypto_pk_free(pk2); - tor_free(dir1); /* XXXX And more !*/ - tor_free(dir2); /* And more !*/ + tor_free(buf2); } #include "failing_routerdescs.inc" From 39ab6c9f7360b77901efefae7dd4b18d5df47b90 Mon Sep 17 00:00:00 2001 From: teor Date: Tue, 19 Feb 2019 15:56:09 +1000 Subject: [PATCH 0508/2557] test_dir: Test descriptor variants Including: * relays and bridges, * no stats, basic stats, and all stats Part of 29017 and 29018. --- src/test/test_dir.c | 81 +++++++++++++++++++++++++++++++++++++++------ 1 file changed, 70 insertions(+), 11 deletions(-) diff --git a/src/test/test_dir.c b/src/test/test_dir.c index cc0ef07fbf..210b09a4e7 100644 --- a/src/test/test_dir.c +++ b/src/test/test_dir.c @@ -411,6 +411,19 @@ get_new_ntor_onion_key_line(const curve25519_public_key_t *ntor_onion_pubkey) return line; } +/* Allocate and return a new string containing a "bridge-distribution-request" + * line for options. + */ +static char * +get_new_bridge_distribution_request_line(const or_options_t *options) +{ + if (options->BridgeRelay) { + return tor_strdup("bridge-distribution-request any\n"); + } else { + return tor_strdup(""); + } +} + static smartlist_t *mocked_configured_ports = NULL; /** Returns mocked_configured_ports */ @@ -577,6 +590,43 @@ setup_mocks_for_fresh_descriptor(const routerinfo_t *r1, MOCK(get_onion_key, mock_get_onion_key); } +/* Set options based on arg. + * + * b: BridgeRelay 1 + * e: ExtraInfoStatistics 1 + * s: sets all the individual statistics options to 1 + * + * Always sets AssumeReachable to 1. + * + * Does not set ServerTransportPlugin, because it's parsed before use. + * + * Does not set BridgeRecordUsageByCountry, because the tests don't have access + * to a GeoIPFile or GeoIPv6File. */ +static void +setup_dir_formats_options(const char *arg, or_options_t *options) +{ + /* Skip reachability checks for DirPort, ORPort, and tunnelled-dir-server */ + options->AssumeReachable = 1; + + if (strchr(arg, 'b')) { + options->BridgeRelay = 1; + } + + if (strchr(arg, 'e')) { + options->ExtraInfoStatistics = 1; + } + + if (strchr(arg, 's')) { + options->DirReqStatistics = 1; + options->HiddenServiceStatistics = 1; + options->EntryStatistics = 1; + options->CellStatistics = 1; + options->ExitPortStatistics = 1; + options->ConnDirectionStatistics = 1; + options->PaddingStatistics = 1; + } +} + /* Check that routerinfos r1 and rp1 are consistent. * Only performs some basic checks. */ @@ -638,8 +688,7 @@ test_dir_formats_rsa(void *arg) int rv = -1; or_options_t *options = get_options_mutable(); - - (void)arg; + setup_dir_formats_options((const char *)arg, options); hibernate_set_state_for_testing_(HIBERNATE_STATE_LIVE); @@ -665,8 +714,6 @@ test_dir_formats_rsa(void *arg) /* XXXX+++ router_dump_to_string should really take this from ri. */ options->ContactInfo = tor_strdup("Magri White " ""); - /* Skip reachability checks for DirPort, ORPort, and tunnelled-dir-server */ - options->AssumeReachable = 1; setup_mock_configured_ports(r1->or_port, r1->dir_port); @@ -697,6 +744,7 @@ test_dir_formats_rsa(void *arg) smartlist_add_strdup(chunks, "contact Magri White " "\n"); + smartlist_add(chunks, get_new_bridge_distribution_request_line(options)); smartlist_add(chunks, get_new_ntor_onion_key_line(&r1_onion_keypair.pubkey)); smartlist_add_strdup(chunks, "reject *:*\n"); smartlist_add_strdup(chunks, "tunnelled-dir-server\n"); @@ -869,8 +917,7 @@ test_dir_formats_rsa_ed25519(void *arg) int rv = -1; or_options_t *options = get_options_mutable(); - - (void)arg; + setup_dir_formats_options((const char *)arg, options); hibernate_set_state_for_testing_(HIBERNATE_STATE_LIVE); @@ -919,9 +966,6 @@ test_dir_formats_rsa_ed25519(void *arg) smartlist_add(r2->exit_policy, ex1); smartlist_add(r2->exit_policy, ex2); - /* Skip reachability checks for ORPort and tunnelled-dir-server */ - options->AssumeReachable = 1; - /* Fake just enough of an ORPort to get by */ setup_mock_configured_ports(r2->or_port, 0); @@ -1001,6 +1045,7 @@ test_dir_formats_rsa_ed25519(void *arg) smartlist_add_strdup(chunks, "hidden-service-dir\n"); + smartlist_add(chunks, get_new_bridge_distribution_request_line(options)); smartlist_add(chunks, get_new_ntor_onion_key_line(&r2_onion_keypair.pubkey)); smartlist_add_strdup(chunks, "accept *:80\nreject 18.0.0.0/8:24\n"); smartlist_add_strdup(chunks, "tunnelled-dir-server\n"); @@ -7119,8 +7164,22 @@ test_dir_format_versions_list(void *arg) struct testcase_t dir_tests[] = { DIR_LEGACY(nicknames), - DIR_LEGACY(formats_rsa), - DIR_LEGACY(formats_rsa_ed25519), + /* extrainfo without any stats */ + DIR_ARG(formats_rsa, TT_FORK, ""), + DIR_ARG(formats_rsa_ed25519, TT_FORK, ""), + /* on a bridge */ + DIR_ARG(formats_rsa, TT_FORK, "b"), + DIR_ARG(formats_rsa_ed25519, TT_FORK, "b"), + /* extrainfo with basic stats */ + DIR_ARG(formats_rsa, TT_FORK, "e"), + DIR_ARG(formats_rsa_ed25519, TT_FORK, "e"), + DIR_ARG(formats_rsa, TT_FORK, "be"), + DIR_ARG(formats_rsa_ed25519, TT_FORK, "be"), + /* extrainfo with all stats */ + DIR_ARG(formats_rsa, TT_FORK, "es"), + DIR_ARG(formats_rsa_ed25519, TT_FORK, "es"), + DIR_ARG(formats_rsa, TT_FORK, "bes"), + DIR_ARG(formats_rsa_ed25519, TT_FORK, "bes"), DIR(routerinfo_parsing, 0), DIR(extrainfo_parsing, 0), DIR(parse_router_list, TT_FORK), From 0c0f21582285bec0fe68f71e293158eeedaeeaa3 Mon Sep 17 00:00:00 2001 From: teor Date: Tue, 19 Feb 2019 18:52:54 +1000 Subject: [PATCH 0509/2557] routerkeys: Log failures at info-level in make_tap_onion_key_crosscert() --- src/feature/relay/routerkeys.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/feature/relay/routerkeys.c b/src/feature/relay/routerkeys.c index bdd7a82b58..d965777ad6 100644 --- a/src/feature/relay/routerkeys.c +++ b/src/feature/relay/routerkeys.c @@ -706,6 +706,8 @@ make_tap_onion_key_crosscert(const crypto_pk_t *onion_key, *len_out = 0; if (crypto_pk_get_digest(rsa_id_key, (char*)signed_data) < 0) { + log_info(LD_OR, "crypto_pk_get_digest failed in " + "make_tap_onion_key_crosscert!"); return NULL; } memcpy(signed_data + DIGEST_LEN, master_id_key->pubkey, ED25519_PUBKEY_LEN); @@ -713,8 +715,12 @@ make_tap_onion_key_crosscert(const crypto_pk_t *onion_key, int r = crypto_pk_private_sign(onion_key, (char*)signature, sizeof(signature), (const char*)signed_data, sizeof(signed_data)); - if (r < 0) + if (r < 0) { + /* It's probably missing the private key */ + log_info(LD_OR, "crypto_pk_private_sign failed in " + "make_tap_onion_key_crosscert!"); return NULL; + } *len_out = r; From 51f59f213e3850d1d77f0e60355d914c6583aeb4 Mon Sep 17 00:00:00 2001 From: teor Date: Tue, 19 Feb 2019 21:23:27 +1000 Subject: [PATCH 0510/2557] router: Add some missing #endif comments --- src/feature/relay/router.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/feature/relay/router.h b/src/feature/relay/router.h index f17e6a9438..55b9ef9e68 100644 --- a/src/feature/relay/router.h +++ b/src/feature/relay/router.h @@ -134,8 +134,8 @@ STATIC extrainfo_t *router_build_fresh_signed_extrainfo( STATIC void router_update_routerinfo_from_extrainfo(routerinfo_t *ri, const extrainfo_t *ei); STATIC int router_dump_and_sign_routerinfo_descriptor_body(routerinfo_t *ri); -#endif +#endif /* defined(TOR_UNIT_TESTS) */ -#endif +#endif /* defined(ROUTER_PRIVATE) */ #endif /* !defined(TOR_ROUTER_H) */ From 9a158a45b1fe9f36cf0d41c5d1dcec7be74f1719 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 19 Feb 2019 09:29:43 -0500 Subject: [PATCH 0511/2557] Bump to 0.4.1.0-alpha-dev --- configure.ac | 4 ++-- contrib/win32build/tor-mingw.nsi.in | 2 +- src/win32/orconfig.h | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/configure.ac b/configure.ac index 7de3c3a46e..6036cdffe5 100644 --- a/configure.ac +++ b/configure.ac @@ -4,7 +4,7 @@ dnl Copyright (c) 2007-2019, The Tor Project, Inc. dnl See LICENSE for licensing information AC_PREREQ([2.63]) -AC_INIT([tor],[0.4.0.1-alpha-dev]) +AC_INIT([tor],[0.4.1.0-alpha-dev]) AC_CONFIG_SRCDIR([src/app/main/tor_main.c]) AC_CONFIG_MACRO_DIR([m4]) @@ -14,7 +14,7 @@ AC_CONFIG_MACRO_DIR([m4]) # version number changes. Tor uses it to make sure that it # only shuts down for missing "required protocols" when those protocols # are listed as required by a consensus after this date. -AC_DEFINE(APPROX_RELEASE_DATE, ["2019-01-18"], # for 0.4.0.1-alpha +AC_DEFINE(APPROX_RELEASE_DATE, ["2019-02-19"], # for 0.4.1.0-alpha-dev [Approximate date when this software was released. (Updated when the version changes.)]) # "foreign" means we don't follow GNU package layout standards diff --git a/contrib/win32build/tor-mingw.nsi.in b/contrib/win32build/tor-mingw.nsi.in index 4f2f82d007..419b5aa58c 100644 --- a/contrib/win32build/tor-mingw.nsi.in +++ b/contrib/win32build/tor-mingw.nsi.in @@ -8,7 +8,7 @@ !include "LogicLib.nsh" !include "FileFunc.nsh" !insertmacro GetParameters -!define VERSION "0.4.0.1-alpha-dev" +!define VERSION "0.4.1.0-alpha-dev" !define INSTALLER "tor-${VERSION}-win32.exe" !define WEBSITE "https://www.torproject.org/" !define LICENSE "LICENSE" diff --git a/src/win32/orconfig.h b/src/win32/orconfig.h index c618c5b49d..892531ceb4 100644 --- a/src/win32/orconfig.h +++ b/src/win32/orconfig.h @@ -218,7 +218,7 @@ #define USING_TWOS_COMPLEMENT /* Version number of package */ -#define VERSION "0.4.0.1-alpha-dev" +#define VERSION "0.4.1.0-alpha-dev" From 6c652eae0a95cae095f5adad9cc51e1f1a183245 Mon Sep 17 00:00:00 2001 From: teor Date: Wed, 20 Feb 2019 00:40:18 +1000 Subject: [PATCH 0512/2557] fixup! test_dir: Refactor common code out of the dir_format unit tests --- src/test/test_dir.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/test/test_dir.c b/src/test/test_dir.c index 210b09a4e7..31471fb7d6 100644 --- a/src/test/test_dir.c +++ b/src/test/test_dir.c @@ -355,8 +355,9 @@ get_new_rsa_key_block(const char *key_name, crypto_pk_t *pk1) "%s\n%s", key_name, pk1_str); - tor_assert(block); + tor_free(pk1_str); + tor_assert(block); return block; } From c9ff6a7f834e25a93c806d0079b789d26b11695a Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 19 Feb 2019 13:14:26 -0500 Subject: [PATCH 0513/2557] Mark map_anon_nofork test as skipped in 0.4.0 This test fails in some environments; since the code isn't used in 0.4.0, let's disable it for now. Band-aid solution for #29534; bug not in any released Tor. --- src/test/test_util.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/test_util.c b/src/test/test_util.c index 913c5e289d..7a2708c541 100644 --- a/src/test/test_util.c +++ b/src/test/test_util.c @@ -6360,6 +6360,6 @@ struct testcase_t util_tests[] = { UTIL_TEST(get_unquoted_path, 0), UTIL_TEST(log_mallinfo, 0), UTIL_TEST(map_anon, 0), - UTIL_TEST(map_anon_nofork, 0), + UTIL_TEST(map_anon_nofork, TT_SKIP /* See bug #29535 */), END_OF_TESTCASES }; From d32e4079761f0076cd9e704a7f9996aa5b599f60 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 19 Feb 2019 14:02:32 -0500 Subject: [PATCH 0514/2557] Downgrade some LOG_ERR messages in the address/* tests to warnings Fixes bug 29530, where the LOG_ERR messages were occurring when we had no configured network, and so we were failing the unit tests because of the recently-merged #28668. Bug not in any released Tor. --- src/test/test_address.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/test/test_address.c b/src/test/test_address.c index d9304a0cfa..bf9ca047dc 100644 --- a/src/test/test_address.c +++ b/src/test/test_address.c @@ -746,7 +746,7 @@ test_address_get_if_addrs_list_internal(void *arg) (void)arg; - results = get_interface_address_list(LOG_ERR, 1); + results = get_interface_address_list(LOG_WARN, 1); tt_ptr_op(results, OP_NE, NULL); /* When the network is down, a system might not have any non-local @@ -777,7 +777,7 @@ test_address_get_if_addrs_list_no_internal(void *arg) (void)arg; - results = get_interface_address_list(LOG_ERR, 0); + results = get_interface_address_list(LOG_WARN, 0); tt_ptr_op(results, OP_NE, NULL); /* Work even on systems with only internal IPv4 addresses */ @@ -988,7 +988,7 @@ test_address_get_if_addrs(void *arg) (void)arg; - rv = get_interface_address(LOG_ERR, &addr_h); + rv = get_interface_address(LOG_WARN, &addr_h); /* When the network is down, a system might not have any non-local * non-multicast IPv4 addresses, not even internal ones. From b25cd5cfe168b85500c160ca538a44d9adba52c1 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 19 Feb 2019 15:23:31 -0500 Subject: [PATCH 0515/2557] Implement code to manage a per-thread instance of crypto_fast_rng() The subsystems API makes this really simple, fortunately. Closes ticket 29536 --- changes/ticket29536 | 9 ++++ src/lib/crypt_ops/crypto_init.c | 7 +++ src/lib/crypt_ops/crypto_rand.h | 9 ++++ src/lib/crypt_ops/crypto_rand_fast.c | 67 +++++++++++++++++++++++++++- 4 files changed, 91 insertions(+), 1 deletion(-) create mode 100644 changes/ticket29536 diff --git a/changes/ticket29536 b/changes/ticket29536 new file mode 100644 index 0000000000..a5ae26b701 --- /dev/null +++ b/changes/ticket29536 @@ -0,0 +1,9 @@ + o Minor features (performance, RNG): + - Tor now constructs a fast secure pseudorandom number generator for + each thread, to use for cases where performance is critical. This PRNG + is based on AES-CTR, using a buffering construction similar to + libottery and the (newer) OpenBSD arc4random() code. It outperforms + OpenSSL 1.1.1a's CSPRNG by roughly a factor of 100 for small outputs. + Although we believe it to be cryptographically strong, we are only + using it when necessary for reasonable performance. Implements tickets + 29023 and 29536. diff --git a/src/lib/crypt_ops/crypto_init.c b/src/lib/crypt_ops/crypto_init.c index 4040085c76..cf491f32d1 100644 --- a/src/lib/crypt_ops/crypto_init.c +++ b/src/lib/crypt_ops/crypto_init.c @@ -12,6 +12,8 @@ #include "orconfig.h" +#define CRYPTO_PRIVATE + #include "lib/crypt_ops/crypto_init.h" #include "lib/crypt_ops/crypto_curve25519.h" @@ -69,6 +71,8 @@ crypto_early_init(void) if (crypto_init_siphash_key() < 0) return -1; + crypto_rand_fast_init(); + curve25519_init(); ed25519_init(); } @@ -111,6 +115,7 @@ crypto_thread_cleanup(void) #ifdef ENABLE_OPENSSL crypto_openssl_thread_cleanup(); #endif + destroy_thread_fast_rng(); } /** @@ -129,6 +134,8 @@ crypto_global_cleanup(void) crypto_nss_global_cleanup(); #endif + crypto_rand_fast_shutdown(); + crypto_early_initialized_ = 0; crypto_global_initialized_ = 0; have_seeded_siphash = 0; diff --git a/src/lib/crypt_ops/crypto_rand.h b/src/lib/crypt_ops/crypto_rand.h index 8a81a4acdc..6eef22ed4d 100644 --- a/src/lib/crypt_ops/crypto_rand.h +++ b/src/lib/crypt_ops/crypto_rand.h @@ -68,6 +68,15 @@ unsigned crypto_fast_rng_get_uint(crypto_fast_rng_t *rng, unsigned limit); uint64_t crypto_fast_rng_get_uint64(crypto_fast_rng_t *rng, uint64_t limit); double crypto_fast_rng_get_double(crypto_fast_rng_t *rng); +crypto_fast_rng_t *get_thread_fast_rng(void); + +#ifdef CRYPTO_PRIVATE +/* These are only used from crypto_init.c */ +void destroy_thread_fast_rng(void); +void crypto_rand_fast_init(void); +void crypto_rand_fast_shutdown(void); +#endif + #if defined(TOR_UNIT_TESTS) /* Used for white-box testing */ size_t crypto_fast_rng_get_bytes_used_per_stream(void); diff --git a/src/lib/crypt_ops/crypto_rand_fast.c b/src/lib/crypt_ops/crypto_rand_fast.c index 34e763bf51..760e1025ed 100644 --- a/src/lib/crypt_ops/crypto_rand_fast.c +++ b/src/lib/crypt_ops/crypto_rand_fast.c @@ -33,6 +33,7 @@ */ #define CRYPTO_RAND_FAST_PRIVATE +#define CRYPTO_PRIVATE #include "lib/crypt_ops/crypto_rand.h" #include "lib/crypt_ops/crypto_cipher.h" @@ -41,6 +42,7 @@ #include "lib/intmath/cmp.h" #include "lib/cc/ctassert.h" #include "lib/malloc/map_anon.h" +#include "lib/thread/threads.h" #include "lib/log/util_bug.h" @@ -122,7 +124,8 @@ crypto_fast_rng_new(void) * long. * * Note that this object is NOT thread-safe. If you need a thread-safe - * prng, use crypto_rand(), or wrap this in a mutex. + * prng, you should probably look at get_thread_fast_rng(). Alternatively, + * use crypto_rand(), wrap this in a mutex. **/ crypto_fast_rng_t * crypto_fast_rng_new_from_seed(const uint8_t *seed) @@ -261,3 +264,65 @@ crypto_fast_rng_get_bytes_used_per_stream(void) return BUFLEN; } #endif + +/** + * Thread-local instance for our fast RNG. + **/ +static tor_threadlocal_t thread_rng; + +/** + * Return a per-thread fast RNG, initializing it if necessary. + * + * You do not need to free this yourself. + * + * It is NOT safe to share this value across threads. + **/ +crypto_fast_rng_t * +get_thread_fast_rng(void) +{ + crypto_fast_rng_t *rng = tor_threadlocal_get(&thread_rng); + + if (PREDICT_UNLIKELY(rng == NULL)) { + rng = crypto_fast_rng_new(); + tor_threadlocal_set(&thread_rng, rng); + } + + return rng; +} + +/** + * Used when a thread is exiting: free the per-thread fast RNG if needed. + * Invoked from the crypto subsystem's thread-cleanup code. + **/ +void +destroy_thread_fast_rng(void) +{ + crypto_fast_rng_t *rng = tor_threadlocal_get(&thread_rng); + if (!rng) + return; + crypto_fast_rng_free(rng); + tor_threadlocal_set(&thread_rng, NULL); +} + +/** + * Initialize the global thread-local key that will be used to keep track + * of per-thread fast RNG instances. Called from the crypto subsystem's + * initialization code. + **/ +void +crypto_rand_fast_init(void) +{ + tor_threadlocal_init(&thread_rng); +} + +/** + * Initialize the global thread-local key that will be used to keep track + * of per-thread fast RNG instances. Called from the crypto subsystem's + * shutdown code. + **/ +void +crypto_rand_fast_shutdown(void) +{ + destroy_thread_fast_rng(); + tor_threadlocal_destroy(&thread_rng); +} From 208f04e9b8241569fe314f6d5cb65521c9b55d8b Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 19 Feb 2019 15:36:11 -0500 Subject: [PATCH 0516/2557] Add a quick test for get_thread_fast_rng() --- src/test/test_crypto_rng.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/test/test_crypto_rng.c b/src/test/test_crypto_rng.c index 23b0c66514..6b7749a889 100644 --- a/src/test/test_crypto_rng.c +++ b/src/test/test_crypto_rng.c @@ -218,6 +218,14 @@ test_crypto_rng_fast(void *arg) tt_int_op(counts[i], OP_GT, 0); } + /* per-thread rand_fast shouldn't crash or leak. */ + crypto_fast_rng_t *t_rng = get_thread_fast_rng(); + for (int i = 0; i < N; ++i) { + uint64_t u64 = crypto_fast_rng_get_uint64(t_rng, UINT64_C(1)<<40); + tt_u64_op(u64, OP_GE, 0); + tt_u64_op(u64, OP_LT, UINT64_C(1)<<40); + } + done: crypto_fast_rng_free(rng); } From 249319ec5d3a17faf9a6060f1b3109f640874863 Mon Sep 17 00:00:00 2001 From: Roger Dingledine Date: Wed, 20 Feb 2019 10:32:47 -0500 Subject: [PATCH 0517/2557] fix typos from #28614 --- changes/ticket28614 | 2 +- src/feature/nodelist/networkstatus.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/changes/ticket28614 b/changes/ticket28614 index 6c65ce49de..3c93313726 100644 --- a/changes/ticket28614 +++ b/changes/ticket28614 @@ -3,6 +3,6 @@ "binary" mode so that we can safely map it into memory later. Fixes part of bug 28614; bugfix on 0.4.0.1-alpha. - When reading a consensus file from disk, detect whether it - was written in text mode, and re-read it in text mode if it + was written in text mode, and re-read it in text mode if so. Fixes part of bug 28614; bugfix on 0.4.0.1-alpha. diff --git a/src/feature/nodelist/networkstatus.c b/src/feature/nodelist/networkstatus.c index 2c34754621..023115978c 100644 --- a/src/feature/nodelist/networkstatus.c +++ b/src/feature/nodelist/networkstatus.c @@ -1743,7 +1743,7 @@ networkstatus_set_current_consensus_from_ns(networkstatus_t *c, #endif /* defined(TOR_UNIT_TESTS) */ /** - * Helper: Read a the current consensus of type flavor from + * Helper: Read the current consensus of type flavor from * fname. Flags and return values are as for * networkstatus_set_current_consensus(). **/ From b7ad8bcaad093a5d043c63f30c196c52741830d2 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Wed, 13 Feb 2019 15:50:09 +0200 Subject: [PATCH 0518/2557] Use setrlimit instead of ulimit -c in backtrace tests --- changes/bug29061 | 4 ++++ src/test/test_bt.sh | 2 -- src/test/test_bt_cl.c | 8 ++++++++ 3 files changed, 12 insertions(+), 2 deletions(-) create mode 100644 changes/bug29061 diff --git a/changes/bug29061 b/changes/bug29061 new file mode 100644 index 0000000000..58fc4f22e9 --- /dev/null +++ b/changes/bug29061 @@ -0,0 +1,4 @@ + o Minor bugfixes (testing): + - Call setrlimit() to disable core dumps in test_bt_cl.c instead of + using `ulimit -c` in test_bt.sh, which violates POSIX shell + compatibility. Fixes bug 29061; bugfix on 0.3.5.1-alpha. diff --git a/src/test/test_bt.sh b/src/test/test_bt.sh index df8bcb8eda..312905a4e2 100755 --- a/src/test/test_bt.sh +++ b/src/test/test_bt.sh @@ -3,8 +3,6 @@ exitcode=0 -ulimit -c 0 - export ASAN_OPTIONS="handle_segv=0:allow_user_segv_handler=1" "${builddir:-.}/src/test/test-bt-cl" backtraces || exit $? "${builddir:-.}/src/test/test-bt-cl" assert 2>&1 | "${PYTHON:-python}" "${abs_top_srcdir:-.}/src/test/bt_test.py" || exitcode="$?" diff --git a/src/test/test_bt_cl.c b/src/test/test_bt_cl.c index 0c15a02ee4..08b08ba423 100644 --- a/src/test/test_bt_cl.c +++ b/src/test/test_bt_cl.c @@ -4,6 +4,9 @@ #include "orconfig.h" #include #include +#ifdef HAVE_SYS_RESOURCE_H +#include +#endif /* To prevent 'assert' from going away. */ #undef TOR_COVERAGE @@ -88,6 +91,11 @@ main(int argc, char **argv) return 1; } +#ifdef HAVE_SYS_RESOURCE_H + struct rlimit rlim = { .rlim_cur = 0, .rlim_max = 0 }; + setrlimit(RLIMIT_CORE, &rlim); +#endif + #if !(defined(HAVE_EXECINFO_H) && defined(HAVE_BACKTRACE) && \ defined(HAVE_BACKTRACE_SYMBOLS_FD) && defined(HAVE_SIGACTION)) puts("Backtrace reporting is not supported on this platform"); From 97b9dfe3052f1fe3b14f47fc1326e235f96305b7 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 20 Feb 2019 12:21:05 -0500 Subject: [PATCH 0519/2557] Add a convenience macro to get a fast one-in-n calculation --- src/lib/crypt_ops/crypto_rand.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/lib/crypt_ops/crypto_rand.h b/src/lib/crypt_ops/crypto_rand.h index 6eef22ed4d..bb424fd691 100644 --- a/src/lib/crypt_ops/crypto_rand.h +++ b/src/lib/crypt_ops/crypto_rand.h @@ -68,6 +68,9 @@ unsigned crypto_fast_rng_get_uint(crypto_fast_rng_t *rng, unsigned limit); uint64_t crypto_fast_rng_get_uint64(crypto_fast_rng_t *rng, uint64_t limit); double crypto_fast_rng_get_double(crypto_fast_rng_t *rng); +#define crypto_fast_rng_one_in_n(rng, n) \ + (0 == (crypto_fast_rng_get_uint((rng), (n)))) + crypto_fast_rng_t *get_thread_fast_rng(void); #ifdef CRYPTO_PRIVATE From b3416476b487304426296173dd177e1277388e48 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 20 Feb 2019 12:21:23 -0500 Subject: [PATCH 0520/2557] Remove all uses of weak_rng. I'm not removing the weak_rng code itself yet, since it is possible that we will want to revert one of these. --- changes/ticket29542 | 7 +++++++ src/app/main/main.c | 2 +- src/core/mainloop/cpuworker.c | 7 ++----- src/core/or/relay.c | 14 ++------------ src/core/or/relay.h | 3 --- src/lib/evloop/workqueue.c | 10 +--------- 6 files changed, 13 insertions(+), 30 deletions(-) create mode 100644 changes/ticket29542 diff --git a/changes/ticket29542 b/changes/ticket29542 new file mode 100644 index 0000000000..465a8e31bc --- /dev/null +++ b/changes/ticket29542 @@ -0,0 +1,7 @@ + o Minor features (defense in depth): + - Tor now uses a fast cryptographically strong PRNG even for decisions + that we do not believe are security-sensitive. Previously, for + performance reasons, we had used a trivially predictable linear + congruential generator algorithm for certain load-balancing and + statistical sampling decisions. Now we use our fast RNG in those cases. + Closes ticket 29542. diff --git a/src/app/main/main.c b/src/app/main/main.c index 0ffc27d456..ec15109f6c 100644 --- a/src/app/main/main.c +++ b/src/app/main/main.c @@ -669,7 +669,7 @@ tor_init(int argc, char *argv[]) log_err(LD_BUG, "Unable to initialize OpenSSL. Exiting."); return -1; } - stream_choice_seed_weak_rng(); + if (tor_init_libevent_rng() < 0) { log_warn(LD_NET, "Problem initializing libevent RNG."); } diff --git a/src/core/mainloop/cpuworker.c b/src/core/mainloop/cpuworker.c index e704d55642..436fcd28c3 100644 --- a/src/core/mainloop/cpuworker.c +++ b/src/core/mainloop/cpuworker.c @@ -34,7 +34,6 @@ #include "core/crypto/onion_crypto.h" #include "core/or/or_circuit_st.h" -#include "lib/intmath/weakrng.h" static void queue_pending_tasks(void); @@ -74,8 +73,6 @@ worker_state_free_void(void *arg) static replyqueue_t *replyqueue = NULL; static threadpool_t *threadpool = NULL; -static tor_weak_rng_t request_sample_rng = TOR_WEAK_RNG_INIT; - static int total_pending_tasks = 0; static int max_pending_tasks = 128; @@ -109,7 +106,6 @@ cpu_init(void) /* Total voodoo. Can we make this more sensible? */ max_pending_tasks = get_num_cpus(get_options()) * 64; - crypto_seed_weak_rng(&request_sample_rng); } /** Magic numbers to make sure our cpuworker_requests don't grow any @@ -235,9 +231,10 @@ should_time_request(uint16_t onionskin_type) * sample */ if (onionskins_n_processed[onionskin_type] < 4096) return 1; + /** Otherwise, measure with P=1/128. We avoid doing this for every * handshake, since the measurement itself can take a little time. */ - return tor_weak_random_one_in_n(&request_sample_rng, 128); + return crypto_fast_rng_one_in_n(get_thread_fast_rng(), 128); } /** Return an estimate of how many microseconds we will need for a single diff --git a/src/core/or/relay.c b/src/core/or/relay.c index 706a6e05cb..7f7fa2fe1f 100644 --- a/src/core/or/relay.c +++ b/src/core/or/relay.c @@ -94,8 +94,6 @@ #include "feature/nodelist/routerinfo_st.h" #include "core/or/socks_request_st.h" -#include "lib/intmath/weakrng.h" - static edge_connection_t *relay_lookup_conn(circuit_t *circ, cell_t *cell, cell_direction_t cell_direction, crypt_path_t *layer_hint); @@ -134,9 +132,6 @@ uint64_t stats_n_relay_cells_delivered = 0; * reached (see append_cell_to_circuit_queue()) */ uint64_t stats_n_circ_max_cell_reached = 0; -/** Used to tell which stream to read from first on a circuit. */ -static tor_weak_rng_t stream_choice_rng = TOR_WEAK_RNG_INIT; - /** * Update channel usage state based on the type of relay cell and * circuit properties. @@ -2180,12 +2175,6 @@ circuit_resume_edge_reading(circuit_t *circ, crypt_path_t *layer_hint) circ, layer_hint); } -void -stream_choice_seed_weak_rng(void) -{ - crypto_seed_weak_rng(&stream_choice_rng); -} - /** A helper function for circuit_resume_edge_reading() above. * The arguments are the same, except that conn is the head * of a linked list of edge streams that should each be considered. @@ -2237,7 +2226,8 @@ circuit_resume_edge_reading_helper(edge_connection_t *first_conn, int num_streams = 0; for (conn = first_conn; conn; conn = conn->next_stream) { num_streams++; - if (tor_weak_random_one_in_n(&stream_choice_rng, num_streams)) { + + if (crypto_fast_rng_one_in_n(get_thread_fast_rng(), num_streams)) { chosen_stream = conn; } /* Invariant: chosen_stream has been chosen uniformly at random from diff --git a/src/core/or/relay.h b/src/core/or/relay.h index 044f6be156..ea1b358ffb 100644 --- a/src/core/or/relay.h +++ b/src/core/or/relay.h @@ -94,8 +94,6 @@ const uint8_t *decode_address_from_payload(tor_addr_t *addr_out, int payload_len); void circuit_clear_cell_queue(circuit_t *circ, channel_t *chan); -void stream_choice_seed_weak_rng(void); - circid_t packed_cell_get_circid(const packed_cell_t *cell, int wide_circ_ids); #ifdef RELAY_PRIVATE @@ -126,4 +124,3 @@ STATIC int connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, #endif /* defined(RELAY_PRIVATE) */ #endif /* !defined(TOR_RELAY_H) */ - diff --git a/src/lib/evloop/workqueue.c b/src/lib/evloop/workqueue.c index b36a02da5e..015b694290 100644 --- a/src/lib/evloop/workqueue.c +++ b/src/lib/evloop/workqueue.c @@ -59,9 +59,6 @@ struct threadpool_s { * p is work[p]. */ work_tailq_t work[WORKQUEUE_N_PRIORITIES]; - /** Weak RNG, used to decide when to ignore priority. */ - tor_weak_rng_t weak_rng; - /** The current 'update generation' of the threadpool. Any thread that is * at an earlier generation needs to run the update function. */ unsigned generation; @@ -238,7 +235,7 @@ worker_thread_extract_next_work(workerthread_t *thread) this_queue = &pool->work[i]; if (!TOR_TAILQ_EMPTY(this_queue)) { queue = this_queue; - if (! tor_weak_random_one_in_n(&pool->weak_rng, + if (! crypto_fast_rng_one_in_n(get_thread_fast_rng(), thread->lower_priority_chance)) { /* Usually we'll just break now, so that we can get out of the loop * and use the queue where we found work. But with a small @@ -555,11 +552,6 @@ threadpool_new(int n_threads, for (i = WORKQUEUE_PRIORITY_FIRST; i <= WORKQUEUE_PRIORITY_LAST; ++i) { TOR_TAILQ_INIT(&pool->work[i]); } - { - unsigned seed; - crypto_rand((void*)&seed, sizeof(seed)); - tor_init_weak_random(&pool->weak_rng, seed); - } pool->new_thread_state_fn = new_thread_state_fn; pool->new_thread_state_arg = arg; From 34183f0d71c2e2d10e3d64291e3bb83a27c2416a Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 21 Feb 2019 09:06:51 -0500 Subject: [PATCH 0521/2557] Fix a goofy change from abcde10fce that broke test-slow linking boklm tracked this down, and it doesn't make sense. It caused This change goes back to the previous LDFLAGS line. --- src/test/include.am | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/include.am b/src/test/include.am index 9bfc626b28..d585c2a38a 100644 --- a/src/test/include.am +++ b/src/test/include.am @@ -259,7 +259,7 @@ src_test_test_LDADD = \ src_test_test_slow_CPPFLAGS = $(src_test_test_CPPFLAGS) src_test_test_slow_CFLAGS = $(src_test_test_CFLAGS) src_test_test_slow_LDADD = $(src_test_test_LDADD) -src_test_test_slow_LDFLAGS = @TOR_LDFLAGS_openssl@ @TOR_LDFLAGS_libevent@ +src_test_test_slow_LDFLAGS = $(src_test_test_LDFLAGS) src_test_test_rng_CPPFLAGS = $(src_test_test_CPPFLAGS) src_test_test_rng_CFLAGS = $(src_test_test_CFLAGS) From df3484b2b01835c075b57c30b2ea3ac260ee157c Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 21 Feb 2019 09:52:27 -0500 Subject: [PATCH 0522/2557] Bump version to 0.4.0.2-alpha --- configure.ac | 4 ++-- contrib/win32build/tor-mingw.nsi.in | 2 +- src/win32/orconfig.h | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/configure.ac b/configure.ac index 7de3c3a46e..6e60489a57 100644 --- a/configure.ac +++ b/configure.ac @@ -4,7 +4,7 @@ dnl Copyright (c) 2007-2019, The Tor Project, Inc. dnl See LICENSE for licensing information AC_PREREQ([2.63]) -AC_INIT([tor],[0.4.0.1-alpha-dev]) +AC_INIT([tor],[0.4.0.2-alpha]) AC_CONFIG_SRCDIR([src/app/main/tor_main.c]) AC_CONFIG_MACRO_DIR([m4]) @@ -14,7 +14,7 @@ AC_CONFIG_MACRO_DIR([m4]) # version number changes. Tor uses it to make sure that it # only shuts down for missing "required protocols" when those protocols # are listed as required by a consensus after this date. -AC_DEFINE(APPROX_RELEASE_DATE, ["2019-01-18"], # for 0.4.0.1-alpha +AC_DEFINE(APPROX_RELEASE_DATE, ["2019-02-21"], # for 0.4.0.2-alpha [Approximate date when this software was released. (Updated when the version changes.)]) # "foreign" means we don't follow GNU package layout standards diff --git a/contrib/win32build/tor-mingw.nsi.in b/contrib/win32build/tor-mingw.nsi.in index 4f2f82d007..eb961e6160 100644 --- a/contrib/win32build/tor-mingw.nsi.in +++ b/contrib/win32build/tor-mingw.nsi.in @@ -8,7 +8,7 @@ !include "LogicLib.nsh" !include "FileFunc.nsh" !insertmacro GetParameters -!define VERSION "0.4.0.1-alpha-dev" +!define VERSION "0.4.0.2-alpha" !define INSTALLER "tor-${VERSION}-win32.exe" !define WEBSITE "https://www.torproject.org/" !define LICENSE "LICENSE" diff --git a/src/win32/orconfig.h b/src/win32/orconfig.h index c618c5b49d..0081bc4553 100644 --- a/src/win32/orconfig.h +++ b/src/win32/orconfig.h @@ -218,7 +218,7 @@ #define USING_TWOS_COMPLEMENT /* Version number of package */ -#define VERSION "0.4.0.1-alpha-dev" +#define VERSION "0.4.0.2-alpha" From 85e1c493788557a3256bbe732208654340c3344b Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 21 Feb 2019 10:32:08 -0500 Subject: [PATCH 0523/2557] Copy today's releases into changelogs and releasenotes. --- ChangeLog | 321 +++++++++++++++++++++++++++++++++++++++++++++++++++ ReleaseNotes | 186 +++++++++++++++++++++++++++++ 2 files changed, 507 insertions(+) diff --git a/ChangeLog b/ChangeLog index e6bcc806e2..dd2a98469d 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,324 @@ +Changes in version 0.3.5.8 - 2019-02-21 + Tor 0.3.5.8 backports serveral fixes from later releases, including fixes + for an annoying SOCKS-parsing bug that affected users in earlier 0.3.5.x + releases. + + It also includes a fix for a medium-severity security bug affecting Tor + 0.3.2.1-alpha and later. All Tor instances running an affected release + should upgrade to 0.3.3.12, 0.3.4.11, 0.3.5.8, or 0.4.0.2-alpha. + + o Major bugfixes (cell scheduler, KIST, security): + - Make KIST consider the outbuf length when computing what it can + put in the outbuf. Previously, KIST acted as though the outbuf + were empty, which could lead to the outbuf becoming too full. It + is possible that an attacker could exploit this bug to cause a Tor + client or relay to run out of memory and crash. Fixes bug 29168; + bugfix on 0.3.2.1-alpha. This issue is also being tracked as + TROVE-2019-001 and CVE-2019-8955. + + o Major bugfixes (networking, backport from 0.4.0.2-alpha): + - Gracefully handle empty username/password fields in SOCKS5 + username/password auth messsage and allow SOCKS5 handshake to + continue. Previously, we had rejected these handshakes, breaking + certain applications. Fixes bug 29175; bugfix on 0.3.5.1-alpha. + + o Minor features (compilation, backport from 0.4.0.2-alpha): + - Compile correctly when OpenSSL is built with engine support + disabled, or with deprecated APIs disabled. Closes ticket 29026. + Patches from "Mangix". + + o Minor features (geoip): + - Update geoip and geoip6 to the February 5 2019 Maxmind GeoLite2 + Country database. Closes ticket 29478. + + o Minor features (testing, backport from 0.4.0.2-alpha): + - Treat all unexpected ERR and BUG messages as test failures. Closes + ticket 28668. + + o Minor bugfixes (onion service v3, client, backport from 0.4.0.1-alpha): + - Stop logging a "BUG()" warning and stacktrace when we find a SOCKS + connection waiting for a descriptor that we actually have in the + cache. It turns out that this can actually happen, though it is + rare. Now, tor will recover and retry the descriptor. Fixes bug + 28669; bugfix on 0.3.2.4-alpha. + + o Minor bugfixes (IPv6, backport from 0.4.0.1-alpha): + - Fix tor_ersatz_socketpair on IPv6-only systems. Previously, the + IPv6 socket was bound using an address family of AF_INET instead + of AF_INET6. Fixes bug 28995; bugfix on 0.3.5.1-alpha. Patch from + Kris Katterjohn. + + o Minor bugfixes (build, compatibility, rust, backport from 0.4.0.2-alpha): + - Update Cargo.lock file to match the version made by the latest + version of Rust, so that "make distcheck" will pass again. Fixes + bug 29244; bugfix on 0.3.3.4-alpha. + + o Minor bugfixes (client, clock skew, backport from 0.4.0.1-alpha): + - Select guards even if the consensus has expired, as long as the + consensus is still reasonably live. Fixes bug 24661; bugfix + on 0.3.0.1-alpha. + + o Minor bugfixes (compilation, backport from 0.4.0.1-alpha): + - Compile correctly on OpenBSD; previously, we were missing some + headers required in order to detect it properly. Fixes bug 28938; + bugfix on 0.3.5.1-alpha. Patch from Kris Katterjohn. + + o Minor bugfixes (documentation, backport from 0.4.0.2-alpha): + - Describe the contents of the v3 onion service client authorization + files correctly: They hold public keys, not private keys. Fixes + bug 28979; bugfix on 0.3.5.1-alpha. Spotted by "Felixix". + + o Minor bugfixes (logging, backport from 0.4.0.1-alpha): + - Rework rep_hist_log_link_protocol_counts() to iterate through all + link protocol versions when logging incoming/outgoing connection + counts. Tor no longer skips version 5, and we won't have to + remember to update this function when new link protocol version is + developed. Fixes bug 28920; bugfix on 0.2.6.10. + + o Minor bugfixes (logging, backport from 0.4.0.2-alpha): + - Log more information at "warning" level when unable to read a + private key; log more information at "info" level when unable to + read a public key. We had warnings here before, but they were lost + during our NSS work. Fixes bug 29042; bugfix on 0.3.5.1-alpha. + + o Minor bugfixes (misc, backport from 0.4.0.2-alpha): + - The amount of total available physical memory is now determined + using the sysctl identifier HW_PHYSMEM (rather than HW_USERMEM) + when it is defined and a 64-bit variant is not available. Fixes + bug 28981; bugfix on 0.2.5.4-alpha. Patch from Kris Katterjohn. + + o Minor bugfixes (onion services, backport from 0.4.0.2-alpha): + - Avoid crashing if ClientOnionAuthDir (incorrectly) contains more + than one private key for a hidden service. Fixes bug 29040; bugfix + on 0.3.5.1-alpha. + - In hs_cache_store_as_client() log an HSDesc we failed to parse at + "debug" level. Tor used to log it as a warning, which caused very + long log lines to appear for some users. Fixes bug 29135; bugfix + on 0.3.2.1-alpha. + - Stop logging "Tried to establish rendezvous on non-OR circuit..." + as a warning. Instead, log it as a protocol warning, because there + is nothing that relay operators can do to fix it. Fixes bug 29029; + bugfix on 0.2.5.7-rc. + + o Minor bugfixes (tests, directory clients, backport from 0.4.0.1-alpha): + - Mark outdated dirservers when Tor only has a reasonably live + consensus. Fixes bug 28569; bugfix on 0.3.2.5-alpha. + + o Minor bugfixes (tests, backport from 0.4.0.2-alpha): + - Detect and suppress "bug" warnings from the util/time test on + Windows. Fixes bug 29161; bugfix on 0.2.9.3-alpha. + - Do not log an error-level message if we fail to find an IPv6 + network interface from the unit tests. Fixes bug 29160; bugfix + on 0.2.7.3-rc. + + o Minor bugfixes (usability, backport from 0.4.0.1-alpha): + - Stop saying "Your Guard ..." in pathbias_measure_{use,close}_rate(). + Some users took this phrasing to mean that the mentioned guard was + under their control or responsibility, which it is not. Fixes bug + 28895; bugfix on Tor 0.3.0.1-alpha. + + +Changes in version 0.3.4.11 - 2019-02-21 + Tor 0.3.4.11 is the third stable release in its series. It includes + a fix for a medium-severity security bug affecting Tor 0.3.2.1-alpha and + later. All Tor instances running an affected release should upgrade to + 0.3.3.12, 0.3.4.11, 0.3.5.8, or 0.4.0.2-alpha. + + o Major bugfixes (cell scheduler, KIST, security): + - Make KIST consider the outbuf length when computing what it can + put in the outbuf. Previously, KIST acted as though the outbuf + were empty, which could lead to the outbuf becoming too full. It + is possible that an attacker could exploit this bug to cause a Tor + client or relay to run out of memory and crash. Fixes bug 29168; + bugfix on 0.3.2.1-alpha. This issue is also being tracked as + TROVE-2019-001 and CVE-2019-8955. + + o Minor features (geoip): + - Update geoip and geoip6 to the February 5 2019 Maxmind GeoLite2 + Country database. Closes ticket 29478. + + o Minor bugfixes (build, compatibility, rust, backport from 0.4.0.2-alpha): + - Update Cargo.lock file to match the version made by the latest + version of Rust, so that "make distcheck" will pass again. Fixes + bug 29244; bugfix on 0.3.3.4-alpha. + + o Minor bugfixes (onion services, backport from 0.4.0.2-alpha): + - Stop logging "Tried to establish rendezvous on non-OR circuit..." + as a warning. Instead, log it as a protocol warning, because there + is nothing that relay operators can do to fix it. Fixes bug 29029; + bugfix on 0.2.5.7-rc. + + +Changes in version 0.3.3.12 - 2019-02-21 + Tor 0.3.3.12 fixes a medium-severity security bug affecting Tor + 0.3.2.1-alpha and later. All Tor instances running an affected release + should upgrade to 0.3.3.12, 0.3.4.11, 0.3.5.8, or 0.4.0.2-alpha. + + This release marks the end of support for the Tor 0.3.3.x series. We + recommend that users switch to either the Tor 0.3.4 series (supported + until at least 10 June 2019), or the Tor 0.3.5 series, which will + receive long-term support until at least 1 Feb 2022. + + o Major bugfixes (cell scheduler, KIST, security): + - Make KIST consider the outbuf length when computing what it can + put in the outbuf. Previously, KIST acted as though the outbuf + were empty, which could lead to the outbuf becoming too full. It + is possible that an attacker could exploit this bug to cause a Tor + client or relay to run out of memory and crash. Fixes bug 29168; + bugfix on 0.3.2.1-alpha. This issue is also being tracked as + TROVE-2019-001 and CVE-2019-8955. + + o Minor features (geoip): + - Update geoip and geoip6 to the February 5 2019 Maxmind GeoLite2 + Country database. Closes ticket 29478. + + o Minor bugfixes (build, compatibility, rust, backport from 0.4.0.2-alpha): + - Update Cargo.lock file to match the version made by the latest + version of Rust, so that "make distcheck" will pass again. Fixes + bug 29244; bugfix on 0.3.3.4-alpha. + + o Minor bugfixes (onion services, backport from 0.4.0.2-alpha): + - Stop logging "Tried to establish rendezvous on non-OR circuit..." + as a warning. Instead, log it as a protocol warning, because there + is nothing that relay operators can do to fix it. Fixes bug 29029; + bugfix on 0.2.5.7-rc. + + +Changes in version 0.4.0.2-alpha - 2019-02-21 + Tor 0.4.0.2-alpha is the second alpha in its series; it fixes several + bugs from earlier versions, including several that had broken + backward compatibility. + + It also includes a fix for a medium-severity security bug affecting Tor + 0.3.2.1-alpha and later. All Tor instances running an affected release + should upgrade to 0.3.3.12, 0.3.4.11, 0.3.5.8, or 0.4.0.2-alpha. + + o Major bugfixes (cell scheduler, KIST, security): + - Make KIST consider the outbuf length when computing what it can + put in the outbuf. Previously, KIST acted as though the outbuf + were empty, which could lead to the outbuf becoming too full. It + is possible that an attacker could exploit this bug to cause a Tor + client or relay to run out of memory and crash. Fixes bug 29168; + bugfix on 0.3.2.1-alpha. This issue is also being tracked as + TROVE-2019-001 and CVE-2019-8955. + + o Major bugfixes (networking): + - Gracefully handle empty username/password fields in SOCKS5 + username/password auth messsage and allow SOCKS5 handshake to + continue. Previously, we had rejected these handshakes, breaking + certain applications. Fixes bug 29175; bugfix on 0.3.5.1-alpha. + + o Major bugfixes (windows, startup): + - When reading a consensus file from disk, detect whether it was + written in text mode, and re-read it in text mode if so. Always + write consensus files in binary mode so that we can map them into + memory later. Previously, we had written in text mode, which + confused us when we tried to map the file on windows. Fixes bug + 28614; bugfix on 0.4.0.1-alpha. + + o Minor features (compilation): + - Compile correctly when OpenSSL is built with engine support + disabled, or with deprecated APIs disabled. Closes ticket 29026. + Patches from "Mangix". + + o Minor features (developer tooling): + - Check that bugfix versions in changes files look like Tor versions + from the versions spec. Warn when bugfixes claim to be on a future + release. Closes ticket 27761. + - Provide a git pre-commit hook that disallows commiting if we have + any failures in our code and changelog formatting checks. It is + now available in scripts/maint/pre-commit.git-hook. Implements + feature 28976. + + o Minor features (directory authority): + - When a directory authority is using a bandwidth file to obtain + bandwidth values, include the digest of that file in the vote. + Closes ticket 26698. + + o Minor features (geoip): + - Update geoip and geoip6 to the February 5 2019 Maxmind GeoLite2 + Country database. Closes ticket 29478. + + o Minor features (testing): + - Treat all unexpected ERR and BUG messages as test failures. Closes + ticket 28668. + + o Minor bugfixes (build, compatibility, rust): + - Update Cargo.lock file to match the version made by the latest + version of Rust, so that "make distcheck" will pass again. Fixes + bug 29244; bugfix on 0.3.3.4-alpha. + + o Minor bugfixes (compilation): + - Fix compilation warnings in test_circuitpadding.c. Fixes bug + 29169; bugfix on 0.4.0.1-alpha. + - Silence a compiler warning in test-memwipe.c on OpenBSD. Fixes bug + 29145; bugfix on 0.2.9.3-alpha. Patch from Kris Katterjohn. + + o Minor bugfixes (documentation): + - Describe the contents of the v3 onion service client authorization + files correctly: They hold public keys, not private keys. Fixes + bug 28979; bugfix on 0.3.5.1-alpha. Spotted by "Felixix". + + o Minor bugfixes (linux seccomp sandbox): + - Fix startup crash when experimental sandbox support is enabled. + Fixes bug 29150; bugfix on 0.4.0.1-alpha. Patch by Peter Gerber. + + o Minor bugfixes (logging): + - Avoid logging that we are relaxing a circuit timeout when that + timeout is fixed. Fixes bug 28698; bugfix on 0.2.4.7-alpha. + - Log more information at "warning" level when unable to read a + private key; log more information at "info" level when unable to + read a public key. We had warnings here before, but they were lost + during our NSS work. Fixes bug 29042; bugfix on 0.3.5.1-alpha. + + o Minor bugfixes (misc): + - The amount of total available physical memory is now determined + using the sysctl identifier HW_PHYSMEM (rather than HW_USERMEM) + when it is defined and a 64-bit variant is not available. Fixes + bug 28981; bugfix on 0.2.5.4-alpha. Patch from Kris Katterjohn. + + o Minor bugfixes (onion services): + - Avoid crashing if ClientOnionAuthDir (incorrectly) contains more + than one private key for a hidden service. Fixes bug 29040; bugfix + on 0.3.5.1-alpha. + - In hs_cache_store_as_client() log an HSDesc we failed to parse at + "debug" level. Tor used to log it as a warning, which caused very + long log lines to appear for some users. Fixes bug 29135; bugfix + on 0.3.2.1-alpha. + - Stop logging "Tried to establish rendezvous on non-OR circuit..." + as a warning. Instead, log it as a protocol warning, because there + is nothing that relay operators can do to fix it. Fixes bug 29029; + bugfix on 0.2.5.7-rc. + + o Minor bugfixes (scheduler): + - When re-adding channels to the pending list, check the correct + channel's sched_heap_idx. This issue has had no effect in mainline + Tor, but could have led to bugs down the road in improved versions + of our circuit scheduling code. Fixes bug 29508; bugfix + on 0.3.2.10. + + o Minor bugfixes (tests): + - Fix intermittent failures on an adaptive padding test. Fixes one + case of bug 29122; bugfix on 0.4.0.1-alpha. + - Disable an unstable circuit-padding test that was failing + intermittently because of an ill-defined small histogram. Such + histograms will be allowed again after 29298 is implemented. Fixes + a second case of bug 29122; bugfix on 0.4.0.1-alpha. + - Detect and suppress "bug" warnings from the util/time test on + Windows. Fixes bug 29161; bugfix on 0.2.9.3-alpha. + - Do not log an error-level message if we fail to find an IPv6 + network interface from the unit tests. Fixes bug 29160; bugfix + on 0.2.7.3-rc. + + o Documentation: + - In the manpage entry describing MapAddress torrc setting, use + example IP addresses from ranges specified for use in documentation + by RFC 5737. Resolves issue 28623. + + o Removed features: + - Remove the old check-tor script. Resolves issue 29072. + + Changes in version 0.4.0.1-alpha - 2019-01-18 Tor 0.4.0.1-alpha is the first release in the new 0.4.0.x series. It introduces improved features for power and bandwidth conservation, diff --git a/ReleaseNotes b/ReleaseNotes index 6c9aa3c294..93dad1673b 100644 --- a/ReleaseNotes +++ b/ReleaseNotes @@ -2,6 +2,192 @@ This document summarizes new features and bugfixes in each stable release of Tor. If you want to see more detailed descriptions of the changes in each development snapshot, see the ChangeLog file. +Changes in version 0.3.5.8 - 2019-02-21 + Tor 0.3.5.8 backports serveral fixes from later releases, including fixes + for an annoying SOCKS-parsing bug that affected users in earlier 0.3.5.x + releases. + + It also includes a fix for a medium-severity security bug affecting Tor + 0.3.2.1-alpha and later. All Tor instances running an affected release + should upgrade to 0.3.3.12, 0.3.4.11, 0.3.5.8, or 0.4.0.2-alpha. + + o Major bugfixes (cell scheduler, KIST, security): + - Make KIST consider the outbuf length when computing what it can + put in the outbuf. Previously, KIST acted as though the outbuf + were empty, which could lead to the outbuf becoming too full. It + is possible that an attacker could exploit this bug to cause a Tor + client or relay to run out of memory and crash. Fixes bug 29168; + bugfix on 0.3.2.1-alpha. This issue is also being tracked as + TROVE-2019-001 and CVE-2019-8955. + + o Major bugfixes (networking, backport from 0.4.0.2-alpha): + - Gracefully handle empty username/password fields in SOCKS5 + username/password auth messsage and allow SOCKS5 handshake to + continue. Previously, we had rejected these handshakes, breaking + certain applications. Fixes bug 29175; bugfix on 0.3.5.1-alpha. + + o Minor features (compilation, backport from 0.4.0.2-alpha): + - Compile correctly when OpenSSL is built with engine support + disabled, or with deprecated APIs disabled. Closes ticket 29026. + Patches from "Mangix". + + o Minor features (geoip): + - Update geoip and geoip6 to the February 5 2019 Maxmind GeoLite2 + Country database. Closes ticket 29478. + + o Minor features (testing, backport from 0.4.0.2-alpha): + - Treat all unexpected ERR and BUG messages as test failures. Closes + ticket 28668. + + o Minor bugfixes (onion service v3, client, backport from 0.4.0.1-alpha): + - Stop logging a "BUG()" warning and stacktrace when we find a SOCKS + connection waiting for a descriptor that we actually have in the + cache. It turns out that this can actually happen, though it is + rare. Now, tor will recover and retry the descriptor. Fixes bug + 28669; bugfix on 0.3.2.4-alpha. + + o Minor bugfixes (IPv6, backport from 0.4.0.1-alpha): + - Fix tor_ersatz_socketpair on IPv6-only systems. Previously, the + IPv6 socket was bound using an address family of AF_INET instead + of AF_INET6. Fixes bug 28995; bugfix on 0.3.5.1-alpha. Patch from + Kris Katterjohn. + + o Minor bugfixes (build, compatibility, rust, backport from 0.4.0.2-alpha): + - Update Cargo.lock file to match the version made by the latest + version of Rust, so that "make distcheck" will pass again. Fixes + bug 29244; bugfix on 0.3.3.4-alpha. + + o Minor bugfixes (client, clock skew, backport from 0.4.0.1-alpha): + - Select guards even if the consensus has expired, as long as the + consensus is still reasonably live. Fixes bug 24661; bugfix + on 0.3.0.1-alpha. + + o Minor bugfixes (compilation, backport from 0.4.0.1-alpha): + - Compile correctly on OpenBSD; previously, we were missing some + headers required in order to detect it properly. Fixes bug 28938; + bugfix on 0.3.5.1-alpha. Patch from Kris Katterjohn. + + o Minor bugfixes (documentation, backport from 0.4.0.2-alpha): + - Describe the contents of the v3 onion service client authorization + files correctly: They hold public keys, not private keys. Fixes + bug 28979; bugfix on 0.3.5.1-alpha. Spotted by "Felixix". + + o Minor bugfixes (logging, backport from 0.4.0.1-alpha): + - Rework rep_hist_log_link_protocol_counts() to iterate through all + link protocol versions when logging incoming/outgoing connection + counts. Tor no longer skips version 5, and we won't have to + remember to update this function when new link protocol version is + developed. Fixes bug 28920; bugfix on 0.2.6.10. + + o Minor bugfixes (logging, backport from 0.4.0.2-alpha): + - Log more information at "warning" level when unable to read a + private key; log more information at "info" level when unable to + read a public key. We had warnings here before, but they were lost + during our NSS work. Fixes bug 29042; bugfix on 0.3.5.1-alpha. + + o Minor bugfixes (misc, backport from 0.4.0.2-alpha): + - The amount of total available physical memory is now determined + using the sysctl identifier HW_PHYSMEM (rather than HW_USERMEM) + when it is defined and a 64-bit variant is not available. Fixes + bug 28981; bugfix on 0.2.5.4-alpha. Patch from Kris Katterjohn. + + o Minor bugfixes (onion services, backport from 0.4.0.2-alpha): + - Avoid crashing if ClientOnionAuthDir (incorrectly) contains more + than one private key for a hidden service. Fixes bug 29040; bugfix + on 0.3.5.1-alpha. + - In hs_cache_store_as_client() log an HSDesc we failed to parse at + "debug" level. Tor used to log it as a warning, which caused very + long log lines to appear for some users. Fixes bug 29135; bugfix + on 0.3.2.1-alpha. + - Stop logging "Tried to establish rendezvous on non-OR circuit..." + as a warning. Instead, log it as a protocol warning, because there + is nothing that relay operators can do to fix it. Fixes bug 29029; + bugfix on 0.2.5.7-rc. + + o Minor bugfixes (tests, directory clients, backport from 0.4.0.1-alpha): + - Mark outdated dirservers when Tor only has a reasonably live + consensus. Fixes bug 28569; bugfix on 0.3.2.5-alpha. + + o Minor bugfixes (tests, backport from 0.4.0.2-alpha): + - Detect and suppress "bug" warnings from the util/time test on + Windows. Fixes bug 29161; bugfix on 0.2.9.3-alpha. + - Do not log an error-level message if we fail to find an IPv6 + network interface from the unit tests. Fixes bug 29160; bugfix + on 0.2.7.3-rc. + + o Minor bugfixes (usability, backport from 0.4.0.1-alpha): + - Stop saying "Your Guard ..." in pathbias_measure_{use,close}_rate(). + Some users took this phrasing to mean that the mentioned guard was + under their control or responsibility, which it is not. Fixes bug + 28895; bugfix on Tor 0.3.0.1-alpha. + + +Changes in version 0.3.4.11 - 2019-02-21 + Tor 0.3.4.11 is the third stable release in its series. It includes + a fix for a medium-severity security bug affecting Tor 0.3.2.1-alpha and + later. All Tor instances running an affected release should upgrade to + 0.3.3.12, 0.3.4.11, 0.3.5.8, or 0.4.0.2-alpha. + + o Major bugfixes (cell scheduler, KIST, security): + - Make KIST consider the outbuf length when computing what it can + put in the outbuf. Previously, KIST acted as though the outbuf + were empty, which could lead to the outbuf becoming too full. It + is possible that an attacker could exploit this bug to cause a Tor + client or relay to run out of memory and crash. Fixes bug 29168; + bugfix on 0.3.2.1-alpha. This issue is also being tracked as + TROVE-2019-001 and CVE-2019-8955. + + o Minor features (geoip): + - Update geoip and geoip6 to the February 5 2019 Maxmind GeoLite2 + Country database. Closes ticket 29478. + + o Minor bugfixes (build, compatibility, rust, backport from 0.4.0.2-alpha): + - Update Cargo.lock file to match the version made by the latest + version of Rust, so that "make distcheck" will pass again. Fixes + bug 29244; bugfix on 0.3.3.4-alpha. + + o Minor bugfixes (onion services, backport from 0.4.0.2-alpha): + - Stop logging "Tried to establish rendezvous on non-OR circuit..." + as a warning. Instead, log it as a protocol warning, because there + is nothing that relay operators can do to fix it. Fixes bug 29029; + bugfix on 0.2.5.7-rc. + + +Changes in version 0.3.3.12 - 2019-02-21 + Tor 0.3.3.12 fixes a medium-severity security bug affecting Tor + 0.3.2.1-alpha and later. All Tor instances running an affected release + should upgrade to 0.3.3.12, 0.3.4.11, 0.3.5.8, or 0.4.0.2-alpha. + + This release marks the end of support for the Tor 0.3.3.x series. We + recommend that users switch to either the Tor 0.3.4 series (supported + until at least 10 June 2019), or the Tor 0.3.5 series, which will + receive long-term support until at least 1 Feb 2022. + + o Major bugfixes (cell scheduler, KIST, security): + - Make KIST consider the outbuf length when computing what it can + put in the outbuf. Previously, KIST acted as though the outbuf + were empty, which could lead to the outbuf becoming too full. It + is possible that an attacker could exploit this bug to cause a Tor + client or relay to run out of memory and crash. Fixes bug 29168; + bugfix on 0.3.2.1-alpha. This issue is also being tracked as + TROVE-2019-001 and CVE-2019-8955. + + o Minor features (geoip): + - Update geoip and geoip6 to the February 5 2019 Maxmind GeoLite2 + Country database. Closes ticket 29478. + + o Minor bugfixes (build, compatibility, rust, backport from 0.4.0.2-alpha): + - Update Cargo.lock file to match the version made by the latest + version of Rust, so that "make distcheck" will pass again. Fixes + bug 29244; bugfix on 0.3.3.4-alpha. + + o Minor bugfixes (onion services, backport from 0.4.0.2-alpha): + - Stop logging "Tried to establish rendezvous on non-OR circuit..." + as a warning. Instead, log it as a protocol warning, because there + is nothing that relay operators can do to fix it. Fixes bug 29029; + bugfix on 0.2.5.7-rc. + + Changes in version 0.3.3.11 - 2019-01-07 Tor 0.3.3.11 backports numerous fixes from later versions of Tor. numerous fixes, including an important fix for anyone using OpenSSL From 658770a0f05810cf9a12f4efe349ecda9b63c969 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 21 Feb 2019 10:33:16 -0500 Subject: [PATCH 0524/2557] Remove changes entries for fixes that appeared in 0.4.0.2-alpha --- changes/bug28698 | 3 --- changes/bug28979 | 4 ---- changes/bug28981 | 5 ----- changes/bug29029 | 5 ----- changes/bug29040 | 4 ---- changes/bug29042 | 5 ----- changes/bug29122 | 3 --- changes/bug29135 | 5 ----- changes/bug29145 | 3 --- changes/bug29150 | 3 --- changes/bug29161 | 3 --- changes/bug29169 | 3 --- changes/bug29175_035 | 4 ---- changes/bug29244 | 4 ---- changes/bug29298 | 5 ----- changes/bug29508 | 3 --- changes/doc28623 | 3 --- changes/feature28976 | 4 ---- changes/geoip-2019-02-05 | 4 ---- changes/ticket26698 | 4 ---- changes/ticket27761 | 4 ---- changes/ticket28614 | 8 -------- changes/ticket28668 | 3 --- changes/ticket29026 | 4 ---- changes/ticket29072 | 2 -- changes/ticket29160 | 4 ---- changes/ticket29168 | 5 ----- 27 files changed, 107 deletions(-) delete mode 100644 changes/bug28698 delete mode 100644 changes/bug28979 delete mode 100644 changes/bug28981 delete mode 100644 changes/bug29029 delete mode 100644 changes/bug29040 delete mode 100644 changes/bug29042 delete mode 100644 changes/bug29122 delete mode 100644 changes/bug29135 delete mode 100644 changes/bug29145 delete mode 100644 changes/bug29150 delete mode 100644 changes/bug29161 delete mode 100644 changes/bug29169 delete mode 100644 changes/bug29175_035 delete mode 100644 changes/bug29244 delete mode 100644 changes/bug29298 delete mode 100644 changes/bug29508 delete mode 100644 changes/doc28623 delete mode 100644 changes/feature28976 delete mode 100644 changes/geoip-2019-02-05 delete mode 100644 changes/ticket26698 delete mode 100644 changes/ticket27761 delete mode 100644 changes/ticket28614 delete mode 100644 changes/ticket28668 delete mode 100644 changes/ticket29026 delete mode 100644 changes/ticket29072 delete mode 100644 changes/ticket29160 delete mode 100644 changes/ticket29168 diff --git a/changes/bug28698 b/changes/bug28698 deleted file mode 100644 index 716aa0c552..0000000000 --- a/changes/bug28698 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor bugfix (logging): - - Avoid logging about relaxing circuits when their time is fixed. - Fixes bug 28698; bugfix on 0.2.4.7-alpha diff --git a/changes/bug28979 b/changes/bug28979 deleted file mode 100644 index 0625fd5d25..0000000000 --- a/changes/bug28979 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (documentation): - - Describe the contents of the v3 onion service client authorization - files correctly: They hold public keys, not private keys. Fixes bug - 28979; bugfix on 0.3.5.1-alpha. Spotted by "Felixix". diff --git a/changes/bug28981 b/changes/bug28981 deleted file mode 100644 index c0ea92ab35..0000000000 --- a/changes/bug28981 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor bugfixes (misc): - - The amount of total available physical memory is now determined - using the sysctl identifier HW_PHYSMEM (rather than HW_USERMEM) - when it is defined and a 64-bit variant is not available. Fixes - bug 28981; bugfix on 0.2.5.4-alpha. Patch from Kris Katterjohn. diff --git a/changes/bug29029 b/changes/bug29029 deleted file mode 100644 index e100a8c2ed..0000000000 --- a/changes/bug29029 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor bugfixes (logging, onion services): - - Stop logging "Tried to establish rendezvous on non-OR circuit..." as - a warning. Instead, log it as a protocol warning, because there is - nothing that relay operators can do to fix it. Fixes bug 29029; - bugfix on 0.2.5.7-rc. diff --git a/changes/bug29040 b/changes/bug29040 deleted file mode 100644 index 0662aaa8a5..0000000000 --- a/changes/bug29040 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (onion services): - - Avoid crashing if ClientOnionAuthDir (incorrectly) contains - more than one private key for a hidden service. Fixes bug 29040; - bugfix on 0.3.5.1-alpha. diff --git a/changes/bug29042 b/changes/bug29042 deleted file mode 100644 index 8d76939cea..0000000000 --- a/changes/bug29042 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor bugfixes (logging): - - Log more information at "warning" level when unable to read a private - key; log more information ad "info" level when unable to read a public - key. We had warnings here before, but they were lost during our - NSS work. Fixes bug 29042; bugfix on 0.3.5.1-alpha. diff --git a/changes/bug29122 b/changes/bug29122 deleted file mode 100644 index 020052ff8f..0000000000 --- a/changes/bug29122 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor bugfixes (unit tests): - - Fix intermittent failures on an adaptive padding unittest. Fixes bug - 29122; bugfix on 0.4.0.1-alpha diff --git a/changes/bug29135 b/changes/bug29135 deleted file mode 100644 index fd7b1ae80e..0000000000 --- a/changes/bug29135 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor bugfixes (onion services, logging): - - In hs_cache_store_as_client() log an HSDesc we failed to parse at Debug - loglevel. Tor used to log it at Warning loglevel, which caused - very long log lines to appear for some users. Fixes bug 29135; bugfix on - 0.3.2.1-alpha. diff --git a/changes/bug29145 b/changes/bug29145 deleted file mode 100644 index 40d3da4b91..0000000000 --- a/changes/bug29145 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor bugfixes (compilation, testing): - - Silence a compiler warning in test-memwipe.c on OpenBSD. Fixes - bug 29145; bugfix on 0.2.9.3-alpha. Patch from Kris Katterjohn. diff --git a/changes/bug29150 b/changes/bug29150 deleted file mode 100644 index 7696b90378..0000000000 --- a/changes/bug29150 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor bugfixes (linux seccomp sandbox): - - Fix startup crash when experimental sandbox support is enabled. - Fixes bug 29150; bugfix on 0.4.0.1-alpha. Patch by Peter Gerber. diff --git a/changes/bug29161 b/changes/bug29161 deleted file mode 100644 index 39a638acf6..0000000000 --- a/changes/bug29161 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor bugfixes (tests): - - Detect and suppress "bug" warnings from the util/time test on Windows. - Fixes bug 29161; bugfix on 0.2.9.3-alpha. diff --git a/changes/bug29169 b/changes/bug29169 deleted file mode 100644 index 41d4b76ef5..0000000000 --- a/changes/bug29169 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor bugfixes (compilation): - - Fix compilation warnings in test_circuitpadding.c. Fixes bug 29169; - bugfix on 0.4.0.1-alpha. diff --git a/changes/bug29175_035 b/changes/bug29175_035 deleted file mode 100644 index 134c1d9529..0000000000 --- a/changes/bug29175_035 +++ /dev/null @@ -1,4 +0,0 @@ - o Major bugfixes (networking): - - Gracefully handle empty username/password fields in SOCKS5 - username/password auth messsage and allow SOCKS5 handshake to - continue. Fixes bug 29175; bugfix on 0.3.5.1-alpha. diff --git a/changes/bug29244 b/changes/bug29244 deleted file mode 100644 index 6206a95463..0000000000 --- a/changes/bug29244 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (build, compatibility): - - Update Cargo.lock file to match the version made by the latest - version of Rust, so that "make distcheck" will pass again. - Fixes bug 29244; bugfix on 0.3.3.4-alpha. diff --git a/changes/bug29298 b/changes/bug29298 deleted file mode 100644 index df12db77d7..0000000000 --- a/changes/bug29298 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor bugfixes (testing, circuit padding): - - Disabled unstable circuit padding unittest that was causing intermittent - test failures because of ill-defined small histogram. Such histograms - will be allowed again after 29298 is implemented. Fixes second case of - bug 29122; bugfix on 0.4.0.1-alpha. \ No newline at end of file diff --git a/changes/bug29508 b/changes/bug29508 deleted file mode 100644 index ee728bbbc9..0000000000 --- a/changes/bug29508 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor bugfixes (scheduler): - - When readding channels to the pending list, check the correct channel's - sched_heap_idx. Fixes bug 29508; bugfix on 0.3.2.10 diff --git a/changes/doc28623 b/changes/doc28623 deleted file mode 100644 index 3c3313abdd..0000000000 --- a/changes/doc28623 +++ /dev/null @@ -1,3 +0,0 @@ - o Documentation: - - In manpage entry describing MapAddress torrc setting, use example - IP addresses from ranges specified by RFC 5737. Resolves issue 28623. diff --git a/changes/feature28976 b/changes/feature28976 deleted file mode 100644 index c7ebc207f7..0000000000 --- a/changes/feature28976 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor features (developer tooling): - - Provide a git pre-commit hook that disallows commiting if we have any - failures in our code and changelog formatting checks. It is now available - in scripts/maint/pre-commit.git-hook. Implements feature 28976. diff --git a/changes/geoip-2019-02-05 b/changes/geoip-2019-02-05 deleted file mode 100644 index 78ee6d4242..0000000000 --- a/changes/geoip-2019-02-05 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor features (geoip): - - Update geoip and geoip6 to the February 5 2019 Maxmind GeoLite2 - Country database. Closes ticket 29478. - diff --git a/changes/ticket26698 b/changes/ticket26698 deleted file mode 100644 index 6b029a1b73..0000000000 --- a/changes/ticket26698 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor features (directory authority): - - When a directory authority is using a bandwidth file to obtain the - bandwidth values, include the digest of the file in the vote. - Closes ticket 26698. diff --git a/changes/ticket27761 b/changes/ticket27761 deleted file mode 100644 index 35106ee9c6..0000000000 --- a/changes/ticket27761 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor features (changelogs): - - Check that bugfix versions in changes files look like Tor versions - from the versions spec. Warn when bugfixes claim to be on a future - release. Closes ticket 27761. diff --git a/changes/ticket28614 b/changes/ticket28614 deleted file mode 100644 index 3c93313726..0000000000 --- a/changes/ticket28614 +++ /dev/null @@ -1,8 +0,0 @@ - o Major bugfixes (windows, startup): - - When writing a consensus file to disk, always write in - "binary" mode so that we can safely map it into memory later. - Fixes part of bug 28614; bugfix on 0.4.0.1-alpha. - - When reading a consensus file from disk, detect whether it - was written in text mode, and re-read it in text mode if so. - Fixes part of bug 28614; bugfix on 0.4.0.1-alpha. - diff --git a/changes/ticket28668 b/changes/ticket28668 deleted file mode 100644 index 6386e0051f..0000000000 --- a/changes/ticket28668 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor features (testing): - - Treat all unexpected ERR and BUG messages as test failures. - Closes ticket 28668. diff --git a/changes/ticket29026 b/changes/ticket29026 deleted file mode 100644 index 1db873dfcf..0000000000 --- a/changes/ticket29026 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor features (compilation): - - Compile correctly when OpenSSL is built with engine support - disabled, or with deprecated APIs disabled. Closes ticket - 29026. Patches from "Mangix". diff --git a/changes/ticket29072 b/changes/ticket29072 deleted file mode 100644 index 3526330f30..0000000000 --- a/changes/ticket29072 +++ /dev/null @@ -1,2 +0,0 @@ - o Removed features: - - Remove check-tor script from repository. Resolves issue 29072. diff --git a/changes/ticket29160 b/changes/ticket29160 deleted file mode 100644 index 8e11183064..0000000000 --- a/changes/ticket29160 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (tests): - - Do not log an error-level message if we fail to find an IPv6 - network interface from the unit tests. Fixes bug 29160; bugfix on - 0.2.7.3-rc. diff --git a/changes/ticket29168 b/changes/ticket29168 deleted file mode 100644 index 65c5232f65..0000000000 --- a/changes/ticket29168 +++ /dev/null @@ -1,5 +0,0 @@ - o Major bugfixes (cell scheduler, KIST): - - Make KIST to always take into account the outbuf length when computing - what we can actually put in the outbuf. This could lead to the outbuf - being filled up and thus a possible memory DoS vector. TROVE-2019-001. - Fixes bug 29168; bugfix on 0.3.2.1-alpha. From 1bff5646e64a32b83dcf5ac5acf4e31b98382aae Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 21 Feb 2019 13:25:33 -0500 Subject: [PATCH 0525/2557] Bump to 0.4.0.2-alpha-dev --- configure.ac | 4 ++-- contrib/win32build/tor-mingw.nsi.in | 2 +- src/win32/orconfig.h | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/configure.ac b/configure.ac index 6e60489a57..91a0498440 100644 --- a/configure.ac +++ b/configure.ac @@ -4,7 +4,7 @@ dnl Copyright (c) 2007-2019, The Tor Project, Inc. dnl See LICENSE for licensing information AC_PREREQ([2.63]) -AC_INIT([tor],[0.4.0.2-alpha]) +AC_INIT([tor],[0.4.0.2-alpha-dev]) AC_CONFIG_SRCDIR([src/app/main/tor_main.c]) AC_CONFIG_MACRO_DIR([m4]) @@ -14,7 +14,7 @@ AC_CONFIG_MACRO_DIR([m4]) # version number changes. Tor uses it to make sure that it # only shuts down for missing "required protocols" when those protocols # are listed as required by a consensus after this date. -AC_DEFINE(APPROX_RELEASE_DATE, ["2019-02-21"], # for 0.4.0.2-alpha +AC_DEFINE(APPROX_RELEASE_DATE, ["2019-02-21"], # for 0.4.0.2-alpha-dev [Approximate date when this software was released. (Updated when the version changes.)]) # "foreign" means we don't follow GNU package layout standards diff --git a/contrib/win32build/tor-mingw.nsi.in b/contrib/win32build/tor-mingw.nsi.in index eb961e6160..c937ec2f75 100644 --- a/contrib/win32build/tor-mingw.nsi.in +++ b/contrib/win32build/tor-mingw.nsi.in @@ -8,7 +8,7 @@ !include "LogicLib.nsh" !include "FileFunc.nsh" !insertmacro GetParameters -!define VERSION "0.4.0.2-alpha" +!define VERSION "0.4.0.2-alpha-dev" !define INSTALLER "tor-${VERSION}-win32.exe" !define WEBSITE "https://www.torproject.org/" !define LICENSE "LICENSE" diff --git a/src/win32/orconfig.h b/src/win32/orconfig.h index 0081bc4553..017ed223e6 100644 --- a/src/win32/orconfig.h +++ b/src/win32/orconfig.h @@ -218,7 +218,7 @@ #define USING_TWOS_COMPLEMENT /* Version number of package */ -#define VERSION "0.4.0.2-alpha" +#define VERSION "0.4.0.2-alpha-dev" From b3b737b8750b2063c5be260287088defbfadd1b9 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 21 Feb 2019 13:35:33 -0500 Subject: [PATCH 0526/2557] Update pre-commit.git-hook for ticket 29553 - handle older source layout - handle empty changes directories - "set -e" so that we exit if there's a problem. --- changes/ticket29553 | 5 +++++ scripts/maint/pre-commit.git-hook | 33 ++++++++++++++++++++++--------- 2 files changed, 29 insertions(+), 9 deletions(-) create mode 100644 changes/ticket29553 diff --git a/changes/ticket29553 b/changes/ticket29553 new file mode 100644 index 0000000000..af441b92b0 --- /dev/null +++ b/changes/ticket29553 @@ -0,0 +1,5 @@ + o Minor bugfixes (developer tools): + - Update our pre-commit.git-hook script to work correctly on older Tor + branches and release branches without any changes files, + and to actually exit when something fails. Fixes bug 29553; bugfix on + 0.4.0.2-alpha. diff --git a/scripts/maint/pre-commit.git-hook b/scripts/maint/pre-commit.git-hook index b4c4ce2061..65fa99f4c4 100755 --- a/scripts/maint/pre-commit.git-hook +++ b/scripts/maint/pre-commit.git-hook @@ -10,16 +10,31 @@ workdir=$(git rev-parse --show-toplevel) cd "$workdir" || exit 1 -python scripts/maint/lintChanges.py ./changes/* +set -e -perl scripts/maint/checkSpace.pl -C \ -src/lib/*/*.[ch] \ -src/core/*/*.[ch] \ -src/feature/*/*.[ch] \ -src/app/*/*.[ch] \ -src/test/*.[ch] \ -src/test/*/*.[ch] \ -src/tools/*.[ch] +if [ ! -z "ls ./changes/*" ]; then + python scripts/maint/lintChanges.py ./changes/* +fi + +if [ -d src/lib ]; then + # This is the layout in 0.3.5 + perl scripts/maint/checkSpace.pl -C \ + src/lib/*/*.[ch] \ + src/core/*/*.[ch] \ + src/feature/*/*.[ch] \ + src/app/*/*.[ch] \ + src/test/*.[ch] \ + src/test/*/*.[ch] \ + src/tools/*.[ch] +elif [ -d src/common]; then + # This was the layout before 0.3.5 + perl scripts/maint/checkSpace.pl -C \ + src/common/*/*.[ch] \ + src/or/*/*.[ch] \ + src/test/*.[ch] \ + src/test/*/*.[ch] \ + src/tools/*.[ch] +fi if test -e scripts/maint/checkIncludes.py; then python scripts/maint/checkIncludes.py From c346eff223e94b5fbeb6e751a99393fc5f7dd4b0 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Thu, 21 Feb 2019 21:09:40 +0200 Subject: [PATCH 0527/2557] Walk back from requiring bash Refrain from using bash array to remember $@. --- src/test/test-network.sh | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/test/test-network.sh b/src/test/test-network.sh index 4d56e83806..372c8cbac3 100755 --- a/src/test/test-network.sh +++ b/src/test/test-network.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/bin/sh # This script calls the equivalent script in chutney/tools @@ -18,10 +18,6 @@ ECHO="${ECHO:-echo}" # Output is prefixed with the name of the script myname=$(basename "$0") -# Save the arguments before we destroy them -# This might not preserve arguments with spaces in them -ORIGINAL_ARGS=( "$@" ) - # We need to find CHUTNEY_PATH, so that we can call the version of this script # in chutney/tools with the same arguments. We also need to respect --quiet. until [ -z "$1" ] @@ -99,7 +95,7 @@ if [ -d "$CHUTNEY_PATH" ] && [ -x "$TEST_NETWORK" ]; then # this may fail if some arguments have spaces in them # if so, set CHUTNEY_PATH before calling test-network.sh, and spaces # will be handled correctly - exec "$TEST_NETWORK" "${ORIGINAL_ARGS[@]}" # $ORIGINAL_ARGS + exec "$TEST_NETWORK" "$@" else $ECHO "$myname: Could not find tools/test-network.sh in CHUTNEY_PATH." $ECHO "$myname: Please update your chutney using 'git pull'." From df8ad6473575e217fe69de7d0d12341a1162b95e Mon Sep 17 00:00:00 2001 From: Neel Chauhan Date: Fri, 22 Feb 2019 13:36:02 -0500 Subject: [PATCH 0528/2557] When a DirAuth checks reachability on itself and has IPv6, mark it as reachable --- changes/bug24338 | 4 ++++ src/feature/dirauth/voteflags.c | 18 +++++++++++++++--- 2 files changed, 19 insertions(+), 3 deletions(-) create mode 100644 changes/bug24338 diff --git a/changes/bug24338 b/changes/bug24338 new file mode 100644 index 0000000000..75984b6329 --- /dev/null +++ b/changes/bug24338 @@ -0,0 +1,4 @@ + o Minor bugfixes (dirauth, ipv6): + - If we are a durauth with IPv6 and are marking relays as running, mark + ourselves as reachable on IPv6. Fixes bug 24338; bugfix on 0.4.0.2-alpha. + Patch by Neel Chauhan diff --git a/src/feature/dirauth/voteflags.c b/src/feature/dirauth/voteflags.c index 4f7593a3e1..0a53c588d6 100644 --- a/src/feature/dirauth/voteflags.c +++ b/src/feature/dirauth/voteflags.c @@ -531,6 +531,20 @@ dirserv_set_router_is_running(routerinfo_t *router, time_t now) node->is_running = answer; } +/* Check node and ri on whether or not we should publish a + * relay's IPv6 addresses. */ +static int +should_publish_node_ipv6(const node_t *node, const routerinfo_t *ri, + time_t now) +{ + const or_options_t *options = get_options(); + + return options->AuthDirHasIPv6Connectivity == 1 && + !tor_addr_is_null(&ri->ipv6_addr) && + ((node->last_reachable6 >= now - REACHABLE_TIMEOUT) || + router_is_me(ri)); +} + /** Extract status information from ri and from other authority * functions and store it in rs. rs is zeroed out before it is * set. @@ -597,9 +611,7 @@ set_routerstatus_from_routerinfo(routerstatus_t *rs, rs->is_staledesc = (ri->cache_info.published_on + DESC_IS_STALE_INTERVAL) < now; - if (options->AuthDirHasIPv6Connectivity == 1 && - !tor_addr_is_null(&ri->ipv6_addr) && - node->last_reachable6 >= now - REACHABLE_TIMEOUT) { + if (should_publish_node_ipv6(node, ri, now)) { /* We're configured as having IPv6 connectivity. There's an IPv6 OR port and it's reachable so copy it to the routerstatus. */ tor_addr_copy(&rs->ipv6_addr, &ri->ipv6_addr); From b7dced893a7de1c0ba303905f69022fee7d05fc9 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Thu, 7 Feb 2019 17:05:14 +0200 Subject: [PATCH 0529/2557] Fix shellcheck SC2006 warnings in test_switch_id.sh --- changes/ticket29065 | 3 +++ src/test/test_switch_id.sh | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 changes/ticket29065 diff --git a/changes/ticket29065 b/changes/ticket29065 new file mode 100644 index 0000000000..edf00ac99c --- /dev/null +++ b/changes/ticket29065 @@ -0,0 +1,3 @@ + o Code simplification and refactoring (shell scripts): + - Cleanup test_switch_id.sh to silence shellcheck warnings. Closes + ticket 29065. diff --git a/src/test/test_switch_id.sh b/src/test/test_switch_id.sh index 79c44f2eb1..b13bf7602f 100755 --- a/src/test/test_switch_id.sh +++ b/src/test/test_switch_id.sh @@ -1,11 +1,11 @@ #!/bin/sh -if test "`id -u`" != '0'; then +if test "$(id -u)" != '0'; then echo "This test only works when run as root. Skipping." >&2 exit 77 fi -if test "`id -u nobody`" = ""; then +if test "$(id -u nobody)" = ""; then echo "This test requires that your system have a 'nobody' user. Sorry." >&2 exit 1 fi From 341cd6ea6631217a5487198a773b56a057afd5f9 Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Mon, 25 Feb 2019 13:28:35 +0200 Subject: [PATCH 0530/2557] histogram_edges is histogram_len long --- src/core/or/circuitpadding.c | 1 - src/core/or/circuitpadding.h | 2 +- src/test/test_circuitpadding.c | 2 -- 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/src/core/or/circuitpadding.c b/src/core/or/circuitpadding.c index ceb9181f94..09dda6da3d 100644 --- a/src/core/or/circuitpadding.c +++ b/src/core/or/circuitpadding.c @@ -2185,7 +2185,6 @@ circpad_circ_responder_machine_init(void) circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram_edges[3]= 4000; circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram_edges[4]= 8000; circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram_edges[5]= 16000; - circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram_edges[6]=1000000; /* Specify histogram tokens */ circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram[0] = 0; circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram[1] = 1; diff --git a/src/core/or/circuitpadding.h b/src/core/or/circuitpadding.h index b9e903b9f0..d11358a159 100644 --- a/src/core/or/circuitpadding.h +++ b/src/core/or/circuitpadding.h @@ -290,7 +290,7 @@ typedef struct circpad_state_t { * bin. The rightmost edge is always infinity and is not specified in this * array. * - * This array must have histogram_len+1 elements. */ + * This array must have histogram_len elements. */ circpad_delay_t histogram_edges[CIRCPAD_MAX_HISTOGRAM_LEN+1]; /** Total number of tokens in this histogram. This is a constant and is *not* * decremented every time we spend a token. It's used for initializing and diff --git a/src/test/test_circuitpadding.c b/src/test/test_circuitpadding.c index 65bf529e42..599b7fb24c 100644 --- a/src/test/test_circuitpadding.c +++ b/src/test/test_circuitpadding.c @@ -425,8 +425,6 @@ helper_create_basic_machine(void) circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_edges[2] = 5000; circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_edges[3] = 10000; circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_edges[4] = 20000; - circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_edges[5] = 400000; - circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_edges[6] = 1000000; circ_client_machine.states[CIRCPAD_STATE_BURST].histogram[0] = 1; circ_client_machine.states[CIRCPAD_STATE_BURST].histogram[1] = 0; From 331a067ae3ce778037650d5283504350979e50ee Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Mon, 25 Feb 2019 13:46:58 +0200 Subject: [PATCH 0531/2557] Fix dist_min_usec documentation and naming. --- src/core/or/circuitpadding.c | 13 +++++++------ src/core/or/circuitpadding.h | 15 +++++++++------ src/test/test_circuitpadding.c | 24 ++++++++++++------------ 3 files changed, 28 insertions(+), 24 deletions(-) diff --git a/src/core/or/circuitpadding.c b/src/core/or/circuitpadding.c index 09dda6da3d..7e76cd7ad5 100644 --- a/src/core/or/circuitpadding.c +++ b/src/core/or/circuitpadding.c @@ -380,17 +380,17 @@ circpad_choose_state_length(circpad_machine_state_t *mi) */ static circpad_delay_t circpad_distribution_sample_iat_delay(const circpad_state_t *state, - circpad_delay_t min_delay) + circpad_delay_t delay_shift) { double val = circpad_distribution_sample(state->iat_dist); /* These comparisons are safe, because the output is in the range * [0, 2**32), and double has a precision of 53 bits. */ val = MAX(0, val); - val = MIN(val, state->dist_max_usec); + val = MIN(val, state->dist_max_sample_usec); /* This addition is exact: val is at most 2**32-1, min_delay * is at most 2**32-1, and doubles have a precision of 53 bits. */ - val += min_delay; + val += delay_shift; /* Clamp the distribution at infinite delay val */ return (circpad_delay_t)MIN(tor_llround(val), CIRCPAD_DELAY_INFINITE); @@ -420,9 +420,10 @@ circpad_machine_sample_delay(circpad_machine_state_t *mi) if (state->iat_dist.type != CIRCPAD_DIST_NONE) { /* Sample from a fixed IAT distribution and return */ - circpad_delay_t min_iat_delay = state->use_rtt_estimate ? - mi->rtt_estimate_usec + state->dist_min_usec : state->dist_min_usec; - return circpad_distribution_sample_iat_delay(state, min_iat_delay); + circpad_delay_t iat_delay_shift = state->use_rtt_estimate ? + mi->rtt_estimate_usec + state->dist_added_shift_usec : + state->dist_added_shift_usec; + return circpad_distribution_sample_iat_delay(state, iat_delay_shift); } else if (state->token_removal != CIRCPAD_TOKEN_REMOVAL_NONE) { /* We have a mutable histogram. Do basic sanity check and apply: */ if (BUG(!mi->histogram) || diff --git a/src/core/or/circuitpadding.h b/src/core/or/circuitpadding.h index d11358a159..4e6454b4a2 100644 --- a/src/core/or/circuitpadding.h +++ b/src/core/or/circuitpadding.h @@ -309,13 +309,16 @@ typedef struct circpad_state_t { * results of sampling from this distribution (range_sec is used as a max). */ circpad_distribution_t iat_dist; - /* If a delay probability distribution is used, this represents the minimum - * delay we can sample from the distribution. */ - circpad_delay_t dist_min_usec; /* If a delay probability distribution is used, this is used as the max - * delay we can sample from the distribution. - */ - circpad_delay_t dist_max_usec; + * value we can sample from the distribution. However, RTT measurements and + * dist_added_shift gets applied on top of this value to derive the final + * padding delay. */ + circpad_delay_t dist_max_sample_usec; + /* If a delay probability distribution is used and this is set, we will add + * this value on top of the value sampled from the IAT distribution to + * derive the final padding delay (We also add the RTT measurement if it's + * enabled.). */ + circpad_delay_t dist_added_shift_usec; /** * The length dist is a parameterized way of encoding how long this diff --git a/src/test/test_circuitpadding.c b/src/test/test_circuitpadding.c index 599b7fb24c..3570b179bd 100644 --- a/src/test/test_circuitpadding.c +++ b/src/test/test_circuitpadding.c @@ -2098,48 +2098,48 @@ helper_circpad_circ_distribution_machine_setup(int min, int max) zero_st->iat_dist.type = CIRCPAD_DIST_UNIFORM; zero_st->iat_dist.param1 = min; zero_st->iat_dist.param2 = max; - zero_st->dist_min_usec = min; - zero_st->dist_max_usec = max; + zero_st->dist_added_shift_usec = min; + zero_st->dist_max_sample_usec = max; circpad_state_t *first_st = &circ_client_machine.states[1]; first_st->next_state[CIRCPAD_EVENT_NONPADDING_RECV] = 2; first_st->iat_dist.type = CIRCPAD_DIST_LOGISTIC; first_st->iat_dist.param1 = min; first_st->iat_dist.param2 = max; - first_st->dist_min_usec = min; - first_st->dist_max_usec = max; + first_st->dist_added_shift_usec = min; + first_st->dist_max_sample_usec = max; circpad_state_t *second_st = &circ_client_machine.states[2]; second_st->next_state[CIRCPAD_EVENT_NONPADDING_RECV] = 3; second_st->iat_dist.type = CIRCPAD_DIST_LOG_LOGISTIC; second_st->iat_dist.param1 = min; second_st->iat_dist.param2 = max; - second_st->dist_min_usec = min; - second_st->dist_max_usec = max; + second_st->dist_added_shift_usec = min; + second_st->dist_max_sample_usec = max; circpad_state_t *third_st = &circ_client_machine.states[3]; third_st->next_state[CIRCPAD_EVENT_NONPADDING_RECV] = 4; third_st->iat_dist.type = CIRCPAD_DIST_GEOMETRIC; third_st->iat_dist.param1 = min; third_st->iat_dist.param2 = max; - third_st->dist_min_usec = min; - third_st->dist_max_usec = max; + third_st->dist_added_shift_usec = min; + third_st->dist_max_sample_usec = max; circpad_state_t *fourth_st = &circ_client_machine.states[4]; fourth_st->next_state[CIRCPAD_EVENT_NONPADDING_RECV] = 5; fourth_st->iat_dist.type = CIRCPAD_DIST_WEIBULL; fourth_st->iat_dist.param1 = min; fourth_st->iat_dist.param2 = max; - fourth_st->dist_min_usec = min; - fourth_st->dist_max_usec = max; + fourth_st->dist_added_shift_usec = min; + fourth_st->dist_max_sample_usec = max; circpad_state_t *fifth_st = &circ_client_machine.states[5]; fifth_st->next_state[CIRCPAD_EVENT_NONPADDING_RECV] = 6; fifth_st->iat_dist.type = CIRCPAD_DIST_PARETO; fifth_st->iat_dist.param1 = min; fifth_st->iat_dist.param2 = max; - fifth_st->dist_min_usec = min; - fifth_st->dist_max_usec = max; + fifth_st->dist_added_shift_usec = min; + fifth_st->dist_max_sample_usec = max; } /** Simple test that the padding delays sampled from a uniform distribution From 71c11d7306dbbcb81c780a7a786429e2d8eaa669 Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Mon, 25 Feb 2019 13:59:18 +0200 Subject: [PATCH 0532/2557] document picking infinity bin --- src/core/or/circuitpadding.c | 4 ++++ src/core/or/circuitpadding.h | 6 ++++++ 2 files changed, 10 insertions(+) diff --git a/src/core/or/circuitpadding.c b/src/core/or/circuitpadding.c index 7e76cd7ad5..29a6189309 100644 --- a/src/core/or/circuitpadding.c +++ b/src/core/or/circuitpadding.c @@ -224,6 +224,10 @@ circpad_machine_current_state(const circpad_machine_state_t *mi) /** * Get the lower bound of a histogram bin. The upper bound is obtained by * calling this function with bin+1, and subtracting 1. + * + * This function can also be called with 'bin' set to a value equal or greater + * than histogram_len in which case the infinity bin is chosen and + * CIRCPAD_DELAY_INFINITE is returned. */ STATIC circpad_delay_t circpad_histogram_bin_to_usec(const circpad_machine_state_t *mi, diff --git a/src/core/or/circuitpadding.h b/src/core/or/circuitpadding.h index 4e6454b4a2..ca19e31ca5 100644 --- a/src/core/or/circuitpadding.h +++ b/src/core/or/circuitpadding.h @@ -276,6 +276,12 @@ typedef struct circpad_state_t { * histogram[] = { 6, 10, 6, 7, 9, 6 } * histogram_edges[] = { 0, 100, 200, 350, 500, 1000 } * + * The final bin is called the "infinity bin" and if it's chosen we don't + * schedule any padding. The infinity bin is strange because its lower edge + * is the max value of possible non-infinite delay allowed by this histogram, + * and its upper edge is CIRCPAD_DELAY_INFINITE. You can tell if the infinity + * bin is chosen by inspecting its bin index or inspecting its upper edge. + * * If a delay probability distribution is used for this state, this is set * to 0. */ circpad_hist_index_t histogram_len; From 18de065cbbcd9863842fc7f773033f295a0000dc Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Mon, 25 Feb 2019 14:01:55 +0200 Subject: [PATCH 0533/2557] Switch an int32_t bin to a circpad_hist_index_t. --- src/core/or/circuitpadding.c | 2 +- src/core/or/circuitpadding.h | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/core/or/circuitpadding.c b/src/core/or/circuitpadding.c index 29a6189309..b7e4b56846 100644 --- a/src/core/or/circuitpadding.c +++ b/src/core/or/circuitpadding.c @@ -293,7 +293,7 @@ circpad_histogram_usec_to_bin(const circpad_machine_state_t *mi, { const circpad_state_t *state = circpad_machine_current_state(mi); circpad_delay_t rtt_add_usec = 0; - int32_t bin; + circpad_hist_index_t bin; /* Our state should have been checked to be non-null by the caller * (circpad_machine_remove_token()) */ diff --git a/src/core/or/circuitpadding.h b/src/core/or/circuitpadding.h index ca19e31ca5..74f9f35c98 100644 --- a/src/core/or/circuitpadding.h +++ b/src/core/or/circuitpadding.h @@ -235,6 +235,9 @@ typedef uint16_t circpad_statenum_t; * * Memory concerns are not so great here since the corresponding histogram and * histogram_edges arrays are global and not per-circuit. + * + * If we ever upgrade this to a value that can't be represented by 8-bits we + * also need to upgrade circpad_hist_index_t. */ #define CIRCPAD_MAX_HISTOGRAM_LEN (100) From 065e7da8e6fdbd9331de8c13344275a8e0fbf32d Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 25 Feb 2019 08:55:25 -0500 Subject: [PATCH 0534/2557] Re-enable and fix unit test for nofork mappings This test was previously written to use the contents of the system headers to decide whether INHERIT_NONE or INHERIT_ZERO was going to work. But that won't work across different environments, such as (for example) when the kernel doesn't match the headers. Instead, we add a testing-only feature to the code to track which of these options actually worked, and verify that it behaved as we expected. Closes ticket 29541; bugfix not on any released version of Tor. --- src/lib/malloc/map_anon.c | 34 ++++++++++++++++++++++++++++++++-- src/lib/malloc/map_anon.h | 4 ++++ src/test/test_util.c | 34 +++++++++++++++++++++++++--------- 3 files changed, 61 insertions(+), 11 deletions(-) diff --git a/src/lib/malloc/map_anon.c b/src/lib/malloc/map_anon.c index 2fc6e89ea2..5dac5256a6 100644 --- a/src/lib/malloc/map_anon.c +++ b/src/lib/malloc/map_anon.c @@ -107,6 +107,29 @@ nodump_mem(void *mem, size_t sz) #endif } +#ifdef TOR_UNIT_TESTS +static unsigned last_anon_map_noinherit = ~0; +/* Testing helper: return the outcome of the last call to noinherit_mem(): + * 0 if it did no good; 1 if it caused the memory not to be inherited, and + * 2 if it caused the memory to be cleared on fork */ +unsigned +get_last_anon_map_noinherit(void) +{ + return last_anon_map_noinherit; +} +static void +set_last_anon_map_noinherit(unsigned f) +{ + last_anon_map_noinherit = f; +} +#else +static void +set_last_anon_map_noinherit(unsigned f) +{ + (void)f; +} +#endif + /** * Helper: try to prevent the sz bytes at mem from being * accessible in child processes -- ideally by having them set to 0 after a @@ -117,13 +140,20 @@ nodump_mem(void *mem, size_t sz) static int noinherit_mem(void *mem, size_t sz) { + set_last_anon_map_noinherit(0); #ifdef FLAG_ZERO int r = MINHERIT(mem, sz, FLAG_ZERO); - if (r == 0) + if (r == 0) { + set_last_anon_map_noinherit(2); return 0; + } #endif #ifdef FLAG_NOINHERIT - return MINHERIT(mem, sz, FLAG_NOINHERIT); + int r2 = MINHERIT(mem, sz, FLAG_NOINHERIT); + if (r2 == 0) { + set_last_anon_map_noinherit(1); + } + return r2; #else (void)mem; (void)sz; diff --git a/src/lib/malloc/map_anon.h b/src/lib/malloc/map_anon.h index cc5797e4ec..395145bd71 100644 --- a/src/lib/malloc/map_anon.h +++ b/src/lib/malloc/map_anon.h @@ -34,4 +34,8 @@ void *tor_mmap_anonymous(size_t sz, unsigned flags); void tor_munmap_anonymous(void *mapping, size_t sz); +#ifdef TOR_UNIT_TESTS +unsigned get_last_anon_map_noinherit(void); +#endif + #endif /* !defined(TOR_MAP_ANON_H) */ diff --git a/src/test/test_util.c b/src/test/test_util.c index 7a2708c541..4990aa709a 100644 --- a/src/test/test_util.c +++ b/src/test/test_util.c @@ -6165,8 +6165,8 @@ static void test_util_map_anon_nofork(void *arg) { (void)arg; -#if !defined(HAVE_MADVISE) && !defined(HAVE_MINHERIT) - /* The operating system doesn't support this. */ +#ifdef _WIN32 + /* The operating system doesn't support forking. */ tt_skip(); done: ; @@ -6182,6 +6182,7 @@ test_util_map_anon_nofork(void *arg) tor_munmap_anonymous(ptr, sz); ptr = tor_mmap_anonymous(sz, ANONMAP_NOINHERIT); + int outcome = get_last_anon_map_noinherit(); tt_ptr_op(ptr, OP_NE, 0); memset(ptr, 0xd0, sz); @@ -6202,15 +6203,30 @@ test_util_map_anon_nofork(void *arg) pipefd[1] = -1; char buf[1]; ssize_t r = read(pipefd[0], buf, 1); -#if defined(INHERIT_ZERO) || defined(MADV_WIPEONFORK) - tt_int_op((int)r, OP_EQ, 1); // child should send us a byte. - tt_int_op(buf[0], OP_EQ, 0); -#else - tt_int_op(r, OP_LE, 0); // child said nothing; it should have crashed. -#endif + + if (outcome == 2) { + // We should be seeing clear-on-fork behavior. + tt_int_op((int)r, OP_EQ, 1); // child should send us a byte. + tt_int_op(buf[0], OP_EQ, 0); // that byte should be zero. + } else if (outcome == 1) { + // We should be seeing noinherit behavior. + tt_int_op(r, OP_LE, 0); // child said nothing; it should have crashed. + } else { + // noinherit isn't implemented. + tt_int_op(outcome, OP_EQ, 0); + tt_int_op((int)r, OP_EQ, 1); // child should send us a byte. + tt_int_op(buf[0], OP_EQ, 0xd0); // that byte should what we set it to. + } + int ws; waitpid(child, &ws, 0); + if (outcome == 0) { + /* Call this test "skipped", not "passed", since noinherit wasn't + * implemented. */ + tt_skip(); + } + done: tor_munmap_anonymous(ptr, sz); if (pipefd[0] >= 0) { @@ -6360,6 +6376,6 @@ struct testcase_t util_tests[] = { UTIL_TEST(get_unquoted_path, 0), UTIL_TEST(log_mallinfo, 0), UTIL_TEST(map_anon, 0), - UTIL_TEST(map_anon_nofork, TT_SKIP /* See bug #29535 */), + UTIL_TEST(map_anon_nofork, 0), END_OF_TESTCASES }; From d731ab45831cfc2104b17b3a623e0c89c3174145 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Mon, 25 Feb 2019 19:07:47 +0200 Subject: [PATCH 0535/2557] Check that all valid values of int and unsigned int can be put into void pointer --- changes/ticket29537 | 3 ++ src/test/include.am | 1 + src/test/test.h | 1 + src/test/test_ptr_slow.c | 67 ++++++++++++++++++++++++++++++++++++++++ src/test/test_slow.c | 1 + 5 files changed, 73 insertions(+) create mode 100644 changes/ticket29537 create mode 100644 src/test/test_ptr_slow.c diff --git a/changes/ticket29537 b/changes/ticket29537 new file mode 100644 index 0000000000..048e13c65f --- /dev/null +++ b/changes/ticket29537 @@ -0,0 +1,3 @@ + o Testing: + - Check that all valid values of `int` and `unsigned int` can be + represented by `void *`. Resolves issue 29537. diff --git a/src/test/include.am b/src/test/include.am index d585c2a38a..e6cebe1d1a 100644 --- a/src/test/include.am +++ b/src/test/include.am @@ -211,6 +211,7 @@ src_test_test_slow_SOURCES += \ src/test/test_crypto_slow.c \ src/test/test_process_slow.c \ src/test/test_prob_distr.c \ + src/test/test_ptr_slow.c \ src/test/testing_common.c \ src/test/testing_rsakeys.c \ src/ext/tinytest.c diff --git a/src/test/test.h b/src/test/test.h index 2564432985..9d9184e2aa 100644 --- a/src/test/test.h +++ b/src/test/test.h @@ -278,6 +278,7 @@ extern struct testcase_t x509_tests[]; extern struct testcase_t slow_crypto_tests[]; extern struct testcase_t slow_process_tests[]; +extern struct testcase_t slow_ptr_tests[]; extern struct testgroup_t testgroups[]; diff --git a/src/test/test_ptr_slow.c b/src/test/test_ptr_slow.c new file mode 100644 index 0000000000..a5914c9120 --- /dev/null +++ b/src/test/test_ptr_slow.c @@ -0,0 +1,67 @@ +/* Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "orconfig.h" +#include "core/or/or.h" +#include "test/test.h" + +#include +#include + +/** Check that all values of int can be cast to void * and back. */ +static void +test_int_voidstar_interop(void *arg) +{ + int a; + (void)arg; + + for (a = INT_MIN; a < INT_MAX; a++) { + intptr_t ap = (intptr_t)a; + void *b = (void *)ap; + intptr_t c = (intptr_t)b; + void *d = (void *)c; + + tt_assert(ap == c); + tt_assert(b == d); + } + + done: + return; +} + +/** Check that all values of unsigned int can be cast to void * and back. */ +static void +test_uint_voidstar_interop(void *arg) +{ + unsigned int a; + (void)arg; + + for (a = 0; a < UINT_MAX; a++) { + intptr_t ap = (intptr_t)a; + void *b = (void *)ap; + intptr_t c = (intptr_t)b; + void *d = (void *)c; + + tt_assert(ap == c); + tt_assert(b == d); + } + + done: + return; +} + +struct testcase_t slow_ptr_tests[] = { + { .name = "int_voidstar_interop", + .fn = test_int_voidstar_interop, + .flags = 0, + .setup = NULL, + .setup_data = NULL }, + { .name = "uint_voidstar_interop", + .fn = test_uint_voidstar_interop, + .flags = 0, + .setup = NULL, + .setup_data = NULL }, + END_OF_TESTCASES +}; diff --git a/src/test/test_slow.c b/src/test/test_slow.c index c3e7edd408..d4d5b755a5 100644 --- a/src/test/test_slow.c +++ b/src/test/test_slow.c @@ -22,6 +22,7 @@ struct testgroup_t testgroups[] = { { "slow/crypto/", slow_crypto_tests }, { "slow/process/", slow_process_tests }, { "slow/prob_distr/", slow_stochastic_prob_distr_tests }, + { "slow/ptr/", slow_ptr_tests }, END_OF_GROUPS }; From aa360b255bc1c262486500655bac70c4f0f00118 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Tue, 26 Feb 2019 15:26:33 +0100 Subject: [PATCH 0536/2557] Fix crash bug in PT subsystem. This patch fixes a crash bug (assertion failure) in the PT subsystem that could get triggered if the user cancels bootstrap via the UI in TorBrowser. This would cause Tor to call `managed_proxy_destroy()` which called `process_free()` after it had called `process_terminate()`. This leads to a crash when the various process callbacks returns with data after the `process_t` have been freed using `process_free()`. We solve this issue by ensuring that everywhere we call `process_terminate()` we make sure to detach the `managed_proxy_t` from the `process_t` (by calling `process_set_data(process, NULL)`) and avoid calling `process_free()` at all in the transports code. Instead we just call `process_terminate()` and let the process exit callback in `managed_proxy_exit_callback()` handle the `process_free()` call by returning true to the process subsystem. See: https://bugs.torproject.org/29562 --- changes/bug29562 | 4 ++++ src/feature/client/transports.c | 29 ++++++++++++++--------------- 2 files changed, 18 insertions(+), 15 deletions(-) create mode 100644 changes/bug29562 diff --git a/changes/bug29562 b/changes/bug29562 new file mode 100644 index 0000000000..0621cd09a0 --- /dev/null +++ b/changes/bug29562 @@ -0,0 +1,4 @@ + o Minor bugfixes (pluggable transports): + - Fix an assertion failure crash bug when a pluggable transport process is + terminated during the bootstrap phase. Fixes bug 29562; bugfix on + 0.4.0.1-alpha. diff --git a/src/feature/client/transports.c b/src/feature/client/transports.c index e247055164..e7ff3bf34a 100644 --- a/src/feature/client/transports.c +++ b/src/feature/client/transports.c @@ -713,10 +713,13 @@ managed_proxy_destroy(managed_proxy_t *mp, tor_free(mp->proxy_uri); /* do we want to terminate our process if it's still running? */ - if (also_terminate_process && mp->process) + if (also_terminate_process && mp->process) { + /* Note that we do not call process_free(mp->process) here because we let + * the exit handler in managed_proxy_exit_callback() return `true` which + * makes the process subsystem deallocate the process_t. */ + process_set_data(mp->process, NULL); process_terminate(mp->process); - - process_free(mp->process); + } tor_free(mp); } @@ -1823,6 +1826,9 @@ managed_proxy_stdout_callback(process_t *process, managed_proxy_t *mp = process_get_data(process); + if (BUG(mp == NULL)) + return; + handle_proxy_line(line, mp); if (proxy_configuration_finished(mp)) { @@ -1846,6 +1852,9 @@ managed_proxy_stderr_callback(process_t *process, managed_proxy_t *mp = process_get_data(process); + if (BUG(mp == NULL)) + return; + log_warn(LD_PT, "Managed proxy at '%s' reported: %s", mp->argv[0], line); } @@ -1862,18 +1871,8 @@ managed_proxy_exit_callback(process_t *process, process_exit_code_t exit_code) "Pluggable Transport process terminated with status code %" PRIu64, exit_code); - /* We detach ourself from the MP (if we are attached) and free ourself. */ - managed_proxy_t *mp = process_get_data(process); - - /* If we are still attached to the process, it is probably because our PT - * process crashed before we got to call process_set_data(p, NULL); */ - if (BUG(mp != NULL)) { - /* FIXME(ahf): Our process stopped without us having told it to stop - * (crashed). Should we restart it here? */ - mp->process = NULL; - process_set_data(process, NULL); - } - + /* Returning true here means that the process subsystem will take care of + * calling process_free() on our process_t. */ return true; } From e1ad22643e414da9802f4f5c2522fde35933fd47 Mon Sep 17 00:00:00 2001 From: David Goulet Date: Thu, 14 Feb 2019 14:27:40 -0500 Subject: [PATCH 0537/2557] maint: Helpful scripts for tor maintenance Closes #29391 Signed-off-by: David Goulet --- changes/ticket29391 | 3 + scripts/maint/git-merge-forward.sh | 194 +++++++++++++++++++++++++++++ scripts/maint/git-pull-all.sh | 178 ++++++++++++++++++++++++++ scripts/maint/git-push-all.sh | 12 ++ 4 files changed, 387 insertions(+) create mode 100644 changes/ticket29391 create mode 100755 scripts/maint/git-merge-forward.sh create mode 100755 scripts/maint/git-pull-all.sh create mode 100755 scripts/maint/git-push-all.sh diff --git a/changes/ticket29391 b/changes/ticket29391 new file mode 100644 index 0000000000..f00fa61c47 --- /dev/null +++ b/changes/ticket29391 @@ -0,0 +1,3 @@ + o Minor feature (maintenance scripts): + - Add to scripts/maint/ helper maintainer scripts used for git maintenance. + Closes ticket 29391. diff --git a/scripts/maint/git-merge-forward.sh b/scripts/maint/git-merge-forward.sh new file mode 100755 index 0000000000..3728067e7a --- /dev/null +++ b/scripts/maint/git-merge-forward.sh @@ -0,0 +1,194 @@ +#!/bin/bash + +############################## +# Configuration (change me!) # +############################## + +# The general setup that is suggested here is: +# +# GIT_PATH = /home//git/ +# ... where the git repository directories resides. +# TOR_MASTER_NAME = "tor" +# ... which means that tor.git was cloned in /home//git/tor +# TOR_WKT_NAME = "tor-wkt" +# ... which means that the tor worktrees are in /home//git/tor-wkt + +# Where are all those git repositories? +GIT_PATH="FULL_PATH_TO_GIT_REPOSITORY_DIRECTORY" +# The tor master git repository directory from which all the worktree have +# been created. +TOR_MASTER_NAME="tor" +# The worktrees location (directory). +TOR_WKT_NAME="tor-wkt" + +######################### +# End of configuration. # +######################### + +# Configuration of the branches that needs merging. The values are in order: +# (1) Branch name that we merge onto. +# (2) Branch name to merge from. In other words, this is merge into (1) +# (3) Full path of the git worktree. +# +# As an example: +# $ cd (3) +# $ git checkout maint-0.3.5 (1) +# $ git pull +# $ git merge maint-0.3.4 (2) +# +# First set of arrays are the maint-* branch and then the release-* branch. +# New arrays need to be in the WORKTREE= array else they aren't considered. +MAINT_033=( "maint-0.3.3" "maint-0.2.9" "$GIT_PATH/$TOR_WKT_NAME/maint-0.3.3" ) +MAINT_034=( "maint-0.3.4" "maint-0.3.3" "$GIT_PATH/$TOR_WKT_NAME/maint-0.3.4" ) +MAINT_035=( "maint-0.3.5" "maint-0.3.4" "$GIT_PATH/$TOR_WKT_NAME/maint-0.3.5" ) +MAINT_040=( "maint-0.4.0" "maint-0.3.5" "$GIT_PATH/$TOR_WKT_NAME/maint-0.4.0" ) +MAINT_MASTER=( "master" "maint-0.4.0" "$GIT_PATH/$TOR_MASTER_NAME" ) + +RELEASE_029=( "release-0.2.9" "maint-0.2.9" "$GIT_PATH/$TOR_WKT_NAME/release-0.2.9" ) +RELEASE_033=( "release-0.3.3" "maint-0.3.3" "$GIT_PATH/$TOR_WKT_NAME/release-0.3.3" ) +RELEASE_034=( "release-0.3.4" "maint-0.3.4" "$GIT_PATH/$TOR_WKT_NAME/release-0.3.4" ) +RELEASE_035=( "release-0.3.5" "maint-0.3.5" "$GIT_PATH/$TOR_WKT_NAME/release-0.3.5" ) +RELEASE_040=( "release-0.4.0" "maint-0.4.0" "$GIT_PATH/$TOR_WKT_NAME/release-0.4.0" ) + +########################## +# Git Worktree to manage # +########################## + +# List of all worktrees to work on. All defined above. Ordering is important. +# Always the maint-* branch BEFORE then the release-*. +WORKTREE=( + RELEASE_029[@] + + MAINT_033[@] + RELEASE_033[@] + + MAINT_034[@] + RELEASE_034[@] + + MAINT_035[@] + RELEASE_035[@] + + MAINT_040[@] + RELEASE_040[@] + + MAINT_MASTER[@] +) +COUNT=${#WORKTREE[@]} + +# Controlled by the -n option. The dry run option will just output the command +# that would have been executed for each worktree. +DRY_RUN=0 + +# Control characters +CNRM=$'\x1b[0;0m' # Clear color + +# Bright color +BGRN=$'\x1b[1;32m' +BBLU=$'\x1b[1;34m' +BRED=$'\x1b[1;31m' +BYEL=$'\x1b[1;33m' +IWTH=$'\x1b[3;37m' + +# Strings for the pretty print. +MARKER="${BBLU}[${BGRN}+${BBLU}]${CNRM}" +SUCCESS="${BGRN}success${CNRM}" +FAILED="${BRED}failed${CNRM}" + +#################### +# Helper functions # +#################### + +# Validate the given returned value (error code), print success or failed. The +# second argument is the error output in case of failure, it is printed out. +# On failure, this function exits. +function validate_ret +{ + if [ "$1" -eq 0 ]; then + printf "%s\\n" "$SUCCESS" + else + printf "%s\\n" "$FAILED" + printf " %s" "$2" + exit 1 + fi +} + +# Switch to the given branch name. +function switch_branch +{ + local cmd="git checkout $1" + printf " %s Switching branch to %s..." "$MARKER" "$1" + if [ $DRY_RUN -eq 0 ]; then + msg=$( eval "$cmd" 2>&1 ) + validate_ret $? "$msg" + else + printf "\\n %s\\n" "${IWTH}$cmd${CNRM}" + fi +} + +# Pull the given branch name. +function pull_branch +{ + local cmd="git pull" + printf " %s Pulling branch %s..." "$MARKER" "$1" + if [ $DRY_RUN -eq 0 ]; then + msg=$( eval "$cmd" 2>&1 ) + validate_ret $? "$msg" + else + printf "\\n %s\\n" "${IWTH}$cmd${CNRM}" + fi +} + +# Merge the given branch name ($2) into the current branch ($1). +function merge_branch +{ + local cmd="git merge --no-edit $1" + printf " %s Merging branch %s into %s..." "$MARKER" "$1" "$2" + if [ $DRY_RUN -eq 0 ]; then + msg=$( eval "$cmd" 2>&1 ) + validate_ret $? "$msg" + else + printf "\\n %s\\n" "${IWTH}$cmd${CNRM}" + fi +} + +# Go into the worktree repository. +function goto_repo +{ + if [ ! -d "$1" ]; then + echo " $1: Not found. Stopping." + exit 1 + fi + cd "$1" || exit +} + +############### +# Entry point # +############### + +while getopts "n" opt; do + case "$opt" in + n) DRY_RUN=1 + echo " *** DRY DRUN MODE ***" + ;; + *) + ;; + esac +done + +# Go over all configured worktree. +for ((i=0; i/git/ +# ... where the git repository directories resides. +# TOR_MASTER_NAME = "tor" +# ... which means that tor.git was cloned in /home//git/tor +# TOR_WKT_NAME = "tor-wkt" +# ... which means that the tor worktrees are in /home//git/tor-wkt + +# Where are all those git repositories? +GIT_PATH="FULL_PATH_TO_GIT_REPOSITORY_DIRECTORY" +# The tor master git repository directory from which all the worktree have +# been created. +TOR_MASTER_NAME="tor" +# The worktrees location (directory). +TOR_WKT_NAME="tor-wkt" + +######################### +# End of configuration. # +######################### + +# Configuration of the branches that needs merging. The values are in order: +# (1) Branch name to pull (update). +# (2) Full path of the git worktree. +# +# As an example: +# $ cd (3) +# $ git checkout maint-0.3.5 (1) +# $ git pull +# +# First set of arrays are the maint-* branch and then the release-* branch. +# New arrays need to be in the WORKTREE= array else they aren't considered. +MAINT_029=( "maint-0.2.9" "$GIT_PATH/$TOR_WKT_NAME/maint-0.2.9" ) +MAINT_033=( "maint-0.3.3" "$GIT_PATH/$TOR_WKT_NAME/maint-0.3.3" ) +MAINT_034=( "maint-0.3.4" "$GIT_PATH/$TOR_WKT_NAME/maint-0.3.4" ) +MAINT_035=( "maint-0.3.5" "$GIT_PATH/$TOR_WKT_NAME/maint-0.3.5" ) +MAINT_040=( "maint-0.4.0" "$GIT_PATH/$TOR_WKT_NAME/maint-0.4.0" ) +MAINT_MASTER=( "master" "$GIT_PATH/$TOR_MASTER_NAME" ) + +RELEASE_029=( "release-0.2.9" "$GIT_PATH/$TOR_WKT_NAME/release-0.2.9" ) +RELEASE_033=( "release-0.3.3" "$GIT_PATH/$TOR_WKT_NAME/release-0.3.3" ) +RELEASE_034=( "release-0.3.4" "$GIT_PATH/$TOR_WKT_NAME/release-0.3.4" ) +RELEASE_035=( "release-0.3.5" "$GIT_PATH/$TOR_WKT_NAME/release-0.3.5" ) +RELEASE_040=( "release-0.4.0" "$GIT_PATH/$TOR_WKT_NAME/release-0.4.0" ) + +########################## +# Git Worktree to manage # +########################## + +# List of all worktrees to work on. All defined above. Ordering is important. +# Always the maint-* branch first then the release-*. +WORKTREE=( + MAINT_029[@] + RELEASE_029[@] + + MAINT_033[@] + RELEASE_033[@] + + MAINT_034[@] + RELEASE_034[@] + + MAINT_035[@] + RELEASE_035[@] + + MAINT_040[@] + RELEASE_040[@] + + MAINT_MASTER[@] +) +COUNT=${#WORKTREE[@]} + +# Controlled by the -n option. The dry run option will just output the command +# that would have been executed for each worktree. +DRY_RUN=0 + +# Control characters +CNRM=$'\x1b[0;0m' # Clear color + +# Bright color +BGRN=$'\x1b[1;32m' +BBLU=$'\x1b[1;34m' +BRED=$'\x1b[1;31m' +BYEL=$'\x1b[1;33m' +IWTH=$'\x1b[3;37m' + +# Strings for the pretty print. +MARKER="${BBLU}[${BGRN}+${BBLU}]${CNRM}" +SUCCESS="${BGRN}ok${CNRM}" +FAILED="${BRED}failed${CNRM}" + +#################### +# Helper functions # +#################### + +# Validate the given returned value (error code), print success or failed. The +# second argument is the error output in case of failure, it is printed out. +# On failure, this function exits. +function validate_ret +{ + if [ "$1" -eq 0 ]; then + printf "%s\\n" "$SUCCESS" + else + printf "%s\\n" "$FAILED" + printf " %s" "$2" + exit 1 + fi +} + +# Switch to the given branch name. +function switch_branch +{ + local cmd="git checkout $1" + printf " %s Switching branch to %s..." "$MARKER" "$1" + if [ $DRY_RUN -eq 0 ]; then + msg=$( eval "$cmd" 2>&1 ) + validate_ret $? "$msg" + else + printf "\\n %s\\n" "${IWTH}$cmd${CNRM}" + fi +} + +# Pull the given branch name. +function pull_branch +{ + local cmd="git pull" + printf " %s Pulling branch %s..." "$MARKER" "$1" + if [ $DRY_RUN -eq 0 ]; then + msg=$( eval "$cmd" 2>&1 ) + validate_ret $? "$msg" + else + printf "\\n %s\\n" "${IWTH}$cmd${CNRM}" + fi +} + +# Go into the worktree repository. +function goto_repo +{ + if [ ! -d "$1" ]; then + echo " $1: Not found. Stopping." + exit 1 + fi + cd "$1" || exit +} + +############### +# Entry point # +############### + +while getopts "n" opt; do + case "$opt" in + n) DRY_RUN=1 + echo " *** DRY DRUN MODE ***" + ;; + *) + ;; + esac +done + +# Go over all configured worktree. +for ((i=0; i Date: Wed, 27 Feb 2019 15:14:19 +0200 Subject: [PATCH 0538/2557] Initial commit of practracker.py . --- scripts/maint/practracker/metrics.py | 31 ++++++ scripts/maint/practracker/practracker.py | 119 +++++++++++++++++++++++ scripts/maint/practracker/util.py | 23 +++++ 3 files changed, 173 insertions(+) create mode 100644 scripts/maint/practracker/metrics.py create mode 100755 scripts/maint/practracker/practracker.py create mode 100644 scripts/maint/practracker/util.py diff --git a/scripts/maint/practracker/metrics.py b/scripts/maint/practracker/metrics.py new file mode 100644 index 0000000000..43ec3f809d --- /dev/null +++ b/scripts/maint/practracker/metrics.py @@ -0,0 +1,31 @@ +#!/usr/bin/python + +import re + +def file_len(f): + """Get file length of file""" + for i, l in enumerate(f): + pass + return i + 1 + +def function_lines(f): + """ + Return iterator which iterates over functions and returns (function name, function lines) + """ + + # XXX Buggy! Doesn't work with MOCK_IMPL and ENABLE_GCC_WARNINGS + in_function = False + for lineno, line in enumerate(f): + if not in_function: + # find the start of a function + m = re.match(r'^([a-zA-Z_][a-zA-Z_0-9]*),?\(', line) + if m: + func_name = m.group(1) + func_start = lineno + in_function = True + else: + # Fund the end of a function + if line.startswith("}"): + n_lines = lineno - func_start + in_function = False + yield (func_name, n_lines) diff --git a/scripts/maint/practracker/practracker.py b/scripts/maint/practracker/practracker.py new file mode 100755 index 0000000000..c95c432c39 --- /dev/null +++ b/scripts/maint/practracker/practracker.py @@ -0,0 +1,119 @@ +#!/usr/bin/python + +""" +Tor code best-practices tracker + +Go through the various .c files and collect metrics about them. If the metrics +violate some of our best practices and they are not found in the optional +exceptions file ("./exceptions.txt"), then log a violation about them. + +The exceptions file is meant to be initialized with the current state of the +source code as follows: ./practracker.py > ./exceptions.txt + +We currently do metrics about file size, function size and number of includes. + +TODO: + - How is this tool supposed to be used? How should the exception file work? + How should the UI work? Does it need special exit codes? + - Fix the function_length function so that practracker_tests.py passes. +""" + +import os, sys + +import metrics +import util + +# We don't want to run metrics for unittests, automatically-generated C files, +# external libraries or git leftovers. +EXCLUDE_SOURCE_DIRS = ["/src/test/", "/src/trunnel/", "/src/ext/", "/.git/"] + +# Where the Tor source code is +TOR_TOPDIR = "../../../" +# An optional exceptions_file +EXCEPTIONS_FILE = "./exceptions.txt" + +# Recommended file size +MAX_FILE_SIZE = 3000 # lines +# Recommended function size +MAX_FUNCTION_SIZE = 100 # lines +# Recommended number of #includes +MAX_INCLUDE_COUNT = 50 + +####################################################### + +def print_violation_if_not_exception(violation_str, exceptions_str): + # Check if this violation is already in the optional exceptions file + if exceptions_str and violation_str in exceptions_str: + return + + print violation_str + +####################################################### + +def consider_file_size(fname, f, exceptions_str): + file_size = metrics.file_len(f) + if file_size > MAX_FILE_SIZE: + violation_str = "violation file-size %s %d" % (fname, file_size) + print_violation_if_not_exception(violation_str, exceptions_str) + +def consider_includes(fname, f, exceptions_str): + include_count = 0 + for _, line in enumerate(f): + if line.startswith("#include "): + include_count += 1 + + if include_count > MAX_INCLUDE_COUNT: + violation_str = "violation include-count %s %d" % (fname, include_count) + print_violation_if_not_exception(violation_str, exceptions_str) + +def consider_function_size(fname, f, exceptions_str): + for name, lines in metrics.function_lines(f): + # Don't worry about functions within our limits + if lines <= MAX_FUNCTION_SIZE: + continue + + # That's a big function! Issue a violation! + canonical_function_name = "%s:%s()" % (fname,name) + violation_str = "violation function-size %s %s" % (lines, canonical_function_name) + print_violation_if_not_exception(violation_str, exceptions_str) + +####################################################### + +def consider_all_metrics(files_list, exceptions_str): + """Consider metrics for all files""" + for fname in files_list: + with open(fname, 'r') as f: + consider_metrics_for_file(fname, f, exceptions_str) + +def consider_metrics_for_file(fname, f, exceptions_str): + """ + Get metrics for file with filename 'fname' and file descriptor 'f'. + """ + # Get file length + consider_file_size(fname, f, exceptions_str) + + # Consider number of #includes + f.seek(0) + consider_includes(fname, f, exceptions_str) + + # Get function length + f.seek(0) + consider_function_size(fname, f, exceptions_str) + +def main(): + # 1) Get all the .c files we care about + files_list = util.get_tor_c_files(TOR_TOPDIR, EXCLUDE_SOURCE_DIRS) + + # 2) Read an optional exceptions file so that we don't warn about the past + exceptions_str = None + try: + with open(EXCEPTIONS_FILE, 'r') as exception_f: + exceptions_str = exception_f.read() + except IOError: + print "No exception file provided" + + # 3) Go through all the files and report violations if they are not exceptions + consider_all_metrics(files_list, exceptions_str) + +if __name__ == '__main__': + main() diff --git a/scripts/maint/practracker/util.py b/scripts/maint/practracker/util.py new file mode 100644 index 0000000000..8e9d95ece4 --- /dev/null +++ b/scripts/maint/practracker/util.py @@ -0,0 +1,23 @@ +import os + +def get_tor_c_files(tor_topdir, exclude_dirs): + """ + Return a list with the .c filenames we want to get metrics of. + """ + files_list = [] + + for root, directories, filenames in os.walk(tor_topdir): + for filename in filenames: + # We only care about .c files + if not filename.endswith(".c"): + continue + + # Exclude the excluded paths + full_path = os.path.join(root,filename) + if any(exclude_dir in full_path for exclude_dir in exclude_dirs): + continue + + files_list.append(full_path) + + return files_list + From a7684fcb57b3d28285b437f20d5eff6ce079cab5 Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Wed, 27 Feb 2019 15:14:29 +0200 Subject: [PATCH 0539/2557] Add some failing unittests for the function length metric. --- .../maint/practracker/practracker_tests.py | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100755 scripts/maint/practracker/practracker_tests.py diff --git a/scripts/maint/practracker/practracker_tests.py b/scripts/maint/practracker/practracker_tests.py new file mode 100755 index 0000000000..f3d225207a --- /dev/null +++ b/scripts/maint/practracker/practracker_tests.py @@ -0,0 +1,46 @@ +"""Some simple tests for practracker metrics""" + +import unittest + +import StringIO + +import metrics + +function_file = """static void +fun(directory_request_t *req, const char *resource) +{ + time_t if_modified_since = 0; + uint8_t or_diff_from[DIGEST256_LEN]; +} + +static void +fun(directory_request_t *req, + const char *resource) +{ + time_t if_modified_since = 0; + uint8_t or_diff_from[DIGEST256_LEN]; +} + +MOCK_IMPL(void, +fun,( + uint8_t dir_purpose, + uint8_t router_purpose, + const char *resource, + int pds_flags, + download_want_authority_t want_authority)) +{ + const routerstatus_t *rs = NULL; + const or_options_t *options = get_options(); +} +""" + +class TestFunctionLength(unittest.TestCase): + def test_function_length(self): + funcs = StringIO.StringIO(function_file) + # All functions should have length 2 + for name, lines in metrics.function_lines(funcs): + self.assertEqual(name, "fun") + self.assertEqual(lines, 2) + +if __name__ == '__main__': + unittest.main() From 371ea65c080471cb6e1a57d9f78e5beaaa4212b8 Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Wed, 27 Feb 2019 17:05:00 +0200 Subject: [PATCH 0540/2557] Improve #include counting func and move it to metrics.py. --- scripts/maint/practracker/metrics.py | 8 ++++++++ scripts/maint/practracker/practracker.py | 5 +---- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/scripts/maint/practracker/metrics.py b/scripts/maint/practracker/metrics.py index 43ec3f809d..525a058405 100644 --- a/scripts/maint/practracker/metrics.py +++ b/scripts/maint/practracker/metrics.py @@ -8,6 +8,14 @@ def file_len(f): pass return i + 1 +def get_include_count(f): + """Get number of #include statements in the file""" + include_count = 0 + for line in f: + if re.match(r' *# *include', line): + include_count += 1 + return include_count + def function_lines(f): """ Return iterator which iterates over functions and returns (function name, function lines) diff --git a/scripts/maint/practracker/practracker.py b/scripts/maint/practracker/practracker.py index c95c432c39..c3b8a9783b 100755 --- a/scripts/maint/practracker/practracker.py +++ b/scripts/maint/practracker/practracker.py @@ -57,10 +57,7 @@ def consider_file_size(fname, f, exceptions_str): print_violation_if_not_exception(violation_str, exceptions_str) def consider_includes(fname, f, exceptions_str): - include_count = 0 - for _, line in enumerate(f): - if line.startswith("#include "): - include_count += 1 + include_count = metrics.get_include_count(f) if include_count > MAX_INCLUDE_COUNT: violation_str = "violation include-count %s %d" % (fname, include_count) From 26c4f6cfd07fea6d49109e60a8b591b92ec59c49 Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Wed, 27 Feb 2019 18:24:10 +0200 Subject: [PATCH 0541/2557] Add more intelligent problem tracking. --- scripts/maint/practracker/practracker.py | 55 +++++++------- scripts/maint/practracker/problem.py | 96 ++++++++++++++++++++++++ 2 files changed, 123 insertions(+), 28 deletions(-) create mode 100644 scripts/maint/practracker/problem.py diff --git a/scripts/maint/practracker/practracker.py b/scripts/maint/practracker/practracker.py index c3b8a9783b..4fbca615c5 100755 --- a/scripts/maint/practracker/practracker.py +++ b/scripts/maint/practracker/practracker.py @@ -5,7 +5,7 @@ Tor code best-practices tracker Go through the various .c files and collect metrics about them. If the metrics violate some of our best practices and they are not found in the optional -exceptions file ("./exceptions.txt"), then log a violation about them. +exceptions file ("./exceptions.txt"), then log a problem about them. The exceptions file is meant to be initialized with the current state of the source code as follows: ./practracker.py > ./exceptions.txt @@ -22,6 +22,7 @@ import os, sys import metrics import util +import problem # We don't want to run metrics for unittests, automatically-generated C files, # external libraries or git leftovers. @@ -41,76 +42,74 @@ MAX_INCLUDE_COUNT = 50 ####################################################### -def print_violation_if_not_exception(violation_str, exceptions_str): - # Check if this violation is already in the optional exceptions file - if exceptions_str and violation_str in exceptions_str: - return - - print violation_str +ProblemVault = None ####################################################### -def consider_file_size(fname, f, exceptions_str): +def consider_file_size(fname, f): file_size = metrics.file_len(f) if file_size > MAX_FILE_SIZE: - violation_str = "violation file-size %s %d" % (fname, file_size) - print_violation_if_not_exception(violation_str, exceptions_str) + v = problem.FileSizeProblem(fname, file_size) + ProblemVault.register_problem(v) -def consider_includes(fname, f, exceptions_str): +def consider_includes(fname, f): include_count = metrics.get_include_count(f) if include_count > MAX_INCLUDE_COUNT: - violation_str = "violation include-count %s %d" % (fname, include_count) - print_violation_if_not_exception(violation_str, exceptions_str) + v = problem.IncludeCountProblem(fname, include_count) + ProblemVault.register_problem(v) -def consider_function_size(fname, f, exceptions_str): +def consider_function_size(fname, f): for name, lines in metrics.function_lines(f): # Don't worry about functions within our limits if lines <= MAX_FUNCTION_SIZE: continue - # That's a big function! Issue a violation! + # That's a big function! Issue a problem! canonical_function_name = "%s:%s()" % (fname,name) - violation_str = "violation function-size %s %s" % (lines, canonical_function_name) - print_violation_if_not_exception(violation_str, exceptions_str) + v = problem.FunctionSizeProblem(canonical_function_name, lines) + ProblemVault.register_problem(v) ####################################################### -def consider_all_metrics(files_list, exceptions_str): +def consider_all_metrics(files_list): """Consider metrics for all files""" for fname in files_list: with open(fname, 'r') as f: - consider_metrics_for_file(fname, f, exceptions_str) + consider_metrics_for_file(fname, f) -def consider_metrics_for_file(fname, f, exceptions_str): +def consider_metrics_for_file(fname, f): """ Get metrics for file with filename 'fname' and file descriptor 'f'. """ # Get file length - consider_file_size(fname, f, exceptions_str) + consider_file_size(fname, f) # Consider number of #includes f.seek(0) - consider_includes(fname, f, exceptions_str) + consider_includes(fname, f) # Get function length f.seek(0) - consider_function_size(fname, f, exceptions_str) + consider_function_size(fname, f) def main(): + global ProblemVault + # 1) Get all the .c files we care about files_list = util.get_tor_c_files(TOR_TOPDIR, EXCLUDE_SOURCE_DIRS) - # 2) Read an optional exceptions file so that we don't warn about the past - exceptions_str = None + # 2) Initialize problem vault and load an optional exceptions file so that + # we don't warn about the past try: with open(EXCEPTIONS_FILE, 'r') as exception_f: - exceptions_str = exception_f.read() + ProblemVault = problem.ProblemVault(exception_f) except IOError: print "No exception file provided" + ProblemVault = problem.ProblemVault(None) - # 3) Go through all the files and report violations if they are not exceptions - consider_all_metrics(files_list, exceptions_str) + # 3) Go through all the files and report problems if they are not exceptions + consider_all_metrics(files_list) if __name__ == '__main__': main() diff --git a/scripts/maint/practracker/problem.py b/scripts/maint/practracker/problem.py new file mode 100644 index 0000000000..fe6b30915b --- /dev/null +++ b/scripts/maint/practracker/problem.py @@ -0,0 +1,96 @@ +""" +In this file we define a ProblemVault class where we store all the +exceptions and all the problems we find with the code. + +The ProblemVault is capable of registering problems and also figuring out if a +problem is worse than a registered exception so that it only warns when things +get worse. +""" + +class ProblemVault(object): + """ + Singleton where we store the various new problems we + found in the code, and also the old problems we read from the exception + file. + """ + def __init__(self, exception_file): + # Exception dictionary: { problem.key() : Problem object } + self.exceptions = {} + + if exception_file: + self.register_exceptions(exception_file) + + def register_exceptions(self, exception_file): + # Register exceptions + for line in exception_file: + problem = get_old_problem_from_exception_str(line) + if problem is None: + continue + + # XXX this might overwrite problems with the same key (e.g. MOCK_IMPL) + self.exceptions[problem.key()] = problem + #print "Registering exception: %s" % problem + + def register_problem(self, problem): + # This is a new problem, print it + if problem.key() not in self.exceptions: + print problem + return + + # If it's an old problem, we don't warn if the situation got better + # (e.g. we went from 4k LoC to 3k LoC), but we do warn if the + # situation worsened (e.g. we went from 60 includes to 80). + if problem.is_worse_than(self.exceptions[problem.key()]): + print problem + return +# else: +# print "OK %s better than %s" % (problem, self.exceptions[problem.key()]) + + +class Problem(object): + def __init__(self, problem_type, problem_location, metric_value): + self.problem_location = problem_location + self.metric_value = int(metric_value) + self.problem_type = problem_type + + def is_worse_than(self, other_problem): + """Return True if this is a worse problem than other_problem""" + if self.metric_value > other_problem.metric_value: + return True + return False + + def key(self): + """Generate a unique key that describes this problem that can be used as a dictionary key""" + return "%s:%s" % (self.problem_location, self.problem_type) + + def __str__(self): + return "problem %s %s %s" % (self.problem_type, self.problem_location, self.metric_value) + +class FileSizeProblem(Problem): + def __init__(self, problem_location, metric_value): + super(FileSizeProblem, self).__init__("file-size", problem_location, metric_value) + +class IncludeCountProblem(Problem): + def __init__(self, problem_location, metric_value): + super(IncludeCountProblem, self).__init__("include-count", problem_location, metric_value) + +class FunctionSizeProblem(Problem): + def __init__(self, problem_location, metric_value): + super(FunctionSizeProblem, self).__init__("function-size", problem_location, metric_value) + +def get_old_problem_from_exception_str(exception_str): + try: + _, problem_type, problem_location, metric_value = exception_str.split(" ") + except ValueError: + return None + + if problem_type == "file-size": + return FileSizeProblem(problem_location, metric_value) + elif problem_type == "include-count": + return IncludeCountProblem(problem_location, metric_value) + elif problem_type == "function-size": + return FunctionSizeProblem(problem_location, metric_value) + else: + print "Unknown exception line %s" % exception_str + return None + From 2a722e2841c9b1631852affd313cc3d0403da7b9 Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Wed, 27 Feb 2019 19:29:27 +0200 Subject: [PATCH 0542/2557] Improve metrics implementation. No longer messes up with MOCK_IMPL. Also update the tests to show that. We are still being innacurate on the line count in some cases, but that doesnt matter so much. --- scripts/maint/practracker/metrics.py | 17 ++++++++++++++--- scripts/maint/practracker/practracker_tests.py | 4 ++++ 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/scripts/maint/practracker/metrics.py b/scripts/maint/practracker/metrics.py index 525a058405..c7d2091d05 100644 --- a/scripts/maint/practracker/metrics.py +++ b/scripts/maint/practracker/metrics.py @@ -1,8 +1,12 @@ #!/usr/bin/python +# Implementation of various source code metrics. +# These are currently ad-hoc string operations and regexps. +# We might want to use a proper static analysis library in the future, if we want to get more advanced metrics. + import re -def file_len(f): +def get_file_len(f): """Get file length of file""" for i, l in enumerate(f): pass @@ -16,14 +20,20 @@ def get_include_count(f): include_count += 1 return include_count -def function_lines(f): +def get_function_lines(f): """ Return iterator which iterates over functions and returns (function name, function lines) """ - # XXX Buggy! Doesn't work with MOCK_IMPL and ENABLE_GCC_WARNINGS + # Skip lines with these terms since they confuse our regexp + REGEXP_CONFUSE_TERMS = ["MOCK_IMPL", "ENABLE_GCC_WARNINGS", "ENABLE_GCC_WARNING", "DUMMY_TYPECHECK_INSTANCE", + "DISABLE_GCC_WARNING", "DISABLE_GCC_WARNINGS"] + in_function = False for lineno, line in enumerate(f): + if any(x in line for x in REGEXP_CONFUSE_TERMS): + continue + if not in_function: # find the start of a function m = re.match(r'^([a-zA-Z_][a-zA-Z_0-9]*),?\(', line) @@ -31,6 +41,7 @@ def function_lines(f): func_name = m.group(1) func_start = lineno in_function = True + else: # Fund the end of a function if line.startswith("}"): diff --git a/scripts/maint/practracker/practracker_tests.py b/scripts/maint/practracker/practracker_tests.py index f3d225207a..cdbab2908e 100755 --- a/scripts/maint/practracker/practracker_tests.py +++ b/scripts/maint/practracker/practracker_tests.py @@ -40,6 +40,10 @@ class TestFunctionLength(unittest.TestCase): # All functions should have length 2 for name, lines in metrics.function_lines(funcs): self.assertEqual(name, "fun") + + funcs.seek(0) + + for name, lines in metrics.function_lines(funcs): self.assertEqual(lines, 2) if __name__ == '__main__': From 31c1d91ffb85671a9e3dc499655c65622d844333 Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Wed, 27 Feb 2019 19:30:39 +0200 Subject: [PATCH 0543/2557] Exit with 1 if new issues were found. Also work with python3. --- scripts/maint/practracker/practracker.py | 69 +++++++++++++++--------- scripts/maint/practracker/problem.py | 27 ++++++---- 2 files changed, 60 insertions(+), 36 deletions(-) diff --git a/scripts/maint/practracker/practracker.py b/scripts/maint/practracker/practracker.py index 4fbca615c5..3c22abd444 100755 --- a/scripts/maint/practracker/practracker.py +++ b/scripts/maint/practracker/practracker.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 """ Tor code best-practices tracker @@ -47,69 +47,88 @@ ProblemVault = None ####################################################### def consider_file_size(fname, f): - file_size = metrics.file_len(f) + """Consider file size issues for 'f' and return True if a new issue was found""" + file_size = metrics.get_file_len(f) if file_size > MAX_FILE_SIZE: - v = problem.FileSizeProblem(fname, file_size) - ProblemVault.register_problem(v) + p = problem.FileSizeProblem(fname, file_size) + return ProblemVault.register_problem(p) + return False def consider_includes(fname, f): + """Consider #include issues for 'f' and return True if a new issue was found""" include_count = metrics.get_include_count(f) if include_count > MAX_INCLUDE_COUNT: - v = problem.IncludeCountProblem(fname, include_count) - ProblemVault.register_problem(v) + p = problem.IncludeCountProblem(fname, include_count) + return ProblemVault.register_problem(p) + return False def consider_function_size(fname, f): - for name, lines in metrics.function_lines(f): + """Consider the function sizes for 'f' and return True if a new issue was found""" + found_new_issues = False + + for name, lines in metrics.get_function_lines(f): # Don't worry about functions within our limits if lines <= MAX_FUNCTION_SIZE: continue # That's a big function! Issue a problem! - canonical_function_name = "%s:%s()" % (fname,name) - v = problem.FunctionSizeProblem(canonical_function_name, lines) - ProblemVault.register_problem(v) + canonical_function_name = "%s:%s()" % (fname, name) + p = problem.FunctionSizeProblem(canonical_function_name, lines) + found_new_issues |= ProblemVault.register_problem(p) + + return found_new_issues ####################################################### def consider_all_metrics(files_list): - """Consider metrics for all files""" + """Consider metrics for all files, and return True if new issues were found""" + found_new_issues = False for fname in files_list: with open(fname, 'r') as f: - consider_metrics_for_file(fname, f) + found_new_issues |= consider_metrics_for_file(fname, f) + return found_new_issues def consider_metrics_for_file(fname, f): """ - Get metrics for file with filename 'fname' and file descriptor 'f'. + Consider the various metrics for file with filename 'fname' and file descriptor 'f'. + Return True if we found new issues. """ + # Strip the useless part of the path + if fname.startswith(TOR_TOPDIR): + fname = fname[len(TOR_TOPDIR):] + + found_new_issues = False + # Get file length - consider_file_size(fname, f) + found_new_issues |= consider_file_size(fname, f) # Consider number of #includes f.seek(0) - consider_includes(fname, f) + found_new_issues |= consider_includes(fname, f) # Get function length f.seek(0) - consider_function_size(fname, f) + found_new_issues |= consider_function_size(fname, f) + + return found_new_issues def main(): - global ProblemVault - # 1) Get all the .c files we care about files_list = util.get_tor_c_files(TOR_TOPDIR, EXCLUDE_SOURCE_DIRS) # 2) Initialize problem vault and load an optional exceptions file so that # we don't warn about the past - try: - with open(EXCEPTIONS_FILE, 'r') as exception_f: - ProblemVault = problem.ProblemVault(exception_f) - except IOError: - print "No exception file provided" - ProblemVault = problem.ProblemVault(None) + global ProblemVault + ProblemVault = problem.ProblemVault(EXCEPTIONS_FILE) # 3) Go through all the files and report problems if they are not exceptions - consider_all_metrics(files_list) + found_new_issues = consider_all_metrics(files_list) + + if found_new_issues: + sys.exit(1) + else: + sys.exit(0) if __name__ == '__main__': main() diff --git a/scripts/maint/practracker/problem.py b/scripts/maint/practracker/problem.py index fe6b30915b..e23d3bbf0e 100644 --- a/scripts/maint/practracker/problem.py +++ b/scripts/maint/practracker/problem.py @@ -13,12 +13,15 @@ class ProblemVault(object): found in the code, and also the old problems we read from the exception file. """ - def __init__(self, exception_file): + def __init__(self, exception_fname): # Exception dictionary: { problem.key() : Problem object } self.exceptions = {} - if exception_file: - self.register_exceptions(exception_file) + try: + with open(exception_fname, 'r') as exception_f: + self.register_exceptions(exception_f) + except IOError: + print("No exception file provided") def register_exceptions(self, exception_file): # Register exceptions @@ -27,25 +30,27 @@ class ProblemVault(object): if problem is None: continue - # XXX this might overwrite problems with the same key (e.g. MOCK_IMPL) self.exceptions[problem.key()] = problem #print "Registering exception: %s" % problem def register_problem(self, problem): + """ + Register this problem to the problem value. Return True if it was a new + problem or it worsens an already existing problem. + """ # This is a new problem, print it if problem.key() not in self.exceptions: - print problem - return + print(problem) + return True # If it's an old problem, we don't warn if the situation got better # (e.g. we went from 4k LoC to 3k LoC), but we do warn if the # situation worsened (e.g. we went from 60 includes to 80). if problem.is_worse_than(self.exceptions[problem.key()]): - print problem - return -# else: -# print "OK %s better than %s" % (problem, self.exceptions[problem.key()]) + print(problem) + return True + return False class Problem(object): def __init__(self, problem_type, problem_location, metric_value): @@ -91,6 +96,6 @@ def get_old_problem_from_exception_str(exception_str): elif problem_type == "function-size": return FunctionSizeProblem(problem_location, metric_value) else: - print "Unknown exception line %s" % exception_str + print("Unknown exception line {}".format(exception_str)) return None From c142e3d1e65c22891c19d71541c8084532099183 Mon Sep 17 00:00:00 2001 From: Neel Chauhan Date: Wed, 27 Feb 2019 13:54:23 -0500 Subject: [PATCH 0544/2557] Set CIRCLAUNCH_NEED_UPTIME in rend_service_relaunch_rendezvous() on a hs_service_requires_uptime_circ() --- changes/bug17357 | 7 +++++++ src/feature/rend/rendservice.c | 26 ++++++++++++++++++++++++-- 2 files changed, 31 insertions(+), 2 deletions(-) create mode 100644 changes/bug17357 diff --git a/changes/bug17357 b/changes/bug17357 new file mode 100644 index 0000000000..1188b65fd7 --- /dev/null +++ b/changes/bug17357 @@ -0,0 +1,7 @@ + o Minor bugfixes (onion services): + - If we are relaunching a circuit to a rendevous service in + rend_service_relaunch_rendezvous() and hs_service_requires_uptime_circ() + is true, the CIRCLAUNCH_NEED_UPTIME flag is added to the circuit. + Previously, we only set this flag when we received a INTRODUCE2 + cell in rend_service_receive_introduction(). Fixes bug 17357; + bugfix on 0.4.0.2-alpha. Patch by Neel Chauhan diff --git a/src/feature/rend/rendservice.c b/src/feature/rend/rendservice.c index 5ee084b0b7..73edcaccf5 100644 --- a/src/feature/rend/rendservice.c +++ b/src/feature/rend/rendservice.c @@ -3012,6 +3012,10 @@ rend_service_relaunch_rendezvous(origin_circuit_t *oldcirc) { origin_circuit_t *newcirc; cpath_build_state_t *newstate, *oldstate; + const char *rend_pk_digest; + rend_service_t *service = NULL; + + int flags = CIRCLAUNCH_NEED_CAPACITY | CIRCLAUNCH_IS_INTERNAL; tor_assert(oldcirc->base_.purpose == CIRCUIT_PURPOSE_S_CONNECT_REND); oldstate = oldcirc->build_state; @@ -3026,13 +3030,31 @@ rend_service_relaunch_rendezvous(origin_circuit_t *oldcirc) log_info(LD_REND,"Reattempting rendezvous circuit to '%s'", safe_str(extend_info_describe(oldstate->chosen_exit))); + /* Look up the service. */ + rend_pk_digest = (char *) rend_data_get_pk_digest(oldcirc->rend_data, NULL); + service = rend_service_get_by_pk_digest(rend_pk_digest); + + if (!service) { + char serviceid[REND_SERVICE_ID_LEN_BASE32+1]; + base32_encode(serviceid, REND_SERVICE_ID_LEN_BASE32+1, + rend_pk_digest, REND_SERVICE_ID_LEN); + + log_warn(LD_BUG, "Internal error: Trying to relaunch a rendezvous circ " + "for an unrecognized service %s.", + safe_str_client(serviceid)); + return; + } + + if (hs_service_requires_uptime_circ(service->ports)) { + flags |= CIRCLAUNCH_NEED_UPTIME; + } + /* You'd think Single Onion Services would want to retry the rendezvous * using a direct connection. But if it's blocked by a firewall, or the * service is IPv6-only, or the rend point avoiding becoming a one-hop * proxy, we need a 3-hop connection. */ newcirc = circuit_launch_by_extend_info(CIRCUIT_PURPOSE_S_CONNECT_REND, - oldstate->chosen_exit, - CIRCLAUNCH_NEED_CAPACITY|CIRCLAUNCH_IS_INTERNAL); + oldstate->chosen_exit, flags); if (!newcirc) { log_warn(LD_REND,"Couldn't relaunch rendezvous circuit to '%s'.", From 9256b02cc8b2cc6ae495f9f2262b546429a89dbd Mon Sep 17 00:00:00 2001 From: David Goulet Date: Thu, 28 Feb 2019 13:16:41 -0500 Subject: [PATCH 0545/2557] maint: Remove 0.3.3 branch from git scripts It is EOL today: Febuary 28th, 2019 No more releases will occur. Closes #29616 Signed-off-by: David Goulet --- scripts/maint/git-merge-forward.sh | 7 +------ scripts/maint/git-pull-all.sh | 5 ----- scripts/maint/git-push-all.sh | 1 - 3 files changed, 1 insertion(+), 12 deletions(-) diff --git a/scripts/maint/git-merge-forward.sh b/scripts/maint/git-merge-forward.sh index 3728067e7a..e22e7befad 100755 --- a/scripts/maint/git-merge-forward.sh +++ b/scripts/maint/git-merge-forward.sh @@ -38,14 +38,12 @@ TOR_WKT_NAME="tor-wkt" # # First set of arrays are the maint-* branch and then the release-* branch. # New arrays need to be in the WORKTREE= array else they aren't considered. -MAINT_033=( "maint-0.3.3" "maint-0.2.9" "$GIT_PATH/$TOR_WKT_NAME/maint-0.3.3" ) -MAINT_034=( "maint-0.3.4" "maint-0.3.3" "$GIT_PATH/$TOR_WKT_NAME/maint-0.3.4" ) +MAINT_034=( "maint-0.3.4" "maint-0.2.9" "$GIT_PATH/$TOR_WKT_NAME/maint-0.3.4" ) MAINT_035=( "maint-0.3.5" "maint-0.3.4" "$GIT_PATH/$TOR_WKT_NAME/maint-0.3.5" ) MAINT_040=( "maint-0.4.0" "maint-0.3.5" "$GIT_PATH/$TOR_WKT_NAME/maint-0.4.0" ) MAINT_MASTER=( "master" "maint-0.4.0" "$GIT_PATH/$TOR_MASTER_NAME" ) RELEASE_029=( "release-0.2.9" "maint-0.2.9" "$GIT_PATH/$TOR_WKT_NAME/release-0.2.9" ) -RELEASE_033=( "release-0.3.3" "maint-0.3.3" "$GIT_PATH/$TOR_WKT_NAME/release-0.3.3" ) RELEASE_034=( "release-0.3.4" "maint-0.3.4" "$GIT_PATH/$TOR_WKT_NAME/release-0.3.4" ) RELEASE_035=( "release-0.3.5" "maint-0.3.5" "$GIT_PATH/$TOR_WKT_NAME/release-0.3.5" ) RELEASE_040=( "release-0.4.0" "maint-0.4.0" "$GIT_PATH/$TOR_WKT_NAME/release-0.4.0" ) @@ -59,9 +57,6 @@ RELEASE_040=( "release-0.4.0" "maint-0.4.0" "$GIT_PATH/$TOR_WKT_NAME/release-0.4 WORKTREE=( RELEASE_029[@] - MAINT_033[@] - RELEASE_033[@] - MAINT_034[@] RELEASE_034[@] diff --git a/scripts/maint/git-pull-all.sh b/scripts/maint/git-pull-all.sh index e5c5d94670..e414ea28f0 100755 --- a/scripts/maint/git-pull-all.sh +++ b/scripts/maint/git-pull-all.sh @@ -37,14 +37,12 @@ TOR_WKT_NAME="tor-wkt" # First set of arrays are the maint-* branch and then the release-* branch. # New arrays need to be in the WORKTREE= array else they aren't considered. MAINT_029=( "maint-0.2.9" "$GIT_PATH/$TOR_WKT_NAME/maint-0.2.9" ) -MAINT_033=( "maint-0.3.3" "$GIT_PATH/$TOR_WKT_NAME/maint-0.3.3" ) MAINT_034=( "maint-0.3.4" "$GIT_PATH/$TOR_WKT_NAME/maint-0.3.4" ) MAINT_035=( "maint-0.3.5" "$GIT_PATH/$TOR_WKT_NAME/maint-0.3.5" ) MAINT_040=( "maint-0.4.0" "$GIT_PATH/$TOR_WKT_NAME/maint-0.4.0" ) MAINT_MASTER=( "master" "$GIT_PATH/$TOR_MASTER_NAME" ) RELEASE_029=( "release-0.2.9" "$GIT_PATH/$TOR_WKT_NAME/release-0.2.9" ) -RELEASE_033=( "release-0.3.3" "$GIT_PATH/$TOR_WKT_NAME/release-0.3.3" ) RELEASE_034=( "release-0.3.4" "$GIT_PATH/$TOR_WKT_NAME/release-0.3.4" ) RELEASE_035=( "release-0.3.5" "$GIT_PATH/$TOR_WKT_NAME/release-0.3.5" ) RELEASE_040=( "release-0.4.0" "$GIT_PATH/$TOR_WKT_NAME/release-0.4.0" ) @@ -59,9 +57,6 @@ WORKTREE=( MAINT_029[@] RELEASE_029[@] - MAINT_033[@] - RELEASE_033[@] - MAINT_034[@] RELEASE_034[@] diff --git a/scripts/maint/git-push-all.sh b/scripts/maint/git-push-all.sh index 0f4f5302a6..0ce951d4bd 100755 --- a/scripts/maint/git-push-all.sh +++ b/scripts/maint/git-push-all.sh @@ -8,5 +8,4 @@ git push $UPSTREAM_BRANCH \ {release,maint}-0.4.0 \ {release,maint}-0.3.5 \ {release,maint}-0.3.4 \ - {release,maint}-0.3.3 \ {release,maint}-0.2.9 From 64f594499a3e8893a6097fa5db7d47962f83d2f5 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 1 Mar 2019 08:20:54 -0500 Subject: [PATCH 0546/2557] Document crypto_fast_rng_one_in_n. --- src/lib/crypt_ops/crypto_rand.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/lib/crypt_ops/crypto_rand.h b/src/lib/crypt_ops/crypto_rand.h index bb424fd691..6f09aedf6a 100644 --- a/src/lib/crypt_ops/crypto_rand.h +++ b/src/lib/crypt_ops/crypto_rand.h @@ -68,6 +68,12 @@ unsigned crypto_fast_rng_get_uint(crypto_fast_rng_t *rng, unsigned limit); uint64_t crypto_fast_rng_get_uint64(crypto_fast_rng_t *rng, uint64_t limit); double crypto_fast_rng_get_double(crypto_fast_rng_t *rng); +/** + * Using the fast_rng rng, yield true with probability + * 1/n. Otherwise yield false. + * + * n must not be zero. + **/ #define crypto_fast_rng_one_in_n(rng, n) \ (0 == (crypto_fast_rng_get_uint((rng), (n)))) From 436e0c8ca4a0cfb5206b404835c86aae45d98c86 Mon Sep 17 00:00:00 2001 From: David Goulet Date: Thu, 28 Feb 2019 13:23:23 -0500 Subject: [PATCH 0547/2557] maint: Make git scripts fetch origin once Replace the "git pull" by one single "git fetch origin" and then "git merge" into each defined branches. This speeds up the process considerably. Closes #29616 Signed-off-by: David Goulet --- scripts/maint/git-merge-forward.sh | 39 ++++++++++++++++++++++++++++-- scripts/maint/git-pull-all.sh | 32 ++++++++++++++++++++---- 2 files changed, 64 insertions(+), 7 deletions(-) diff --git a/scripts/maint/git-merge-forward.sh b/scripts/maint/git-merge-forward.sh index e22e7befad..4b294f4945 100755 --- a/scripts/maint/git-merge-forward.sh +++ b/scripts/maint/git-merge-forward.sh @@ -48,6 +48,11 @@ RELEASE_034=( "release-0.3.4" "maint-0.3.4" "$GIT_PATH/$TOR_WKT_NAME/release-0.3 RELEASE_035=( "release-0.3.5" "maint-0.3.5" "$GIT_PATH/$TOR_WKT_NAME/release-0.3.5" ) RELEASE_040=( "release-0.4.0" "maint-0.4.0" "$GIT_PATH/$TOR_WKT_NAME/release-0.4.0" ) +# The master branch path has to be the main repository thus contains the +# origin that will be used to fetch the updates. All the worktrees are created +# from that repository. +ORIGIN_PATH="$GIT_PATH/$TOR_MASTER_NAME" + ########################## # Git Worktree to manage # ########################## @@ -146,6 +151,19 @@ function merge_branch fi } +# Pull the given branch name. +function merge_branch_origin +{ + local cmd="git merge --ff-only origin/$1" + printf " %s Merging branch origin/%s..." "$MARKER" "$1" + if [ $DRY_RUN -eq 0 ]; then + msg=$( eval "$cmd" 2>&1 ) + validate_ret $? "$msg" + else + printf "\\n %s\\n" "${IWTH}$cmd${CNRM}" + fi +} + # Go into the worktree repository. function goto_repo { @@ -156,6 +174,19 @@ function goto_repo cd "$1" || exit } +# Fetch the origin. No arguments. +function fetch_origin +{ + local cmd="git fetch origin" + printf " %s Fetching origin..." "$MARKER" + if [ $DRY_RUN -eq 0 ]; then + msg=$( eval "$cmd" 2>&1 ) + validate_ret $? "$msg" + else + printf "\\n %s\\n" "${IWTH}$cmd${CNRM}" + fi +} + ############### # Entry point # ############### @@ -170,6 +201,10 @@ while getopts "n" opt; do esac done +# First, fetch the origin. +goto_repo "$ORIGIN_PATH" +fetch_origin + # Go over all configured worktree. for ((i=0; i&1 ) validate_ret $? "$msg" @@ -143,6 +148,19 @@ function goto_repo cd "$1" || exit } +# Fetch the origin. No arguments. +function fetch_origin +{ + local cmd="git fetch origin" + printf " %s Fetching origin..." "$MARKER" + if [ $DRY_RUN -eq 0 ]; then + msg=$( eval "$cmd" 2>&1 ) + validate_ret $? "$msg" + else + printf "\\n %s\\n" "${IWTH}$cmd${CNRM}" + fi +} + ############### # Entry point # ############### @@ -157,6 +175,10 @@ while getopts "n" opt; do esac done +# First, fetch the origin. +goto_repo "$ORIGIN_PATH" +fetch_origin + # Go over all configured worktree. for ((i=0; i Date: Fri, 18 Jan 2019 12:26:13 +0200 Subject: [PATCH 0548/2557] Split crypto_digest.c * Move out code that depends on NSS to crypto_digest_nss.c * Move out code that depends on OpenSSL to crypto_digest_openssl.c * Keep the general code that is not specific to any of the above in crypto_digest.c --- changes/ticket29108 | 5 + src/lib/crypt_ops/crypto_digest.c | 736 ---------------------- src/lib/crypt_ops/crypto_digest_nss.c | 560 ++++++++++++++++ src/lib/crypt_ops/crypto_digest_openssl.c | 522 +++++++++++++++ src/lib/crypt_ops/include.am | 2 + 5 files changed, 1089 insertions(+), 736 deletions(-) create mode 100644 changes/ticket29108 create mode 100644 src/lib/crypt_ops/crypto_digest_nss.c create mode 100644 src/lib/crypt_ops/crypto_digest_openssl.c diff --git a/changes/ticket29108 b/changes/ticket29108 new file mode 100644 index 0000000000..7adb08ecb1 --- /dev/null +++ b/changes/ticket29108 @@ -0,0 +1,5 @@ + o Code simplification and refactoring: + - Split crypto_digest.c into three parts: 1) general code that does not + depend on either NSS or OpenSSL (stays in crypto_digest.c); 2) code that + depends on NSS API (moved to crypto_digest_nss.c); 3) code that depends + on OpenSSL API (moved to crypto_digest_openssl.c). Resolves ticket 29108. diff --git a/src/lib/crypt_ops/crypto_digest.c b/src/lib/crypt_ops/crypto_digest.c index 1e64100f2f..9da135e9c4 100644 --- a/src/lib/crypt_ops/crypto_digest.c +++ b/src/lib/crypt_ops/crypto_digest.c @@ -23,187 +23,6 @@ #include "lib/arch/bytes.h" -#ifdef ENABLE_NSS -DISABLE_GCC_WARNING(strict-prototypes) -#include -ENABLE_GCC_WARNING(strict-prototypes) -#else - -#include "lib/crypt_ops/crypto_openssl_mgt.h" - -DISABLE_GCC_WARNING(redundant-decls) - -#include -#include - -ENABLE_GCC_WARNING(redundant-decls) - -#ifdef HAVE_EVP_SHA3_256 -#define OPENSSL_HAS_SHA3 -#include -#endif - -#endif - -#ifdef ENABLE_NSS -/** - * Convert a digest_algorithm_t (used by tor) to a HashType (used by NSS). - * On failure, return SEC_OID_UNKNOWN. */ -static SECOidTag -digest_alg_to_nss_oid(digest_algorithm_t alg) -{ - switch (alg) { - case DIGEST_SHA1: return SEC_OID_SHA1; - case DIGEST_SHA256: return SEC_OID_SHA256; - case DIGEST_SHA512: return SEC_OID_SHA512; - case DIGEST_SHA3_256: /* Fall through */ - case DIGEST_SHA3_512: /* Fall through */ - default: - return SEC_OID_UNKNOWN; - } -} - -/* Helper: get an unkeyed digest via pk11wrap */ -static int -digest_nss_internal(SECOidTag alg, - char *digest, unsigned len_out, - const char *msg, size_t msg_len) -{ - if (alg == SEC_OID_UNKNOWN) - return -1; - tor_assert(msg_len <= UINT_MAX); - - int rv = -1; - SECStatus s; - PK11Context *ctx = PK11_CreateDigestContext(alg); - if (!ctx) - return -1; - - s = PK11_DigestBegin(ctx); - if (s != SECSuccess) - goto done; - - s = PK11_DigestOp(ctx, (const unsigned char *)msg, (unsigned int)msg_len); - if (s != SECSuccess) - goto done; - - unsigned int len = 0; - s = PK11_DigestFinal(ctx, (unsigned char *)digest, &len, len_out); - if (s != SECSuccess) - goto done; - - rv = 0; - done: - PK11_DestroyContext(ctx, PR_TRUE); - return rv; -} - -/** True iff alg is implemented in our crypto library, and we want to use that - * implementation */ -static bool -library_supports_digest(digest_algorithm_t alg) -{ - switch (alg) { - case DIGEST_SHA1: /* Fall through */ - case DIGEST_SHA256: /* Fall through */ - case DIGEST_SHA512: /* Fall through */ - return true; - case DIGEST_SHA3_256: /* Fall through */ - case DIGEST_SHA3_512: /* Fall through */ - default: - return false; - } -} -#endif - -/* Crypto digest functions */ - -/** Compute the SHA1 digest of the len bytes on data stored in - * m. Write the DIGEST_LEN byte result into digest. - * Return 0 on success, -1 on failure. - */ -MOCK_IMPL(int, -crypto_digest,(char *digest, const char *m, size_t len)) -{ - tor_assert(m); - tor_assert(digest); -#ifdef ENABLE_NSS - return digest_nss_internal(SEC_OID_SHA1, digest, DIGEST_LEN, m, len); -#else - if (SHA1((const unsigned char*)m,len,(unsigned char*)digest) == NULL) { - return -1; - } -#endif - return 0; -} - -/** Compute a 256-bit digest of len bytes in data stored in m, - * using the algorithm algorithm. Write the DIGEST_LEN256-byte result - * into digest. Return 0 on success, -1 on failure. */ -int -crypto_digest256(char *digest, const char *m, size_t len, - digest_algorithm_t algorithm) -{ - tor_assert(m); - tor_assert(digest); - tor_assert(algorithm == DIGEST_SHA256 || algorithm == DIGEST_SHA3_256); - - int ret = 0; - if (algorithm == DIGEST_SHA256) { -#ifdef ENABLE_NSS - return digest_nss_internal(SEC_OID_SHA256, digest, DIGEST256_LEN, m, len); -#else - ret = (SHA256((const uint8_t*)m,len,(uint8_t*)digest) != NULL); -#endif - } else { -#ifdef OPENSSL_HAS_SHA3 - unsigned int dlen = DIGEST256_LEN; - ret = EVP_Digest(m, len, (uint8_t*)digest, &dlen, EVP_sha3_256(), NULL); -#else - ret = (sha3_256((uint8_t *)digest, DIGEST256_LEN,(const uint8_t *)m, len) - > -1); -#endif - } - - if (!ret) - return -1; - return 0; -} - -/** Compute a 512-bit digest of len bytes in data stored in m, - * using the algorithm algorithm. Write the DIGEST_LEN512-byte result - * into digest. Return 0 on success, -1 on failure. */ -int -crypto_digest512(char *digest, const char *m, size_t len, - digest_algorithm_t algorithm) -{ - tor_assert(m); - tor_assert(digest); - tor_assert(algorithm == DIGEST_SHA512 || algorithm == DIGEST_SHA3_512); - - int ret = 0; - if (algorithm == DIGEST_SHA512) { -#ifdef ENABLE_NSS - return digest_nss_internal(SEC_OID_SHA512, digest, DIGEST512_LEN, m, len); -#else - ret = (SHA512((const unsigned char*)m,len,(unsigned char*)digest) - != NULL); -#endif - } else { -#ifdef OPENSSL_HAS_SHA3 - unsigned int dlen = DIGEST512_LEN; - ret = EVP_Digest(m, len, (uint8_t*)digest, &dlen, EVP_sha3_512(), NULL); -#else - ret = (sha3_512((uint8_t*)digest, DIGEST512_LEN, (const uint8_t*)m, len) - > -1); -#endif - } - - if (!ret) - return -1; - return 0; -} - /** Set the common_digests_t in ds_out to contain every digest on the * len bytes in m that we know how to compute. Return 0 on * success, -1 on failure. */ @@ -283,561 +102,6 @@ crypto_digest_algorithm_get_length(digest_algorithm_t alg) } } -/** Intermediate information about the digest of a stream of data. */ -struct crypto_digest_t { - digest_algorithm_t algorithm; /**< Which algorithm is in use? */ - /** State for the digest we're using. Only one member of the - * union is usable, depending on the value of algorithm. Note also - * that space for other members might not even be allocated! - */ - union { -#ifdef ENABLE_NSS - PK11Context *ctx; -#else - SHA_CTX sha1; /**< state for SHA1 */ - SHA256_CTX sha2; /**< state for SHA256 */ - SHA512_CTX sha512; /**< state for SHA512 */ -#endif -#ifdef OPENSSL_HAS_SHA3 - EVP_MD_CTX *md; -#else - keccak_state sha3; /**< state for SHA3-[256,512] */ -#endif - } d; -}; - -#ifdef TOR_UNIT_TESTS - -digest_algorithm_t -crypto_digest_get_algorithm(crypto_digest_t *digest) -{ - tor_assert(digest); - - return digest->algorithm; -} - -#endif /* defined(TOR_UNIT_TESTS) */ - -/** - * Return the number of bytes we need to malloc in order to get a - * crypto_digest_t for alg, or the number of bytes we need to wipe - * when we free one. - */ -static size_t -crypto_digest_alloc_bytes(digest_algorithm_t alg) -{ - /* Helper: returns the number of bytes in the 'f' field of 'st' */ -#define STRUCT_FIELD_SIZE(st, f) (sizeof( ((st*)0)->f )) - /* Gives the length of crypto_digest_t through the end of the field 'd' */ -#define END_OF_FIELD(f) (offsetof(crypto_digest_t, f) + \ - STRUCT_FIELD_SIZE(crypto_digest_t, f)) - switch (alg) { -#ifdef ENABLE_NSS - case DIGEST_SHA1: /* Fall through */ - case DIGEST_SHA256: /* Fall through */ - case DIGEST_SHA512: - return END_OF_FIELD(d.ctx); -#else - case DIGEST_SHA1: - return END_OF_FIELD(d.sha1); - case DIGEST_SHA256: - return END_OF_FIELD(d.sha2); - case DIGEST_SHA512: - return END_OF_FIELD(d.sha512); -#endif -#ifdef OPENSSL_HAS_SHA3 - case DIGEST_SHA3_256: /* Fall through */ - case DIGEST_SHA3_512: - return END_OF_FIELD(d.md); -#else - case DIGEST_SHA3_256: /* Fall through */ - case DIGEST_SHA3_512: - return END_OF_FIELD(d.sha3); -#endif - default: - tor_assert(0); // LCOV_EXCL_LINE - return 0; // LCOV_EXCL_LINE - } -#undef END_OF_FIELD -#undef STRUCT_FIELD_SIZE -} - -/** - * Internal function: create and return a new digest object for 'algorithm'. - * Does not typecheck the algorithm. - */ -static crypto_digest_t * -crypto_digest_new_internal(digest_algorithm_t algorithm) -{ - crypto_digest_t *r = tor_malloc(crypto_digest_alloc_bytes(algorithm)); - r->algorithm = algorithm; - - switch (algorithm) - { -#ifdef ENABLE_NSS - case DIGEST_SHA1: /* fall through */ - case DIGEST_SHA256: /* fall through */ - case DIGEST_SHA512: - r->d.ctx = PK11_CreateDigestContext(digest_alg_to_nss_oid(algorithm)); - if (BUG(!r->d.ctx)) { - tor_free(r); - return NULL; - } - if (BUG(SECSuccess != PK11_DigestBegin(r->d.ctx))) { - crypto_digest_free(r); - return NULL; - } - break; -#else - case DIGEST_SHA1: - SHA1_Init(&r->d.sha1); - break; - case DIGEST_SHA256: - SHA256_Init(&r->d.sha2); - break; - case DIGEST_SHA512: - SHA512_Init(&r->d.sha512); - break; -#endif -#ifdef OPENSSL_HAS_SHA3 - case DIGEST_SHA3_256: - r->d.md = EVP_MD_CTX_new(); - if (!EVP_DigestInit(r->d.md, EVP_sha3_256())) { - crypto_digest_free(r); - return NULL; - } - break; - case DIGEST_SHA3_512: - r->d.md = EVP_MD_CTX_new(); - if (!EVP_DigestInit(r->d.md, EVP_sha3_512())) { - crypto_digest_free(r); - return NULL; - } - break; -#else - case DIGEST_SHA3_256: - keccak_digest_init(&r->d.sha3, 256); - break; - case DIGEST_SHA3_512: - keccak_digest_init(&r->d.sha3, 512); - break; -#endif - default: - tor_assert_unreached(); - } - - return r; -} - -/** Allocate and return a new digest object to compute SHA1 digests. - */ -crypto_digest_t * -crypto_digest_new(void) -{ - return crypto_digest_new_internal(DIGEST_SHA1); -} - -/** Allocate and return a new digest object to compute 256-bit digests - * using algorithm. - * - * C_RUST_COUPLED: `external::crypto_digest::crypto_digest256_new` - * C_RUST_COUPLED: `crypto::digest::Sha256::default` - */ -crypto_digest_t * -crypto_digest256_new(digest_algorithm_t algorithm) -{ - tor_assert(algorithm == DIGEST_SHA256 || algorithm == DIGEST_SHA3_256); - return crypto_digest_new_internal(algorithm); -} - -/** Allocate and return a new digest object to compute 512-bit digests - * using algorithm. */ -crypto_digest_t * -crypto_digest512_new(digest_algorithm_t algorithm) -{ - tor_assert(algorithm == DIGEST_SHA512 || algorithm == DIGEST_SHA3_512); - return crypto_digest_new_internal(algorithm); -} - -/** Deallocate a digest object. - */ -void -crypto_digest_free_(crypto_digest_t *digest) -{ - if (!digest) - return; -#ifdef ENABLE_NSS - if (library_supports_digest(digest->algorithm)) { - PK11_DestroyContext(digest->d.ctx, PR_TRUE); - } -#endif -#ifdef OPENSSL_HAS_SHA3 - if (digest->algorithm == DIGEST_SHA3_256 || - digest->algorithm == DIGEST_SHA3_512) { - if (digest->d.md) { - EVP_MD_CTX_free(digest->d.md); - } - } -#endif - size_t bytes = crypto_digest_alloc_bytes(digest->algorithm); - memwipe(digest, 0, bytes); - tor_free(digest); -} - -/** Add len bytes from data to the digest object. - * - * C_RUST_COUPLED: `external::crypto_digest::crypto_digest_add_bytess` - * C_RUST_COUPLED: `crypto::digest::Sha256::process` - */ -void -crypto_digest_add_bytes(crypto_digest_t *digest, const char *data, - size_t len) -{ - tor_assert(digest); - tor_assert(data); - /* Using the SHA*_*() calls directly means we don't support doing - * SHA in hardware. But so far the delay of getting the question - * to the hardware, and hearing the answer, is likely higher than - * just doing it ourselves. Hashes are fast. - */ - switch (digest->algorithm) { -#ifdef ENABLE_NSS - case DIGEST_SHA1: /* fall through */ - case DIGEST_SHA256: /* fall through */ - case DIGEST_SHA512: - tor_assert(len <= UINT_MAX); - SECStatus s = PK11_DigestOp(digest->d.ctx, - (const unsigned char *)data, - (unsigned int)len); - tor_assert(s == SECSuccess); - break; -#else - case DIGEST_SHA1: - SHA1_Update(&digest->d.sha1, (void*)data, len); - break; - case DIGEST_SHA256: - SHA256_Update(&digest->d.sha2, (void*)data, len); - break; - case DIGEST_SHA512: - SHA512_Update(&digest->d.sha512, (void*)data, len); - break; -#endif -#ifdef OPENSSL_HAS_SHA3 - case DIGEST_SHA3_256: /* FALLSTHROUGH */ - case DIGEST_SHA3_512: { - int r = EVP_DigestUpdate(digest->d.md, data, len); - tor_assert(r); - } - break; -#else - case DIGEST_SHA3_256: /* FALLSTHROUGH */ - case DIGEST_SHA3_512: - keccak_digest_update(&digest->d.sha3, (const uint8_t *)data, len); - break; -#endif - default: - /* LCOV_EXCL_START */ - tor_fragile_assert(); - break; - /* LCOV_EXCL_STOP */ - } -} - -/** Compute the hash of the data that has been passed to the digest - * object; write the first out_len bytes of the result to out. - * out_len must be \<= DIGEST512_LEN. - * - * C_RUST_COUPLED: `external::crypto_digest::crypto_digest_get_digest` - * C_RUST_COUPLED: `impl digest::FixedOutput for Sha256` - */ -void -crypto_digest_get_digest(crypto_digest_t *digest, - char *out, size_t out_len) -{ - unsigned char r[DIGEST512_LEN]; - tor_assert(digest); - tor_assert(out); - tor_assert(out_len <= crypto_digest_algorithm_get_length(digest->algorithm)); - - if (digest->algorithm == DIGEST_SHA3_256 || - digest->algorithm == DIGEST_SHA3_512) { -#ifdef OPENSSL_HAS_SHA3 - unsigned dlen = (unsigned) - crypto_digest_algorithm_get_length(digest->algorithm); - EVP_MD_CTX *tmp = EVP_MD_CTX_new(); - EVP_MD_CTX_copy(tmp, digest->d.md); - memset(r, 0xff, sizeof(r)); - int res = EVP_DigestFinal(tmp, r, &dlen); - EVP_MD_CTX_free(tmp); - tor_assert(res == 1); - goto done; -#else - /* Tiny-Keccak handles copying into a temporary ctx, and also can handle - * short output buffers by truncating appropriately. */ - keccak_digest_sum(&digest->d.sha3, (uint8_t *)out, out_len); - return; -#endif - } - -#ifdef ENABLE_NSS - /* Copy into a temporary buffer since DigestFinal (alters) the context */ - unsigned char buf[1024]; - unsigned int saved_len = 0; - unsigned rlen; - unsigned char *saved = PK11_SaveContextAlloc(digest->d.ctx, - buf, sizeof(buf), - &saved_len); - tor_assert(saved); - SECStatus s = PK11_DigestFinal(digest->d.ctx, r, &rlen, sizeof(r)); - tor_assert(s == SECSuccess); - tor_assert(rlen >= out_len); - s = PK11_RestoreContext(digest->d.ctx, saved, saved_len); - tor_assert(s == SECSuccess); - if (saved != buf) { - PORT_ZFree(saved, saved_len); - } -#else - const size_t alloc_bytes = crypto_digest_alloc_bytes(digest->algorithm); - crypto_digest_t tmpenv; - /* memcpy into a temporary ctx, since SHA*_Final clears the context */ - memcpy(&tmpenv, digest, alloc_bytes); - switch (digest->algorithm) { - case DIGEST_SHA1: - SHA1_Final(r, &tmpenv.d.sha1); - break; - case DIGEST_SHA256: - SHA256_Final(r, &tmpenv.d.sha2); - break; - case DIGEST_SHA512: - SHA512_Final(r, &tmpenv.d.sha512); - break; -//LCOV_EXCL_START - case DIGEST_SHA3_256: /* FALLSTHROUGH */ - case DIGEST_SHA3_512: - default: - log_warn(LD_BUG, "Handling unexpected algorithm %d", digest->algorithm); - /* This is fatal, because it should never happen. */ - tor_assert_unreached(); - break; -//LCOV_EXCL_STOP - } -#endif - -#ifdef OPENSSL_HAS_SHA3 - done: -#endif - memcpy(out, r, out_len); - memwipe(r, 0, sizeof(r)); -} - -/** Allocate and return a new digest object with the same state as - * digest - * - * C_RUST_COUPLED: `external::crypto_digest::crypto_digest_dup` - * C_RUST_COUPLED: `impl Clone for crypto::digest::Sha256` - */ -crypto_digest_t * -crypto_digest_dup(const crypto_digest_t *digest) -{ - tor_assert(digest); - const size_t alloc_bytes = crypto_digest_alloc_bytes(digest->algorithm); - crypto_digest_t *result = tor_memdup(digest, alloc_bytes); -#ifdef ENABLE_NSS - if (library_supports_digest(digest->algorithm)) { - result->d.ctx = PK11_CloneContext(digest->d.ctx); - } -#endif -#ifdef OPENSSL_HAS_SHA3 - if (digest->algorithm == DIGEST_SHA3_256 || - digest->algorithm == DIGEST_SHA3_512) { - result->d.md = EVP_MD_CTX_new(); - EVP_MD_CTX_copy(result->d.md, digest->d.md); - } -#endif - return result; -} - -/** Temporarily save the state of digest in checkpoint. - * Asserts that digest is a SHA1 digest object. - */ -void -crypto_digest_checkpoint(crypto_digest_checkpoint_t *checkpoint, - const crypto_digest_t *digest) -{ - const size_t bytes = crypto_digest_alloc_bytes(digest->algorithm); - tor_assert(bytes <= sizeof(checkpoint->mem)); -#ifdef ENABLE_NSS - if (library_supports_digest(digest->algorithm)) { - unsigned char *allocated; - allocated = PK11_SaveContextAlloc(digest->d.ctx, - (unsigned char *)checkpoint->mem, - sizeof(checkpoint->mem), - &checkpoint->bytes_used); - /* No allocation is allowed here. */ - tor_assert(allocated == checkpoint->mem); - return; - } -#endif - memcpy(checkpoint->mem, digest, bytes); -} - -/** Restore the state of digest from checkpoint. - * Asserts that digest is a SHA1 digest object. Requires that the - * state was previously stored with crypto_digest_checkpoint() */ -void -crypto_digest_restore(crypto_digest_t *digest, - const crypto_digest_checkpoint_t *checkpoint) -{ - const size_t bytes = crypto_digest_alloc_bytes(digest->algorithm); -#ifdef ENABLE_NSS - if (library_supports_digest(digest->algorithm)) { - SECStatus s = PK11_RestoreContext(digest->d.ctx, - (unsigned char *)checkpoint->mem, - checkpoint->bytes_used); - tor_assert(s == SECSuccess); - return; - } -#endif - memcpy(digest, checkpoint->mem, bytes); -} - -/** Replace the state of the digest object into with the state - * of the digest object from. Requires that 'into' and 'from' - * have the same digest type. - */ -void -crypto_digest_assign(crypto_digest_t *into, - const crypto_digest_t *from) -{ - tor_assert(into); - tor_assert(from); - tor_assert(into->algorithm == from->algorithm); - const size_t alloc_bytes = crypto_digest_alloc_bytes(from->algorithm); -#ifdef ENABLE_NSS - if (library_supports_digest(from->algorithm)) { - PK11_DestroyContext(into->d.ctx, PR_TRUE); - into->d.ctx = PK11_CloneContext(from->d.ctx); - return; - } -#endif - -#ifdef OPENSSL_HAS_SHA3 - if (from->algorithm == DIGEST_SHA3_256 || - from->algorithm == DIGEST_SHA3_512) { - EVP_MD_CTX_copy(into->d.md, from->d.md); - return; - } -#endif - - memcpy(into,from,alloc_bytes); -} - -/** Given a list of strings in lst, set the len_out-byte digest - * at digest_out to the hash of the concatenation of those strings, - * plus the optional string append, computed with the algorithm - * alg. - * out_len must be \<= DIGEST512_LEN. */ -void -crypto_digest_smartlist(char *digest_out, size_t len_out, - const smartlist_t *lst, - const char *append, - digest_algorithm_t alg) -{ - crypto_digest_smartlist_prefix(digest_out, len_out, NULL, lst, append, alg); -} - -/** Given a list of strings in lst, set the len_out-byte digest - * at digest_out to the hash of the concatenation of: the - * optional string prepend, those strings, - * and the optional string append, computed with the algorithm - * alg. - * len_out must be \<= DIGEST512_LEN. */ -void -crypto_digest_smartlist_prefix(char *digest_out, size_t len_out, - const char *prepend, - const smartlist_t *lst, - const char *append, - digest_algorithm_t alg) -{ - crypto_digest_t *d = crypto_digest_new_internal(alg); - if (prepend) - crypto_digest_add_bytes(d, prepend, strlen(prepend)); - SMARTLIST_FOREACH(lst, const char *, cp, - crypto_digest_add_bytes(d, cp, strlen(cp))); - if (append) - crypto_digest_add_bytes(d, append, strlen(append)); - crypto_digest_get_digest(d, digest_out, len_out); - crypto_digest_free(d); -} - -/** Compute the HMAC-SHA-256 of the msg_len bytes in msg, using - * the key of length key_len. Store the DIGEST256_LEN-byte - * result in hmac_out. Asserts on failure. - */ -void -crypto_hmac_sha256(char *hmac_out, - const char *key, size_t key_len, - const char *msg, size_t msg_len) -{ - /* If we've got OpenSSL >=0.9.8 we can use its hmac implementation. */ - tor_assert(key_len < INT_MAX); - tor_assert(msg_len < INT_MAX); - tor_assert(hmac_out); -#ifdef ENABLE_NSS - PK11SlotInfo *slot = NULL; - PK11SymKey *symKey = NULL; - PK11Context *hmac = NULL; - - int ok = 0; - SECStatus s; - SECItem keyItem, paramItem; - keyItem.data = (unsigned char *)key; - keyItem.len = (unsigned)key_len; - paramItem.type = siBuffer; - paramItem.data = NULL; - paramItem.len = 0; - - slot = PK11_GetBestSlot(CKM_SHA256_HMAC, NULL); - if (!slot) - goto done; - symKey = PK11_ImportSymKey(slot, CKM_SHA256_HMAC, - PK11_OriginUnwrap, CKA_SIGN, &keyItem, NULL); - if (!symKey) - goto done; - - hmac = PK11_CreateContextBySymKey(CKM_SHA256_HMAC, CKA_SIGN, symKey, - ¶mItem); - if (!hmac) - goto done; - s = PK11_DigestBegin(hmac); - if (s != SECSuccess) - goto done; - s = PK11_DigestOp(hmac, (const unsigned char *)msg, (unsigned int)msg_len); - if (s != SECSuccess) - goto done; - unsigned int len=0; - s = PK11_DigestFinal(hmac, (unsigned char *)hmac_out, &len, DIGEST256_LEN); - if (s != SECSuccess || len != DIGEST256_LEN) - goto done; - ok = 1; - - done: - if (hmac) - PK11_DestroyContext(hmac, PR_TRUE); - if (symKey) - PK11_FreeSymKey(symKey); - if (slot) - PK11_FreeSlot(slot); - - tor_assert(ok); -#else - unsigned char *rv = NULL; - rv = HMAC(EVP_sha256(), key, (int)key_len, (unsigned char*)msg, (int)msg_len, - (unsigned char*)hmac_out, NULL); - tor_assert(rv); -#endif -} - /** Compute a MAC using SHA3-256 of msg_len bytes in msg using a * key of length key_len and a salt of length * salt_len. Store the result of len_out bytes in in diff --git a/src/lib/crypt_ops/crypto_digest_nss.c b/src/lib/crypt_ops/crypto_digest_nss.c new file mode 100644 index 0000000000..b73f0736fd --- /dev/null +++ b/src/lib/crypt_ops/crypto_digest_nss.c @@ -0,0 +1,560 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file crypto_digest_nss.c + * \brief Block of functions related with digest and xof utilities and + * operations (NSS specific implementations). + **/ + +#include "lib/container/smartlist.h" +#include "lib/crypt_ops/crypto_digest.h" +#include "lib/crypt_ops/crypto_util.h" +#include "lib/log/log.h" +#include "lib/log/util_bug.h" + +#include "keccak-tiny/keccak-tiny.h" + +#include +#include + +#include "lib/arch/bytes.h" + +DISABLE_GCC_WARNING(strict-prototypes) +#include +ENABLE_GCC_WARNING(strict-prototypes) + +/** + * Convert a digest_algorithm_t (used by tor) to a HashType (used by NSS). + * On failure, return SEC_OID_UNKNOWN. */ +static SECOidTag +digest_alg_to_nss_oid(digest_algorithm_t alg) +{ + switch (alg) { + case DIGEST_SHA1: return SEC_OID_SHA1; + case DIGEST_SHA256: return SEC_OID_SHA256; + case DIGEST_SHA512: return SEC_OID_SHA512; + case DIGEST_SHA3_256: /* Fall through */ + case DIGEST_SHA3_512: /* Fall through */ + default: + return SEC_OID_UNKNOWN; + } +} + +/* Helper: get an unkeyed digest via pk11wrap */ +static int +digest_nss_internal(SECOidTag alg, + char *digest, unsigned len_out, + const char *msg, size_t msg_len) +{ + if (alg == SEC_OID_UNKNOWN) + return -1; + tor_assert(msg_len <= UINT_MAX); + + int rv = -1; + SECStatus s; + PK11Context *ctx = PK11_CreateDigestContext(alg); + if (!ctx) + return -1; + + s = PK11_DigestBegin(ctx); + if (s != SECSuccess) + goto done; + + s = PK11_DigestOp(ctx, (const unsigned char *)msg, (unsigned int)msg_len); + if (s != SECSuccess) + goto done; + + unsigned int len = 0; + s = PK11_DigestFinal(ctx, (unsigned char *)digest, &len, len_out); + if (s != SECSuccess) + goto done; + + rv = 0; + done: + PK11_DestroyContext(ctx, PR_TRUE); + return rv; +} + +/** True iff alg is implemented in our crypto library, and we want to use that + * implementation */ +static bool +library_supports_digest(digest_algorithm_t alg) +{ + switch (alg) { + case DIGEST_SHA1: /* Fall through */ + case DIGEST_SHA256: /* Fall through */ + case DIGEST_SHA512: /* Fall through */ + return true; + case DIGEST_SHA3_256: /* Fall through */ + case DIGEST_SHA3_512: /* Fall through */ + default: + return false; + } +} + +/* Crypto digest functions */ + +/** Compute the SHA1 digest of the len bytes on data stored in + * m. Write the DIGEST_LEN byte result into digest. + * Return 0 on success, -1 on failure. + */ +MOCK_IMPL(int, +crypto_digest,(char *digest, const char *m, size_t len)) +{ + tor_assert(m); + tor_assert(digest); + return digest_nss_internal(SEC_OID_SHA1, digest, DIGEST_LEN, m, len); +} + +/** Compute a 256-bit digest of len bytes in data stored in m, + * using the algorithm algorithm. Write the DIGEST_LEN256-byte result + * into digest. Return 0 on success, -1 on failure. */ +int +crypto_digest256(char *digest, const char *m, size_t len, + digest_algorithm_t algorithm) +{ + tor_assert(m); + tor_assert(digest); + tor_assert(algorithm == DIGEST_SHA256 || algorithm == DIGEST_SHA3_256); + + int ret = 0; + if (algorithm == DIGEST_SHA256) { + return digest_nss_internal(SEC_OID_SHA256, digest, DIGEST256_LEN, m, len); + } else { + ret = (sha3_256((uint8_t *)digest, DIGEST256_LEN,(const uint8_t *)m, len) + > -1); + } + + if (!ret) + return -1; + return 0; +} + +/** Compute a 512-bit digest of len bytes in data stored in m, + * using the algorithm algorithm. Write the DIGEST_LEN512-byte result + * into digest. Return 0 on success, -1 on failure. */ +int +crypto_digest512(char *digest, const char *m, size_t len, + digest_algorithm_t algorithm) +{ + tor_assert(m); + tor_assert(digest); + tor_assert(algorithm == DIGEST_SHA512 || algorithm == DIGEST_SHA3_512); + + int ret = 0; + if (algorithm == DIGEST_SHA512) { + return digest_nss_internal(SEC_OID_SHA512, digest, DIGEST512_LEN, m, len); + } else { + ret = (sha3_512((uint8_t*)digest, DIGEST512_LEN, (const uint8_t*)m, len) + > -1); + } + + if (!ret) + return -1; + return 0; +} + +/** Intermediate information about the digest of a stream of data. */ +struct crypto_digest_t { + digest_algorithm_t algorithm; /**< Which algorithm is in use? */ + /** State for the digest we're using. Only one member of the + * union is usable, depending on the value of algorithm. Note also + * that space for other members might not even be allocated! + */ + union { + PK11Context *ctx; + keccak_state sha3; /**< state for SHA3-[256,512] */ + } d; +}; + +#ifdef TOR_UNIT_TESTS + +digest_algorithm_t +crypto_digest_get_algorithm(crypto_digest_t *digest) +{ + tor_assert(digest); + + return digest->algorithm; +} + +#endif /* defined(TOR_UNIT_TESTS) */ + +/** + * Return the number of bytes we need to malloc in order to get a + * crypto_digest_t for alg, or the number of bytes we need to wipe + * when we free one. + */ +static size_t +crypto_digest_alloc_bytes(digest_algorithm_t alg) +{ + /* Helper: returns the number of bytes in the 'f' field of 'st' */ +#define STRUCT_FIELD_SIZE(st, f) (sizeof( ((st*)0)->f )) + /* Gives the length of crypto_digest_t through the end of the field 'd' */ +#define END_OF_FIELD(f) (offsetof(crypto_digest_t, f) + \ + STRUCT_FIELD_SIZE(crypto_digest_t, f)) + switch (alg) { + case DIGEST_SHA1: /* Fall through */ + case DIGEST_SHA256: /* Fall through */ + case DIGEST_SHA512: + return END_OF_FIELD(d.ctx); + case DIGEST_SHA3_256: + case DIGEST_SHA3_512: + return END_OF_FIELD(d.sha3); + default: + tor_assert(0); // LCOV_EXCL_LINE + return 0; // LCOV_EXCL_LINE + } +#undef END_OF_FIELD +#undef STRUCT_FIELD_SIZE +} + +/** + * Internal function: create and return a new digest object for 'algorithm'. + * Does not typecheck the algorithm. + */ +static crypto_digest_t * +crypto_digest_new_internal(digest_algorithm_t algorithm) +{ + crypto_digest_t *r = tor_malloc(crypto_digest_alloc_bytes(algorithm)); + r->algorithm = algorithm; + + switch (algorithm) + { + case DIGEST_SHA1: /* fall through */ + case DIGEST_SHA256: /* fall through */ + case DIGEST_SHA512: + r->d.ctx = PK11_CreateDigestContext(digest_alg_to_nss_oid(algorithm)); + if (BUG(!r->d.ctx)) { + tor_free(r); + return NULL; + } + if (BUG(SECSuccess != PK11_DigestBegin(r->d.ctx))) { + crypto_digest_free(r); + return NULL; + } + break; + case DIGEST_SHA3_256: + keccak_digest_init(&r->d.sha3, 256); + break; + case DIGEST_SHA3_512: + keccak_digest_init(&r->d.sha3, 512); + break; + default: + tor_assert_unreached(); + } + + return r; +} + +/** Allocate and return a new digest object to compute SHA1 digests. + */ +crypto_digest_t * +crypto_digest_new(void) +{ + return crypto_digest_new_internal(DIGEST_SHA1); +} + +/** Allocate and return a new digest object to compute 256-bit digests + * using algorithm. + * + * C_RUST_COUPLED: `external::crypto_digest::crypto_digest256_new` + * C_RUST_COUPLED: `crypto::digest::Sha256::default` + */ +crypto_digest_t * +crypto_digest256_new(digest_algorithm_t algorithm) +{ + tor_assert(algorithm == DIGEST_SHA256 || algorithm == DIGEST_SHA3_256); + return crypto_digest_new_internal(algorithm); +} + +/** Allocate and return a new digest object to compute 512-bit digests + * using algorithm. */ +crypto_digest_t * +crypto_digest512_new(digest_algorithm_t algorithm) +{ + tor_assert(algorithm == DIGEST_SHA512 || algorithm == DIGEST_SHA3_512); + return crypto_digest_new_internal(algorithm); +} + +/** Deallocate a digest object. + */ +void +crypto_digest_free_(crypto_digest_t *digest) +{ + if (!digest) + return; + if (library_supports_digest(digest->algorithm)) { + PK11_DestroyContext(digest->d.ctx, PR_TRUE); + } + size_t bytes = crypto_digest_alloc_bytes(digest->algorithm); + memwipe(digest, 0, bytes); + tor_free(digest); +} + +/** Add len bytes from data to the digest object. + * + * C_RUST_COUPLED: `external::crypto_digest::crypto_digest_add_bytess` + * C_RUST_COUPLED: `crypto::digest::Sha256::process` + */ +void +crypto_digest_add_bytes(crypto_digest_t *digest, const char *data, + size_t len) +{ + tor_assert(digest); + tor_assert(data); + /* Using the SHA*_*() calls directly means we don't support doing + * SHA in hardware. But so far the delay of getting the question + * to the hardware, and hearing the answer, is likely higher than + * just doing it ourselves. Hashes are fast. + */ + switch (digest->algorithm) { + case DIGEST_SHA1: /* fall through */ + case DIGEST_SHA256: /* fall through */ + case DIGEST_SHA512: + tor_assert(len <= UINT_MAX); + SECStatus s = PK11_DigestOp(digest->d.ctx, + (const unsigned char *)data, + (unsigned int)len); + tor_assert(s == SECSuccess); + break; + case DIGEST_SHA3_256: /* FALLSTHROUGH */ + case DIGEST_SHA3_512: + keccak_digest_update(&digest->d.sha3, (const uint8_t *)data, len); + break; + default: + /* LCOV_EXCL_START */ + tor_fragile_assert(); + break; + /* LCOV_EXCL_STOP */ + } +} + +/** Compute the hash of the data that has been passed to the digest + * object; write the first out_len bytes of the result to out. + * out_len must be \<= DIGEST512_LEN. + * + * C_RUST_COUPLED: `external::crypto_digest::crypto_digest_get_digest` + * C_RUST_COUPLED: `impl digest::FixedOutput for Sha256` + */ +void +crypto_digest_get_digest(crypto_digest_t *digest, + char *out, size_t out_len) +{ + unsigned char r[DIGEST512_LEN]; + tor_assert(digest); + tor_assert(out); + tor_assert(out_len <= crypto_digest_algorithm_get_length(digest->algorithm)); + + /* The SHA-3 code handles copying into a temporary ctx, and also can handle + * short output buffers by truncating appropriately. */ + if (digest->algorithm == DIGEST_SHA3_256 || + digest->algorithm == DIGEST_SHA3_512) { + keccak_digest_sum(&digest->d.sha3, (uint8_t *)out, out_len); + return; + } + + /* Copy into a temporary buffer since DigestFinal (alters) the context */ + unsigned char buf[1024]; + unsigned int saved_len = 0; + unsigned rlen; + unsigned char *saved = PK11_SaveContextAlloc(digest->d.ctx, + buf, sizeof(buf), + &saved_len); + tor_assert(saved); + SECStatus s = PK11_DigestFinal(digest->d.ctx, r, &rlen, sizeof(r)); + tor_assert(s == SECSuccess); + tor_assert(rlen >= out_len); + s = PK11_RestoreContext(digest->d.ctx, saved, saved_len); + tor_assert(s == SECSuccess); + + if (saved != buf) { + PORT_ZFree(saved, saved_len); + } + memcpy(out, r, out_len); + memwipe(r, 0, sizeof(r)); +} + +/** Allocate and return a new digest object with the same state as + * digest + * + * C_RUST_COUPLED: `external::crypto_digest::crypto_digest_dup` + * C_RUST_COUPLED: `impl Clone for crypto::digest::Sha256` + */ +crypto_digest_t * +crypto_digest_dup(const crypto_digest_t *digest) +{ + tor_assert(digest); + const size_t alloc_bytes = crypto_digest_alloc_bytes(digest->algorithm); + crypto_digest_t *result = tor_memdup(digest, alloc_bytes); + + if (library_supports_digest(digest->algorithm)) { + result->d.ctx = PK11_CloneContext(digest->d.ctx); + } + + return result; +} + +/** Temporarily save the state of digest in checkpoint. + * Asserts that digest is a SHA1 digest object. + */ +void +crypto_digest_checkpoint(crypto_digest_checkpoint_t *checkpoint, + const crypto_digest_t *digest) +{ + const size_t bytes = crypto_digest_alloc_bytes(digest->algorithm); + tor_assert(bytes <= sizeof(checkpoint->mem)); + if (library_supports_digest(digest->algorithm)) { + unsigned char *allocated; + allocated = PK11_SaveContextAlloc(digest->d.ctx, + (unsigned char *)checkpoint->mem, + sizeof(checkpoint->mem), + &checkpoint->bytes_used); + /* No allocation is allowed here. */ + tor_assert(allocated == checkpoint->mem); + return; + } + memcpy(checkpoint->mem, digest, bytes); +} + +/** Restore the state of digest from checkpoint. + * Asserts that digest is a SHA1 digest object. Requires that the + * state was previously stored with crypto_digest_checkpoint() */ +void +crypto_digest_restore(crypto_digest_t *digest, + const crypto_digest_checkpoint_t *checkpoint) +{ + const size_t bytes = crypto_digest_alloc_bytes(digest->algorithm); + if (library_supports_digest(digest->algorithm)) { + SECStatus s = PK11_RestoreContext(digest->d.ctx, + (unsigned char *)checkpoint->mem, + checkpoint->bytes_used); + tor_assert(s == SECSuccess); + return; + } + memcpy(digest, checkpoint->mem, bytes); +} + +/** Replace the state of the digest object into with the state + * of the digest object from. Requires that 'into' and 'from' + * have the same digest type. + */ +void +crypto_digest_assign(crypto_digest_t *into, + const crypto_digest_t *from) +{ + tor_assert(into); + tor_assert(from); + tor_assert(into->algorithm == from->algorithm); + const size_t alloc_bytes = crypto_digest_alloc_bytes(from->algorithm); + if (library_supports_digest(from->algorithm)) { + PK11_DestroyContext(into->d.ctx, PR_TRUE); + into->d.ctx = PK11_CloneContext(from->d.ctx); + return; + } + memcpy(into,from,alloc_bytes); +} + +/** Given a list of strings in lst, set the len_out-byte digest + * at digest_out to the hash of the concatenation of those strings, + * plus the optional string append, computed with the algorithm + * alg. + * out_len must be \<= DIGEST512_LEN. */ +void +crypto_digest_smartlist(char *digest_out, size_t len_out, + const smartlist_t *lst, + const char *append, + digest_algorithm_t alg) +{ + crypto_digest_smartlist_prefix(digest_out, len_out, NULL, lst, append, alg); +} + +/** Given a list of strings in lst, set the len_out-byte digest + * at digest_out to the hash of the concatenation of: the + * optional string prepend, those strings, + * and the optional string append, computed with the algorithm + * alg. + * len_out must be \<= DIGEST512_LEN. */ +void +crypto_digest_smartlist_prefix(char *digest_out, size_t len_out, + const char *prepend, + const smartlist_t *lst, + const char *append, + digest_algorithm_t alg) +{ + crypto_digest_t *d = crypto_digest_new_internal(alg); + if (prepend) + crypto_digest_add_bytes(d, prepend, strlen(prepend)); + SMARTLIST_FOREACH(lst, const char *, cp, + crypto_digest_add_bytes(d, cp, strlen(cp))); + if (append) + crypto_digest_add_bytes(d, append, strlen(append)); + crypto_digest_get_digest(d, digest_out, len_out); + crypto_digest_free(d); +} + +/** Compute the HMAC-SHA-256 of the msg_len bytes in msg, using + * the key of length key_len. Store the DIGEST256_LEN-byte + * result in hmac_out. Asserts on failure. + */ +void +crypto_hmac_sha256(char *hmac_out, + const char *key, size_t key_len, + const char *msg, size_t msg_len) +{ + /* If we've got OpenSSL >=0.9.8 we can use its hmac implementation. */ + tor_assert(key_len < INT_MAX); + tor_assert(msg_len < INT_MAX); + tor_assert(hmac_out); + + PK11SlotInfo *slot = NULL; + PK11SymKey *symKey = NULL; + PK11Context *hmac = NULL; + + int ok = 0; + SECStatus s; + SECItem keyItem, paramItem; + keyItem.data = (unsigned char *)key; + keyItem.len = (unsigned)key_len; + paramItem.type = siBuffer; + paramItem.data = NULL; + paramItem.len = 0; + + slot = PK11_GetBestSlot(CKM_SHA256_HMAC, NULL); + if (!slot) + goto done; + symKey = PK11_ImportSymKey(slot, CKM_SHA256_HMAC, + PK11_OriginUnwrap, CKA_SIGN, &keyItem, NULL); + if (!symKey) + goto done; + + hmac = PK11_CreateContextBySymKey(CKM_SHA256_HMAC, CKA_SIGN, symKey, + ¶mItem); + if (!hmac) + goto done; + s = PK11_DigestBegin(hmac); + if (s != SECSuccess) + goto done; + s = PK11_DigestOp(hmac, (const unsigned char *)msg, (unsigned int)msg_len); + if (s != SECSuccess) + goto done; + unsigned int len=0; + s = PK11_DigestFinal(hmac, (unsigned char *)hmac_out, &len, DIGEST256_LEN); + if (s != SECSuccess || len != DIGEST256_LEN) + goto done; + ok = 1; + + done: + if (hmac) + PK11_DestroyContext(hmac, PR_TRUE); + if (symKey) + PK11_FreeSymKey(symKey); + if (slot) + PK11_FreeSlot(slot); + + tor_assert(ok); +} + diff --git a/src/lib/crypt_ops/crypto_digest_openssl.c b/src/lib/crypt_ops/crypto_digest_openssl.c new file mode 100644 index 0000000000..a1c92351fc --- /dev/null +++ b/src/lib/crypt_ops/crypto_digest_openssl.c @@ -0,0 +1,522 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file crypto_digest_openssl.c + * \brief Block of functions related with digest and xof utilities and + * operations (OpenSSL specific implementations). + **/ + +#include "lib/container/smartlist.h" +#include "lib/crypt_ops/crypto_digest.h" +#include "lib/crypt_ops/crypto_util.h" +#include "lib/log/log.h" +#include "lib/log/util_bug.h" + +#include "keccak-tiny/keccak-tiny.h" + +#include +#include + +#include "lib/arch/bytes.h" + +#include "lib/crypt_ops/crypto_openssl_mgt.h" + +DISABLE_GCC_WARNING(redundant-decls) + +#include +#include + +ENABLE_GCC_WARNING(redundant-decls) + +/* Crypto digest functions */ + +/** Compute the SHA1 digest of the len bytes on data stored in + * m. Write the DIGEST_LEN byte result into digest. + * Return 0 on success, -1 on failure. + */ +MOCK_IMPL(int, +crypto_digest,(char *digest, const char *m, size_t len)) +{ + tor_assert(m); + tor_assert(digest); + if (SHA1((const unsigned char*)m,len,(unsigned char*)digest) == NULL) { + return -1; + } + return 0; +} + +/** Compute a 256-bit digest of len bytes in data stored in m, + * using the algorithm algorithm. Write the DIGEST_LEN256-byte result + * into digest. Return 0 on success, -1 on failure. */ +int +crypto_digest256(char *digest, const char *m, size_t len, + digest_algorithm_t algorithm) +{ + tor_assert(m); + tor_assert(digest); + tor_assert(algorithm == DIGEST_SHA256 || algorithm == DIGEST_SHA3_256); + + int ret = 0; + if (algorithm == DIGEST_SHA256) { + ret = (SHA256((const uint8_t*)m,len,(uint8_t*)digest) != NULL); + } else { +#ifdef OPENSSL_HAS_SHA3 + unsigned int dlen = DIGEST256_LEN; + ret = EVP_Digest(m, len, (uint8_t*)digest, &dlen, EVP_sha3_256(), NULL); +#else + ret = (sha3_256((uint8_t *)digest, DIGEST256_LEN,(const uint8_t *)m, len) + > -1); +#endif + } + + if (!ret) + return -1; + return 0; +} + +/** Compute a 512-bit digest of len bytes in data stored in m, + * using the algorithm algorithm. Write the DIGEST_LEN512-byte result + * into digest. Return 0 on success, -1 on failure. */ +int +crypto_digest512(char *digest, const char *m, size_t len, + digest_algorithm_t algorithm) +{ + tor_assert(m); + tor_assert(digest); + tor_assert(algorithm == DIGEST_SHA512 || algorithm == DIGEST_SHA3_512); + + int ret = 0; + if (algorithm == DIGEST_SHA512) { + ret = (SHA512((const unsigned char*)m,len,(unsigned char*)digest) + != NULL); + } else { +#ifdef OPENSSL_HAS_SHA3 + unsigned int dlen = DIGEST512_LEN; + ret = EVP_Digest(m, len, (uint8_t*)digest, &dlen, EVP_sha3_512(), NULL); +#else + ret = (sha3_512((uint8_t*)digest, DIGEST512_LEN, (const uint8_t*)m, len) + > -1); +#endif + } + + if (!ret) + return -1; + return 0; +} + +/** Intermediate information about the digest of a stream of data. */ +struct crypto_digest_t { + digest_algorithm_t algorithm; /**< Which algorithm is in use? */ + /** State for the digest we're using. Only one member of the + * union is usable, depending on the value of algorithm. Note also + * that space for other members might not even be allocated! + */ + union { + SHA_CTX sha1; /**< state for SHA1 */ + SHA256_CTX sha2; /**< state for SHA256 */ + SHA512_CTX sha512; /**< state for SHA512 */ +#ifdef OPENSSL_HAS_SHA3 + EVP_MD_CTX *md; +#else + keccak_state sha3; /**< state for SHA3-[256,512] */ +#endif + } d; +}; + +#ifdef TOR_UNIT_TESTS + +digest_algorithm_t +crypto_digest_get_algorithm(crypto_digest_t *digest) +{ + tor_assert(digest); + + return digest->algorithm; +} + +#endif /* defined(TOR_UNIT_TESTS) */ + +/** + * Return the number of bytes we need to malloc in order to get a + * crypto_digest_t for alg, or the number of bytes we need to wipe + * when we free one. + */ +static size_t +crypto_digest_alloc_bytes(digest_algorithm_t alg) +{ + /* Helper: returns the number of bytes in the 'f' field of 'st' */ +#define STRUCT_FIELD_SIZE(st, f) (sizeof( ((st*)0)->f )) + /* Gives the length of crypto_digest_t through the end of the field 'd' */ +#define END_OF_FIELD(f) (offsetof(crypto_digest_t, f) + \ + STRUCT_FIELD_SIZE(crypto_digest_t, f)) + switch (alg) { + case DIGEST_SHA1: + return END_OF_FIELD(d.sha1); + case DIGEST_SHA256: + return END_OF_FIELD(d.sha2); + case DIGEST_SHA512: + return END_OF_FIELD(d.sha512); +#ifdef OPENSSL_HAS_SHA3 + case DIGEST_SHA3_256: /* Fall through */ + case DIGEST_SHA3_512: + return END_OF_FIELD(d.md); +#else + case DIGEST_SHA3_256: /* Fall through */ + case DIGEST_SHA3_512: + return END_OF_FIELD(d.sha3); +#endif + default: + tor_assert(0); // LCOV_EXCL_LINE + return 0; // LCOV_EXCL_LINE + } +#undef END_OF_FIELD +#undef STRUCT_FIELD_SIZE +} + +/** + * Internal function: create and return a new digest object for 'algorithm'. + * Does not typecheck the algorithm. + */ +static crypto_digest_t * +crypto_digest_new_internal(digest_algorithm_t algorithm) +{ + crypto_digest_t *r = tor_malloc(crypto_digest_alloc_bytes(algorithm)); + r->algorithm = algorithm; + + switch (algorithm) + { + case DIGEST_SHA1: + SHA1_Init(&r->d.sha1); + break; + case DIGEST_SHA256: + SHA256_Init(&r->d.sha2); + break; + case DIGEST_SHA512: + SHA512_Init(&r->d.sha512); + break; +#ifdef OPENSSL_HAS_SHA3 + case DIGEST_SHA3_256: + r->d.md = EVP_MD_CTX_new(); + if (!EVP_DigestInit(r->d.md, EVP_sha3_256())) { + crypto_digest_free(r); + return NULL; + } + break; + case DIGEST_SHA3_512: + r->d.md = EVP_MD_CTX_new(); + if (!EVP_DigestInit(r->d.md, EVP_sha3_512())) { + crypto_digest_free(r); + return NULL; + } + break; +#else + case DIGEST_SHA3_256: + keccak_digest_init(&r->d.sha3, 256); + break; + case DIGEST_SHA3_512: + keccak_digest_init(&r->d.sha3, 512); + break; +#endif + default: + tor_assert_unreached(); + } + + return r; +} + +/** Allocate and return a new digest object to compute SHA1 digests. + */ +crypto_digest_t * +crypto_digest_new(void) +{ + return crypto_digest_new_internal(DIGEST_SHA1); +} + +/** Allocate and return a new digest object to compute 256-bit digests + * using algorithm. + * + * C_RUST_COUPLED: `external::crypto_digest::crypto_digest256_new` + * C_RUST_COUPLED: `crypto::digest::Sha256::default` + */ +crypto_digest_t * +crypto_digest256_new(digest_algorithm_t algorithm) +{ + tor_assert(algorithm == DIGEST_SHA256 || algorithm == DIGEST_SHA3_256); + return crypto_digest_new_internal(algorithm); +} + +/** Allocate and return a new digest object to compute 512-bit digests + * using algorithm. */ +crypto_digest_t * +crypto_digest512_new(digest_algorithm_t algorithm) +{ + tor_assert(algorithm == DIGEST_SHA512 || algorithm == DIGEST_SHA3_512); + return crypto_digest_new_internal(algorithm); +} + +/** Deallocate a digest object. + */ +void +crypto_digest_free_(crypto_digest_t *digest) +{ + if (!digest) + return; +#ifdef OPENSSL_HAS_SHA3 + if (digest->algorithm == DIGEST_SHA3_256 || + digest->algorithm == DIGEST_SHA3_512) { + if (digest->d.md) { + EVP_MD_CTX_free(digest->d.md); + } + } +#endif + size_t bytes = crypto_digest_alloc_bytes(digest->algorithm); + memwipe(digest, 0, bytes); + tor_free(digest); +} + +/** Add len bytes from data to the digest object. + * + * C_RUST_COUPLED: `external::crypto_digest::crypto_digest_add_bytess` + * C_RUST_COUPLED: `crypto::digest::Sha256::process` + */ +void +crypto_digest_add_bytes(crypto_digest_t *digest, const char *data, + size_t len) +{ + tor_assert(digest); + tor_assert(data); + /* Using the SHA*_*() calls directly means we don't support doing + * SHA in hardware. But so far the delay of getting the question + * to the hardware, and hearing the answer, is likely higher than + * just doing it ourselves. Hashes are fast. + */ + switch (digest->algorithm) { + case DIGEST_SHA1: + SHA1_Update(&digest->d.sha1, (void*)data, len); + break; + case DIGEST_SHA256: + SHA256_Update(&digest->d.sha2, (void*)data, len); + break; + case DIGEST_SHA512: + SHA512_Update(&digest->d.sha512, (void*)data, len); + break; +#ifdef OPENSSL_HAS_SHA3 + case DIGEST_SHA3_256: /* FALLSTHROUGH */ + case DIGEST_SHA3_512: { + int r = EVP_DigestUpdate(digest->d.md, data, len); + tor_assert(r); + } + break; +#else + case DIGEST_SHA3_256: /* FALLSTHROUGH */ + case DIGEST_SHA3_512: + keccak_digest_update(&digest->d.sha3, (const uint8_t *)data, len); + break; +#endif + default: + /* LCOV_EXCL_START */ + tor_fragile_assert(); + break; + /* LCOV_EXCL_STOP */ + } +} + +/** Compute the hash of the data that has been passed to the digest + * object; write the first out_len bytes of the result to out. + * out_len must be \<= DIGEST512_LEN. + * + * C_RUST_COUPLED: `external::crypto_digest::crypto_digest_get_digest` + * C_RUST_COUPLED: `impl digest::FixedOutput for Sha256` + */ +void +crypto_digest_get_digest(crypto_digest_t *digest, + char *out, size_t out_len) +{ + unsigned char r[DIGEST512_LEN]; + tor_assert(digest); + tor_assert(out); + tor_assert(out_len <= crypto_digest_algorithm_get_length(digest->algorithm)); + + /* The SHA-3 code handles copying into a temporary ctx, and also can handle + * short output buffers by truncating appropriately. */ + if (digest->algorithm == DIGEST_SHA3_256 || + digest->algorithm == DIGEST_SHA3_512) { +#ifdef OPENSSL_HAS_SHA3 + unsigned dlen = (unsigned) + crypto_digest_algorithm_get_length(digest->algorithm); + EVP_MD_CTX *tmp = EVP_MD_CTX_new(); + EVP_MD_CTX_copy(tmp, digest->d.md); + memset(r, 0xff, sizeof(r)); + int res = EVP_DigestFinal(tmp, r, &dlen); + EVP_MD_CTX_free(tmp); + tor_assert(res == 1); + goto done; +#else + /* Tiny-Keccak handles copying into a temporary ctx, and also can handle + * short output buffers by truncating appropriately. */ + keccak_digest_sum(&digest->d.sha3, (uint8_t *)out, out_len); + return; +#endif + } + + const size_t alloc_bytes = crypto_digest_alloc_bytes(digest->algorithm); + crypto_digest_t tmpenv; + /* memcpy into a temporary ctx, since SHA*_Final clears the context */ + memcpy(&tmpenv, digest, alloc_bytes); + switch (digest->algorithm) { + case DIGEST_SHA1: + SHA1_Final(r, &tmpenv.d.sha1); + break; + case DIGEST_SHA256: + SHA256_Final(r, &tmpenv.d.sha2); + break; + case DIGEST_SHA512: + SHA512_Final(r, &tmpenv.d.sha512); + break; +//LCOV_EXCL_START + case DIGEST_SHA3_256: /* FALLSTHROUGH */ + case DIGEST_SHA3_512: + default: + log_warn(LD_BUG, "Handling unexpected algorithm %d", digest->algorithm); + /* This is fatal, because it should never happen. */ + tor_assert_unreached(); + break; +//LCOV_EXCL_STOP + } +#ifdef OPENSSL_HAS_SHA3 + done: +#endif + memcpy(out, r, out_len); + memwipe(r, 0, sizeof(r)); +} + +/** Allocate and return a new digest object with the same state as + * digest + * + * C_RUST_COUPLED: `external::crypto_digest::crypto_digest_dup` + * C_RUST_COUPLED: `impl Clone for crypto::digest::Sha256` + */ +crypto_digest_t * +crypto_digest_dup(const crypto_digest_t *digest) +{ + tor_assert(digest); + const size_t alloc_bytes = crypto_digest_alloc_bytes(digest->algorithm); + crypto_digest_t *result = tor_memdup(digest, alloc_bytes); + +#ifdef OPENSSL_HAS_SHA3 + if (digest->algorithm == DIGEST_SHA3_256 || + digest->algorithm == DIGEST_SHA3_512) { + result->d.md = EVP_MD_CTX_new(); + EVP_MD_CTX_copy(result->d.md, digest->d.md); + } +#endif + return result; +} + +/** Temporarily save the state of digest in checkpoint. + * Asserts that digest is a SHA1 digest object. + */ +void +crypto_digest_checkpoint(crypto_digest_checkpoint_t *checkpoint, + const crypto_digest_t *digest) +{ + const size_t bytes = crypto_digest_alloc_bytes(digest->algorithm); + tor_assert(bytes <= sizeof(checkpoint->mem)); + memcpy(checkpoint->mem, digest, bytes); +} + +/** Restore the state of digest from checkpoint. + * Asserts that digest is a SHA1 digest object. Requires that the + * state was previously stored with crypto_digest_checkpoint() */ +void +crypto_digest_restore(crypto_digest_t *digest, + const crypto_digest_checkpoint_t *checkpoint) +{ + const size_t bytes = crypto_digest_alloc_bytes(digest->algorithm); + memcpy(digest, checkpoint->mem, bytes); +} + +/** Replace the state of the digest object into with the state + * of the digest object from. Requires that 'into' and 'from' + * have the same digest type. + */ +void +crypto_digest_assign(crypto_digest_t *into, + const crypto_digest_t *from) +{ + tor_assert(into); + tor_assert(from); + tor_assert(into->algorithm == from->algorithm); + const size_t alloc_bytes = crypto_digest_alloc_bytes(from->algorithm); + +#ifdef OPENSSL_HAS_SHA3 + if (from->algorithm == DIGEST_SHA3_256 || + from->algorithm == DIGEST_SHA3_512) { + EVP_MD_CTX_copy(into->d.md, from->d.md); + return; + } +#endif + + memcpy(into,from,alloc_bytes); +} + +/** Given a list of strings in lst, set the len_out-byte digest + * at digest_out to the hash of the concatenation of those strings, + * plus the optional string append, computed with the algorithm + * alg. + * out_len must be \<= DIGEST512_LEN. */ +void +crypto_digest_smartlist(char *digest_out, size_t len_out, + const smartlist_t *lst, + const char *append, + digest_algorithm_t alg) +{ + crypto_digest_smartlist_prefix(digest_out, len_out, NULL, lst, append, alg); +} + +/** Given a list of strings in lst, set the len_out-byte digest + * at digest_out to the hash of the concatenation of: the + * optional string prepend, those strings, + * and the optional string append, computed with the algorithm + * alg. + * len_out must be \<= DIGEST512_LEN. */ +void +crypto_digest_smartlist_prefix(char *digest_out, size_t len_out, + const char *prepend, + const smartlist_t *lst, + const char *append, + digest_algorithm_t alg) +{ + crypto_digest_t *d = crypto_digest_new_internal(alg); + if (prepend) + crypto_digest_add_bytes(d, prepend, strlen(prepend)); + SMARTLIST_FOREACH(lst, const char *, cp, + crypto_digest_add_bytes(d, cp, strlen(cp))); + if (append) + crypto_digest_add_bytes(d, append, strlen(append)); + crypto_digest_get_digest(d, digest_out, len_out); + crypto_digest_free(d); +} + +/** Compute the HMAC-SHA-256 of the msg_len bytes in msg, using + * the key of length key_len. Store the DIGEST256_LEN-byte + * result in hmac_out. Asserts on failure. + */ +void +crypto_hmac_sha256(char *hmac_out, + const char *key, size_t key_len, + const char *msg, size_t msg_len) +{ + /* If we've got OpenSSL >=0.9.8 we can use its hmac implementation. */ + tor_assert(key_len < INT_MAX); + tor_assert(msg_len < INT_MAX); + tor_assert(hmac_out); + unsigned char *rv = NULL; + rv = HMAC(EVP_sha256(), key, (int)key_len, (unsigned char*)msg, (int)msg_len, + (unsigned char*)hmac_out, NULL); + tor_assert(rv); +} + diff --git a/src/lib/crypt_ops/include.am b/src/lib/crypt_ops/include.am index 4730440143..c90ef6eca8 100644 --- a/src/lib/crypt_ops/include.am +++ b/src/lib/crypt_ops/include.am @@ -27,12 +27,14 @@ src_lib_libtor_crypt_ops_a_SOURCES = \ if USE_NSS src_lib_libtor_crypt_ops_a_SOURCES += \ src/lib/crypt_ops/aes_nss.c \ + src/lib/crypt_ops/crypto_digest_nss.c \ src/lib/crypt_ops/crypto_dh_nss.c \ src/lib/crypt_ops/crypto_nss_mgt.c \ src/lib/crypt_ops/crypto_rsa_nss.c else src_lib_libtor_crypt_ops_a_SOURCES += \ src/lib/crypt_ops/aes_openssl.c \ + src/lib/crypt_ops/crypto_digest_openssl.c \ src/lib/crypt_ops/crypto_rsa_openssl.c endif From 27eec505270ba6378e8ff732da5432b30ac107f7 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Sun, 3 Mar 2019 11:51:08 +0200 Subject: [PATCH 0549/2557] manpage: fix formatting of example on quoting options with spaces --- changes/ticket29635 | 3 +++ doc/tor.1.txt | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 changes/ticket29635 diff --git a/changes/ticket29635 b/changes/ticket29635 new file mode 100644 index 0000000000..cbadbf648a --- /dev/null +++ b/changes/ticket29635 @@ -0,0 +1,3 @@ + o Minor bugfixes (documentation, manpage): + - Use proper formatting when providing an example on quoting options that + contain whitespace. Fixes bug 29635; bugfix on 0.2.3.18-rc. diff --git a/doc/tor.1.txt b/doc/tor.1.txt index 52f5bfa0c3..ee91976066 100644 --- a/doc/tor.1.txt +++ b/doc/tor.1.txt @@ -147,8 +147,8 @@ instance, you can tell Tor to start listening for SOCKS connections on port 9999 by passing --SocksPort 9999 or SocksPort 9999 to it on the command line, or by putting "SocksPort 9999" in the configuration file. You will need to quote options with spaces in them: if you want Tor to log all debugging -messages to debug.log, you will probably need to say --Log 'debug file -debug.log'. +messages to debug.log, you will probably need to say **--Log** `"debug file +debug.log"`. Options on the command line override those in configuration files. See the next section for more information. From e52d725977ffc7d47992375772b5936d36bb9d7d Mon Sep 17 00:00:00 2001 From: teor Date: Mon, 4 Mar 2019 11:21:15 +1000 Subject: [PATCH 0550/2557] doc: Improve the monotonic time module and function documentation Explain what "monotonic" actually means, and document some results that have surprised people. Fixes bug 29640; bugfix on 0.2.9.1-alpha. --- changes/bug29640 | 4 +++ src/lib/time/compat_time.c | 10 +++++++- src/lib/time/compat_time.h | 50 ++++++++++++++++++++++++++++++++++++-- 3 files changed, 61 insertions(+), 3 deletions(-) create mode 100644 changes/bug29640 diff --git a/changes/bug29640 b/changes/bug29640 new file mode 100644 index 0000000000..81adeae32a --- /dev/null +++ b/changes/bug29640 @@ -0,0 +1,4 @@ + o Minor bugfixes (documentation): + - Improve the monotonic time module and function documentation. Explain + what "monotonic" actually means, and document some results that have + surprised people. Fixes bug 29640; bugfix on 0.2.9.1-alpha. diff --git a/src/lib/time/compat_time.c b/src/lib/time/compat_time.c index 33e077a587..94823b8a0a 100644 --- a/src/lib/time/compat_time.c +++ b/src/lib/time/compat_time.c @@ -164,6 +164,8 @@ static int64_t last_tick_count = 0; * to be monotonic; increments them as appropriate so that they actually * _are_ monotonic. * + * The returned time may be the same as the previous returned time. + * * Caller must hold lock. */ STATIC int64_t ratchet_performance_counter(int64_t count_raw) @@ -202,6 +204,8 @@ static struct timeval timeofday_offset = { 0, 0 }; * supposed to be monotonic; increments them as appropriate so that they * actually _are_ monotonic. * + * The returned time may be the same as the previous returned time. + * * Caller must hold lock. */ STATIC void ratchet_timeval(const struct timeval *timeval_raw, struct timeval *out) @@ -270,7 +274,9 @@ monotime_init_internal(void) } /** - * Set "out" to the most recent monotonic time value + * Set "out" to the most recent monotonic time value. + * + * The returned time may be the same as the previous returned time. */ void monotime_get(monotime_t *out) @@ -302,6 +308,8 @@ monotime_coarse_get(monotime_coarse_t *out) /** * Return the number of nanoseconds between start and end. + * + * The returned value may be equal to zero. */ int64_t monotime_diff_nsec(const monotime_t *start, diff --git a/src/lib/time/compat_time.h b/src/lib/time/compat_time.h index 2cd4b3bee3..360d92e5c9 100644 --- a/src/lib/time/compat_time.h +++ b/src/lib/time/compat_time.h @@ -15,11 +15,29 @@ * of tens of milliseconds. */ -/* Q: Should you use monotime or monotime_coarse as your source? +/* Q: When should I use monotonic time? + * + * A: If you need a time that never decreases, use monotonic time. If you need + * to send a time to a user or another process, or store a time, use the + * wall-clock time. + * + * Q: Should you use monotime or monotime_coarse as your source? * * A: Generally, you get better precision with monotime, but better * performance with monotime_coarse. * + * Q: What is a "monotonic" time, exactly? + * + * A: Monotonic times are strictly non-decreasing. The difference between any + * previous monotonic time, and the current monotonic time, is always greater + * than *or equal to* zero. + * Zero deltas happen more often: + * - on Windows (due to an OS bug), + * - when using monotime_coarse, or on systems with low-resolution timers, + * - on platforms where we emulate monotonic time using wall-clock time, and + * - when using time units that are larger than nanoseconds (due to + * truncation on division). + * * Q: Should you use monotime_t or monotime_coarse_t directly? Should you use * usec? msec? "stamp units?" * @@ -95,7 +113,7 @@ * All, "timestamp units": Cheap everywhere: it never divides. * * Q: This is only somewhat related, but how much precision could I hope for - * from a libevent time.? + * from a libevent time? * * A: Actually, it's _very_ related if you're timing in order to have a * timeout happen. @@ -182,26 +200,36 @@ void monotime_init(void); void monotime_get(monotime_t *out); /** * Return the number of nanoseconds between start and end. + * The returned value may be equal to zero. */ int64_t monotime_diff_nsec(const monotime_t *start, const monotime_t *end); /** * Return the number of microseconds between start and end. + * The returned value may be equal to zero. + * Fractional units are truncated, not rounded. */ int64_t monotime_diff_usec(const monotime_t *start, const monotime_t *end); /** * Return the number of milliseconds between start and end. + * The returned value may be equal to zero. + * Fractional units are truncated, not rounded. */ int64_t monotime_diff_msec(const monotime_t *start, const monotime_t *end); /** * Return the number of nanoseconds since the timer system was initialized. + * The returned value may be equal to zero. */ uint64_t monotime_absolute_nsec(void); /** * Return the number of microseconds since the timer system was initialized. + * The returned value may be equal to zero. + * Fractional units are truncated, not rounded. */ MOCK_DECL(uint64_t, monotime_absolute_usec,(void)); /** * Return the number of milliseconds since the timer system was initialized. + * The returned value may be equal to zero. + * Fractional units are truncated, not rounded. */ uint64_t monotime_absolute_msec(void); @@ -225,6 +253,9 @@ void monotime_add_msec(monotime_t *out, const monotime_t *val, uint32_t msec); * Set out to the current coarse time. */ void monotime_coarse_get(monotime_coarse_t *out); +/** + * Like monotime_absolute_*(), but faster on some platforms. + */ uint64_t monotime_coarse_absolute_nsec(void); uint64_t monotime_coarse_absolute_usec(void); uint64_t monotime_coarse_absolute_msec(void); @@ -248,18 +279,27 @@ uint32_t monotime_coarse_to_stamp(const monotime_coarse_t *t); /** * Convert a difference, expressed in the units of monotime_coarse_to_stamp, * into an approximate number of milliseconds. + * + * The returned value may be equal to zero. + * Fractional units are truncated, not rounded. */ uint64_t monotime_coarse_stamp_units_to_approx_msec(uint64_t units); uint64_t monotime_msec_to_approx_coarse_stamp_units(uint64_t msec); uint32_t monotime_coarse_get_stamp(void); #if defined(MONOTIME_COARSE_TYPE_IS_DIFFERENT) +/** + * Like monotime_diff_*(), but faster on some platforms. + */ int64_t monotime_coarse_diff_nsec(const monotime_coarse_t *start, const monotime_coarse_t *end); int64_t monotime_coarse_diff_usec(const monotime_coarse_t *start, const monotime_coarse_t *end); int64_t monotime_coarse_diff_msec(const monotime_coarse_t *start, const monotime_coarse_t *end); +/** + * Like monotime_*(), but faster on some platforms. + */ void monotime_coarse_zero(monotime_coarse_t *out); int monotime_coarse_is_zero(const monotime_coarse_t *val); void monotime_coarse_add_msec(monotime_coarse_t *out, @@ -278,6 +318,9 @@ void monotime_coarse_add_msec(monotime_coarse_t *out, * * Requires that the difference fit into an int32_t; not for use with * large time differences. + * + * The returned value may be equal to zero. + * Fractional units are truncated, not rounded. */ int32_t monotime_coarse_diff_msec32_(const monotime_coarse_t *start, const monotime_coarse_t *end); @@ -287,6 +330,9 @@ int32_t monotime_coarse_diff_msec32_(const monotime_coarse_t *start, * * Requires that the difference fit into an int32_t; not for use with * large time differences. + * + * The returned value may be equal to zero. + * Fractional units are truncated, not rounded. */ static inline int32_t monotime_coarse_diff_msec32(const monotime_coarse_t *start, From f186f21a4efb9dafe2c29f850126d509079c0c37 Mon Sep 17 00:00:00 2001 From: teor Date: Mon, 4 Mar 2019 11:22:02 +1000 Subject: [PATCH 0551/2557] doc: Fix an incorrect comment about calling FreeLibrary() on Windows There's an incorrect comment in compat_time.c that suggests we call FreeLibrary() before we're done using the library's functions. See 29642 for background. Closes ticket 29643. --- src/lib/time/compat_time.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/lib/time/compat_time.c b/src/lib/time/compat_time.c index 33e077a587..c6625c7806 100644 --- a/src/lib/time/compat_time.c +++ b/src/lib/time/compat_time.c @@ -522,7 +522,9 @@ monotime_init_internal(void) GetTickCount64_fn = (GetTickCount64_fn_t) GetProcAddress(h, "GetTickCount64"); } - // FreeLibrary(h) ? + // We can't call FreeLibrary(h) here, because freeing the handle may + // unload the library, and cause future calls to GetTickCount64_fn() + // to fail. See 29642 for details. } void From 4578c3eb219e93da9b1aaa37c4f3c5f61c7106f5 Mon Sep 17 00:00:00 2001 From: David Fifield Date: Fri, 22 Feb 2019 23:54:17 -0700 Subject: [PATCH 0552/2557] Set TOR_PT_EXIT_ON_STDIN_CLOSE=1 for client transports too. Closes #25614. --- changes/ticket25614 | 3 +++ src/feature/client/transports.c | 10 +++++----- 2 files changed, 8 insertions(+), 5 deletions(-) create mode 100644 changes/ticket25614 diff --git a/changes/ticket25614 b/changes/ticket25614 new file mode 100644 index 0000000000..ad05549021 --- /dev/null +++ b/changes/ticket25614 @@ -0,0 +1,3 @@ + o Minor bugfixes (pluggable transports): + - Tor now sets TOR_PT_EXIT_ON_STDIN_CLOSE=1 for client transports as + well as servers. Closes ticket 25614. diff --git a/src/feature/client/transports.c b/src/feature/client/transports.c index e7ff3bf34a..6fb357b466 100644 --- a/src/feature/client/transports.c +++ b/src/feature/client/transports.c @@ -1424,11 +1424,6 @@ create_managed_proxy_environment(const managed_proxy_t *mp) } else { smartlist_add_asprintf(envs, "TOR_PT_EXTENDED_SERVER_PORT="); } - - /* All new versions of tor will keep stdin open, so PTs can use it - * as a reliable termination detection mechanism. - */ - smartlist_add_asprintf(envs, "TOR_PT_EXIT_ON_STDIN_CLOSE=1"); } else { /* If ClientTransportPlugin has a HTTPS/SOCKS proxy configured, set the * TOR_PT_PROXY line. @@ -1439,6 +1434,11 @@ create_managed_proxy_environment(const managed_proxy_t *mp) } } + /* All new versions of tor will keep stdin open, so PTs can use it + * as a reliable termination detection mechanism. + */ + smartlist_add_asprintf(envs, "TOR_PT_EXIT_ON_STDIN_CLOSE=1"); + SMARTLIST_FOREACH_BEGIN(envs, const char *, env_var) { set_environment_variable_in_smartlist(merged_env_vars, env_var, tor_free_, 1); From f236c9e7f91f8844d004164d6202915a5c78e6d6 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Sun, 18 Mar 2018 14:56:59 +0100 Subject: [PATCH 0553/2557] Introduce tor_assertf() to allow logging extra error message on assert failure With format string support! --- src/lib/log/util_bug.c | 17 ++++++++++++++--- src/lib/log/util_bug.h | 16 ++++++++++++---- 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/src/lib/log/util_bug.c b/src/lib/log/util_bug.c index f42d2d2ab4..93a460156d 100644 --- a/src/lib/log/util_bug.c +++ b/src/lib/log/util_bug.c @@ -70,14 +70,25 @@ tor_set_failed_assertion_callback(void (*fn)(void)) /** Helper for tor_assert: report the assertion failure. */ void tor_assertion_failed_(const char *fname, unsigned int line, - const char *func, const char *expr) + const char *func, const char *expr, + const char *fmt, ...) { char buf[256]; + char *extra = NULL; + va_list ap; + + if (fmt) { + va_start(ap,fmt); + tor_vasprintf(&extra, fmt, ap); + va_end(ap); + } + log_err(LD_BUG, "%s:%u: %s: Assertion %s failed; aborting.", fname, line, func, expr); tor_snprintf(buf, sizeof(buf), - "Assertion %s failed in %s at %s:%u", - expr, func, fname, line); + "Assertion %s failed in %s at %s:%u: %s", + expr, func, fname, line, extra ? extra : ""); + tor_free(extra); log_backtrace(LOG_ERR, LD_BUG, buf); } diff --git a/src/lib/log/util_bug.h b/src/lib/log/util_bug.h index 18d40bbf39..c0670a50a4 100644 --- a/src/lib/log/util_bug.h +++ b/src/lib/log/util_bug.h @@ -92,13 +92,20 @@ #define tor_assert(a) STMT_BEGIN \ (void)(a); \ STMT_END +#define tor_assertf(a, fmt, ...) STMT_BEGIN \ + (void)(a); \ + (void)(fmt); \ + STMT_END #else /** Like assert(3), but send assertion failures to the log as well as to * stderr. */ -#define tor_assert(expr) STMT_BEGIN \ +#define tor_assert(expr) tor_assertf(expr, NULL) + +#define tor_assertf(expr, fmt, ...) STMT_BEGIN \ if (ASSERT_PREDICT_LIKELY_(expr)) { \ } else { \ - tor_assertion_failed_(SHORT_FILE__, __LINE__, __func__, #expr); \ + tor_assertion_failed_(SHORT_FILE__, __LINE__, __func__, #expr, \ + fmt, ##__VA_ARGS__); \ abort(); \ } STMT_END #endif /* defined(TOR_UNIT_TESTS) && defined(DISABLE_ASSERTS_IN_UNIT_TESTS) */ @@ -106,7 +113,7 @@ #define tor_assert_unreached() \ STMT_BEGIN { \ tor_assertion_failed_(SHORT_FILE__, __LINE__, __func__, \ - "line should be unreached"); \ + "line should be unreached", NULL); \ abort(); \ } STMT_END @@ -221,7 +228,8 @@ #define tor_fragile_assert() tor_assert_nonfatal_unreached_once() void tor_assertion_failed_(const char *fname, unsigned int line, - const char *func, const char *expr); + const char *func, const char *expr, + const char *fmt, ...); void tor_bug_occurred_(const char *fname, unsigned int line, const char *func, const char *expr, int once); From b6813845cf3140475f72f1f52638fa04b44c3a1b Mon Sep 17 00:00:00 2001 From: rl1987 Date: Tue, 5 Mar 2019 16:37:14 +0200 Subject: [PATCH 0554/2557] Also add tor_assertf_nonfatal() --- src/lib/log/util_bug.c | 17 ++++++++++++++--- src/lib/log/util_bug.h | 28 ++++++++++++++++++++-------- 2 files changed, 34 insertions(+), 11 deletions(-) diff --git a/src/lib/log/util_bug.c b/src/lib/log/util_bug.c index 93a460156d..bbc744f0cc 100644 --- a/src/lib/log/util_bug.c +++ b/src/lib/log/util_bug.c @@ -96,7 +96,7 @@ tor_assertion_failed_(const char *fname, unsigned int line, void tor_bug_occurred_(const char *fname, unsigned int line, const char *func, const char *expr, - int once) + int once, const char *fmt, ...) { char buf[256]; const char *once_str = once ? @@ -116,11 +116,22 @@ tor_bug_occurred_(const char *fname, unsigned int line, add_captured_bug(expr); return; } + + va_list ap; + char *extra = NULL; + + if (fmt) { + va_start(ap,fmt); + tor_vasprintf(&extra, fmt, ap); + va_end(ap); + } + log_warn(LD_BUG, "%s:%u: %s: Non-fatal assertion %s failed.%s", fname, line, func, expr, once_str); tor_snprintf(buf, sizeof(buf), - "Non-fatal assertion %s failed in %s at %s:%u", - expr, func, fname, line); + "Non-fatal assertion %s failed in %s at %s:%u%s%s", + expr, func, fname, line, fmt ? " : " : "", extra); + tor_free(extra); } log_backtrace(LOG_WARN, LD_BUG, buf); diff --git a/src/lib/log/util_bug.h b/src/lib/log/util_bug.h index c0670a50a4..63c5309c98 100644 --- a/src/lib/log/util_bug.h +++ b/src/lib/log/util_bug.h @@ -143,6 +143,7 @@ #ifdef ALL_BUGS_ARE_FATAL #define tor_assert_nonfatal_unreached() tor_assert(0) #define tor_assert_nonfatal(cond) tor_assert((cond)) +#define tor_assertf_nonfatal(cond, fmt, ...) tor_assertf(cond, fmt, ...) #define tor_assert_nonfatal_unreached_once() tor_assert(0) #define tor_assert_nonfatal_once(cond) tor_assert((cond)) #define BUG(cond) \ @@ -153,24 +154,35 @@ #elif defined(TOR_UNIT_TESTS) && defined(DISABLE_ASSERTS_IN_UNIT_TESTS) #define tor_assert_nonfatal_unreached() STMT_NIL #define tor_assert_nonfatal(cond) ((void)(cond)) +#define tor_assertf_nonfatal(cond, fmt, ...) STMT_BEGIN \ + (void)cond; \ + (void)fmt; \ + STMT_END #define tor_assert_nonfatal_unreached_once() STMT_NIL #define tor_assert_nonfatal_once(cond) ((void)(cond)) #define BUG(cond) (ASSERT_PREDICT_UNLIKELY_(cond) ? 1 : 0) #else /* Normal case, !ALL_BUGS_ARE_FATAL, !DISABLE_ASSERTS_IN_UNIT_TESTS */ #define tor_assert_nonfatal_unreached() STMT_BEGIN \ - tor_bug_occurred_(SHORT_FILE__, __LINE__, __func__, NULL, 0); \ + tor_bug_occurred_(SHORT_FILE__, __LINE__, __func__, NULL, 0, NULL); \ STMT_END #define tor_assert_nonfatal(cond) STMT_BEGIN \ if (ASSERT_PREDICT_LIKELY_(cond)) { \ } else { \ - tor_bug_occurred_(SHORT_FILE__, __LINE__, __func__, #cond, 0); \ + tor_bug_occurred_(SHORT_FILE__, __LINE__, __func__, #cond, 0, NULL);\ + } \ + STMT_END +#define tor_assertf_nonfatal(cond, fmt, ...) STMT_BEGIN \ + if (ASSERT_PREDICT_UNLIKELY_(cond)) { \ + } else { \ + tor_bug_occurred_(SHORT_FILE__, __LINE__, __func__, #cond, 0, \ + fmt, ##__VA_ARGS__); \ } \ STMT_END #define tor_assert_nonfatal_unreached_once() STMT_BEGIN \ static int warning_logged__ = 0; \ if (!warning_logged__) { \ warning_logged__ = 1; \ - tor_bug_occurred_(SHORT_FILE__, __LINE__, __func__, NULL, 1); \ + tor_bug_occurred_(SHORT_FILE__, __LINE__, __func__, NULL, 1, NULL); \ } \ STMT_END #define tor_assert_nonfatal_once(cond) STMT_BEGIN \ @@ -178,12 +190,12 @@ if (ASSERT_PREDICT_LIKELY_(cond)) { \ } else if (!warning_logged__) { \ warning_logged__ = 1; \ - tor_bug_occurred_(SHORT_FILE__, __LINE__, __func__, #cond, 1); \ + tor_bug_occurred_(SHORT_FILE__, __LINE__, __func__, #cond, 1, NULL);\ } \ STMT_END #define BUG(cond) \ (ASSERT_PREDICT_UNLIKELY_(cond) ? \ - (tor_bug_occurred_(SHORT_FILE__,__LINE__,__func__,"!("#cond")",0), 1) \ + (tor_bug_occurred_(SHORT_FILE__,__LINE__,__func__,"!("#cond")",1,NULL),1) \ : 0) #endif /* defined(ALL_BUGS_ARE_FATAL) || ... */ @@ -195,7 +207,7 @@ if (bool_result && !var) { \ var = 1; \ tor_bug_occurred_(SHORT_FILE__, __LINE__, __func__, \ - "!("#cond")", 1); \ + "!("#cond")", 1, NULL); \ } \ bool_result; } )) #else /* !(defined(__GNUC__)) */ @@ -205,7 +217,7 @@ (var ? 1 : \ (var=1, \ tor_bug_occurred_(SHORT_FILE__, __LINE__, __func__, \ - "!("#cond")", 1), \ + "!("#cond")", 1, NULL), \ 1)) \ : 0) #endif /* defined(__GNUC__) */ @@ -232,7 +244,7 @@ void tor_assertion_failed_(const char *fname, unsigned int line, const char *fmt, ...); void tor_bug_occurred_(const char *fname, unsigned int line, const char *func, const char *expr, - int once); + int once, const char *fmt, ...); #ifdef _WIN32 #define SHORT_FILE__ (tor_fix_source_file(__FILE__)) From c9a9de120f4287ccb5cd3998916cbe7b81dba89a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Tue, 5 Mar 2019 15:50:30 +0100 Subject: [PATCH 0555/2557] Fix changelog for ticket 25614 to make it pass check-changes. Yawning's commit in fda61e030e12cbd66a2be9905e6855ec6c6a3c63 was used to find which version this is a bugfix on. --- changes/ticket25614 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changes/ticket25614 b/changes/ticket25614 index ad05549021..82988eeace 100644 --- a/changes/ticket25614 +++ b/changes/ticket25614 @@ -1,3 +1,3 @@ o Minor bugfixes (pluggable transports): - Tor now sets TOR_PT_EXIT_ON_STDIN_CLOSE=1 for client transports as - well as servers. Closes ticket 25614. + well as servers. Fixes bug 25614; bugfix on 0.2.7.1-alpha. From a999cb43df3dc4e64820347486a2d861636bfc2e Mon Sep 17 00:00:00 2001 From: David Goulet Date: Fri, 1 Mar 2019 13:50:00 -0500 Subject: [PATCH 0556/2557] protover: Add missing Padding to translate_to_rust This commit also explicitly set the value of the PRT enum so we can match/pin the C enum values to the Rust one in protover/ffi.rs. Fixes #29631 Signed-off-by: David Goulet --- changes/ticket29631 | 4 ++++ src/core/or/protover.h | 22 +++++++++++----------- src/rust/protover/ffi.rs | 1 + 3 files changed, 16 insertions(+), 11 deletions(-) create mode 100644 changes/ticket29631 diff --git a/changes/ticket29631 b/changes/ticket29631 new file mode 100644 index 0000000000..9fc194ba96 --- /dev/null +++ b/changes/ticket29631 @@ -0,0 +1,4 @@ + o Minor bugfixes (Rust, protover): + - The Rust implementation of protover was missing the "Padding" value in + the translate function from C to Rust. Fixes bug 29631; bugfix on + 0.4.0.1-alpha. diff --git a/src/core/or/protover.h b/src/core/or/protover.h index 27106d4bec..567b94a168 100644 --- a/src/core/or/protover.h +++ b/src/core/or/protover.h @@ -33,17 +33,17 @@ struct smartlist_t; /// C_RUST_COUPLED: src/rust/protover/ffi.rs `translate_to_rust` /// C_RUST_COUPLED: src/rust/protover/protover.rs `Proto` typedef enum protocol_type_t { - PRT_LINK, - PRT_LINKAUTH, - PRT_RELAY, - PRT_DIRCACHE, - PRT_HSDIR, - PRT_HSINTRO, - PRT_HSREND, - PRT_DESC, - PRT_MICRODESC, - PRT_CONS, - PRT_PADDING, + PRT_LINK = 0, + PRT_LINKAUTH = 1, + PRT_RELAY = 2, + PRT_DIRCACHE = 3, + PRT_HSDIR = 4, + PRT_HSINTRO = 5, + PRT_HSREND = 6, + PRT_DESC = 7, + PRT_MICRODESC = 8, + PRT_CONS = 9, + PRT_PADDING = 10, } protocol_type_t; bool protover_contains_long_protocol_names(const char *s); diff --git a/src/rust/protover/ffi.rs b/src/rust/protover/ffi.rs index 6ee63adb10..066b08eddb 100644 --- a/src/rust/protover/ffi.rs +++ b/src/rust/protover/ffi.rs @@ -30,6 +30,7 @@ fn translate_to_rust(c_proto: uint32_t) -> Result { 7 => Ok(Protocol::Desc), 8 => Ok(Protocol::Microdesc), 9 => Ok(Protocol::Cons), + 10 => Ok(Protocol::Padding), _ => Err(ProtoverError::UnknownProtocol), } } From ae5a0f39cd3d0098ca684c7dfb83ebfe3acca8b6 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Wed, 20 Feb 2019 09:35:27 +0200 Subject: [PATCH 0557/2557] Update git pre-push hook so that only upstream branches can get pushed to origin --- changes/feature29532 | 4 ++++ scripts/maint/pre-push.git-hook | 21 ++++++++++++++++++--- 2 files changed, 22 insertions(+), 3 deletions(-) create mode 100644 changes/feature29532 diff --git a/changes/feature29532 b/changes/feature29532 new file mode 100644 index 0000000000..4d95e6bca8 --- /dev/null +++ b/changes/feature29532 @@ -0,0 +1,4 @@ + o Minor features (developer tooling): + - Modify git pre-push hook script to disallow pushing branches other than + master, release-* and maint-* to origin remote. Implements feature + 29532. diff --git a/scripts/maint/pre-push.git-hook b/scripts/maint/pre-push.git-hook index 26296023fb..78d62527e0 100755 --- a/scripts/maint/pre-push.git-hook +++ b/scripts/maint/pre-push.git-hook @@ -1,17 +1,32 @@ #!/bin/bash +# git pre-push hook script to prevent "fixup!" and "squash!" commit +# from ending up in master, or in any branch if CUR_BRANCH check is removed. +# Furthermore, it disallows pushing branches other than master, release-* +# and maint-* to origin (e.g. gitweb.torproject.org). +# # To install this script, copy it into .git/hooks/pre-push path in your # local copy of git repository. Make sure it has permission to execute. # -# This is git pre-push hook script to prevent "fixup!" and "squash!" commits -# from ending up in upstream branches (master, release-* or maint-*). -# # The following sample script was used as starting point: # https://github.com/git/git/blob/master/templates/hooks--pre-push.sample z40=0000000000000000000000000000000000000000 +remote="$1" CUR_BRANCH=$(git rev-parse --abbrev-ref HEAD) + +# Only allow pushing master, release-* and maint-* branches to origin. +if [ "$remote" == "origin" ] +then + if [ "$CUR_BRANCH" != "master" ] && [[ $CUR_BRANCH != release-* ]] && + [[ $CUR_BRANCH != maint-* ]] + then + echo >&2 "Not pushing $CUR_BRANCH to origin" + exit 1 + fi +fi + if [ "$CUR_BRANCH" != "master" ] && [[ $CUR_BRANCH != release-* ]] && [[ $CUR_BRANCH != maint-* ]] then From f3eac74ed9c68212a9df1a1acbdf146662469a5a Mon Sep 17 00:00:00 2001 From: rl1987 Date: Wed, 20 Feb 2019 19:48:52 +0200 Subject: [PATCH 0558/2557] In pre-push hook script, actually check local and remote refs --- scripts/maint/pre-push.git-hook | 38 ++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/scripts/maint/pre-push.git-hook b/scripts/maint/pre-push.git-hook index 78d62527e0..14bc0690c1 100755 --- a/scripts/maint/pre-push.git-hook +++ b/scripts/maint/pre-push.git-hook @@ -11,29 +11,20 @@ # The following sample script was used as starting point: # https://github.com/git/git/blob/master/templates/hooks--pre-push.sample +echo "Running pre-push hook" + z40=0000000000000000000000000000000000000000 remote="$1" -CUR_BRANCH=$(git rev-parse --abbrev-ref HEAD) -# Only allow pushing master, release-* and maint-* branches to origin. -if [ "$remote" == "origin" ] -then - if [ "$CUR_BRANCH" != "master" ] && [[ $CUR_BRANCH != release-* ]] && - [[ $CUR_BRANCH != maint-* ]] +ref_is_upstream_branch() { + if [ "$1" == "refs/heads/master" ] || + [[ "$1" == refs/heads/release-* ]] || + [[ "$1" == refs/heads/maint-* ]] then - echo >&2 "Not pushing $CUR_BRANCH to origin" - exit 1 + return 1 fi -fi - -if [ "$CUR_BRANCH" != "master" ] && [[ $CUR_BRANCH != release-* ]] && - [[ $CUR_BRANCH != maint-* ]] -then - exit 0 -fi - -echo "Running pre-push hook" +} # shellcheck disable=SC2034 while read -r local_ref local_sha remote_ref remote_sha @@ -52,6 +43,19 @@ do range="$remote_sha..$local_sha" fi + if ref_is_upstream_branch "$local_ref" == 0 || + ref_is_upstream_branch "$remote_ref" == 0 + then + if [ "$remote" == "origin" ] + then + echo >&2 "Not pushing: $local_ref to $remote_ref" + echo >&2 "If you really want to push this, use --no-verify." + exit 1 + else + continue + fi + fi + # Check for fixup! commit commit=$(git rev-list -n 1 --grep '^fixup!' "$range") if [ -n "$commit" ] From 0deea98d021d105eeb5dc5c43c144847ad188ea5 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Wed, 20 Feb 2019 19:56:37 +0200 Subject: [PATCH 0559/2557] Improve pre-push.git-hook description --- scripts/maint/pre-push.git-hook | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/scripts/maint/pre-push.git-hook b/scripts/maint/pre-push.git-hook index 14bc0690c1..24043978b6 100755 --- a/scripts/maint/pre-push.git-hook +++ b/scripts/maint/pre-push.git-hook @@ -1,9 +1,10 @@ #!/bin/bash -# git pre-push hook script to prevent "fixup!" and "squash!" commit -# from ending up in master, or in any branch if CUR_BRANCH check is removed. -# Furthermore, it disallows pushing branches other than master, release-* -# and maint-* to origin (e.g. gitweb.torproject.org). +# git pre-push hook script to: +# 1) prevent "fixup!" and "squash!" commit from ending up in master, release-* +# or maint-* +# 2) Disallow pushing branches other than master, release-* +# and maint-* to origin (e.g. gitweb.torproject.org). # # To install this script, copy it into .git/hooks/pre-push path in your # local copy of git repository. Make sure it has permission to execute. From 7f0516022bb84378df13136273b42019e54157f9 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Fri, 22 Feb 2019 17:05:07 +0200 Subject: [PATCH 0560/2557] Also disallow pushing to/from upstream branch when branch names do not match --- scripts/maint/pre-push.git-hook | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/maint/pre-push.git-hook b/scripts/maint/pre-push.git-hook index 24043978b6..e7a72efa08 100755 --- a/scripts/maint/pre-push.git-hook +++ b/scripts/maint/pre-push.git-hook @@ -44,8 +44,9 @@ do range="$remote_sha..$local_sha" fi - if ref_is_upstream_branch "$local_ref" == 0 || - ref_is_upstream_branch "$remote_ref" == 0 + if (ref_is_upstream_branch "$local_ref" == 0 || + ref_is_upstream_branch "$remote_ref" == 0) && + [ "$local_ref" != "$remote_ref" ] then if [ "$remote" == "origin" ] then From cd67911033e83e9b83a5bb5d62832fb0c6077ca7 Mon Sep 17 00:00:00 2001 From: Mike Perry Date: Thu, 21 Feb 2019 01:34:55 +0000 Subject: [PATCH 0561/2557] Bug 29204: Inspect circuit queues before sending padding. Mitigates OOM conditions at relays. --- changes/bug29204 | 4 ++++ src/core/or/circuitpadding.c | 21 +++++++++++++++++---- 2 files changed, 21 insertions(+), 4 deletions(-) create mode 100644 changes/bug29204 diff --git a/changes/bug29204 b/changes/bug29204 new file mode 100644 index 0000000000..ec2cf67b2f --- /dev/null +++ b/changes/bug29204 @@ -0,0 +1,4 @@ + o Minor bugfixes (circuitpadding): + - Inspect circuit-level cell queue before sending padding, to avoid + sending padding while too much data is queued. Fixes bug 29204; + bugfix on 0.4.0.1-alpha. diff --git a/src/core/or/circuitpadding.c b/src/core/or/circuitpadding.c index 0dadc52139..ba6bfe1f53 100644 --- a/src/core/or/circuitpadding.c +++ b/src/core/or/circuitpadding.c @@ -61,6 +61,7 @@ #include "core/or/crypt_path_st.h" #include "core/or/circuit_st.h" #include "core/or/origin_circuit_st.h" +#include "core/or/or_circuit_st.h" #include "feature/nodelist/routerstatus_st.h" #include "feature/nodelist/node_st.h" #include "core/or/cell_st.h" @@ -81,6 +82,7 @@ static double circpad_distribution_sample(circpad_distribution_t dist); /** Cached consensus params */ static uint8_t circpad_global_max_padding_percent; static uint16_t circpad_global_allowed_cells; +static uint16_t circpad_max_circ_queued_cells; /** Global cell counts, for rate limiting */ static uint64_t circpad_global_padding_sent; @@ -1027,10 +1029,17 @@ circpad_send_padding_cell_for_callback(circpad_machine_state_t *mi) } else { // If we're a non-origin circ, we can just send from here as if we're the // edge. - log_fn(LOG_INFO,LD_CIRC, - "Callback: Sending padding to non-origin circuit."); - relay_send_command_from_edge(0, mi->on_circ, RELAY_COMMAND_DROP, NULL, - 0, NULL); + if (TO_OR_CIRCUIT(circ)->p_chan_cells.n <= circpad_max_circ_queued_cells) { + log_fn(LOG_INFO,LD_CIRC, + "Callback: Sending padding to non-origin circuit."); + relay_send_command_from_edge(0, mi->on_circ, RELAY_COMMAND_DROP, NULL, + 0, NULL); + } else { + static ratelim_t cell_lim = RATELIM_INIT(600); + log_fn_ratelim(&cell_lim,LOG_NOTICE,LD_CIRC, + "Too many cells (%d) in circ queue to send padding.", + TO_OR_CIRCUIT(circ)->p_chan_cells.n); + } } rep_hist_padding_count_write(PADDING_TYPE_DROP); @@ -1093,6 +1102,10 @@ circpad_new_consensus_params(const networkstatus_t *ns) circpad_global_max_padding_percent = networkstatus_get_param(ns, "circpad_global_max_padding_pct", 0, 0, 100); + + circpad_max_circ_queued_cells = + networkstatus_get_param(ns, "circpad_max_circ_queued_cells", + CIRCWINDOW_START_MAX, 0, 50*CIRCWINDOW_START_MAX); } /** From ff410edec04793d6d72f9961acd2f13a5ee3b9b5 Mon Sep 17 00:00:00 2001 From: Mike Perry Date: Thu, 21 Feb 2019 01:34:55 +0000 Subject: [PATCH 0562/2557] Bug 29204: Inspect circuit queues before sending padding. Mitigates OOM conditions at relays. --- changes/bug29204 | 4 ++++ src/core/or/circuitpadding.c | 21 +++++++++++++++++---- 2 files changed, 21 insertions(+), 4 deletions(-) create mode 100644 changes/bug29204 diff --git a/changes/bug29204 b/changes/bug29204 new file mode 100644 index 0000000000..ec2cf67b2f --- /dev/null +++ b/changes/bug29204 @@ -0,0 +1,4 @@ + o Minor bugfixes (circuitpadding): + - Inspect circuit-level cell queue before sending padding, to avoid + sending padding while too much data is queued. Fixes bug 29204; + bugfix on 0.4.0.1-alpha. diff --git a/src/core/or/circuitpadding.c b/src/core/or/circuitpadding.c index 0dadc52139..ba6bfe1f53 100644 --- a/src/core/or/circuitpadding.c +++ b/src/core/or/circuitpadding.c @@ -61,6 +61,7 @@ #include "core/or/crypt_path_st.h" #include "core/or/circuit_st.h" #include "core/or/origin_circuit_st.h" +#include "core/or/or_circuit_st.h" #include "feature/nodelist/routerstatus_st.h" #include "feature/nodelist/node_st.h" #include "core/or/cell_st.h" @@ -81,6 +82,7 @@ static double circpad_distribution_sample(circpad_distribution_t dist); /** Cached consensus params */ static uint8_t circpad_global_max_padding_percent; static uint16_t circpad_global_allowed_cells; +static uint16_t circpad_max_circ_queued_cells; /** Global cell counts, for rate limiting */ static uint64_t circpad_global_padding_sent; @@ -1027,10 +1029,17 @@ circpad_send_padding_cell_for_callback(circpad_machine_state_t *mi) } else { // If we're a non-origin circ, we can just send from here as if we're the // edge. - log_fn(LOG_INFO,LD_CIRC, - "Callback: Sending padding to non-origin circuit."); - relay_send_command_from_edge(0, mi->on_circ, RELAY_COMMAND_DROP, NULL, - 0, NULL); + if (TO_OR_CIRCUIT(circ)->p_chan_cells.n <= circpad_max_circ_queued_cells) { + log_fn(LOG_INFO,LD_CIRC, + "Callback: Sending padding to non-origin circuit."); + relay_send_command_from_edge(0, mi->on_circ, RELAY_COMMAND_DROP, NULL, + 0, NULL); + } else { + static ratelim_t cell_lim = RATELIM_INIT(600); + log_fn_ratelim(&cell_lim,LOG_NOTICE,LD_CIRC, + "Too many cells (%d) in circ queue to send padding.", + TO_OR_CIRCUIT(circ)->p_chan_cells.n); + } } rep_hist_padding_count_write(PADDING_TYPE_DROP); @@ -1093,6 +1102,10 @@ circpad_new_consensus_params(const networkstatus_t *ns) circpad_global_max_padding_percent = networkstatus_get_param(ns, "circpad_global_max_padding_pct", 0, 0, 100); + + circpad_max_circ_queued_cells = + networkstatus_get_param(ns, "circpad_max_circ_queued_cells", + CIRCWINDOW_START_MAX, 0, 50*CIRCWINDOW_START_MAX); } /** From 26e6f56023e749800d95bbc5669c60197d481314 Mon Sep 17 00:00:00 2001 From: teor Date: Sat, 9 Mar 2019 10:50:55 +1000 Subject: [PATCH 0563/2557] sr: Free SRVs before replacing them in state_query_put_() Refactor the shared random state's memory management so that it actually takes ownership of the shared random value pointers. Fixes bug 29706; bugfix on 0.2.9.1-alpha. --- changes/bug29706_refactor | 4 ++++ src/or/shared_random.c | 2 +- src/or/shared_random.h | 3 ++- src/or/shared_random_state.c | 20 ++++++++++++++------ src/test/test_shared_random.c | 25 +++++++++++++++---------- 5 files changed, 36 insertions(+), 18 deletions(-) create mode 100644 changes/bug29706_refactor diff --git a/changes/bug29706_refactor b/changes/bug29706_refactor new file mode 100644 index 0000000000..ba1d0c7edd --- /dev/null +++ b/changes/bug29706_refactor @@ -0,0 +1,4 @@ + o Minor bugfixes (memory management): + - Refactor the shared random state's memory management so that it actually + takes ownership of the shared random value pointers. + Fixes bug 29706; bugfix on 0.2.9.1-alpha. diff --git a/src/or/shared_random.c b/src/or/shared_random.c index 5f6b03f1ba..c3e6409771 100644 --- a/src/or/shared_random.c +++ b/src/or/shared_random.c @@ -112,7 +112,7 @@ static const char sr_flag_ns_str[] = "shared-rand-participate"; static int32_t num_srv_agreements_from_vote; /* Return a heap allocated copy of the SRV orig. */ -STATIC sr_srv_t * +sr_srv_t * srv_dup(const sr_srv_t *orig) { sr_srv_t *duplicate = NULL; diff --git a/src/or/shared_random.h b/src/or/shared_random.h index 9885934cc7..68a1019794 100644 --- a/src/or/shared_random.h +++ b/src/or/shared_random.h @@ -129,6 +129,8 @@ const char *sr_commit_get_rsa_fpr(const sr_commit_t *commit) void sr_compute_srv(void); sr_commit_t *sr_generate_our_commit(time_t timestamp, const authority_cert_t *my_rsa_cert); +sr_srv_t *srv_dup(const sr_srv_t *orig); + #ifdef SHARED_RANDOM_PRIVATE /* Encode */ @@ -146,7 +148,6 @@ STATIC sr_srv_t *get_majority_srv_from_votes(const smartlist_t *votes, int current); STATIC void save_commit_to_state(sr_commit_t *commit); -STATIC sr_srv_t *srv_dup(const sr_srv_t *orig); STATIC int commitments_are_the_same(const sr_commit_t *commit_one, const sr_commit_t *commit_two); STATIC int commit_is_authoritative(const sr_commit_t *commit, diff --git a/src/or/shared_random_state.c b/src/or/shared_random_state.c index f27eccafc7..e99817106c 100644 --- a/src/or/shared_random_state.c +++ b/src/or/shared_random_state.c @@ -92,6 +92,8 @@ static const config_format_t state_format = { &state_extra_var, }; +static void state_query_del_(sr_state_object_t obj_type, void *data); + /* Return a string representation of a protocol phase. */ STATIC const char * get_phase_str(sr_phase_t phase) @@ -883,7 +885,8 @@ state_query_get_(sr_state_object_t obj_type, const void *data) } /* Helper function: This handles the PUT state action using an - * obj_type and data needed for the action. */ + * obj_type and data needed for the action. + * PUT frees the previous data before replacing it, if needed. */ static void state_query_put_(sr_state_object_t obj_type, void *data) { @@ -892,13 +895,18 @@ state_query_put_(sr_state_object_t obj_type, void *data) { sr_commit_t *commit = data; tor_assert(commit); + /* commit_add_to_state() frees the old commit, if there is one */ commit_add_to_state(commit, sr_state); break; } case SR_STATE_OBJ_CURSRV: + /* Always free the old SRV */ + state_query_del_(SR_STATE_OBJ_CURSRV, NULL); sr_state->current_srv = (sr_srv_t *) data; break; case SR_STATE_OBJ_PREVSRV: + /* Always free the old SRV */ + state_query_del_(SR_STATE_OBJ_PREVSRV, NULL); sr_state->previous_srv = (sr_srv_t *) data; break; case SR_STATE_OBJ_VALID_AFTER: @@ -1021,16 +1029,16 @@ state_del_previous_srv(void) state_query(SR_STATE_ACTION_DEL, SR_STATE_OBJ_PREVSRV, NULL, NULL); } -/* Rotate SRV value by freeing the previous value, assigning the current - * value to the previous one and nullifying the current one. */ +/* Rotate SRV value by setting the previous SRV to the current SRV, and + * clearing the current SRV. */ STATIC void state_rotate_srv(void) { /* First delete previous SRV from the state. Object will be freed. */ state_del_previous_srv(); - /* Set previous SRV with the current one. */ - sr_state_set_previous_srv(sr_state_get_current_srv()); - /* Nullify the current srv. */ + /* Set previous SRV to a copy of the current one. */ + sr_state_set_previous_srv(srv_dup(sr_state_get_current_srv())); + /* Free and NULL the current srv. */ sr_state_set_current_srv(NULL); } diff --git a/src/test/test_shared_random.c b/src/test/test_shared_random.c index 0a3c2e119b..5d3e574fac 100644 --- a/src/test/test_shared_random.c +++ b/src/test/test_shared_random.c @@ -1013,6 +1013,7 @@ test_state_transition(void *arg) { sr_state_t *state = NULL; time_t now = time(NULL); + sr_srv_t *cur = NULL; (void) arg; @@ -1051,44 +1052,47 @@ test_state_transition(void *arg) /* Test SRV rotation in our state. */ { - const sr_srv_t *cur, *prev; test_sr_setup_srv(1); - cur = sr_state_get_current_srv(); + tt_assert(sr_state_get_current_srv()); + /* Take a copy of the data, because the state owns the pointer */ + cur = srv_dup(sr_state_get_current_srv()); tt_assert(cur); - /* After, current srv should be the previous and then set to NULL. */ + /* After, the previous SRV should be the same as the old current SRV, and + * the current SRV should be set to NULL */ state_rotate_srv(); - prev = sr_state_get_previous_srv(); - tt_assert(prev == cur); + tt_mem_op(sr_state_get_previous_srv(), OP_EQ, cur, sizeof(sr_srv_t)); tt_assert(!sr_state_get_current_srv()); sr_state_clean_srvs(); + tor_free(cur); } /* New protocol run. */ { - const sr_srv_t *cur; /* Setup some new SRVs so we can confirm that a new protocol run * actually makes them rotate and compute new ones. */ test_sr_setup_srv(1); - cur = sr_state_get_current_srv(); - tt_assert(cur); + tt_assert(sr_state_get_current_srv()); + /* Take a copy of the data, because the state owns the pointer */ + cur = srv_dup(sr_state_get_current_srv()); set_sr_phase(SR_PHASE_REVEAL); MOCK(get_my_v3_authority_cert, get_my_v3_authority_cert_m); new_protocol_run(now); UNMOCK(get_my_v3_authority_cert); /* Rotation happened. */ - tt_assert(sr_state_get_previous_srv() == cur); + tt_mem_op(sr_state_get_previous_srv(), OP_EQ, cur, sizeof(sr_srv_t)); /* We are going into COMMIT phase so we had to rotate our SRVs. Usually * our current SRV would be NULL but a new protocol run should make us * compute a new SRV. */ tt_assert(sr_state_get_current_srv()); /* Also, make sure we did change the current. */ - tt_assert(sr_state_get_current_srv() != cur); + tt_mem_op(sr_state_get_current_srv(), OP_NE, cur, sizeof(sr_srv_t)); /* We should have our commitment alone. */ tt_int_op(digestmap_size(state->commits), ==, 1); tt_int_op(state->n_reveal_rounds, ==, 0); tt_int_op(state->n_commit_rounds, ==, 0); /* 46 here since we were at 45 just before. */ tt_u64_op(state->n_protocol_runs, ==, 46); + tor_free(cur); } /* Cleanup of SRVs. */ @@ -1099,6 +1103,7 @@ test_state_transition(void *arg) } done: + tor_free(cur); sr_state_free(); } From 593fde930fa52e0f4ed7aaf70c4ea9e4615d2a0e Mon Sep 17 00:00:00 2001 From: teor Date: Sat, 9 Mar 2019 11:48:05 +1000 Subject: [PATCH 0564/2557] sr: rename srv_dup() to sr_srv_dup() --- src/or/shared_random.c | 6 +++--- src/or/shared_random.h | 2 +- src/or/shared_random_state.c | 2 +- src/test/test_shared_random.c | 8 ++++---- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/or/shared_random.c b/src/or/shared_random.c index c3e6409771..ddb6de1a59 100644 --- a/src/or/shared_random.c +++ b/src/or/shared_random.c @@ -113,7 +113,7 @@ static int32_t num_srv_agreements_from_vote; /* Return a heap allocated copy of the SRV orig. */ sr_srv_t * -srv_dup(const sr_srv_t *orig) +sr_srv_dup(const sr_srv_t *orig) { sr_srv_t *duplicate = NULL; @@ -1318,8 +1318,8 @@ sr_act_post_consensus(const networkstatus_t *consensus) * decided by the majority. */ sr_state_unset_fresh_srv(); /* Set the SR values from the given consensus. */ - sr_state_set_previous_srv(srv_dup(consensus->sr_info.previous_srv)); - sr_state_set_current_srv(srv_dup(consensus->sr_info.current_srv)); + sr_state_set_previous_srv(sr_srv_dup(consensus->sr_info.previous_srv)); + sr_state_set_current_srv(sr_srv_dup(consensus->sr_info.current_srv)); } /* Prepare our state so that it's ready for the next voting period. */ diff --git a/src/or/shared_random.h b/src/or/shared_random.h index 68a1019794..c640470c16 100644 --- a/src/or/shared_random.h +++ b/src/or/shared_random.h @@ -129,7 +129,7 @@ const char *sr_commit_get_rsa_fpr(const sr_commit_t *commit) void sr_compute_srv(void); sr_commit_t *sr_generate_our_commit(time_t timestamp, const authority_cert_t *my_rsa_cert); -sr_srv_t *srv_dup(const sr_srv_t *orig); +sr_srv_t *sr_srv_dup(const sr_srv_t *orig); #ifdef SHARED_RANDOM_PRIVATE diff --git a/src/or/shared_random_state.c b/src/or/shared_random_state.c index e99817106c..50c6d3dd7a 100644 --- a/src/or/shared_random_state.c +++ b/src/or/shared_random_state.c @@ -1037,7 +1037,7 @@ state_rotate_srv(void) /* First delete previous SRV from the state. Object will be freed. */ state_del_previous_srv(); /* Set previous SRV to a copy of the current one. */ - sr_state_set_previous_srv(srv_dup(sr_state_get_current_srv())); + sr_state_set_previous_srv(sr_srv_dup(sr_state_get_current_srv())); /* Free and NULL the current srv. */ sr_state_set_current_srv(NULL); } diff --git a/src/test/test_shared_random.c b/src/test/test_shared_random.c index 5d3e574fac..93125e5135 100644 --- a/src/test/test_shared_random.c +++ b/src/test/test_shared_random.c @@ -932,7 +932,7 @@ test_utils(void *arg) { (void) arg; - /* Testing srv_dup(). */ + /* Testing sr_srv_dup(). */ { sr_srv_t *srv = NULL, *dup_srv = NULL; const char *srv_value = @@ -940,7 +940,7 @@ test_utils(void *arg) srv = tor_malloc_zero(sizeof(*srv)); srv->num_reveals = 42; memcpy(srv->value, srv_value, sizeof(srv->value)); - dup_srv = srv_dup(srv); + dup_srv = sr_srv_dup(srv); tt_assert(dup_srv); tt_u64_op(dup_srv->num_reveals, ==, srv->num_reveals); tt_mem_op(dup_srv->value, OP_EQ, srv->value, sizeof(srv->value)); @@ -1055,7 +1055,7 @@ test_state_transition(void *arg) test_sr_setup_srv(1); tt_assert(sr_state_get_current_srv()); /* Take a copy of the data, because the state owns the pointer */ - cur = srv_dup(sr_state_get_current_srv()); + cur = sr_srv_dup(sr_state_get_current_srv()); tt_assert(cur); /* After, the previous SRV should be the same as the old current SRV, and * the current SRV should be set to NULL */ @@ -1073,7 +1073,7 @@ test_state_transition(void *arg) test_sr_setup_srv(1); tt_assert(sr_state_get_current_srv()); /* Take a copy of the data, because the state owns the pointer */ - cur = srv_dup(sr_state_get_current_srv()); + cur = sr_srv_dup(sr_state_get_current_srv()); set_sr_phase(SR_PHASE_REVEAL); MOCK(get_my_v3_authority_cert, get_my_v3_authority_cert_m); new_protocol_run(now); From 4773fa647434eba582f775cb371894b7e36cdb34 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Sun, 10 Mar 2019 18:16:58 +0200 Subject: [PATCH 0565/2557] Revert "Walk back from requiring bash" This reverts commit c346eff223e94b5fbeb6e751a99393fc5f7dd4b0. --- src/test/test-network.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/test/test-network.sh b/src/test/test-network.sh index 372c8cbac3..4d56e83806 100755 --- a/src/test/test-network.sh +++ b/src/test/test-network.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/usr/bin/env bash # This script calls the equivalent script in chutney/tools @@ -18,6 +18,10 @@ ECHO="${ECHO:-echo}" # Output is prefixed with the name of the script myname=$(basename "$0") +# Save the arguments before we destroy them +# This might not preserve arguments with spaces in them +ORIGINAL_ARGS=( "$@" ) + # We need to find CHUTNEY_PATH, so that we can call the version of this script # in chutney/tools with the same arguments. We also need to respect --quiet. until [ -z "$1" ] @@ -95,7 +99,7 @@ if [ -d "$CHUTNEY_PATH" ] && [ -x "$TEST_NETWORK" ]; then # this may fail if some arguments have spaces in them # if so, set CHUTNEY_PATH before calling test-network.sh, and spaces # will be handled correctly - exec "$TEST_NETWORK" "$@" + exec "$TEST_NETWORK" "${ORIGINAL_ARGS[@]}" # $ORIGINAL_ARGS else $ECHO "$myname: Could not find tools/test-network.sh in CHUTNEY_PATH." $ECHO "$myname: Please update your chutney using 'git pull'." From 2d3ef34dcebf360a98d875639484c8e92274b19b Mon Sep 17 00:00:00 2001 From: rl1987 Date: Fri, 1 Mar 2019 17:38:37 +0200 Subject: [PATCH 0566/2557] Add post-merge git hook to warn about git hooks being updated in the repo --- scripts/maint/post-merge.git-hook | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100755 scripts/maint/post-merge.git-hook diff --git a/scripts/maint/post-merge.git-hook b/scripts/maint/post-merge.git-hook new file mode 100755 index 0000000000..eb98257f0b --- /dev/null +++ b/scripts/maint/post-merge.git-hook @@ -0,0 +1,24 @@ +#!/bin/sh + +git_toplevel=$(git rev-parse --show-toplevel) + +check_for_diffs() { + installed="$git_toplevel/.git/hooks/$1" + latest="$git_toplevel/scripts/maint/$1.git-hook" + + if [ -e "$installed" ] + then + if ! cmp "$installed" "$latest" >/dev/null 2>&1 + then + echo "ATTENTION: $1 hook has changed:" + echo "===============================" + diff "$installed" "$latest" + exit 1 + fi + fi +} + +check_for_diffs "pre-push" +check_for_diffs "pre-commit" +check_for_diffs "post-merge" + From bb8b2f47d0f63b761bdffeceaec9b5eac1b375f5 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Fri, 1 Mar 2019 17:54:54 +0200 Subject: [PATCH 0567/2557] Also print changes in git helper scripts, if any --- scripts/maint/post-merge.git-hook | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/scripts/maint/post-merge.git-hook b/scripts/maint/post-merge.git-hook index eb98257f0b..475a10f590 100755 --- a/scripts/maint/post-merge.git-hook +++ b/scripts/maint/post-merge.git-hook @@ -13,12 +13,20 @@ check_for_diffs() { echo "ATTENTION: $1 hook has changed:" echo "===============================" diff "$installed" "$latest" - exit 1 fi fi } +check_for_script_update() { + fullpath="$git_toplevel/scripts/maint/$1" + + git diff ORIG_HEAD HEAD --exit-code -- "$fullpath" +} + check_for_diffs "pre-push" check_for_diffs "pre-commit" check_for_diffs "post-merge" +check_for_script_update "git-merge-forward.sh" +check_for_script_update "git-pull-all.sh" +check_for_script_update "git-push-all.sh" From 88633fad5b1805e6d4e1ed1f41eb591bcd0add13 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Fri, 1 Mar 2019 17:58:10 +0200 Subject: [PATCH 0568/2557] Write a comment for post-merge.git-hook --- scripts/maint/post-merge.git-hook | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/scripts/maint/post-merge.git-hook b/scripts/maint/post-merge.git-hook index 475a10f590..5c3efe8ba3 100755 --- a/scripts/maint/post-merge.git-hook +++ b/scripts/maint/post-merge.git-hook @@ -1,5 +1,13 @@ #!/bin/sh +# This is post-merge git hook script to check for changes in: +# * git hook scripts +# * helper scripts for using git efficiently. +# If any changes are detected, a diff of them is printed. +# +# To install this script, copy it to .git/hooks/post-merge in local copy of +# tor git repo and make sure it has permission to execute. + git_toplevel=$(git rev-parse --show-toplevel) check_for_diffs() { From 73fed3ee1c93b9a49ce474fdaac74fd10ec4bd55 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Fri, 1 Mar 2019 18:01:24 +0200 Subject: [PATCH 0569/2557] Add changes file --- changes/ticket29588 | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 changes/ticket29588 diff --git a/changes/ticket29588 b/changes/ticket29588 new file mode 100644 index 0000000000..c81bccb00d --- /dev/null +++ b/changes/ticket29588 @@ -0,0 +1,4 @@ + o Minor features (developer tools): + - Introduce a post-merge git hook script to check if we're pulling in any + changes to our git workspace management scripts from upstream. Resolves + issue 29588. From 7b5f31f2d6bea00d6e67d0f387dbc79398192c66 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Wed, 6 Mar 2019 19:42:29 +0200 Subject: [PATCH 0570/2557] Mention what file has changed --- scripts/maint/post-merge.git-hook | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/maint/post-merge.git-hook b/scripts/maint/post-merge.git-hook index 5c3efe8ba3..300684a9b6 100755 --- a/scripts/maint/post-merge.git-hook +++ b/scripts/maint/post-merge.git-hook @@ -28,7 +28,11 @@ check_for_diffs() { check_for_script_update() { fullpath="$git_toplevel/scripts/maint/$1" - git diff ORIG_HEAD HEAD --exit-code -- "$fullpath" + if ! git diff ORIG_HEAD HEAD --exit-code -- "$fullpath" >/dev/null + then + echo "ATTENTION: $1 has changed:" + git diff ORIG_HEAD HEAD -- "$fullpath" + fi } check_for_diffs "pre-push" From 888bb9508b7a89550d3b2d33236073fc14868a98 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Wed, 6 Mar 2019 19:45:58 +0200 Subject: [PATCH 0571/2557] Move all git maintenance scripts to separate directory --- scripts/{maint => git}/git-merge-forward.sh | 0 scripts/{maint => git}/git-pull-all.sh | 0 scripts/{maint => git}/git-push-all.sh | 0 scripts/{maint => git}/post-merge.git-hook | 0 scripts/{maint => git}/pre-commit.git-hook | 0 scripts/{maint => git}/pre-push.git-hook | 0 6 files changed, 0 insertions(+), 0 deletions(-) rename scripts/{maint => git}/git-merge-forward.sh (100%) rename scripts/{maint => git}/git-pull-all.sh (100%) rename scripts/{maint => git}/git-push-all.sh (100%) rename scripts/{maint => git}/post-merge.git-hook (100%) rename scripts/{maint => git}/pre-commit.git-hook (100%) rename scripts/{maint => git}/pre-push.git-hook (100%) diff --git a/scripts/maint/git-merge-forward.sh b/scripts/git/git-merge-forward.sh similarity index 100% rename from scripts/maint/git-merge-forward.sh rename to scripts/git/git-merge-forward.sh diff --git a/scripts/maint/git-pull-all.sh b/scripts/git/git-pull-all.sh similarity index 100% rename from scripts/maint/git-pull-all.sh rename to scripts/git/git-pull-all.sh diff --git a/scripts/maint/git-push-all.sh b/scripts/git/git-push-all.sh similarity index 100% rename from scripts/maint/git-push-all.sh rename to scripts/git/git-push-all.sh diff --git a/scripts/maint/post-merge.git-hook b/scripts/git/post-merge.git-hook similarity index 100% rename from scripts/maint/post-merge.git-hook rename to scripts/git/post-merge.git-hook diff --git a/scripts/maint/pre-commit.git-hook b/scripts/git/pre-commit.git-hook similarity index 100% rename from scripts/maint/pre-commit.git-hook rename to scripts/git/pre-commit.git-hook diff --git a/scripts/maint/pre-push.git-hook b/scripts/git/pre-push.git-hook similarity index 100% rename from scripts/maint/pre-push.git-hook rename to scripts/git/pre-push.git-hook From 5f253f6a4727b31242cd5998f1b3326cc9e7ed33 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Wed, 6 Mar 2019 19:53:50 +0200 Subject: [PATCH 0572/2557] Iterate over contents of scripts/git with check_for_script_update function --- scripts/git/post-merge.git-hook | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/scripts/git/post-merge.git-hook b/scripts/git/post-merge.git-hook index 300684a9b6..ed0e16279c 100755 --- a/scripts/git/post-merge.git-hook +++ b/scripts/git/post-merge.git-hook @@ -12,7 +12,7 @@ git_toplevel=$(git rev-parse --show-toplevel) check_for_diffs() { installed="$git_toplevel/.git/hooks/$1" - latest="$git_toplevel/scripts/maint/$1.git-hook" + latest="$git_toplevel/scripts/git/$1.git-hook" if [ -e "$installed" ] then @@ -26,7 +26,7 @@ check_for_diffs() { } check_for_script_update() { - fullpath="$git_toplevel/scripts/maint/$1" + fullpath="$1" if ! git diff ORIG_HEAD HEAD --exit-code -- "$fullpath" >/dev/null then @@ -39,6 +39,7 @@ check_for_diffs "pre-push" check_for_diffs "pre-commit" check_for_diffs "post-merge" -check_for_script_update "git-merge-forward.sh" -check_for_script_update "git-pull-all.sh" -check_for_script_update "git-push-all.sh" +for file in "$git_toplevel"/scripts/git/* ; do + check_for_script_update "$file" +done + From 537692c1e37cf40decbe4a93e6a47b69fae9885d Mon Sep 17 00:00:00 2001 From: rl1987 Date: Wed, 6 Mar 2019 19:55:38 +0200 Subject: [PATCH 0573/2557] Using diff -u in check_for_diffs --- scripts/git/post-merge.git-hook | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/git/post-merge.git-hook b/scripts/git/post-merge.git-hook index ed0e16279c..31f44f84ab 100755 --- a/scripts/git/post-merge.git-hook +++ b/scripts/git/post-merge.git-hook @@ -20,7 +20,7 @@ check_for_diffs() { then echo "ATTENTION: $1 hook has changed:" echo "===============================" - diff "$installed" "$latest" + diff -u "$installed" "$latest" fi fi } From 0befdb8a35f26a1423107d181ecf08c48fb05229 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Fri, 8 Mar 2019 18:50:49 +0200 Subject: [PATCH 0574/2557] Disable git diff pagination --- scripts/git/post-merge.git-hook | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/git/post-merge.git-hook b/scripts/git/post-merge.git-hook index 31f44f84ab..176b7c9bbd 100755 --- a/scripts/git/post-merge.git-hook +++ b/scripts/git/post-merge.git-hook @@ -31,7 +31,7 @@ check_for_script_update() { if ! git diff ORIG_HEAD HEAD --exit-code -- "$fullpath" >/dev/null then echo "ATTENTION: $1 has changed:" - git diff ORIG_HEAD HEAD -- "$fullpath" + git --no-pager diff ORIG_HEAD HEAD -- "$fullpath" fi } From 134a640a91117e1f45fba49839e6ecb10f9729a0 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Sun, 10 Mar 2019 19:12:47 +0200 Subject: [PATCH 0575/2557] Remove linux-tor-prio.sh script --- changes/ticket29434 | 3 + contrib/operator-tools/linux-tor-prio.sh | 192 ----------------------- 2 files changed, 3 insertions(+), 192 deletions(-) create mode 100644 changes/ticket29434 delete mode 100644 contrib/operator-tools/linux-tor-prio.sh diff --git a/changes/ticket29434 b/changes/ticket29434 new file mode 100644 index 0000000000..8037044f0b --- /dev/null +++ b/changes/ticket29434 @@ -0,0 +1,3 @@ + o Removed features: + - Remove linux-tor-prio.sh script from contrib/operator-tools directory. + Resolves issue 29434. diff --git a/contrib/operator-tools/linux-tor-prio.sh b/contrib/operator-tools/linux-tor-prio.sh deleted file mode 100644 index 30ea5fc659..0000000000 --- a/contrib/operator-tools/linux-tor-prio.sh +++ /dev/null @@ -1,192 +0,0 @@ -#!/bin/bash -# Written by Marco Bonetti & Mike Perry -# Based on instructions from Dan Singletary's ADSL BW Management HOWTO: -# http://www.faqs.org/docs/Linux-HOWTO/ADSL-Bandwidth-Management-HOWTO.html -# This script is Public Domain. - -############################### README ################################# - -# This script provides prioritization of Tor traffic below other -# traffic on a Linux server. It has two modes of operation: UID based -# and IP based. - -# UID BASED PRIORITIZATION -# -# The UID based method requires that Tor be launched from -# a specific user ID. The "User" Tor config setting is -# insufficient, as it sets the UID after the socket is created. -# Here is a C wrapper you can use to execute Tor and drop privs before -# it creates any sockets. -# -# Compile with: -# gcc -DUID=`id -u tor` -DGID=`id -g tor` tor_wrap.c -o tor_wrap -# -# #include -# int main(int argc, char **argv) { -# if(initgroups("tor", GID) == -1) { perror("initgroups"); return 1; } -# if(setresgid(GID, GID, GID) == -1) { perror("setresgid"); return 1; } -# if(setresuid(UID, UID, UID) == -1) { perror("setresuid"); return 1; } -# execl("/bin/tor", "/bin/tor", "-f", "/etc/tor/torrc", NULL); -# perror("execl"); return 1; -# } - -# IP BASED PRIORITIZATION -# -# The IP setting requires that a separate IP address be dedicated to Tor. -# Your Torrc should be set to bind to this IP for "OutboundBindAddress", -# "ListenAddress", and "Address". - -# GENERAL USAGE -# -# You should also tune the individual connection rate parameters below -# to your individual connection. In particular, you should leave *some* -# minimum amount of bandwidth for Tor, so that Tor users are not -# completely choked out when you use your server's bandwidth. 30% is -# probably a reasonable choice. More is better of course. -# -# To start the shaping, run it as: -# ./linux-tor-prio.sh -# -# To get status information (useful to verify packets are getting marked -# and prioritized), run: -# ./linux-tor-prio.sh status -# -# And to stop prioritization: -# ./linux-tor-prio.sh stop -# -######################################################################## - -# BEGIN USER TUNABLE PARAMETERS - -DEV=eth0 - -# NOTE! You must START Tor under this UID. Using the Tor User -# config setting is NOT sufficient. See above. -TOR_UID=$(id -u tor) - -# If the UID mechanism doesn't work for you, you can set this parameter -# instead. If set, it will take precedence over the UID setting. Note that -# you need multiple IPs with one specifically devoted to Tor for this to -# work. -#TOR_IP="42.42.42.42" - -# Average ping to most places on the net, milliseconds -RTT_LATENCY=40 - -# RATE_UP must be less than your connection's upload capacity in -# kbits/sec. If it is larger, then the bottleneck will be at your -# router's queue, which you do not control. This will cause congestion -# and a revert to normal TCP fairness no matter what the queing -# priority is. -RATE_UP=5000 - -# RATE_UP_TOR is the minimum speed your Tor connections will have in -# kbits/sec. They will have at least this much bandwidth for upload. -# In general, you probably shouldn't set this too low, or else Tor -# users who use your node will be completely choked out whenever your -# machine does any other network activity. That is not very fun. -RATE_UP_TOR=1500 - -# RATE_UP_TOR_CEIL is the maximum rate allowed for all Tor traffic in -# kbits/sec. -RATE_UP_TOR_CEIL=5000 - -CHAIN=OUTPUT -#CHAIN=PREROUTING -#CHAIN=POSTROUTING - -MTU=1500 -AVG_PKT=900 # should be more like 600 for non-exit nodes - -# END USER TUNABLE PARAMETERS - - - -# The queue size should be no larger than your bandwidth-delay -# product. This is RT latency*bandwidth/MTU/2 - -BDP=$(expr $RTT_LATENCY \* $RATE_UP / $AVG_PKT) - -# Further research indicates that the BDP calculations should use -# RTT/sqrt(n) where n is the expected number of active connections.. - -BDP=$(expr $BDP / 4) - -if [ "$1" = "status" ] -then - echo "[qdisc]" - tc -s qdisc show dev $DEV - tc -s qdisc show dev imq0 - echo "[class]" - tc -s class show dev $DEV - tc -s class show dev imq0 - echo "[filter]" - tc -s filter show dev $DEV - tc -s filter show dev imq0 - echo "[iptables]" - iptables -t mangle -L TORSHAPER-OUT -v -x 2> /dev/null - exit -fi - - -# Reset everything to a known state (cleared) -tc qdisc del dev $DEV root 2> /dev/null > /dev/null -tc qdisc del dev imq0 root 2> /dev/null > /dev/null -iptables -t mangle -D POSTROUTING -o $DEV -j TORSHAPER-OUT 2> /dev/null > /dev/null -iptables -t mangle -D PREROUTING -o $DEV -j TORSHAPER-OUT 2> /dev/null > /dev/null -iptables -t mangle -D OUTPUT -o $DEV -j TORSHAPER-OUT 2> /dev/null > /dev/null -iptables -t mangle -F TORSHAPER-OUT 2> /dev/null > /dev/null -iptables -t mangle -X TORSHAPER-OUT 2> /dev/null > /dev/null -ip link set imq0 down 2> /dev/null > /dev/null -rmmod imq 2> /dev/null > /dev/null - -if [ "$1" = "stop" ] -then - echo "Shaping removed on $DEV." - exit -fi - -# Outbound Shaping (limits total bandwidth to RATE_UP) - -ip link set dev $DEV qlen $BDP - -# Add HTB root qdisc, default is high prio -tc qdisc add dev $DEV root handle 1: htb default 20 - -# Add main rate limit class -tc class add dev $DEV parent 1: classid 1:1 htb rate ${RATE_UP}kbit - -# Create the two classes, giving Tor at least RATE_UP_TOR kbit and capping -# total upstream at RATE_UP so the queue is under our control. -tc class add dev $DEV parent 1:1 classid 1:20 htb rate $(expr $RATE_UP - $RATE_UP_TOR)kbit ceil ${RATE_UP}kbit prio 0 -tc class add dev $DEV parent 1:1 classid 1:21 htb rate $[$RATE_UP_TOR]kbit ceil ${RATE_UP_TOR_CEIL}kbit prio 10 - -# Start up pfifo -tc qdisc add dev $DEV parent 1:20 handle 20: pfifo limit $BDP -tc qdisc add dev $DEV parent 1:21 handle 21: pfifo limit $BDP - -# filter traffic into classes by fwmark -tc filter add dev $DEV parent 1:0 prio 0 protocol ip handle 20 fw flowid 1:20 -tc filter add dev $DEV parent 1:0 prio 0 protocol ip handle 21 fw flowid 1:21 - -# add TORSHAPER-OUT chain to the mangle table in iptables -iptables -t mangle -N TORSHAPER-OUT -iptables -t mangle -I $CHAIN -o $DEV -j TORSHAPER-OUT - - -# Set firewall marks -# Low priority to Tor -if [ ""$TOR_IP == "" ] -then - echo "Using UID-based QoS. UID $TOR_UID marked as low priority." - iptables -t mangle -A TORSHAPER-OUT -m owner --uid-owner $TOR_UID -j MARK --set-mark 21 -else - echo "Using IP-based QoS. $TOR_IP marked as low priority." - iptables -t mangle -A TORSHAPER-OUT -s $TOR_IP -j MARK --set-mark 21 -fi - -# High prio for everything else -iptables -t mangle -A TORSHAPER-OUT -m mark --mark 0 -j MARK --set-mark 20 - -echo "Outbound shaping added to $DEV. Rate for Tor upload at least: ${RATE_UP_TOR}Kbyte/sec." - From 0cca55411074a78096dab1b84a4781e790e9aece Mon Sep 17 00:00:00 2001 From: teor Date: Tue, 12 Mar 2019 10:24:34 +1000 Subject: [PATCH 0576/2557] sr: Check for replacing a SRV pointer with the same pointer Check if the new pointer is the same as the old one: if it is, it's probably a bug: * the caller may have confused current and previous, or * they may have forgotten to sr_srv_dup(). Putting NULL multiple times is allowed. Part of 29706. --- src/or/shared_random_state.c | 32 +++-- src/test/test_shared_random.c | 212 +++++++++++++++++++++++++++++++++- 2 files changed, 230 insertions(+), 14 deletions(-) diff --git a/src/or/shared_random_state.c b/src/or/shared_random_state.c index 50c6d3dd7a..22a503b89b 100644 --- a/src/or/shared_random_state.c +++ b/src/or/shared_random_state.c @@ -900,14 +900,26 @@ state_query_put_(sr_state_object_t obj_type, void *data) break; } case SR_STATE_OBJ_CURSRV: - /* Always free the old SRV */ - state_query_del_(SR_STATE_OBJ_CURSRV, NULL); - sr_state->current_srv = (sr_srv_t *) data; + /* Check if the new pointer is the same as the old one: if it is, it's + * probably a bug. The caller may have confused current and previous, + * or they may have forgotten to sr_srv_dup(). + * Putting NULL multiple times is allowed. */ + if (!BUG(data && sr_state->current_srv == (sr_srv_t *) data)) { + /* We own the old SRV, so we need to free it. */ + state_query_del_(SR_STATE_OBJ_CURSRV, NULL); + sr_state->current_srv = (sr_srv_t *) data; + } break; case SR_STATE_OBJ_PREVSRV: - /* Always free the old SRV */ - state_query_del_(SR_STATE_OBJ_PREVSRV, NULL); - sr_state->previous_srv = (sr_srv_t *) data; + /* Check if the new pointer is the same as the old one: if it is, it's + * probably a bug. The caller may have confused current and previous, + * or they may have forgotten to sr_srv_dup(). + * Putting NULL multiple times is allowed. */ + if (!BUG(data && sr_state->previous_srv == (sr_srv_t *) data)) { + /* We own the old SRV, so we need to free it. */ + state_query_del_(SR_STATE_OBJ_PREVSRV, NULL); + sr_state->previous_srv = (sr_srv_t *) data; + } break; case SR_STATE_OBJ_VALID_AFTER: sr_state->valid_after = *((time_t *) data); @@ -1059,7 +1071,9 @@ sr_state_get_phase(void) return *(sr_phase_t *) ptr; } -/* Return the previous SRV value from our state. Value CAN be NULL. */ +/* Return the previous SRV value from our state. Value CAN be NULL. + * The state object owns the SRV, so the calling code should not free the SRV. + * Use sr_srv_dup() if you want to keep a copy of the SRV. */ const sr_srv_t * sr_state_get_previous_srv(void) { @@ -1078,7 +1092,9 @@ sr_state_set_previous_srv(const sr_srv_t *srv) NULL); } -/* Return the current SRV value from our state. Value CAN be NULL. */ +/* Return the current SRV value from our state. Value CAN be NULL. + * The state object owns the SRV, so the calling code should not free the SRV. + * Use sr_srv_dup() if you want to keep a copy of the SRV. */ const sr_srv_t * sr_state_get_current_srv(void) { diff --git a/src/test/test_shared_random.c b/src/test/test_shared_random.c index 93125e5135..c81b6ec2d6 100644 --- a/src/test/test_shared_random.c +++ b/src/test/test_shared_random.c @@ -38,7 +38,8 @@ trusteddirserver_get_by_v3_auth_digest_m(const char *digest) } /* Setup a minimal dirauth environment by initializing the SR state and - * making sure the options are set to be an authority directory. */ + * making sure the options are set to be an authority directory. + * You must only call this function once per process. */ static void init_authority_state(void) { @@ -451,7 +452,9 @@ test_encoding(void *arg) /** Setup some SRVs in our SR state. If also_current is set, then set * both current and previous SRVs. - * Helper of test_vote() and test_sr_compute_srv(). */ + * Helper of test_vote() and test_sr_compute_srv(). + * You must call sr_state_free() to free the state at the end of each test + * function (on pass or fail). */ static void test_sr_setup_srv(int also_current) { @@ -927,8 +930,9 @@ test_sr_get_majority_srv_from_votes(void *arg) smartlist_free(votes); } +/* Test utils that don't depend on authority state */ static void -test_utils(void *arg) +test_utils_general(void *arg) { (void) arg; @@ -991,9 +995,19 @@ test_utils(void *arg) tt_str_op(get_phase_str(SR_PHASE_COMMIT), ==, "commit"); } + done: + return; +} + +/* Test utils that depend on authority state */ +static void +test_utils_auth(void *arg) +{ + (void)arg; + init_authority_state(); + /* Testing phase transition */ { - init_authority_state(); set_sr_phase(SR_PHASE_COMMIT); tt_int_op(is_phase_transition(SR_PHASE_REVEAL), ==, 1); tt_int_op(is_phase_transition(SR_PHASE_COMMIT), ==, 0); @@ -1004,8 +1018,193 @@ test_utils(void *arg) tt_int_op(is_phase_transition(42), ==, 1); } + /* Testing get, set, delete, clean SRVs */ + + { + /* Just set the previous SRV */ + test_sr_setup_srv(0); + tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL); + state_del_previous_srv(); + tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL); + } + + { + /* Delete the SRVs one at a time */ + test_sr_setup_srv(1); + tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL); + state_del_current_srv(); + tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL); + state_del_previous_srv(); + tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL); + + /* And in the opposite order */ + test_sr_setup_srv(1); + tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL); + state_del_previous_srv(); + tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL); + state_del_current_srv(); + tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL); + + /* And both at once */ + test_sr_setup_srv(1); + tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL); + sr_state_clean_srvs(); + tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL); + + /* And do the gets and sets multiple times */ + test_sr_setup_srv(1); + tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL); + tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL); + state_del_previous_srv(); + tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL); + state_del_previous_srv(); + tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL); + tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL); + sr_state_clean_srvs(); + tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL); + state_del_current_srv(); + tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL); + sr_state_clean_srvs(); + tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL); + state_del_current_srv(); + tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL); + tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL); + } + + { + /* Now set the SRVs to NULL instead */ + test_sr_setup_srv(1); + tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL); + sr_state_set_current_srv(NULL); + tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL); + sr_state_set_previous_srv(NULL); + tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL); + + /* And in the opposite order */ + test_sr_setup_srv(1); + tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL); + sr_state_set_previous_srv(NULL); + tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL); + sr_state_set_current_srv(NULL); + tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL); + + /* And both at once */ + test_sr_setup_srv(1); + tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL); + sr_state_clean_srvs(); + tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL); + + /* And do the gets and sets multiple times */ + test_sr_setup_srv(1); + tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL); + sr_state_set_previous_srv(NULL); + sr_state_set_previous_srv(NULL); + tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL); + tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL); + sr_state_set_current_srv(NULL); + sr_state_set_previous_srv(NULL); + sr_state_set_current_srv(NULL); + tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL); + } + + { + /* Now copy the values across */ + test_sr_setup_srv(1); + /* Check that the pointers are non-NULL, and different from each other */ + tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL); + tt_ptr_op(sr_state_get_previous_srv(), OP_NE, + sr_state_get_current_srv()); + /* Check that the content is different */ + tt_mem_op(sr_state_get_previous_srv(), OP_NE, + sr_state_get_current_srv(), sizeof(sr_srv_t)); + /* Set the current to the previous: the protocol goes the other way */ + sr_state_set_current_srv(sr_srv_dup(sr_state_get_previous_srv())); + /* Check that the pointers are non-NULL, and different from each other */ + tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL); + tt_ptr_op(sr_state_get_previous_srv(), OP_NE, + sr_state_get_current_srv()); + /* Check that the content is the same */ + tt_mem_op(sr_state_get_previous_srv(), OP_EQ, + sr_state_get_current_srv(), sizeof(sr_srv_t)); + } + + { + /* Now copy a value onto itself */ + test_sr_setup_srv(1); + /* Check that the pointers are non-NULL, and different from each other */ + tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL); + tt_ptr_op(sr_state_get_previous_srv(), OP_NE, + sr_state_get_current_srv()); + /* Take a copy of the old value */ + sr_srv_t old_current_srv; + memcpy(&old_current_srv, sr_state_get_current_srv(), sizeof(sr_srv_t)); + /* Check that the content is different */ + tt_mem_op(sr_state_get_previous_srv(), OP_NE, + sr_state_get_current_srv(), sizeof(sr_srv_t)); + /* Set the current to the current: the protocol never replaces an SRV with + * the same value */ + sr_state_set_current_srv(sr_srv_dup(sr_state_get_current_srv())); + /* Check that the pointers are non-NULL, and different from each other */ + tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL); + tt_ptr_op(sr_state_get_previous_srv(), OP_NE, + sr_state_get_current_srv()); + /* Check that the content is different between current and previous */ + tt_mem_op(sr_state_get_previous_srv(), OP_NE, + sr_state_get_current_srv(), sizeof(sr_srv_t)); + /* Check that the content is the same as the old content */ + tt_mem_op(&old_current_srv, OP_EQ, + sr_state_get_current_srv(), sizeof(sr_srv_t)); + } + + /* I don't think we can say "expect a BUG()" in our tests. */ +#if 0 + { + /* Now copy a value onto itself without sr_srv_dup(). + * This should fail with a BUG() warning. */ + test_sr_setup_srv(1); + sr_state_set_current_srv(sr_state_get_current_srv()); + sr_state_set_previous_srv(sr_state_get_previous_srv()); + } +#endif + done: - return; + sr_state_free(); } static void @@ -1293,7 +1492,8 @@ struct testcase_t sr_tests[] = { { "sr_compute_srv", test_sr_compute_srv, TT_FORK, NULL, NULL }, { "sr_get_majority_srv_from_votes", test_sr_get_majority_srv_from_votes, TT_FORK, NULL, NULL }, - { "utils", test_utils, TT_FORK, NULL, NULL }, + { "utils_general", test_utils_general, TT_FORK, NULL, NULL }, + { "utils_auth", test_utils_auth, TT_FORK, NULL, NULL }, { "state_transition", test_state_transition, TT_FORK, NULL, NULL }, { "state_update", test_state_update, TT_FORK, NULL, NULL }, From 9eeff921ae7b786d960ea4286d5bba56edaef47f Mon Sep 17 00:00:00 2001 From: teor Date: Tue, 12 Mar 2019 11:09:48 +1000 Subject: [PATCH 0577/2557] sr: BUG() on NULL sr_state before doing a state_query_*() Part of #29706. --- src/or/shared_random_state.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/or/shared_random_state.c b/src/or/shared_random_state.c index 22a503b89b..5011b5c7f3 100644 --- a/src/or/shared_random_state.c +++ b/src/or/shared_random_state.c @@ -857,6 +857,9 @@ state_query_get_commit(const char *rsa_fpr) static void * state_query_get_(sr_state_object_t obj_type, const void *data) { + if (BUG(!sr_state)) + return NULL; + void *obj = NULL; switch (obj_type) { @@ -890,6 +893,9 @@ state_query_get_(sr_state_object_t obj_type, const void *data) static void state_query_put_(sr_state_object_t obj_type, void *data) { + if (BUG(!sr_state)) + return; + switch (obj_type) { case SR_STATE_OBJ_COMMIT: { @@ -939,6 +945,9 @@ state_query_put_(sr_state_object_t obj_type, void *data) static void state_query_del_all_(sr_state_object_t obj_type) { + if (BUG(!sr_state)) + return; + switch (obj_type) { case SR_STATE_OBJ_COMMIT: { @@ -967,6 +976,9 @@ state_query_del_(sr_state_object_t obj_type, void *data) { (void) data; + if (BUG(!sr_state)) + return; + switch (obj_type) { case SR_STATE_OBJ_PREVSRV: tor_free(sr_state->previous_srv); From dfc3e552a3b7617add0bd576a5c34dca59f42649 Mon Sep 17 00:00:00 2001 From: teor Date: Tue, 12 Mar 2019 11:34:52 +1000 Subject: [PATCH 0578/2557] test/sr: update sr_state_free() to sr_state_free_all() The function name changed between 0.2.9 and 0.3.4. --- src/test/test_shared_random.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/test_shared_random.c b/src/test/test_shared_random.c index 45f0e1db7f..991d73128e 100644 --- a/src/test/test_shared_random.c +++ b/src/test/test_shared_random.c @@ -548,7 +548,7 @@ test_encoding(void *arg) /** Setup some SRVs in our SR state. If also_current is set, then set * both current and previous SRVs. * Helper of test_vote() and test_sr_compute_srv(). - * You must call sr_state_free() to free the state at the end of each test + * You must call sr_state_free_all() to free the state at the end of each test * function (on pass or fail). */ static void test_sr_setup_srv(int also_current) @@ -1299,7 +1299,7 @@ test_utils_auth(void *arg) #endif done: - sr_state_free(); + sr_state_free_all(); } static void From 052ec08a085d69ac2ffb3475e61631b0aef26562 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Tue, 12 Mar 2019 12:01:26 +0200 Subject: [PATCH 0579/2557] Refrain from doing exhaustive iteration over all values of integers --- src/test/test_ptr_slow.c | 59 ++++++++++++++++++++++++++-------------- 1 file changed, 39 insertions(+), 20 deletions(-) diff --git a/src/test/test_ptr_slow.c b/src/test/test_ptr_slow.c index a5914c9120..632a304177 100644 --- a/src/test/test_ptr_slow.c +++ b/src/test/test_ptr_slow.c @@ -10,46 +10,65 @@ #include #include -/** Check that all values of int can be cast to void * and back. */ +/** Assert that a can be cast to void * and back. */ +static void +assert_int_voidptr_roundtrip(int a) +{ + intptr_t ap = (intptr_t)a; + void *b = (void *)ap; + intptr_t c = (intptr_t)b; + void *d = (void *)c; + + tt_assert(ap == c); + tt_assert(b == d); + + done: + return; +} + static void test_int_voidstar_interop(void *arg) { int a; (void)arg; - for (a = INT_MIN; a < INT_MAX; a++) { - intptr_t ap = (intptr_t)a; - void *b = (void *)ap; - intptr_t c = (intptr_t)b; - void *d = (void *)c; - - tt_assert(ap == c); - tt_assert(b == d); + for (a = 0; a <= 1024; a++) { + assert_int_voidptr_roundtrip(a); } + for (a = INT_MAX-1024; a < INT_MAX; a++) { + assert_int_voidptr_roundtrip(a); + } +} + +static void +assert_uint_voidptr_roundtrip(unsigned int a) +{ + intptr_t ap = (intptr_t)a; + void *b = (void *)ap; + intptr_t c = (intptr_t)b; + void *d = (void *)c; + + tt_assert(ap == c); + tt_assert(b == d); + done: return; } -/** Check that all values of unsigned int can be cast to void * and back. */ static void test_uint_voidstar_interop(void *arg) { unsigned int a; (void)arg; - for (a = 0; a < UINT_MAX; a++) { - intptr_t ap = (intptr_t)a; - void *b = (void *)ap; - intptr_t c = (intptr_t)b; - void *d = (void *)c; - - tt_assert(ap == c); - tt_assert(b == d); + for (a = 0; a <= 1024; a++) { + assert_uint_voidptr_roundtrip(a); } - done: - return; + for (a = UINT_MAX-1024; a < UINT_MAX; a++) { + assert_uint_voidptr_roundtrip(a); + } } struct testcase_t slow_ptr_tests[] = { From e52653e01a2ce6655975c2f893a1d1ff7bef2af7 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Tue, 12 Mar 2019 12:14:22 +0200 Subject: [PATCH 0580/2557] USe uintptr_t for unsigned ints --- src/test/test_ptr_slow.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/test_ptr_slow.c b/src/test/test_ptr_slow.c index 632a304177..f064a3e7c2 100644 --- a/src/test/test_ptr_slow.c +++ b/src/test/test_ptr_slow.c @@ -44,9 +44,9 @@ test_int_voidstar_interop(void *arg) static void assert_uint_voidptr_roundtrip(unsigned int a) { - intptr_t ap = (intptr_t)a; + uintptr_t ap = (uintptr_t)a; void *b = (void *)ap; - intptr_t c = (intptr_t)b; + uintptr_t c = (uintptr_t)b; void *d = (void *)c; tt_assert(ap == c); From 34b562b78301b4e4663eeb98fe56157c94cd1f37 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Tue, 12 Mar 2019 12:24:23 +0200 Subject: [PATCH 0581/2557] Remove linux-tor-prio.sh from include.am --- contrib/include.am | 1 - 1 file changed, 1 deletion(-) diff --git a/contrib/include.am b/contrib/include.am index a23e82d6da..742bc58163 100644 --- a/contrib/include.am +++ b/contrib/include.am @@ -7,7 +7,6 @@ EXTRA_DIST+= \ contrib/dist/tor.sh \ contrib/dist/torctl \ contrib/dist/tor.service.in \ - contrib/operator-tools/linux-tor-prio.sh \ contrib/operator-tools/tor-exit-notice.html \ contrib/or-tools/exitlist \ contrib/win32build/tor-mingw.nsi.in \ From a6dd893e7687c3634f22cc0b1b706a53cc2330f5 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 12 Mar 2019 10:50:54 -0400 Subject: [PATCH 0582/2557] Fix shellcheck warnings in pull-all/merge-all scripts This appears at first glance to be a shellcheck bug. Closes 29747. Bugfix not in any released Tor. --- scripts/maint/git-merge-forward.sh | 12 ++++++++++++ scripts/maint/git-pull-all.sh | 13 +++++++++++++ 2 files changed, 25 insertions(+) diff --git a/scripts/maint/git-merge-forward.sh b/scripts/maint/git-merge-forward.sh index 4b294f4945..67af7e98bf 100755 --- a/scripts/maint/git-merge-forward.sh +++ b/scripts/maint/git-merge-forward.sh @@ -53,6 +53,18 @@ RELEASE_040=( "release-0.4.0" "maint-0.4.0" "$GIT_PATH/$TOR_WKT_NAME/release-0.4 # from that repository. ORIGIN_PATH="$GIT_PATH/$TOR_MASTER_NAME" +# SC2034 -- shellcheck thinks that these are unused. We know better. +ACTUALLY_THESE_ARE_USED=< Date: Thu, 23 Aug 2018 19:21:26 +1000 Subject: [PATCH 0583/2557] hs: abolish hs_desc_link_specifier_t The previous commits for 23576 confused hs_desc_link_specifier_t and link_specifier_t. Removing hs_desc_link_specifier_t fixes this confusion. Fixes bug 22781; bugfix on 0.3.2.1-alpha. --- changes/bug22781 | 4 + src/feature/hs/hs_cell.c | 9 +- src/feature/hs/hs_client.c | 21 ++-- src/feature/hs/hs_common.c | 39 ++++++++ src/feature/hs/hs_common.h | 2 + src/feature/hs/hs_descriptor.c | 178 +++------------------------------ src/feature/hs/hs_descriptor.h | 29 +----- src/feature/hs/hs_intropoint.c | 4 +- src/feature/hs/hs_service.c | 50 +++++---- src/test/hs_test_helpers.c | 85 +++++++++++----- src/test/test_hs_client.c | 4 + src/test/test_hs_descriptor.c | 111 -------------------- src/test/test_hs_service.c | 20 ++-- 13 files changed, 175 insertions(+), 381 deletions(-) create mode 100644 changes/bug22781 diff --git a/changes/bug22781 b/changes/bug22781 new file mode 100644 index 0000000000..5606dfa5e2 --- /dev/null +++ b/changes/bug22781 @@ -0,0 +1,4 @@ + o Code simplification and refactoring: + - Replace hs_desc_link_specifier_t with link_specifier_t, + and remove all hs_desc_link_specifier_t-specific code. + Fixes bug 22781; bugfix on 0.3.2.1-alpha. diff --git a/src/feature/hs/hs_cell.c b/src/feature/hs/hs_cell.c index 597982b34e..e24520da47 100644 --- a/src/feature/hs/hs_cell.c +++ b/src/feature/hs/hs_cell.c @@ -756,7 +756,14 @@ hs_cell_parse_introduce2(hs_cell_introduce2_data_t *data, idx < trn_cell_introduce_encrypted_get_nspec(enc_cell); idx++) { link_specifier_t *lspec = trn_cell_introduce_encrypted_get_nspecs(enc_cell, idx); - smartlist_add(data->link_specifiers, hs_link_specifier_dup(lspec)); + if (BUG(!lspec)) { + goto done; + } + link_specifier_t *lspec_dup = hs_link_specifier_dup(lspec); + if (BUG(!lspec_dup)) { + goto done; + } + smartlist_add(data->link_specifiers, lspec_dup); } /* Success. */ diff --git a/src/feature/hs/hs_client.c b/src/feature/hs/hs_client.c index d4eee50bb7..9d0dee7124 100644 --- a/src/feature/hs/hs_client.c +++ b/src/feature/hs/hs_client.c @@ -528,13 +528,15 @@ find_desc_intro_point_by_legacy_id(const char *legacy_id, SMARTLIST_FOREACH_BEGIN(desc->encrypted_data.intro_points, hs_desc_intro_point_t *, ip) { SMARTLIST_FOREACH_BEGIN(ip->link_specifiers, - const hs_desc_link_specifier_t *, lspec) { + const link_specifier_t *, lspec) { /* Not all tor node have an ed25519 identity key so we still rely on the * legacy identity digest. */ - if (lspec->type != LS_LEGACY_ID) { + if (link_specifier_get_ls_type(lspec) != LS_LEGACY_ID) { continue; } - if (fast_memneq(legacy_id, lspec->u.legacy_id, DIGEST_LEN)) { + if (fast_memneq(legacy_id, + link_specifier_getconstarray_un_legacy_id(lspec), + DIGEST_LEN)) { break; } /* Found it. */ @@ -753,24 +755,13 @@ STATIC extend_info_t * desc_intro_point_to_extend_info(const hs_desc_intro_point_t *ip) { extend_info_t *ei; - smartlist_t *lspecs = smartlist_new(); tor_assert(ip); - /* We first encode the descriptor link specifiers into the binary - * representation which is a trunnel object. */ - SMARTLIST_FOREACH_BEGIN(ip->link_specifiers, - const hs_desc_link_specifier_t *, desc_lspec) { - link_specifier_t *lspec = hs_desc_lspec_to_trunnel(desc_lspec); - smartlist_add(lspecs, lspec); - } SMARTLIST_FOREACH_END(desc_lspec); - /* Explicitly put the direct connection option to 0 because this is client * side and there is no such thing as a non anonymous client. */ - ei = hs_get_extend_info_from_lspecs(lspecs, &ip->onion_key, 0); + ei = hs_get_extend_info_from_lspecs(ip->link_specifiers, &ip->onion_key, 0); - SMARTLIST_FOREACH(lspecs, link_specifier_t *, ls, link_specifier_free(ls)); - smartlist_free(lspecs); return ei; } diff --git a/src/feature/hs/hs_common.c b/src/feature/hs/hs_common.c index 5d8f54230f..a1dc0a6290 100644 --- a/src/feature/hs/hs_common.c +++ b/src/feature/hs/hs_common.c @@ -1843,3 +1843,42 @@ hs_inc_rdv_stream_counter(origin_circuit_t *circ) tor_assert_nonfatal_unreached(); } } + +/* Return a newly allocated link specifier object that is a copy of dst. */ +link_specifier_t * +link_specifier_dup(const link_specifier_t *src) +{ + link_specifier_t *dup = NULL; + uint8_t *buf = NULL; + + if (BUG(!src)) { + goto err; + } + + ssize_t encoded_len_alloc = link_specifier_encoded_len(src); + if (BUG(encoded_len_alloc < 0)) { + goto err; + } + + buf = tor_malloc_zero(encoded_len_alloc); + ssize_t encoded_len_data = link_specifier_encode(buf, + encoded_len_alloc, + src); + if (BUG(encoded_len_data < 0)) { + goto err; + } + + ssize_t parsed_len = link_specifier_parse(&dup, buf, encoded_len_alloc); + if (BUG(parsed_len < 0)) { + goto err; + } + + goto done; + + err: + dup = NULL; + + done: + tor_free(buf); + return dup; +} diff --git a/src/feature/hs/hs_common.h b/src/feature/hs/hs_common.h index a44505930a..544e63e596 100644 --- a/src/feature/hs/hs_common.h +++ b/src/feature/hs/hs_common.h @@ -262,6 +262,8 @@ extend_info_t *hs_get_extend_info_from_lspecs(const smartlist_t *lspecs, const struct curve25519_public_key_t *onion_key, int direct_conn); +link_specifier_t *link_specifier_dup(const link_specifier_t *src); + #ifdef HS_COMMON_PRIVATE STATIC void get_disaster_srv(uint64_t time_period_num, uint8_t *srv_out); diff --git a/src/feature/hs/hs_descriptor.c b/src/feature/hs/hs_descriptor.c index b09d50e010..8f7bdf86ef 100644 --- a/src/feature/hs/hs_descriptor.c +++ b/src/feature/hs/hs_descriptor.c @@ -324,12 +324,11 @@ encode_link_specifiers(const smartlist_t *specs) link_specifier_list_set_n_spec(lslist, smartlist_len(specs)); - SMARTLIST_FOREACH_BEGIN(specs, const hs_desc_link_specifier_t *, + SMARTLIST_FOREACH_BEGIN(specs, const link_specifier_t *, spec) { - link_specifier_t *ls = hs_desc_lspec_to_trunnel(spec); - if (ls) { - link_specifier_list_add_spec(lslist, ls); - } + link_specifier_t *ls = link_specifier_dup(spec); + tor_assert(ls); + link_specifier_list_add_spec(lslist, ls); } SMARTLIST_FOREACH_END(spec); { @@ -1190,52 +1189,22 @@ decode_link_specifiers(const char *encoded) results = smartlist_new(); for (i = 0; i < link_specifier_list_getlen_spec(specs); i++) { - hs_desc_link_specifier_t *hs_spec; link_specifier_t *ls = link_specifier_list_get_spec(specs, i); - tor_assert(ls); - - hs_spec = tor_malloc_zero(sizeof(*hs_spec)); - hs_spec->type = link_specifier_get_ls_type(ls); - switch (hs_spec->type) { - case LS_IPV4: - tor_addr_from_ipv4h(&hs_spec->u.ap.addr, - link_specifier_get_un_ipv4_addr(ls)); - hs_spec->u.ap.port = link_specifier_get_un_ipv4_port(ls); - break; - case LS_IPV6: - tor_addr_from_ipv6_bytes(&hs_spec->u.ap.addr, (const char *) - link_specifier_getarray_un_ipv6_addr(ls)); - hs_spec->u.ap.port = link_specifier_get_un_ipv6_port(ls); - break; - case LS_LEGACY_ID: - /* Both are known at compile time so let's make sure they are the same - * else we can copy memory out of bound. */ - tor_assert(link_specifier_getlen_un_legacy_id(ls) == - sizeof(hs_spec->u.legacy_id)); - memcpy(hs_spec->u.legacy_id, link_specifier_getarray_un_legacy_id(ls), - sizeof(hs_spec->u.legacy_id)); - break; - case LS_ED25519_ID: - /* Both are known at compile time so let's make sure they are the same - * else we can copy memory out of bound. */ - tor_assert(link_specifier_getlen_un_ed25519_id(ls) == - sizeof(hs_spec->u.ed25519_id)); - memcpy(hs_spec->u.ed25519_id, - link_specifier_getconstarray_un_ed25519_id(ls), - sizeof(hs_spec->u.ed25519_id)); - break; - default: - tor_free(hs_spec); + if (BUG(!ls)) { goto err; } - - smartlist_add(results, hs_spec); + link_specifier_t *ls_dup = link_specifier_dup(ls); + if (BUG(!ls_dup)) { + goto err; + } + smartlist_add(results, ls_dup); } goto done; err: if (results) { - SMARTLIST_FOREACH(results, hs_desc_link_specifier_t *, s, tor_free(s)); + SMARTLIST_FOREACH(results, link_specifier_t *, s, + link_specifier_free(s)); smartlist_free(results); results = NULL; } @@ -2878,8 +2847,8 @@ hs_desc_intro_point_free_(hs_desc_intro_point_t *ip) return; } if (ip->link_specifiers) { - SMARTLIST_FOREACH(ip->link_specifiers, hs_desc_link_specifier_t *, - ls, hs_desc_link_specifier_free(ls)); + SMARTLIST_FOREACH(ip->link_specifiers, link_specifier_t *, + ls, link_specifier_free(ls)); smartlist_free(ip->link_specifiers); } tor_cert_free(ip->auth_key_cert); @@ -2972,69 +2941,6 @@ hs_desc_authorized_client_free_(hs_desc_authorized_client_t *client) tor_free(client); } -/* Free the given descriptor link specifier. */ -void -hs_desc_link_specifier_free_(hs_desc_link_specifier_t *ls) -{ - if (ls == NULL) { - return; - } - tor_free(ls); -} - -/* Return a newly allocated descriptor link specifier using the given extend - * info and requested type. Return NULL on error. */ -hs_desc_link_specifier_t * -hs_desc_link_specifier_new(const extend_info_t *info, uint8_t type) -{ - hs_desc_link_specifier_t *ls = NULL; - - tor_assert(info); - - ls = tor_malloc_zero(sizeof(*ls)); - ls->type = type; - switch (ls->type) { - case LS_IPV4: - if (info->addr.family != AF_INET) { - goto err; - } - tor_addr_copy(&ls->u.ap.addr, &info->addr); - ls->u.ap.port = info->port; - break; - case LS_IPV6: - if (info->addr.family != AF_INET6) { - goto err; - } - tor_addr_copy(&ls->u.ap.addr, &info->addr); - ls->u.ap.port = info->port; - break; - case LS_LEGACY_ID: - /* Bug out if the identity digest is not set */ - if (BUG(tor_mem_is_zero(info->identity_digest, - sizeof(info->identity_digest)))) { - goto err; - } - memcpy(ls->u.legacy_id, info->identity_digest, sizeof(ls->u.legacy_id)); - break; - case LS_ED25519_ID: - /* ed25519 keys are optional for intro points */ - if (ed25519_public_key_is_zero(&info->ed_identity)) { - goto err; - } - memcpy(ls->u.ed25519_id, info->ed_identity.pubkey, - sizeof(ls->u.ed25519_id)); - break; - default: - /* Unknown type is code flow error. */ - tor_assert(0); - } - - return ls; - err: - tor_free(ls); - return NULL; -} - /* From the given descriptor, remove and free every introduction point. */ void hs_descriptor_clear_intro_points(hs_descriptor_t *desc) @@ -3050,59 +2956,3 @@ hs_descriptor_clear_intro_points(hs_descriptor_t *desc) smartlist_clear(ips); } } - -/* From a descriptor link specifier object spec, returned a newly allocated - * link specifier object that is the encoded representation of spec. Return - * NULL on error. */ -link_specifier_t * -hs_desc_lspec_to_trunnel(const hs_desc_link_specifier_t *spec) -{ - tor_assert(spec); - - link_specifier_t *ls = link_specifier_new(); - link_specifier_set_ls_type(ls, spec->type); - - switch (spec->type) { - case LS_IPV4: - link_specifier_set_un_ipv4_addr(ls, - tor_addr_to_ipv4h(&spec->u.ap.addr)); - link_specifier_set_un_ipv4_port(ls, spec->u.ap.port); - /* Four bytes IPv4 and two bytes port. */ - link_specifier_set_ls_len(ls, sizeof(spec->u.ap.addr.addr.in_addr) + - sizeof(spec->u.ap.port)); - break; - case LS_IPV6: - { - size_t addr_len = link_specifier_getlen_un_ipv6_addr(ls); - const uint8_t *in6_addr = tor_addr_to_in6_addr8(&spec->u.ap.addr); - uint8_t *ipv6_array = link_specifier_getarray_un_ipv6_addr(ls); - memcpy(ipv6_array, in6_addr, addr_len); - link_specifier_set_un_ipv6_port(ls, spec->u.ap.port); - /* Sixteen bytes IPv6 and two bytes port. */ - link_specifier_set_ls_len(ls, addr_len + sizeof(spec->u.ap.port)); - break; - } - case LS_LEGACY_ID: - { - size_t legacy_id_len = link_specifier_getlen_un_legacy_id(ls); - uint8_t *legacy_id_array = link_specifier_getarray_un_legacy_id(ls); - memcpy(legacy_id_array, spec->u.legacy_id, legacy_id_len); - link_specifier_set_ls_len(ls, legacy_id_len); - break; - } - case LS_ED25519_ID: - { - size_t ed25519_id_len = link_specifier_getlen_un_ed25519_id(ls); - uint8_t *ed25519_id_array = link_specifier_getarray_un_ed25519_id(ls); - memcpy(ed25519_id_array, spec->u.ed25519_id, ed25519_id_len); - link_specifier_set_ls_len(ls, ed25519_id_len); - break; - } - default: - tor_assert_nonfatal_unreached(); - link_specifier_free(ls); - ls = NULL; - } - - return ls; -} diff --git a/src/feature/hs/hs_descriptor.h b/src/feature/hs/hs_descriptor.h index 04a8e16d63..dbe0cb1c94 100644 --- a/src/feature/hs/hs_descriptor.h +++ b/src/feature/hs/hs_descriptor.h @@ -69,28 +69,10 @@ typedef enum { HS_DESC_AUTH_ED25519 = 1 } hs_desc_auth_type_t; -/* Link specifier object that contains information on how to extend to the - * relay that is the address, port and handshake type. */ -typedef struct hs_desc_link_specifier_t { - /* Indicate the type of link specifier. See trunnel ed25519_cert - * specification. */ - uint8_t type; - - /* It must be one of these types, can't be more than one. */ - union { - /* IP address and port of the relay use to extend. */ - tor_addr_port_t ap; - /* Legacy identity. A 20-byte SHA1 identity fingerprint. */ - uint8_t legacy_id[DIGEST_LEN]; - /* ed25519 identity. A 32-byte key. */ - uint8_t ed25519_id[ED25519_PUBKEY_LEN]; - } u; -} hs_desc_link_specifier_t; - /* Introduction point information located in a descriptor. */ typedef struct hs_desc_intro_point_t { /* Link specifier(s) which details how to extend to the relay. This list - * contains hs_desc_link_specifier_t object. It MUST have at least one. */ + * contains link_specifier_t objects. It MUST have at least one. */ smartlist_t *link_specifiers; /* Onion key of the introduction point used to extend to it for the ntor @@ -261,12 +243,6 @@ void hs_desc_encrypted_data_free_(hs_desc_encrypted_data_t *desc); #define hs_desc_encrypted_data_free(desc) \ FREE_AND_NULL(hs_desc_encrypted_data_t, hs_desc_encrypted_data_free_, (desc)) -void hs_desc_link_specifier_free_(hs_desc_link_specifier_t *ls); -#define hs_desc_link_specifier_free(ls) \ - FREE_AND_NULL(hs_desc_link_specifier_t, hs_desc_link_specifier_free_, (ls)) - -hs_desc_link_specifier_t *hs_desc_link_specifier_new( - const extend_info_t *info, uint8_t type); void hs_descriptor_clear_intro_points(hs_descriptor_t *desc); MOCK_DECL(int, @@ -299,9 +275,6 @@ void hs_desc_authorized_client_free_(hs_desc_authorized_client_t *client); FREE_AND_NULL(hs_desc_authorized_client_t, \ hs_desc_authorized_client_free_, (client)) -link_specifier_t *hs_desc_lspec_to_trunnel( - const hs_desc_link_specifier_t *spec); - hs_desc_authorized_client_t *hs_desc_build_fake_authorized_client(void); void hs_desc_build_authorized_client(const uint8_t *subcredential, const curve25519_public_key_t * diff --git a/src/feature/hs/hs_intropoint.c b/src/feature/hs/hs_intropoint.c index b28a5c2b80..c9cd3a0419 100644 --- a/src/feature/hs/hs_intropoint.c +++ b/src/feature/hs/hs_intropoint.c @@ -601,8 +601,8 @@ hs_intropoint_clear(hs_intropoint_t *ip) return; } tor_cert_free(ip->auth_key_cert); - SMARTLIST_FOREACH(ip->link_specifiers, hs_desc_link_specifier_t *, ls, - hs_desc_link_specifier_free(ls)); + SMARTLIST_FOREACH(ip->link_specifiers, link_specifier_t *, ls, + link_specifier_free(ls)); smartlist_free(ip->link_specifiers); memset(ip, 0, sizeof(hs_intropoint_t)); } diff --git a/src/feature/hs/hs_service.c b/src/feature/hs/hs_service.c index 6e4da8856a..8cd863eef4 100644 --- a/src/feature/hs/hs_service.c +++ b/src/feature/hs/hs_service.c @@ -280,9 +280,10 @@ describe_intro_point(const hs_service_intro_point_t *ip) const char *legacy_id = NULL; SMARTLIST_FOREACH_BEGIN(ip->base.link_specifiers, - const hs_desc_link_specifier_t *, lspec) { - if (lspec->type == LS_LEGACY_ID) { - legacy_id = (const char *) lspec->u.legacy_id; + const link_specifier_t *, lspec) { + if (link_specifier_get_ls_type(lspec) == LS_LEGACY_ID) { + legacy_id = (const char *) + link_specifier_getconstarray_un_legacy_id(lspec); break; } } SMARTLIST_FOREACH_END(lspec); @@ -623,16 +624,16 @@ get_objects_from_ident(const hs_ident_circuit_t *ident, * encountered in the link specifier list. Return NULL if it can't be found. * * The caller does NOT have ownership of the object, the intro point does. */ -static hs_desc_link_specifier_t * +static link_specifier_t * get_link_spec_by_type(const hs_service_intro_point_t *ip, uint8_t type) { - hs_desc_link_specifier_t *lnk_spec = NULL; + link_specifier_t *lnk_spec = NULL; tor_assert(ip); SMARTLIST_FOREACH_BEGIN(ip->base.link_specifiers, - hs_desc_link_specifier_t *, ls) { - if (ls->type == type) { + link_specifier_t *, ls) { + if (link_specifier_get_ls_type(ls) == type) { lnk_spec = ls; goto end; } @@ -648,7 +649,7 @@ get_link_spec_by_type(const hs_service_intro_point_t *ip, uint8_t type) STATIC const node_t * get_node_from_intro_point(const hs_service_intro_point_t *ip) { - const hs_desc_link_specifier_t *ls; + const link_specifier_t *ls; tor_assert(ip); @@ -657,7 +658,8 @@ get_node_from_intro_point(const hs_service_intro_point_t *ip) return NULL; } /* XXX In the future, we want to only use the ed25519 ID (#22173). */ - return node_get_by_id((const char *) ls->u.legacy_id); + return node_get_by_id( + (const char *) link_specifier_getconstarray_un_legacy_id(ls)); } /* Given a service intro point, return the extend_info_t for it. This can @@ -1523,7 +1525,7 @@ remember_failing_intro_point(const hs_service_intro_point_t *ip, hs_service_descriptor_t *desc, time_t now) { time_t *time_of_failure, *prev_ptr; - const hs_desc_link_specifier_t *legacy_ls; + const link_specifier_t *legacy_ls; tor_assert(ip); tor_assert(desc); @@ -1532,22 +1534,13 @@ remember_failing_intro_point(const hs_service_intro_point_t *ip, *time_of_failure = now; legacy_ls = get_link_spec_by_type(ip, LS_LEGACY_ID); tor_assert(legacy_ls); - prev_ptr = digestmap_set(desc->intro_points.failed_id, - (const char *) legacy_ls->u.legacy_id, - time_of_failure); + prev_ptr = digestmap_set( + desc->intro_points.failed_id, + (const char *) link_specifier_getconstarray_un_legacy_id(legacy_ls), + time_of_failure); tor_free(prev_ptr); } -/* Copy the descriptor link specifier object from src to dst. */ -static void -link_specifier_copy(hs_desc_link_specifier_t *dst, - const hs_desc_link_specifier_t *src) -{ - tor_assert(dst); - tor_assert(src); - memcpy(dst, src, sizeof(hs_desc_link_specifier_t)); -} - /* Using a given descriptor signing keypair signing_kp, a service intro point * object ip and the time now, setup the content of an already allocated * descriptor intro desc_ip. @@ -1582,9 +1575,14 @@ setup_desc_intro_point(const ed25519_keypair_t *signing_kp, /* Copy link specifier(s). */ SMARTLIST_FOREACH_BEGIN(ip->base.link_specifiers, - const hs_desc_link_specifier_t *, ls) { - hs_desc_link_specifier_t *copy = tor_malloc_zero(sizeof(*copy)); - link_specifier_copy(copy, ls); + const link_specifier_t *, ls) { + if (BUG(!ls)) { + goto done; + } + link_specifier_t *copy = link_specifier_dup(ls); + if (BUG(!copy)) { + goto done; + } smartlist_add(desc_ip->link_specifiers, copy); } SMARTLIST_FOREACH_END(ls); diff --git a/src/test/hs_test_helpers.c b/src/test/hs_test_helpers.c index f2ae8398df..1c47c7d7d4 100644 --- a/src/test/hs_test_helpers.c +++ b/src/test/hs_test_helpers.c @@ -21,22 +21,33 @@ hs_helper_build_intro_point(const ed25519_keypair_t *signing_kp, time_t now, /* For a usable intro point we need at least two link specifiers: One legacy * keyid and one ipv4 */ { - hs_desc_link_specifier_t *ls_legacy = tor_malloc_zero(sizeof(*ls_legacy)); - hs_desc_link_specifier_t *ls_v4 = tor_malloc_zero(sizeof(*ls_v4)); - ls_legacy->type = LS_LEGACY_ID; - memcpy(ls_legacy->u.legacy_id, "0299F268FCA9D55CD157976D39AE92B4B455B3A8", - DIGEST_LEN); - ls_v4->u.ap.port = 9001; - int family = tor_addr_parse(&ls_v4->u.ap.addr, addr); + tor_addr_t a; + tor_addr_make_unspec(&a); + link_specifier_t *ls_legacy = link_specifier_new(); + /* TODO: this name is wrong: it is sometimes an IPv6 address */ + link_specifier_t *ls_v4 = link_specifier_new(); + link_specifier_set_ls_type(ls_legacy, LS_LEGACY_ID); + memcpy(link_specifier_getarray_un_legacy_id(ls_legacy), + /* TODO: this code is wrong: it copies hex into binary */ + "0299F268FCA9D55CD157976D39AE92B4B455B3A8", + link_specifier_getlen_un_legacy_id(ls_legacy)); + int family = tor_addr_parse(&a, addr); switch (family) { case AF_INET: - ls_v4->type = LS_IPV4; + link_specifier_set_ls_type(ls_v4, LS_IPV4); + link_specifier_set_un_ipv4_addr(ls_v4, tor_addr_to_ipv4h(&a)); + link_specifier_set_un_ipv4_port(ls_v4, 9001); break; case AF_INET6: - ls_v4->type = LS_IPV6; + link_specifier_set_ls_type(ls_v4, LS_IPV6); + memcpy(link_specifier_getarray_un_ipv6_addr(ls_v4), + tor_addr_to_in6_addr8(&a), + link_specifier_getlen_un_ipv6_addr(ls_v4)); + link_specifier_set_un_ipv6_port(ls_v4, 9001); break; default: /* Stop the test, not suppose to have an error. */ + /* TODO: just tt_fail(), because this code is confusing */ tt_int_op(family, OP_EQ, AF_INET); } smartlist_add(ip->link_specifiers, ls_legacy); @@ -153,8 +164,11 @@ hs_helper_build_hs_desc_impl(unsigned int no_ip, /* Add four intro points. */ smartlist_add(desc->encrypted_data.intro_points, hs_helper_build_intro_point(signing_kp, now, "1.2.3.4", 0)); +/* IPv6-only introduction points are not supported yet, see #23588 */ +#if 0 smartlist_add(desc->encrypted_data.intro_points, hs_helper_build_intro_point(signing_kp, now, "[2600::1]", 0)); +#endif smartlist_add(desc->encrypted_data.intro_points, hs_helper_build_intro_point(signing_kp, now, "3.2.1.4", 1)); smartlist_add(desc->encrypted_data.intro_points, @@ -202,7 +216,6 @@ void hs_helper_desc_equal(const hs_descriptor_t *desc1, const hs_descriptor_t *desc2) { - char *addr1 = NULL, *addr2 = NULL; /* Plaintext data section. */ tt_int_op(desc1->plaintext_data.version, OP_EQ, desc2->plaintext_data.version); @@ -291,35 +304,57 @@ hs_helper_desc_equal(const hs_descriptor_t *desc1, tt_int_op(smartlist_len(ip1->link_specifiers), ==, smartlist_len(ip2->link_specifiers)); for (int j = 0; j < smartlist_len(ip1->link_specifiers); j++) { - hs_desc_link_specifier_t *ls1 = smartlist_get(ip1->link_specifiers, j), - *ls2 = smartlist_get(ip2->link_specifiers, j); - tt_int_op(ls1->type, ==, ls2->type); - switch (ls1->type) { + link_specifier_t *ls1 = smartlist_get(ip1->link_specifiers, j), + *ls2 = smartlist_get(ip2->link_specifiers, j); + tt_int_op(link_specifier_get_ls_type(ls1), ==, + link_specifier_get_ls_type(ls2)); + switch (link_specifier_get_ls_type(ls1)) { case LS_IPV4: + { + uint32_t addr1 = link_specifier_get_un_ipv4_addr(ls1); + uint32_t addr2 = link_specifier_get_un_ipv4_addr(ls2); + tt_int_op(addr1, OP_EQ, addr2); + uint16_t port1 = link_specifier_get_un_ipv4_port(ls1); + uint16_t port2 = link_specifier_get_un_ipv4_port(ls2); + tt_int_op(port1, ==, port2); + } + break; case LS_IPV6: { - addr1 = tor_addr_to_str_dup(&ls1->u.ap.addr); - addr2 = tor_addr_to_str_dup(&ls2->u.ap.addr); - tt_str_op(addr1, OP_EQ, addr2); - tor_free(addr1); - tor_free(addr2); - tt_int_op(ls1->u.ap.port, ==, ls2->u.ap.port); + const uint8_t *addr1 = + link_specifier_getconstarray_un_ipv6_addr(ls1); + const uint8_t *addr2 = + link_specifier_getconstarray_un_ipv6_addr(ls2); + tt_int_op(link_specifier_getlen_un_ipv6_addr(ls1), OP_EQ, + link_specifier_getlen_un_ipv6_addr(ls2)); + tt_mem_op(addr1, OP_EQ, addr2, + link_specifier_getlen_un_ipv6_addr(ls1)); + uint16_t port1 = link_specifier_get_un_ipv6_port(ls1); + uint16_t port2 = link_specifier_get_un_ipv6_port(ls2); + tt_int_op(port1, ==, port2); } break; case LS_LEGACY_ID: - tt_mem_op(ls1->u.legacy_id, OP_EQ, ls2->u.legacy_id, - sizeof(ls1->u.legacy_id)); + { + const uint8_t *id1 = + link_specifier_getconstarray_un_legacy_id(ls1); + const uint8_t *id2 = + link_specifier_getconstarray_un_legacy_id(ls2); + tt_int_op(link_specifier_getlen_un_legacy_id(ls1), OP_EQ, + link_specifier_getlen_un_legacy_id(ls2)); + tt_mem_op(id1, OP_EQ, id2, + link_specifier_getlen_un_legacy_id(ls1)); + } break; default: /* Unknown type, caught it and print its value. */ - tt_int_op(ls1->type, OP_EQ, -1); + tt_int_op(link_specifier_get_ls_type(ls1), OP_EQ, -1); } } } } done: - tor_free(addr1); - tor_free(addr2); + ; } diff --git a/src/test/test_hs_client.c b/src/test/test_hs_client.c index 2f2bb45581..8362b6cbda 100644 --- a/src/test/test_hs_client.c +++ b/src/test/test_hs_client.c @@ -403,6 +403,9 @@ test_client_pick_intro(void *arg) /* 2) Mark all intro points except _the chosen one_ as failed. Then query the * desc and get a random intro: check that we got _the chosen one_. */ { + /* Tell hs_get_extend_info_from_lspecs() to skip the private address check. + */ + get_options_mutable()->ExtendAllowPrivateAddresses = 1; /* Pick the chosen intro point and get its ei */ hs_desc_intro_point_t *chosen_intro_point = smartlist_get(desc->encrypted_data.intro_points, 0); @@ -476,6 +479,7 @@ test_client_pick_intro(void *arg) SMARTLIST_FOREACH_BEGIN(desc->encrypted_data.intro_points, hs_desc_intro_point_t *, ip) { extend_info_t *intro_ei = desc_intro_point_to_extend_info(ip); + tt_assert(intro_ei); if (intro_ei) { const char *ptr; char ip_addr[TOR_ADDR_BUF_LEN]; diff --git a/src/test/test_hs_descriptor.c b/src/test/test_hs_descriptor.c index de584ed47a..09c6c3e700 100644 --- a/src/test/test_hs_descriptor.c +++ b/src/test/test_hs_descriptor.c @@ -178,115 +178,6 @@ test_descriptor_padding(void *arg) return; } -static void -test_link_specifier(void *arg) -{ - ssize_t ret; - hs_desc_link_specifier_t spec; - smartlist_t *link_specifiers = smartlist_new(); - char buf[256]; - char *b64 = NULL; - link_specifier_t *ls = NULL; - - (void) arg; - - /* Always this port. */ - spec.u.ap.port = 42; - smartlist_add(link_specifiers, &spec); - - /* Test IPv4 for starter. */ - { - uint32_t ipv4; - - spec.type = LS_IPV4; - ret = tor_addr_parse(&spec.u.ap.addr, "1.2.3.4"); - tt_int_op(ret, OP_EQ, AF_INET); - b64 = encode_link_specifiers(link_specifiers); - tt_assert(b64); - - /* Decode it and validate the format. */ - ret = base64_decode(buf, sizeof(buf), b64, strlen(b64)); - tt_int_op(ret, OP_GT, 0); - /* First byte is the number of link specifier. */ - tt_int_op(get_uint8(buf), OP_EQ, 1); - ret = link_specifier_parse(&ls, (uint8_t *) buf + 1, ret - 1); - tt_int_op(ret, OP_EQ, 8); - /* Should be 2 bytes for port and 4 bytes for IPv4. */ - tt_int_op(link_specifier_get_ls_len(ls), OP_EQ, 6); - ipv4 = link_specifier_get_un_ipv4_addr(ls); - tt_int_op(tor_addr_to_ipv4h(&spec.u.ap.addr), OP_EQ, ipv4); - tt_int_op(link_specifier_get_un_ipv4_port(ls), OP_EQ, spec.u.ap.port); - - link_specifier_free(ls); - ls = NULL; - tor_free(b64); - } - - /* Test IPv6. */ - { - uint8_t ipv6[16]; - - spec.type = LS_IPV6; - ret = tor_addr_parse(&spec.u.ap.addr, "[1:2:3:4::]"); - tt_int_op(ret, OP_EQ, AF_INET6); - b64 = encode_link_specifiers(link_specifiers); - tt_assert(b64); - - /* Decode it and validate the format. */ - ret = base64_decode(buf, sizeof(buf), b64, strlen(b64)); - tt_int_op(ret, OP_GT, 0); - /* First byte is the number of link specifier. */ - tt_int_op(get_uint8(buf), OP_EQ, 1); - ret = link_specifier_parse(&ls, (uint8_t *) buf + 1, ret - 1); - tt_int_op(ret, OP_EQ, 20); - /* Should be 2 bytes for port and 16 bytes for IPv6. */ - tt_int_op(link_specifier_get_ls_len(ls), OP_EQ, 18); - for (unsigned int i = 0; i < sizeof(ipv6); i++) { - ipv6[i] = link_specifier_get_un_ipv6_addr(ls, i); - } - tt_mem_op(tor_addr_to_in6_addr8(&spec.u.ap.addr), OP_EQ, ipv6, - sizeof(ipv6)); - tt_int_op(link_specifier_get_un_ipv6_port(ls), OP_EQ, spec.u.ap.port); - - link_specifier_free(ls); - ls = NULL; - tor_free(b64); - } - - /* Test legacy. */ - { - uint8_t *id; - - spec.type = LS_LEGACY_ID; - memset(spec.u.legacy_id, 'Y', sizeof(spec.u.legacy_id)); - b64 = encode_link_specifiers(link_specifiers); - tt_assert(b64); - - /* Decode it and validate the format. */ - ret = base64_decode(buf, sizeof(buf), b64, strlen(b64)); - tt_int_op(ret, OP_GT, 0); - /* First byte is the number of link specifier. */ - tt_int_op(get_uint8(buf), OP_EQ, 1); - ret = link_specifier_parse(&ls, (uint8_t *) buf + 1, ret - 1); - /* 20 bytes digest + 1 byte type + 1 byte len. */ - tt_int_op(ret, OP_EQ, 22); - tt_int_op(link_specifier_getlen_un_legacy_id(ls), OP_EQ, DIGEST_LEN); - /* Digest length is 20 bytes. */ - tt_int_op(link_specifier_get_ls_len(ls), OP_EQ, DIGEST_LEN); - id = link_specifier_getarray_un_legacy_id(ls); - tt_mem_op(spec.u.legacy_id, OP_EQ, id, DIGEST_LEN); - - link_specifier_free(ls); - ls = NULL; - tor_free(b64); - } - - done: - link_specifier_free(ls); - tor_free(b64); - smartlist_free(link_specifiers); -} - static void test_encode_descriptor(void *arg) { @@ -932,8 +823,6 @@ struct testcase_t hs_descriptor[] = { /* Encoding tests. */ { "cert_encoding", test_cert_encoding, TT_FORK, NULL, NULL }, - { "link_specifier", test_link_specifier, TT_FORK, - NULL, NULL }, { "encode_descriptor", test_encode_descriptor, TT_FORK, NULL, NULL }, { "descriptor_padding", test_descriptor_padding, TT_FORK, diff --git a/src/test/test_hs_service.c b/src/test/test_hs_service.c index e8bdcd86e2..57132e6197 100644 --- a/src/test/test_hs_service.c +++ b/src/test/test_hs_service.c @@ -328,17 +328,18 @@ helper_create_service_with_clients(int num_clients) static hs_service_intro_point_t * helper_create_service_ip(void) { - hs_desc_link_specifier_t *ls; + link_specifier_t *ls; hs_service_intro_point_t *ip = service_intro_point_new(NULL); tor_assert(ip); /* Add a first unused link specifier. */ - ls = tor_malloc_zero(sizeof(*ls)); - ls->type = LS_IPV4; + ls = link_specifier_new(); + link_specifier_set_ls_type(ls, LS_IPV4); smartlist_add(ip->base.link_specifiers, ls); /* Add a second link specifier used by a test. */ - ls = tor_malloc_zero(sizeof(*ls)); - ls->type = LS_LEGACY_ID; - memset(ls->u.legacy_id, 'A', sizeof(ls->u.legacy_id)); + ls = link_specifier_new(); + link_specifier_set_ls_type(ls, LS_LEGACY_ID); + memset(link_specifier_getarray_un_legacy_id(ls), 'A', + link_specifier_getlen_un_legacy_id(ls)); smartlist_add(ip->base.link_specifiers, ls); return ip; @@ -811,10 +812,11 @@ test_helper_functions(void *arg) const node_t *node = get_node_from_intro_point(ip); tt_ptr_op(node, OP_EQ, &mock_node); SMARTLIST_FOREACH_BEGIN(ip->base.link_specifiers, - hs_desc_link_specifier_t *, ls) { - if (ls->type == LS_LEGACY_ID) { + link_specifier_t *, ls) { + if (link_specifier_get_ls_type(ls) == LS_LEGACY_ID) { /* Change legacy id in link specifier which is not the mock node. */ - memset(ls->u.legacy_id, 'B', sizeof(ls->u.legacy_id)); + memset(link_specifier_getarray_un_legacy_id(ls), 'B', + link_specifier_getlen_un_legacy_id(ls)); } } SMARTLIST_FOREACH_END(ls); node = get_node_from_intro_point(ip); From 257cea8876e00b18c6791f86ebe5c24bab3b72bf Mon Sep 17 00:00:00 2001 From: teor Date: Wed, 30 Jan 2019 21:52:19 +0100 Subject: [PATCH 0584/2557] test/hs: minor hs test fixes Cleanup some bugs discovered during 23576: * stop copying the first 20 characters of a 40-character hex string to a binary fingerprint * stop putting IPv6 addresses in a variable called "ipv4" * explain why we do a duplicate tt_int_op() to deliberately fail and print a value Fixes bug 29243; bugfix on 0.3.2.1-alpha. --- changes/bug29243 | 3 +++ src/test/hs_test_helpers.c | 30 ++++++++++++++---------------- 2 files changed, 17 insertions(+), 16 deletions(-) create mode 100644 changes/bug29243 diff --git a/changes/bug29243 b/changes/bug29243 new file mode 100644 index 0000000000..b5694f7568 --- /dev/null +++ b/changes/bug29243 @@ -0,0 +1,3 @@ + o Minor bugfixes (testing, v3 onion services): + - Fix some incorrect code in the v3 onion service unit tests. + Fixes bug 29243; bugfix on 0.3.2.1-alpha. diff --git a/src/test/hs_test_helpers.c b/src/test/hs_test_helpers.c index 1c47c7d7d4..c57bdc730b 100644 --- a/src/test/hs_test_helpers.c +++ b/src/test/hs_test_helpers.c @@ -24,34 +24,32 @@ hs_helper_build_intro_point(const ed25519_keypair_t *signing_kp, time_t now, tor_addr_t a; tor_addr_make_unspec(&a); link_specifier_t *ls_legacy = link_specifier_new(); - /* TODO: this name is wrong: it is sometimes an IPv6 address */ - link_specifier_t *ls_v4 = link_specifier_new(); + link_specifier_t *ls_ip = link_specifier_new(); link_specifier_set_ls_type(ls_legacy, LS_LEGACY_ID); - memcpy(link_specifier_getarray_un_legacy_id(ls_legacy), - /* TODO: this code is wrong: it copies hex into binary */ - "0299F268FCA9D55CD157976D39AE92B4B455B3A8", + memset(link_specifier_getarray_un_legacy_id(ls_legacy), 'C', link_specifier_getlen_un_legacy_id(ls_legacy)); int family = tor_addr_parse(&a, addr); switch (family) { case AF_INET: - link_specifier_set_ls_type(ls_v4, LS_IPV4); - link_specifier_set_un_ipv4_addr(ls_v4, tor_addr_to_ipv4h(&a)); - link_specifier_set_un_ipv4_port(ls_v4, 9001); + link_specifier_set_ls_type(ls_ip, LS_IPV4); + link_specifier_set_un_ipv4_addr(ls_ip, tor_addr_to_ipv4h(&a)); + link_specifier_set_un_ipv4_port(ls_ip, 9001); break; case AF_INET6: - link_specifier_set_ls_type(ls_v4, LS_IPV6); - memcpy(link_specifier_getarray_un_ipv6_addr(ls_v4), + link_specifier_set_ls_type(ls_ip, LS_IPV6); + memcpy(link_specifier_getarray_un_ipv6_addr(ls_ip), tor_addr_to_in6_addr8(&a), - link_specifier_getlen_un_ipv6_addr(ls_v4)); - link_specifier_set_un_ipv6_port(ls_v4, 9001); + link_specifier_getlen_un_ipv6_addr(ls_ip)); + link_specifier_set_un_ipv6_port(ls_ip, 9001); break; default: - /* Stop the test, not suppose to have an error. */ - /* TODO: just tt_fail(), because this code is confusing */ - tt_int_op(family, OP_EQ, AF_INET); + /* Stop the test, not supposed to have an error. + * Compare with -1 to show the actual family. + */ + tt_int_op(family, OP_EQ, -1); } smartlist_add(ip->link_specifiers, ls_legacy); - smartlist_add(ip->link_specifiers, ls_v4); + smartlist_add(ip->link_specifiers, ls_ip); } ret = ed25519_keypair_generate(&auth_kp, 0); From 680b2afd84a318e11f79526bfedff29870307ed0 Mon Sep 17 00:00:00 2001 From: teor Date: Fri, 8 Mar 2019 16:47:20 +1000 Subject: [PATCH 0585/2557] hs: abolish hs_desc_link_specifier_dup() The previous commits introduced link_specifier_dup(), which is implemented using trunnel's opaque interfaces. So we can now remove hs_desc_link_specifier_dup(). Cleanup after bug 22781. --- src/feature/hs/hs_cell.c | 2 +- src/feature/hs/hs_common.c | 18 ------------------ src/feature/hs/hs_common.h | 2 -- src/trunnel/ed25519_cert.trunnel | 6 ------ 4 files changed, 1 insertion(+), 27 deletions(-) diff --git a/src/feature/hs/hs_cell.c b/src/feature/hs/hs_cell.c index e24520da47..1dae9c79c1 100644 --- a/src/feature/hs/hs_cell.c +++ b/src/feature/hs/hs_cell.c @@ -759,7 +759,7 @@ hs_cell_parse_introduce2(hs_cell_introduce2_data_t *data, if (BUG(!lspec)) { goto done; } - link_specifier_t *lspec_dup = hs_link_specifier_dup(lspec); + link_specifier_t *lspec_dup = link_specifier_dup(lspec); if (BUG(!lspec_dup)) { goto done; } diff --git a/src/feature/hs/hs_common.c b/src/feature/hs/hs_common.c index a1dc0a6290..2cc23a4184 100644 --- a/src/feature/hs/hs_common.c +++ b/src/feature/hs/hs_common.c @@ -1009,24 +1009,6 @@ hs_build_address(const ed25519_public_key_t *key, uint8_t version, tor_assert(hs_address_is_valid(addr_out)); } -/* Return a newly allocated copy of lspec. */ -link_specifier_t * -hs_link_specifier_dup(const link_specifier_t *lspec) -{ - link_specifier_t *result = link_specifier_new(); - memcpy(result, lspec, sizeof(*result)); - /* The unrecognized field is a dynamic array so make sure to copy its - * content and not the pointer. */ - link_specifier_setlen_un_unrecognized( - result, link_specifier_getlen_un_unrecognized(lspec)); - if (link_specifier_getlen_un_unrecognized(result)) { - memcpy(link_specifier_getarray_un_unrecognized(result), - link_specifier_getconstarray_un_unrecognized(lspec), - link_specifier_getlen_un_unrecognized(result)); - } - return result; -} - /* From a given ed25519 public key pk and an optional secret, compute a * blinded public key and put it in blinded_pk_out. This is only useful to * the client side because the client only has access to the identity public diff --git a/src/feature/hs/hs_common.h b/src/feature/hs/hs_common.h index 544e63e596..abf39fa431 100644 --- a/src/feature/hs/hs_common.h +++ b/src/feature/hs/hs_common.h @@ -217,8 +217,6 @@ uint64_t hs_get_time_period_num(time_t now); uint64_t hs_get_next_time_period_num(time_t now); time_t hs_get_start_time_of_next_time_period(time_t now); -link_specifier_t *hs_link_specifier_dup(const link_specifier_t *lspec); - MOCK_DECL(int, hs_in_period_between_tp_and_srv, (const networkstatus_t *consensus, time_t now)); diff --git a/src/trunnel/ed25519_cert.trunnel b/src/trunnel/ed25519_cert.trunnel index 8d6483d558..e424ce5464 100644 --- a/src/trunnel/ed25519_cert.trunnel +++ b/src/trunnel/ed25519_cert.trunnel @@ -28,12 +28,6 @@ const LS_IPV6 = 0x01; const LS_LEGACY_ID = 0x02; const LS_ED25519_ID = 0x03; -// XXX hs_link_specifier_dup() violates the opaqueness of link_specifier_t by -// taking its sizeof(). If we ever want to turn on TRUNNEL_OPAQUE, or -// if we ever make link_specifier contain other types, we will -// need to refactor that function to do the copy by encoding and decoding the -// object. - // amended from tor.trunnel struct link_specifier { u8 ls_type; From ec2a2a6b7afd5f37cfdc147eb12fb0c64d9ae0cf Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Tue, 12 Mar 2019 20:11:51 +0200 Subject: [PATCH 0586/2557] Fix #28525 changes file that is breaking CI. --- changes/bug28525 | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/changes/bug28525 b/changes/bug28525 index 392a9265e5..988ffb2192 100644 --- a/changes/bug28525 +++ b/changes/bug28525 @@ -1,8 +1,7 @@ - o Minor bugfixes (address selection): + o Minor features (address selection): - Make Tor aware of the RFC 6598 (Carrier Grade NAT) IP range, which is the subnet 100.64.0.0/10. This is deployed by many ISPs as an alternative to RFC 1918 that does not break existing internal networks. This patch fixes security issues caused by RFC 6518 by blocking control ports on these addresses and warns users if client ports or ExtORPorts are listening on - a RFC 6598 address. Fixes bug 28525; bugfix on 0.4.1.1-alpha. Patch by - Neel Chauhan. + a RFC 6598 address. Closes ticket 28525. Patch by Neel Chauhan. From 95209be861738d15fdb8d3f8da9a6164ed5f3903 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 12 Mar 2019 15:19:28 -0400 Subject: [PATCH 0587/2557] Make checkSpace.pl check guard macros: - every .h file needs an #ifndef/#define pair. - They must refer to the same macro. - The guard macros that they refer to must be unique across all headers. --- changes/ticket29756 | 3 +++ scripts/maint/checkSpace.pl | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+) create mode 100644 changes/ticket29756 diff --git a/changes/ticket29756 b/changes/ticket29756 new file mode 100644 index 0000000000..79995b4995 --- /dev/null +++ b/changes/ticket29756 @@ -0,0 +1,3 @@ + o Minor features (developer tools): + - Add a script to check that each header has a well-formed and unique + guard marco. Closes ticket 29756. diff --git a/scripts/maint/checkSpace.pl b/scripts/maint/checkSpace.pl index 633b47e314..433ae62807 100755 --- a/scripts/maint/checkSpace.pl +++ b/scripts/maint/checkSpace.pl @@ -18,6 +18,8 @@ if ($ARGV[0] =~ /^-/) { our %basenames = (); +our %guardnames = (); + for my $fn (@ARGV) { open(F, "$fn"); my $lastnil = 0; @@ -31,6 +33,10 @@ for my $fn (@ARGV) { } else { $basenames{$basename} = $fn; } + my $isheader = ($fn =~ /\.h/); + my $seenguard = 0; + my $guardname = ""; + while () { ## Warn about windows-style newlines. # (We insist on lines that end with a single LF character, not @@ -112,6 +118,23 @@ for my $fn (@ARGV) { next; } } + + if ($isheader) { + if ($seenguard == 0) { + if (/ifndef\s+(\S+)/) { + ++$seenguard; + $guardname = $1; + } + } elsif ($seenguard == 1) { + if (/^\#define (\S+)/) { + ++$seenguard; + if ($1 ne $guardname) { + msg "GUARD:$fn:$.: Header guard macro mismatch.\n"; + } + } + } + } + if (m!/\*.*?\*/!) { s!\s*/\*.*?\*/!!; } elsif (m!/\*!) { @@ -201,6 +224,15 @@ for my $fn (@ARGV) { } } } + if ($isheader && $C) { + if ($seenguard < 2) { + msg "$fn:No #ifndef/#define header guard pair found.\n"; + } elsif ($guardnames{$guardname}) { + msg "$fn:Guard macro $guardname also used in $guardnames{$guardname}\n"; + } else { + $guardnames{$guardname} = $fn; + } + } close(F); } From 065b74fa36344295a12f53e7399e7b20c6d887e0 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 12 Mar 2019 15:20:22 -0400 Subject: [PATCH 0588/2557] Fix all nonconformant headers' guard macros. --- src/feature/control/btrack_orconn_cevent.h | 1 + src/feature/control/btrack_orconn_maps.h | 1 + src/feature/hs/hs_stats.h | 4 ++++ src/lib/log/log.h | 2 +- src/test/test_connection.h | 4 ++++ src/test/test_dir_common.h | 4 ++++ 6 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/feature/control/btrack_orconn_cevent.h b/src/feature/control/btrack_orconn_cevent.h index f9d24633aa..954b452451 100644 --- a/src/feature/control/btrack_orconn_cevent.h +++ b/src/feature/control/btrack_orconn_cevent.h @@ -7,6 +7,7 @@ **/ #ifndef TOR_BTRACK_ORCONN_CEVENT_H +#define TOR_BTRACK_ORCONN_CEVENT_H #include "feature/control/btrack_orconn.h" diff --git a/src/feature/control/btrack_orconn_maps.h b/src/feature/control/btrack_orconn_maps.h index 3ead40984c..2065eb61b2 100644 --- a/src/feature/control/btrack_orconn_maps.h +++ b/src/feature/control/btrack_orconn_maps.h @@ -7,6 +7,7 @@ **/ #ifndef TOR_BTRACK_ORCONN_MAPS_H +#define TOR_BTRACK_ORCONN_MAPS_H void bto_delete(uint64_t); bt_orconn_t *bto_find_or_new(uint64_t, uint64_t); diff --git a/src/feature/hs/hs_stats.h b/src/feature/hs/hs_stats.h index d89440faca..ca048e2123 100644 --- a/src/feature/hs/hs_stats.h +++ b/src/feature/hs/hs_stats.h @@ -6,9 +6,13 @@ * \brief Header file for hs_stats.c **/ +#ifndef TOR_HS_STATS_H +#define TOR_HS_STATS_H + void hs_stats_note_introduce2_cell(int is_hsv3); uint32_t hs_stats_get_n_introduce2_v3_cells(void); uint32_t hs_stats_get_n_introduce2_v2_cells(void); void hs_stats_note_service_rendezvous_launch(void); uint32_t hs_stats_get_n_rendezvous_launches(void); +#endif diff --git a/src/lib/log/log.h b/src/lib/log/log.h index dbc1c47021..0420f35ee0 100644 --- a/src/lib/log/log.h +++ b/src/lib/log/log.h @@ -11,6 +11,7 @@ **/ #ifndef TOR_TORLOG_H +#define TOR_TORLOG_H #include #include "lib/cc/torint.h" @@ -278,5 +279,4 @@ MOCK_DECL(STATIC void, logv, (int severity, log_domain_mask_t domain, va_list ap) CHECK_PRINTF(5,0)); #endif -# define TOR_TORLOG_H #endif /* !defined(TOR_TORLOG_H) */ diff --git a/src/test/test_connection.h b/src/test/test_connection.h index 47a5599e5f..027e405d89 100644 --- a/src/test/test_connection.h +++ b/src/test/test_connection.h @@ -1,6 +1,9 @@ /* Copyright (c) 2014-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ +#ifndef TOR_TEST_CONNECTION_H +#define TOR_TEST_CONNECTION_H + /** Some constants used by test_connection and helpers */ #define TEST_CONN_FAMILY (AF_INET) #define TEST_CONN_ADDRESS "127.0.0.1" @@ -11,3 +14,4 @@ void test_conn_lookup_addr_helper(const char *address, int family, tor_addr_t *addr); +#endif diff --git a/src/test/test_dir_common.h b/src/test/test_dir_common.h index d6c5241b14..ab99ed36f4 100644 --- a/src/test/test_dir_common.h +++ b/src/test/test_dir_common.h @@ -3,6 +3,9 @@ * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ +#ifndef TOR_TEST_DIR_COMMON_H +#define TOR_TEST_DIR_COMMON_H + #include "core/or/or.h" #include "feature/nodelist/networkstatus.h" @@ -49,3 +52,4 @@ int dir_common_construct_vote_3(networkstatus_t **vote, networkstatus_t **vote_out, int *n_vrs, time_t now, int clear_rl); +#endif From a9c3101e2140ceacc47498a8d15c8b54ad2616c5 Mon Sep 17 00:00:00 2001 From: teor Date: Wed, 13 Mar 2019 15:27:48 +1000 Subject: [PATCH 0589/2557] test/sr: Clear SRVs after init, and before setup Part of 29706. --- src/test/test_shared_random.c | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/test/test_shared_random.c b/src/test/test_shared_random.c index 991d73128e..6324fa68b5 100644 --- a/src/test/test_shared_random.c +++ b/src/test/test_shared_random.c @@ -61,6 +61,9 @@ init_authority_state(void) * the phase we are currently in which uses "now" as the starting * timestamp. Delete it before we do any testing below. */ sr_state_delete_commits(); + /* It's also possible that a current SRV has been generated, if we are at + * state transition time. But let's just forget about that SRV. */ + sr_state_clean_srvs(); done: UNMOCK(get_my_v3_authority_cert); @@ -545,22 +548,24 @@ test_encoding(void *arg) ; } -/** Setup some SRVs in our SR state. If also_current is set, then set - * both current and previous SRVs. - * Helper of test_vote() and test_sr_compute_srv(). +/** Setup some SRVs in our SR state. + * If also_current is set, then set both current and previous SRVs. + * Otherwise, just set the previous SRV. (And clear the current SRV.) + * * You must call sr_state_free_all() to free the state at the end of each test * function (on pass or fail). */ static void test_sr_setup_srv(int also_current) { + /* Clear both SRVs before starting. */ + sr_state_clean_srvs(); + sr_srv_t *srv = tor_malloc_zero(sizeof(sr_srv_t)); srv->num_reveals = 42; memcpy(srv->value, "ZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZ", sizeof(srv->value)); - /* sr_state_set_previous_srv() does not free() the old previous srv. */ - state_del_previous_srv(); sr_state_set_previous_srv(srv); if (also_current) { @@ -570,8 +575,6 @@ test_sr_setup_srv(int also_current) "NNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN", sizeof(srv->value)); - /* sr_state_set_previous_srv() does not free() the old current srv. */ - state_del_current_srv(); sr_state_set_current_srv(srv); } } From 32c821c47b56471802a6c6c0f2286eb41d07238b Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Tue, 12 Mar 2019 18:38:55 +0200 Subject: [PATCH 0590/2557] circpad: Documentation improvements after Nick's review. --- src/core/or/circuitpadding.c | 10 ++++++---- src/core/or/circuitpadding.h | 3 ++- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/core/or/circuitpadding.c b/src/core/or/circuitpadding.c index b7e4b56846..106bb4ffa1 100644 --- a/src/core/or/circuitpadding.c +++ b/src/core/or/circuitpadding.c @@ -222,8 +222,9 @@ circpad_machine_current_state(const circpad_machine_state_t *mi) } /** - * Get the lower bound of a histogram bin. The upper bound is obtained by - * calling this function with bin+1, and subtracting 1. + * Get the lower bound of a histogram bin. + * + * You can obtain the upper bound using histogram_get_bin_upper_bound(). * * This function can also be called with 'bin' set to a value equal or greater * than histogram_len in which case the infinity bin is chosen and @@ -273,8 +274,7 @@ circpad_get_histogram_bin_midpoint(const circpad_machine_state_t *mi, int bin_index) { circpad_delay_t left_bound = circpad_histogram_bin_to_usec(mi, bin_index); - circpad_delay_t right_bound = - circpad_histogram_bin_to_usec(mi, bin_index+1)-1; + circpad_delay_t right_bound = histogram_get_bin_upper_bound(mi, bin_index); return left_bound + (right_bound - left_bound)/2; } @@ -381,6 +381,8 @@ circpad_choose_state_length(circpad_machine_state_t *mi) /** * Sample a value from our iat_dist, and clamp it safely * to circpad_delay_t. + * + * Before returning, add delay_shift (can be zero) to the sampled value. */ static circpad_delay_t circpad_distribution_sample_iat_delay(const circpad_state_t *state, diff --git a/src/core/or/circuitpadding.h b/src/core/or/circuitpadding.h index 74f9f35c98..2c763f46ef 100644 --- a/src/core/or/circuitpadding.h +++ b/src/core/or/circuitpadding.h @@ -291,7 +291,8 @@ typedef struct circpad_state_t { /** The histogram itself: an array of uint16s of tokens, whose * widths are exponentially spaced, in microseconds. * - * This array must have histogram_len elements. */ + * This array must have histogram_len elements that are (non-strictly) + * monotonically increasing. */ circpad_hist_token_t histogram[CIRCPAD_MAX_HISTOGRAM_LEN]; /* The histogram bin edges in usec. * From cdaff26f91ff6e3dceca6054157bbcf976713398 Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Wed, 13 Mar 2019 13:21:52 +0200 Subject: [PATCH 0591/2557] circpad: Machines MUST have strictly increasing histogram edges. Add a basic validation function for the histograms. It can be a building block for the future --- src/core/or/circuitpadding.c | 98 ++++++++++++++++++++++++++++++---- src/core/or/circuitpadding.h | 6 ++- src/test/test_circuitpadding.c | 15 +++--- 3 files changed, 99 insertions(+), 20 deletions(-) diff --git a/src/core/or/circuitpadding.c b/src/core/or/circuitpadding.c index 106bb4ffa1..2ee233cc58 100644 --- a/src/core/or/circuitpadding.c +++ b/src/core/or/circuitpadding.c @@ -494,12 +494,13 @@ circpad_machine_sample_delay(circpad_machine_state_t *mi) * function below samples from [bin_start, bin_end) */ bin_end = circpad_histogram_bin_to_usec(mi, curr_bin+1); - // Sample uniformly between histogram[i] to histogram[i+1]-1, - // but no need to sample if they are the same timeval (aka bin 0 or bin 1). - if (bin_end <= bin_start+1) + /* Bin edges are monotonically increasing so this is a bug. Handle it. */ + if (BUG(bin_start > bin_end)) { return bin_start; - else - return (circpad_delay_t)crypto_rand_uint64_range(bin_start, bin_end); + } + + /* Sample randomly from within the bin width */ + return (circpad_delay_t)crypto_rand_uint64_range(bin_start, bin_end); } /** @@ -2050,16 +2051,92 @@ circpad_setup_machine_on_circ(circuit_t *on_circ, on_circ->padding_machine[machine->machine_index] = machine; } -/* These padding machines are only used for tests pending #28634. */ #ifdef TOR_UNIT_TESTS +/** Validate a single state of a padding machine */ +static bool +padding_machine_state_is_valid(const circpad_state_t *state) +{ + int b; + uint32_t tokens_count = 0; + circpad_delay_t prev_bin_edge = 0; + + /* We only validate histograms */ + if (!state->histogram_len) { + return true; + } + + /* We need at least two bins in a histogram */ + if (state->histogram_len < 2) { + log_warn(LD_GENERAL, "You can't have a histogram with less than 2 bins"); + return false; + } + + /* For each machine state, if it's a histogram, make sure all the + * histogram edges are well defined (i.e. are strictly monotonic). */ + for (b = 0 ; b < state->histogram_len ; b++) { + /* Check that histogram edges are strictly increasing. Ignore the first + * edge since it can be zero. */ + if (prev_bin_edge >= state->histogram_edges[b] && b > 0) { + log_warn(LD_GENERAL, "Histogram edges are not increasing [%u/%u]", + prev_bin_edge, state->histogram_edges[b]); + return false; + } + + prev_bin_edge = state->histogram_edges[b]; + + /* Also count the number of tokens as we go through the histogram states */ + tokens_count += state->histogram[b]; + } + /* Verify that the total number of tokens is correct */ + if (tokens_count != state->histogram_total_tokens) { + log_warn(LD_GENERAL, "Histogram token count is wrong [%u/%u]", + tokens_count, state->histogram_total_tokens); + return false; + } + + return true; +} + +/** Basic validation of padding machine */ +static bool +padding_machine_is_valid(const circpad_machine_spec_t *machine) +{ + int i; + + /* Validate the histograms of the padding machine */ + for (i = 0 ; i < machine->num_states ; i++) { + if (!padding_machine_state_is_valid(&machine->states[i])) { + return false; + } + } + + return true; +} + +/* Validate and register machine into machine_list. If + * machine_list is NULL, then just validate. */ +STATIC void +register_padding_machine(circpad_machine_spec_t *machine, + smartlist_t *machine_list) +{ + if (!padding_machine_is_valid(machine)) { + log_warn(LD_GENERAL, "Machine #%u is invalid. Ignoring.", + machine->machine_num); + return; + } + + if (machine_list) { + smartlist_add(machine_list, machine); + } +} + +/* These padding machines are only used for tests pending #28634. */ static void circpad_circ_client_machine_init(void) { circpad_machine_spec_t *circ_client_machine = tor_malloc_zero(sizeof(circpad_machine_spec_t)); - // XXX: Better conditions for merge.. Or disable this machine in - // merge? circ_client_machine->conditions.min_hops = 2; circ_client_machine->conditions.state_mask = CIRCPAD_CIRC_BUILDING|CIRCPAD_CIRC_OPENED|CIRCPAD_CIRC_HAS_RELAY_EARLY; @@ -2091,7 +2168,6 @@ circpad_circ_client_machine_init(void) circ_client_machine->states[CIRCPAD_STATE_BURST].token_removal = CIRCPAD_TOKEN_REMOVAL_CLOSEST; - // FIXME: Tune this histogram circ_client_machine->states[CIRCPAD_STATE_BURST].histogram_len = 2; circ_client_machine->states[CIRCPAD_STATE_BURST].histogram_edges[0]= 500; circ_client_machine->states[CIRCPAD_STATE_BURST].histogram_edges[1]= 1000000; @@ -2105,7 +2181,7 @@ circpad_circ_client_machine_init(void) circ_client_machine->states[CIRCPAD_STATE_BURST].histogram_total_tokens = 5; circ_client_machine->machine_num = smartlist_len(origin_padding_machines); - smartlist_add(origin_padding_machines, circ_client_machine); + register_padding_machine(circ_client_machine, origin_padding_machines); } static void @@ -2205,7 +2281,7 @@ circpad_circ_responder_machine_init(void) CIRCPAD_TOKEN_REMOVAL_CLOSEST_USEC; circ_responder_machine->machine_num = smartlist_len(relay_padding_machines); - smartlist_add(relay_padding_machines, circ_responder_machine); + register_padding_machine(circ_responder_machine, relay_padding_machines); } #endif diff --git a/src/core/or/circuitpadding.h b/src/core/or/circuitpadding.h index 2c763f46ef..54039fa59a 100644 --- a/src/core/or/circuitpadding.h +++ b/src/core/or/circuitpadding.h @@ -291,7 +291,7 @@ typedef struct circpad_state_t { /** The histogram itself: an array of uint16s of tokens, whose * widths are exponentially spaced, in microseconds. * - * This array must have histogram_len elements that are (non-strictly) + * This array must have histogram_len elements that are strictly * monotonically increasing. */ circpad_hist_token_t histogram[CIRCPAD_MAX_HISTOGRAM_LEN]; /* The histogram bin edges in usec. @@ -730,6 +730,10 @@ histogram_get_bin_upper_bound(const circpad_machine_state_t *mi, #ifdef TOR_UNIT_TESTS extern smartlist_t *origin_padding_machines; extern smartlist_t *relay_padding_machines; + +STATIC void +register_padding_machine(circpad_machine_spec_t *machine, + smartlist_t *machine_list); #endif #endif diff --git a/src/test/test_circuitpadding.c b/src/test/test_circuitpadding.c index 3570b179bd..a62b2b5164 100644 --- a/src/test/test_circuitpadding.c +++ b/src/test/test_circuitpadding.c @@ -794,7 +794,7 @@ test_circuitpadding_closest_token_removal(void *arg) tt_int_op(client_side->padding_info[0]->current_state, OP_EQ, CIRCPAD_STATE_BURST); circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_edges[0] = 100; - circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_edges[1] = 100; + circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_edges[1] = 101; circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_edges[2] = 120; mi->padding_scheduled_at_usec = current_time - 102; mi->histogram[0] = 0; @@ -903,7 +903,7 @@ test_circuitpadding_closest_token_removal_usec(void *arg) tt_int_op(client_side->padding_info[0]->current_state, OP_EQ, CIRCPAD_STATE_BURST); circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_edges[0] = 100; - circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_edges[1] = 100; + circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_edges[1] = 101; circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_edges[2] = 120; mi->padding_scheduled_at_usec = current_time - 102; mi->histogram[0] = 0; @@ -1649,7 +1649,7 @@ helper_create_conditional_machine(void) ret->states[CIRCPAD_STATE_BURST].histogram_len = 3; ret->states[CIRCPAD_STATE_BURST].histogram_edges[0] = 0; - ret->states[CIRCPAD_STATE_BURST].histogram_edges[1] = 0; + ret->states[CIRCPAD_STATE_BURST].histogram_edges[1] = 1; ret->states[CIRCPAD_STATE_BURST].histogram_edges[2] = 1000000; ret->states[CIRCPAD_STATE_BURST].histogram[0] = 6; @@ -1686,8 +1686,7 @@ helper_create_conditional_machines(void) add->conditions.state_mask = CIRCPAD_CIRC_BUILDING| CIRCPAD_CIRC_NO_STREAMS|CIRCPAD_CIRC_HAS_RELAY_EARLY; add->conditions.purpose_mask = CIRCPAD_PURPOSE_ALL; - - smartlist_add(origin_padding_machines, add); + register_padding_machine(add, origin_padding_machines); add = helper_create_conditional_machine(); add->machine_num = 3; @@ -1706,15 +1705,15 @@ helper_create_conditional_machines(void) add->conditions.state_mask = CIRCPAD_CIRC_OPENED| CIRCPAD_CIRC_STREAMS|CIRCPAD_CIRC_HAS_NO_RELAY_EARLY; add->conditions.purpose_mask = CIRCPAD_PURPOSE_ALL; - smartlist_add(origin_padding_machines, add); + register_padding_machine(add, origin_padding_machines); add = helper_create_conditional_machine(); add->machine_num = 2; - smartlist_add(relay_padding_machines, add); + register_padding_machine(add, relay_padding_machines); add = helper_create_conditional_machine(); add->machine_num = 3; - smartlist_add(relay_padding_machines, add); + register_padding_machine(add, relay_padding_machines); } void From 58de565988160edec1fc5f0fbc2a794a765bce6f Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Thu, 28 Feb 2019 12:09:10 +0200 Subject: [PATCH 0592/2557] Call practracker as part of check-local. - Introduce 'make check-best-practices'. - Fix up Tor topdir etc to work with the way 'make check-local' gets called. - Make practracker less likely to print useless stuff. --- Makefile.am | 11 ++++- scripts/maint/practracker/practracker.py | 54 ++++++++++++++---------- scripts/maint/practracker/problem.py | 2 +- scripts/maint/practracker/util.py | 8 +++- 4 files changed, 47 insertions(+), 28 deletions(-) diff --git a/Makefile.am b/Makefile.am index b40b2e51bc..415c2f2b4e 100644 --- a/Makefile.am +++ b/Makefile.am @@ -161,7 +161,9 @@ EXTRA_DIST+= \ README \ ReleaseNotes \ scripts/maint/checkIncludes.py \ - scripts/maint/checkSpace.pl + scripts/maint/checkSpace.pl \ + scripts/maint/practracker + ## This tells etags how to find mockable function definitions. AM_ETAGSFLAGS=--regex='{c}/MOCK_IMPL([^,]+,\W*\([a-zA-Z0-9_]+\)\W*,/\1/s' @@ -224,7 +226,7 @@ shellcheck: fi; \ fi -check-local: check-spaces check-changes check-includes shellcheck +check-local: check-spaces check-changes check-includes check-best-practices shellcheck need-chutney-path: @if test ! -d "$$CHUTNEY_PATH"; then \ @@ -345,6 +347,11 @@ if USEPYTHON $(PYTHON) $(top_srcdir)/scripts/maint/checkIncludes.py endif +check-best-practices: +if USEPYTHON + $(PYTHON) $(top_srcdir)/scripts/maint/practracker/practracker.py $(top_srcdir) +endif + check-docs: all $(PERL) $(top_builddir)/scripts/maint/checkOptionDocs.pl diff --git a/scripts/maint/practracker/practracker.py b/scripts/maint/practracker/practracker.py index 3c22abd444..541c5cbfb3 100755 --- a/scripts/maint/practracker/practracker.py +++ b/scripts/maint/practracker/practracker.py @@ -1,21 +1,21 @@ #!/usr/bin/python3 """ -Tor code best-practices tracker +Best-practices tracker for Tor source code. Go through the various .c files and collect metrics about them. If the metrics violate some of our best practices and they are not found in the optional -exceptions file ("./exceptions.txt"), then log a problem about them. - -The exceptions file is meant to be initialized with the current state of the -source code as follows: ./practracker.py > ./exceptions.txt +exceptions file, then log a problem about them. We currently do metrics about file size, function size and number of includes. -TODO: - - How is this tool supposed to be used? How should the exception file work? - How should the UI work? Does it need special exit codes? - - Fix the function_length function so that practracker_tests.py passes. +practracker.py should be run with its second argument pointing to the Tor +top-level source directory like this: + $ python3 ./scripts/maint/practracker/practracker.py . + +The exceptions file is meant to be initialized once with the current state of +the source code and then get saved in the repository for ever after: + $ python3 ./scripts/maint/practracker/practracker.py . > ./scripts/maint/practracker/exceptions.txt """ import os, sys @@ -24,14 +24,8 @@ import metrics import util import problem -# We don't want to run metrics for unittests, automatically-generated C files, -# external libraries or git leftovers. -EXCLUDE_SOURCE_DIRS = ["/src/test/", "/src/trunnel/", "/src/ext/", "/.git/"] - -# Where the Tor source code is -TOR_TOPDIR = "../../../" -# An optional exceptions_file -EXCEPTIONS_FILE = "./exceptions.txt" +# The filename of the exceptions file (it should be placed in the practracker directory) +EXCEPTIONS_FNAME = "./exceptions.txt" # Recommended file size MAX_FILE_SIZE = 3000 # lines @@ -42,8 +36,12 @@ MAX_INCLUDE_COUNT = 50 ####################################################### +# ProblemVault singleton ProblemVault = None +# The Tor source code topdir +TOR_TOPDIR = None + ####################################################### def consider_file_size(fname, f): @@ -114,21 +112,31 @@ def consider_metrics_for_file(fname, f): return found_new_issues def main(): + if (len(sys.argv) != 2): + print("Usage:\n\t$ practracker.py \n\t(e.g. $ practracker.py ~/tor/)") + return + + global TOR_TOPDIR + TOR_TOPDIR = sys.argv[1] + exceptions_file = os.path.join(TOR_TOPDIR, "scripts/maint/practracker", EXCEPTIONS_FNAME) + # 1) Get all the .c files we care about - files_list = util.get_tor_c_files(TOR_TOPDIR, EXCLUDE_SOURCE_DIRS) + files_list = util.get_tor_c_files(TOR_TOPDIR) # 2) Initialize problem vault and load an optional exceptions file so that # we don't warn about the past global ProblemVault - ProblemVault = problem.ProblemVault(EXCEPTIONS_FILE) + ProblemVault = problem.ProblemVault(exceptions_file) # 3) Go through all the files and report problems if they are not exceptions found_new_issues = consider_all_metrics(files_list) - if found_new_issues: - sys.exit(1) - else: - sys.exit(0) + # If new issues were found, try to give out some advice to the developer on how to resolve it. + if (found_new_issues): + new_issues_str = "practracker FAILED as indicated by the problem lines above. Please use the exceptions file ({}) to find any previous state of these problems. If you are unable to fix the underlying best-practices issue right now then you need to either update the relevant exception line or add a new one.".format(exceptions_file) + print(new_issues_str) + + sys.exit(found_new_issues) if __name__ == '__main__': main() diff --git a/scripts/maint/practracker/problem.py b/scripts/maint/practracker/problem.py index e23d3bbf0e..9f5771678e 100644 --- a/scripts/maint/practracker/problem.py +++ b/scripts/maint/practracker/problem.py @@ -96,6 +96,6 @@ def get_old_problem_from_exception_str(exception_str): elif problem_type == "function-size": return FunctionSizeProblem(problem_location, metric_value) else: - print("Unknown exception line {}".format(exception_str)) +# print("Unknown exception line '{}'".format(exception_str)) return None diff --git a/scripts/maint/practracker/util.py b/scripts/maint/practracker/util.py index 8e9d95ece4..905cb8a5e0 100644 --- a/scripts/maint/practracker/util.py +++ b/scripts/maint/practracker/util.py @@ -1,6 +1,10 @@ import os -def get_tor_c_files(tor_topdir, exclude_dirs): +# We don't want to run metrics for unittests, automatically-generated C files, +# external libraries or git leftovers. +EXCLUDE_SOURCE_DIRS = ["/src/test/", "/src/trunnel/", "/src/ext/", "/.git/"] + +def get_tor_c_files(tor_topdir): """ Return a list with the .c filenames we want to get metrics of. """ @@ -14,7 +18,7 @@ def get_tor_c_files(tor_topdir, exclude_dirs): # Exclude the excluded paths full_path = os.path.join(root,filename) - if any(exclude_dir in full_path for exclude_dir in exclude_dirs): + if any(exclude_dir in full_path for exclude_dir in EXCLUDE_SOURCE_DIRS): continue files_list.append(full_path) From 4795f2d3a0df72d0e51a51eb1fbafdc68bfbc77c Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Thu, 28 Feb 2019 12:18:30 +0200 Subject: [PATCH 0593/2557] Fold in an initial practracker exceptions file. --- scripts/maint/practracker/exceptions.txt | 260 +++++++++++++++++++++++ 1 file changed, 260 insertions(+) create mode 100644 scripts/maint/practracker/exceptions.txt diff --git a/scripts/maint/practracker/exceptions.txt b/scripts/maint/practracker/exceptions.txt new file mode 100644 index 0000000000..a42db81a06 --- /dev/null +++ b/scripts/maint/practracker/exceptions.txt @@ -0,0 +1,260 @@ +problem function-size /src/core/proto/proto_socks.c:parse_socks_client() 112 +problem file-size /src/core/or/channel.c 3476 +problem function-size /src/core/or/scheduler_kist.c:kist_scheduler_run() 171 +problem function-size /src/core/or/scheduler_vanilla.c:vanilla_scheduler_run() 109 +problem function-size /src/core/or/command.c:command_process_create_cell() 156 +problem function-size /src/core/or/command.c:command_process_relay_cell() 132 +problem function-size /src/core/or/protover.c:protover_all_supported() 116 +problem function-size /src/core/or/circuitmux.c:circuitmux_detach_all_circuits() 109 +problem function-size /src/core/or/circuitmux.c:circuitmux_set_policy() 110 +problem function-size /src/core/or/circuitmux.c:circuitmux_attach_circuit() 129 +problem include-count /src/core/or/circuitlist.c 54 +problem function-size /src/core/or/circuitlist.c:HT_PROTOTYPE() 128 +problem function-size /src/core/or/circuitlist.c:circuit_free_() 137 +problem function-size /src/core/or/circuitlist.c:circuit_find_to_cannibalize() 102 +problem function-size /src/core/or/circuitlist.c:circuit_about_to_free() 120 +problem function-size /src/core/or/circuitlist.c:circuits_handle_oom() 117 +problem file-size /src/core/or/connection_or.c 3111 +problem include-count /src/core/or/connection_or.c 51 +problem function-size /src/core/or/connection_or.c:connection_or_group_set_badness_() 105 +problem function-size /src/core/or/connection_or.c:connection_or_client_learned_peer_id() 144 +problem function-size /src/core/or/connection_or.c:connection_or_compute_authenticate_cell_body() 235 +problem file-size /src/core/or/policies.c 3163 +problem function-size /src/core/or/policies.c:policy_summarize() 107 +problem function-size /src/core/or/channeltls.c:channel_tls_handle_var_cell() 160 +problem function-size /src/core/or/channeltls.c:channel_tls_process_versions_cell() 170 +problem function-size /src/core/or/channeltls.c:channel_tls_process_netinfo_cell() 214 +problem function-size /src/core/or/channeltls.c:channel_tls_process_certs_cell() 246 +problem function-size /src/core/or/channeltls.c:channel_tls_process_authenticate_cell() 202 +problem file-size /src/core/or/circuituse.c 3146 +problem function-size /src/core/or/circuituse.c:circuit_is_acceptable() 129 +problem function-size /src/core/or/circuituse.c:circuit_expire_building() 394 +problem function-size /src/core/or/circuituse.c:circuit_log_ancient_one_hop_circuits() 126 +problem function-size /src/core/or/circuituse.c:circuit_build_failed() 149 +problem function-size /src/core/or/circuituse.c:circuit_launch_by_extend_info() 110 +problem function-size /src/core/or/circuituse.c:circuit_get_open_circ_or_launch() 354 +problem function-size /src/core/or/circuituse.c:connection_ap_handshake_attach_circuit() 244 +problem file-size /src/core/or/connection_edge.c 4550 +problem include-count /src/core/or/connection_edge.c 64 +problem function-size /src/core/or/connection_edge.c:connection_ap_expire_beginning() 117 +problem function-size /src/core/or/connection_edge.c:connection_ap_handshake_rewrite() 192 +problem function-size /src/core/or/connection_edge.c:connection_ap_handle_onion() 188 +problem function-size /src/core/or/connection_edge.c:connection_ap_handshake_rewrite_and_attach() 423 +problem function-size /src/core/or/connection_edge.c:connection_ap_handshake_send_begin() 111 +problem function-size /src/core/or/connection_edge.c:connection_ap_handshake_socks_resolved() 106 +problem function-size /src/core/or/connection_edge.c:connection_exit_begin_conn() 184 +problem function-size /src/core/or/connection_edge.c:connection_exit_connect() 102 +problem function-size /src/core/or/circuitstats.c:circuit_build_times_parse_state() 124 +problem function-size /src/core/or/versions.c:tor_version_parse() 104 +problem file-size /src/core/or/circuitbuild.c 3060 +problem include-count /src/core/or/circuitbuild.c 53 +problem function-size /src/core/or/circuitbuild.c:get_unique_circ_id_by_chan() 128 +problem function-size /src/core/or/circuitbuild.c:circuit_extend() 147 +problem function-size /src/core/or/circuitbuild.c:choose_good_exit_server_general() 206 +problem file-size /src/core/or/relay.c 3183 +problem function-size /src/core/or/relay.c:circuit_receive_relay_cell() 123 +problem function-size /src/core/or/relay.c:relay_send_command_from_edge_() 101 +problem function-size /src/core/or/relay.c:connection_ap_process_end_not_open() 194 +problem function-size /src/core/or/relay.c:connection_edge_process_relay_cell_not_open() 139 +problem function-size /src/core/or/relay.c:connection_edge_process_relay_cell() 520 +problem function-size /src/core/or/relay.c:connection_edge_package_raw_inbuf() 130 +problem function-size /src/core/or/relay.c:circuit_resume_edge_reading_helper() 147 +problem file-size /src/core/mainloop/mainloop.c 3050 +problem include-count /src/core/mainloop/mainloop.c 65 +problem function-size /src/core/mainloop/mainloop.c:conn_close_if_marked() 108 +problem function-size /src/core/mainloop/mainloop.c:run_connection_housekeeping() 123 +problem function-size /src/core/mainloop/mainloop.c:CALLBACK() 116 +problem file-size /src/core/mainloop/connection.c 5547 +problem include-count /src/core/mainloop/connection.c 60 +problem function-size /src/core/mainloop/connection.c:connection_free_minimal() 184 +problem function-size /src/core/mainloop/connection.c:connection_listener_new() 328 +problem function-size /src/core/mainloop/connection.c:connection_handle_listener_read() 161 +problem function-size /src/core/mainloop/connection.c:connection_connect_sockaddr() 103 +problem function-size /src/core/mainloop/connection.c:connection_proxy_connect() 148 +problem function-size /src/core/mainloop/connection.c:connection_read_proxy_handshake() 153 +problem function-size /src/core/mainloop/connection.c:retry_listener_ports() 116 +problem function-size /src/core/mainloop/connection.c:connection_handle_read_impl() 111 +problem function-size /src/core/mainloop/connection.c:connection_buf_read_from_socket() 177 +problem function-size /src/core/mainloop/connection.c:connection_handle_write_impl() 241 +problem function-size /src/core/mainloop/connection.c:assert_connection_ok() 143 +problem function-size /src/app/config/confparse.c:config_assign_value() 205 +problem function-size /src/app/config/confparse.c:config_get_assigned_option() 129 +problem file-size /src/app/config/config.c 8488 +problem include-count /src/app/config/config.c 84 +problem function-size /src/app/config/config.c:options_act_reversible() 296 +problem function-size /src/app/config/config.c:options_act() 588 +problem function-size /src/app/config/config.c:resolve_my_address() 192 +problem function-size /src/app/config/config.c:options_validate() 1207 +problem function-size /src/app/config/config.c:options_init_from_torrc() 202 +problem function-size /src/app/config/config.c:options_init_from_string() 173 +problem function-size /src/app/config/config.c:options_init_logs() 146 +problem function-size /src/app/config/config.c:parse_bridge_line() 104 +problem function-size /src/app/config/config.c:parse_transport_line() 191 +problem function-size /src/app/config/config.c:parse_dir_authority_line() 151 +problem function-size /src/app/config/config.c:parse_dir_fallback_line() 102 +problem function-size /src/app/config/config.c:parse_port_config() 452 +problem function-size /src/app/config/config.c:parse_ports() 170 +problem function-size /src/app/config/config.c:getinfo_helper_config() 116 +problem function-size /src/app/main/ntmain.c:nt_service_install() 125 +problem include-count /src/app/main/main.c 83 +problem function-size /src/app/main/main.c:dumpstats() 102 +problem function-size /src/app/main/main.c:tor_init() 136 +problem function-size /src/app/main/main.c:sandbox_init_filter() 291 +problem function-size /src/app/main/main.c:run_tor_main_loop() 105 +problem function-size /src/tools/tor-resolve.c:build_socks5_resolve_request() 104 +problem function-size /src/tools/tor-resolve.c:do_resolve() 173 +problem function-size /src/tools/tor-resolve.c:main() 112 +problem function-size /src/tools/tor-gencert.c:parse_commandline() 111 +problem function-size /src/feature/keymgt/loadkey.c:ed_key_init_from_file() 333 +problem function-size /src/feature/dircommon/consdiff.c:gen_ed_diff() 204 +problem function-size /src/feature/dircommon/consdiff.c:apply_ed_diff() 159 +problem file-size /src/feature/control/control.c 7592 +problem include-count /src/feature/control/control.c 83 +problem function-size /src/feature/control/control.c:handle_control_authenticate() 188 +problem function-size /src/feature/control/control.c:getinfo_helper_misc() 109 +problem function-size /src/feature/control/control.c:getinfo_helper_dir() 304 +problem function-size /src/feature/control/control.c:getinfo_helper_events() 236 +problem function-size /src/feature/control/control.c:handle_control_extendcircuit() 151 +problem function-size /src/feature/control/control.c:handle_control_authchallenge() 115 +problem function-size /src/feature/control/control.c:handle_control_hsfetch() 114 +problem function-size /src/feature/control/control.c:handle_control_hspost() 117 +problem function-size /src/feature/control/control.c:handle_control_add_onion() 293 +problem function-size /src/feature/control/control.c:add_onion_helper_keyarg() 125 +problem function-size /src/feature/control/control.c:connection_control_process_inbuf() 239 +problem function-size /src/feature/control/control.c:control_event_stream_status() 119 +problem function-size /src/feature/stats/rephist.c:rep_hist_load_mtbf_data() 185 +problem function-size /src/feature/stats/rephist.c:rep_hist_format_exit_stats() 148 +problem function-size /src/feature/dircache/consdiffmgr.c:consdiffmgr_cleanup() 115 +problem function-size /src/feature/dircache/consdiffmgr.c:consdiffmgr_rescan_flavor_() 111 +problem function-size /src/feature/dircache/consdiffmgr.c:consensus_diff_worker_threadfn() 132 +problem function-size /src/feature/dircache/dircache.c:handle_get_current_consensus() 166 +problem function-size /src/feature/dircache/dircache.c:directory_handle_command_post() 120 +problem function-size /src/feature/hibernate/hibernate.c:accounting_parse_options() 109 +problem function-size /src/feature/relay/routerkeys.c:load_ed_keys() 294 +problem file-size /src/feature/relay/router.c 3221 +problem include-count /src/feature/relay/router.c 56 +problem function-size /src/feature/relay/router.c:init_keys() 252 +problem function-size /src/feature/relay/router.c:get_my_declared_family() 114 +problem function-size /src/feature/relay/router.c:router_build_fresh_descriptor() 190 +problem function-size /src/feature/relay/router.c:router_dump_router_to_string() 375 +problem function-size /src/feature/relay/router.c:extrainfo_dump_to_string() 208 +problem function-size /src/feature/relay/dns.c:dns_resolve_impl() 134 +problem function-size /src/feature/relay/dns.c:configure_nameservers() 161 +problem function-size /src/feature/relay/dns.c:evdns_callback() 109 +problem function-size /src/feature/dirparse/authcert_parse.c:authority_cert_parse_from_string() 182 +problem function-size /src/feature/dirparse/ns_parse.c:routerstatus_parse_entry_from_string() 286 +problem function-size /src/feature/dirparse/ns_parse.c:networkstatus_verify_bw_weights() 389 +problem function-size /src/feature/dirparse/ns_parse.c:networkstatus_parse_vote_from_string() 638 +problem function-size /src/feature/dirparse/parsecommon.c:tokenize_string() 103 +problem function-size /src/feature/dirparse/parsecommon.c:get_next_token() 159 +problem function-size /src/feature/dirparse/routerparse.c:router_parse_entry_from_string() 554 +problem function-size /src/feature/dirparse/routerparse.c:extrainfo_parse_entry_from_string() 210 +problem function-size /src/feature/dirparse/microdesc_parse.c:microdescs_parse_from_string() 154 +problem file-size /src/feature/dirauth/dirvote.c 4729 +problem include-count /src/feature/dirauth/dirvote.c 53 +problem function-size /src/feature/dirauth/dirvote.c:format_networkstatus_vote() 251 +problem function-size /src/feature/dirauth/dirvote.c:networkstatus_compute_bw_weights_v10() 235 +problem function-size /src/feature/dirauth/dirvote.c:networkstatus_compute_consensus() 962 +problem function-size /src/feature/dirauth/dirvote.c:networkstatus_add_detached_signatures() 123 +problem function-size /src/feature/dirauth/dirvote.c:dirvote_add_vote() 162 +problem function-size /src/feature/dirauth/dirvote.c:dirvote_compute_consensuses() 164 +problem function-size /src/feature/dirauth/dirvote.c:dirserv_generate_networkstatus_vote_obj() 293 +problem function-size /src/feature/dirauth/guardfraction.c:dirserv_read_guardfraction_file_from_str() 110 +problem function-size /src/feature/dirauth/shared_random.c:should_keep_commit() 110 +problem function-size /src/feature/dirauth/process_descs.c:dirserv_add_descriptor() 125 +problem function-size /src/feature/dirauth/bwauth.c:dirserv_read_measured_bandwidths() 127 +problem function-size /src/feature/dirauth/dsigs_parse.c:networkstatus_parse_detached_signatures() 196 +problem function-size /src/feature/dirauth/voteflags.c:dirserv_compute_performance_thresholds() 172 +problem file-size /src/feature/hs/hs_service.c 4171 +problem function-size /src/feature/hs/hs_common.c:hs_get_responsible_hsdirs() 104 +problem function-size /src/feature/hs/hs_client.c:send_introduce1() 104 +problem function-size /src/feature/hs/hs_client.c:hs_config_client_authorization() 108 +problem function-size /src/feature/hs/hs_config.c:config_generic_service() 149 +problem function-size /src/feature/hs/hs_cell.c:hs_cell_build_establish_intro() 115 +problem function-size /src/feature/hs/hs_cell.c:hs_cell_parse_introduce2() 147 +problem file-size /src/feature/hs/hs_descriptor.c 3108 +problem function-size /src/feature/hs/hs_descriptor.c:desc_encode_v3() 108 +problem function-size /src/feature/hs/hs_descriptor.c:decrypt_desc_layer() 110 +problem function-size /src/feature/hs/hs_descriptor.c:decode_introduction_point() 122 +problem function-size /src/feature/hs/hs_descriptor.c:desc_decode_superencrypted_v3() 109 +problem function-size /src/feature/hs/hs_descriptor.c:desc_decode_encrypted_v3() 109 +problem file-size /src/feature/dirclient/dirclient.c 3215 +problem include-count /src/feature/dirclient/dirclient.c 51 +problem function-size /src/feature/dirclient/dirclient.c:directory_get_from_dirserver() 131 +problem function-size /src/feature/dirclient/dirclient.c:directory_initiate_request() 201 +problem function-size /src/feature/dirclient/dirclient.c:directory_send_command() 241 +problem function-size /src/feature/dirclient/dirclient.c:dir_client_decompress_response_body() 114 +problem function-size /src/feature/dirclient/dirclient.c:connection_dir_client_reached_eof() 189 +problem function-size /src/feature/dirclient/dirclient.c:handle_response_fetch_consensus() 105 +problem function-size /src/feature/rend/rendmid.c:rend_mid_establish_intro_legacy() 104 +problem function-size /src/feature/rend/rendparse.c:rend_parse_v2_service_descriptor() 187 +problem function-size /src/feature/rend/rendparse.c:rend_decrypt_introduction_points() 104 +problem function-size /src/feature/rend/rendparse.c:rend_parse_introduction_points() 131 +problem file-size /src/feature/rend/rendservice.c 4487 +problem function-size /src/feature/rend/rendservice.c:rend_service_prune_list_impl_() 107 +problem function-size /src/feature/rend/rendservice.c:rend_config_service() 164 +problem function-size /src/feature/rend/rendservice.c:rend_service_load_auth_keys() 178 +problem function-size /src/feature/rend/rendservice.c:rend_service_receive_introduction() 332 +problem function-size /src/feature/rend/rendservice.c:rend_service_parse_intro_for_v3() 115 +problem function-size /src/feature/rend/rendservice.c:rend_service_decrypt_intro() 115 +problem function-size /src/feature/rend/rendservice.c:rend_service_intro_has_opened() 126 +problem function-size /src/feature/rend/rendservice.c:rend_service_rendezvous_has_opened() 117 +problem function-size /src/feature/rend/rendservice.c:directory_post_to_hs_dir() 108 +problem function-size /src/feature/rend/rendservice.c:upload_service_descriptor() 111 +problem function-size /src/feature/rend/rendservice.c:rend_consider_services_intro_points() 169 +problem function-size /src/feature/rend/rendcache.c:rend_cache_store_v2_desc_as_client() 193 +problem function-size /src/feature/rend/rendclient.c:rend_client_send_introduction() 220 +problem function-size /src/feature/rend/rendcommon.c:rend_encode_v2_descriptors() 225 +problem function-size /src/feature/nodelist/authcert.c:trusted_dirs_load_certs_from_string() 124 +problem function-size /src/feature/nodelist/authcert.c:authority_certs_fetch_missing() 296 +problem function-size /src/feature/nodelist/microdesc.c:microdesc_cache_rebuild() 134 +problem function-size /src/feature/nodelist/fmt_routerstatus.c:routerstatus_format_entry() 166 +problem include-count /src/feature/nodelist/networkstatus.c 61 +problem function-size /src/feature/nodelist/networkstatus.c:networkstatus_check_consensus_signature() 176 +problem function-size /src/feature/nodelist/networkstatus.c:networkstatus_set_current_consensus() 293 +problem file-size /src/feature/nodelist/routerlist.c 3234 +problem function-size /src/feature/nodelist/routerlist.c:router_rebuild_store() 148 +problem function-size /src/feature/nodelist/routerlist.c:router_add_to_routerlist() 169 +problem function-size /src/feature/nodelist/routerlist.c:routerlist_remove_old_routers() 121 +problem function-size /src/feature/nodelist/routerlist.c:update_consensus_router_descriptor_downloads() 136 +problem function-size /src/feature/nodelist/routerlist.c:update_extrainfo_downloads() 103 +problem function-size /src/feature/nodelist/nodelist.c:compute_frac_paths_available() 187 +problem function-size /src/feature/nodelist/node_select.c:router_pick_directory_server_impl() 123 +problem function-size /src/feature/nodelist/node_select.c:compute_weighted_bandwidths() 205 +problem function-size /src/feature/nodelist/node_select.c:router_pick_trusteddirserver_impl() 114 +problem function-size /src/feature/client/dnsserv.c:evdns_server_callback() 153 +problem function-size /src/feature/client/transports.c:handle_proxy_line() 108 +problem function-size /src/feature/client/transports.c:parse_method_line_helper() 112 +problem function-size /src/feature/client/transports.c:create_managed_proxy_environment() 109 +problem function-size /src/feature/client/bridges.c:rewrite_node_address_for_bridge() 126 +problem function-size /src/feature/client/addressmap.c:addressmap_rewrite() 112 +problem file-size /src/feature/client/entrynodes.c 3817 +problem function-size /src/feature/client/entrynodes.c:entry_guards_upgrade_waiting_circuits() 153 +problem function-size /src/feature/client/entrynodes.c:entry_guard_parse_from_state() 246 +problem function-size /src/feature/client/circpathbias.c:pathbias_measure_close_rate() 108 +problem function-size /src/lib/log/log.c:parse_log_severity_config() 101 +problem function-size /src/lib/math/prob_distr.c:sample_uniform_interval() 145 +problem function-size /src/lib/process/process_win32.c:process_win32_exec() 138 +problem function-size /src/lib/process/process_win32.c:process_win32_create_pipe() 112 +problem function-size /src/lib/process/setuid.c:switch_id() 156 +problem function-size /src/lib/process/restrict.c:set_max_file_descriptors() 102 +problem function-size /src/lib/process/process_unix.c:process_unix_exec() 220 +problem function-size /src/lib/container/smartlist.c:smartlist_bsearch_idx() 109 +problem function-size /src/lib/sandbox/sandbox.c:prot_strings() 104 +problem function-size /src/lib/compress/compress_zstd.c:tor_zstd_compress_process() 126 +problem function-size /src/lib/compress/compress.c:tor_compress_impl() 133 +problem function-size /src/lib/crypt_ops/crypto_rand.c:crypto_strongest_rand_syscall() 102 +problem function-size /src/lib/string/scanf.c:tor_vsscanf() 112 +problem function-size /src/lib/encoding/confline.c:parse_config_line_from_str_verbose() 119 +problem function-size /src/lib/encoding/binascii.c:base64_encode() 107 +problem function-size /src/lib/encoding/cstring.c:unescape_string() 108 +problem function-size /src/lib/tls/tortls_openssl.c:tor_tls_context_new() 171 +problem function-size /src/lib/tls/x509_nss.c:tor_tls_create_certificate_internal() 126 +problem function-size /src/lib/tls/tortls_nss.c:tor_tls_context_new() 147 +problem function-size /src/lib/osinfo/uname.c:get_uname() 116 +problem function-size /src/lib/net/address.c:tor_addr_parse_mask_ports() 198 +problem function-size /src/lib/net/address.c:tor_addr_compare_masked() 111 +problem function-size /src/lib/net/resolve.c:tor_addr_lookup() 110 +problem function-size /src/lib/net/inaddr.c:tor_inet_pton() 107 +problem function-size /src/lib/net/socketpair.c:tor_ersatz_socketpair() 102 +problem function-size /src/lib/fs/dir.c:check_private_dir() 231 From 8bacc1dad170a9a29633d95c91a4139af6a2bfe7 Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Tue, 5 Mar 2019 15:01:07 +0200 Subject: [PATCH 0594/2557] practracker: Improve documentation in problem.py . --- scripts/maint/practracker/problem.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/scripts/maint/practracker/problem.py b/scripts/maint/practracker/problem.py index 9f5771678e..2330098ef3 100644 --- a/scripts/maint/practracker/problem.py +++ b/scripts/maint/practracker/problem.py @@ -53,6 +53,10 @@ class ProblemVault(object): return False class Problem(object): + """ + A generic problem in our source code. See the subclasses below for the + specific problems we are trying to tackle. + """ def __init__(self, problem_type, problem_location, metric_value): self.problem_location = problem_location self.metric_value = int(metric_value) @@ -72,14 +76,35 @@ class Problem(object): return "problem %s %s %s" % (self.problem_type, self.problem_location, self.metric_value) class FileSizeProblem(Problem): + """ + Denotes a problem with the size of a .c file. + + The 'problem_location' is the filesystem path of the .c file, and the + 'metric_value' is the number of lines in the .c file. + """ def __init__(self, problem_location, metric_value): super(FileSizeProblem, self).__init__("file-size", problem_location, metric_value) class IncludeCountProblem(Problem): + """ + Denotes a problem with the number of #includes in a .c file. + + The 'problem_location' is the filesystem path of the .c file, and the + 'metric_value' is the number of #includes in the .c file. + """ def __init__(self, problem_location, metric_value): super(IncludeCountProblem, self).__init__("include-count", problem_location, metric_value) class FunctionSizeProblem(Problem): + """ + Denotes a problem with a size of a function in a .c file. + + The 'problem_location' is ":()" where is the + filesystem path of the .c file and is the name of the offending + function. + + The 'metric_value' is the size of the offending function in lines. + """ def __init__(self, problem_location, metric_value): super(FunctionSizeProblem, self).__init__("function-size", problem_location, metric_value) From 8c9835c6e5de6f17affb9b7d9f5802dee54a9e37 Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Tue, 5 Mar 2019 15:03:27 +0200 Subject: [PATCH 0595/2557] practracker: Normalize filesystem paths across Windows and Posix. This was causing issues because the exceptions file is written using Posix paths, whereas practracker in Windows was trying to match Windows paths ("\" instead of "/"). --- scripts/maint/practracker/problem.py | 7 ++++++- scripts/maint/practracker/util.py | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/scripts/maint/practracker/problem.py b/scripts/maint/practracker/problem.py index 2330098ef3..00e029d1c4 100644 --- a/scripts/maint/practracker/problem.py +++ b/scripts/maint/practracker/problem.py @@ -7,6 +7,8 @@ problem is worse than a registered exception so that it only warns when things get worse. """ +import os.path + class ProblemVault(object): """ Singleton where we store the various new problems we @@ -70,7 +72,10 @@ class Problem(object): def key(self): """Generate a unique key that describes this problem that can be used as a dictionary key""" - return "%s:%s" % (self.problem_location, self.problem_type) + # Problem location is a filesystem path, so we need to normalize this + # across platforms otherwise same paths are not gonna match. + canonical_location = os.path.normcase(self.problem_location) + return "%s:%s" % (canonical_location, self.problem_type) def __str__(self): return "problem %s %s %s" % (self.problem_type, self.problem_location, self.metric_value) diff --git a/scripts/maint/practracker/util.py b/scripts/maint/practracker/util.py index 905cb8a5e0..966c625158 100644 --- a/scripts/maint/practracker/util.py +++ b/scripts/maint/practracker/util.py @@ -18,7 +18,7 @@ def get_tor_c_files(tor_topdir): # Exclude the excluded paths full_path = os.path.join(root,filename) - if any(exclude_dir in full_path for exclude_dir in EXCLUDE_SOURCE_DIRS): + if any(os.path.normcase(exclude_dir) in full_path for exclude_dir in EXCLUDE_SOURCE_DIRS): continue files_list.append(full_path) From 157f7ba93ed8800dd34eafd4184a893b37965829 Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Tue, 5 Mar 2019 15:17:54 +0200 Subject: [PATCH 0596/2557] practracker: Update exceptions file :) --- scripts/maint/practracker/exceptions.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/maint/practracker/exceptions.txt b/scripts/maint/practracker/exceptions.txt index a42db81a06..da354d8f83 100644 --- a/scripts/maint/practracker/exceptions.txt +++ b/scripts/maint/practracker/exceptions.txt @@ -58,7 +58,7 @@ problem function-size /src/core/or/relay.c:connection_ap_process_end_not_open() problem function-size /src/core/or/relay.c:connection_edge_process_relay_cell_not_open() 139 problem function-size /src/core/or/relay.c:connection_edge_process_relay_cell() 520 problem function-size /src/core/or/relay.c:connection_edge_package_raw_inbuf() 130 -problem function-size /src/core/or/relay.c:circuit_resume_edge_reading_helper() 147 +problem function-size /src/core/or/relay.c:circuit_resume_edge_reading_helper() 148 problem file-size /src/core/mainloop/mainloop.c 3050 problem include-count /src/core/mainloop/mainloop.c 65 problem function-size /src/core/mainloop/mainloop.c:conn_close_if_marked() 108 @@ -190,7 +190,7 @@ problem function-size /src/feature/rend/rendmid.c:rend_mid_establish_intro_legac problem function-size /src/feature/rend/rendparse.c:rend_parse_v2_service_descriptor() 187 problem function-size /src/feature/rend/rendparse.c:rend_decrypt_introduction_points() 104 problem function-size /src/feature/rend/rendparse.c:rend_parse_introduction_points() 131 -problem file-size /src/feature/rend/rendservice.c 4487 +problem file-size /src/feature/rend/rendservice.c 4509 problem function-size /src/feature/rend/rendservice.c:rend_service_prune_list_impl_() 107 problem function-size /src/feature/rend/rendservice.c:rend_config_service() 164 problem function-size /src/feature/rend/rendservice.c:rend_service_load_auth_keys() 178 From ec8c5b3fea78b2596d3b68571fd47b8e41d4a3ea Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Tue, 12 Mar 2019 14:35:26 +0200 Subject: [PATCH 0597/2557] practracker: Improvements based on last Nick's review. --- scripts/maint/practracker/metrics.py | 6 +++--- scripts/maint/practracker/problem.py | 3 ++- scripts/maint/practracker/util.py | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/scripts/maint/practracker/metrics.py b/scripts/maint/practracker/metrics.py index c7d2091d05..56b9e8383e 100644 --- a/scripts/maint/practracker/metrics.py +++ b/scripts/maint/practracker/metrics.py @@ -26,8 +26,8 @@ def get_function_lines(f): """ # Skip lines with these terms since they confuse our regexp - REGEXP_CONFUSE_TERMS = ["MOCK_IMPL", "ENABLE_GCC_WARNINGS", "ENABLE_GCC_WARNING", "DUMMY_TYPECHECK_INSTANCE", - "DISABLE_GCC_WARNING", "DISABLE_GCC_WARNINGS"] + REGEXP_CONFUSE_TERMS = {"MOCK_IMPL", "ENABLE_GCC_WARNINGS", "ENABLE_GCC_WARNING", "DUMMY_TYPECHECK_INSTANCE", + "DISABLE_GCC_WARNING", "DISABLE_GCC_WARNINGS"} in_function = False for lineno, line in enumerate(f): @@ -43,7 +43,7 @@ def get_function_lines(f): in_function = True else: - # Fund the end of a function + # Find the end of a function if line.startswith("}"): n_lines = lineno - func_start in_function = False diff --git a/scripts/maint/practracker/problem.py b/scripts/maint/practracker/problem.py index 00e029d1c4..60ee3eb242 100644 --- a/scripts/maint/practracker/problem.py +++ b/scripts/maint/practracker/problem.py @@ -8,6 +8,7 @@ get worse. """ import os.path +import sys class ProblemVault(object): """ @@ -23,7 +24,7 @@ class ProblemVault(object): with open(exception_fname, 'r') as exception_f: self.register_exceptions(exception_f) except IOError: - print("No exception file provided") + print("No exception file provided", file=sys.stderr) def register_exceptions(self, exception_file): # Register exceptions diff --git a/scripts/maint/practracker/util.py b/scripts/maint/practracker/util.py index 966c625158..63de72d5a3 100644 --- a/scripts/maint/practracker/util.py +++ b/scripts/maint/practracker/util.py @@ -2,7 +2,7 @@ import os # We don't want to run metrics for unittests, automatically-generated C files, # external libraries or git leftovers. -EXCLUDE_SOURCE_DIRS = ["/src/test/", "/src/trunnel/", "/src/ext/", "/.git/"] +EXCLUDE_SOURCE_DIRS = {"/src/test/", "/src/trunnel/", "/src/ext/", "/.git/"} def get_tor_c_files(tor_topdir): """ From f0302d51abf2f4c6d7660b82ad5038d1cbf9e5d7 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 12 Mar 2019 09:22:53 -0400 Subject: [PATCH 0598/2557] practracker: Be more careful about excluding "confusing terms" Previously we excluded any line containing one of these terms from consideration as the start or end of a function. Now we're more careful, and we only ignore these terms when they appear to be starting a function definition. --- scripts/maint/practracker/metrics.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/maint/practracker/metrics.py b/scripts/maint/practracker/metrics.py index 56b9e8383e..5fa305a868 100644 --- a/scripts/maint/practracker/metrics.py +++ b/scripts/maint/practracker/metrics.py @@ -25,20 +25,20 @@ def get_function_lines(f): Return iterator which iterates over functions and returns (function name, function lines) """ - # Skip lines with these terms since they confuse our regexp + # Skip lines that look like they are defining functions with these + # names: they aren't real function definitions. REGEXP_CONFUSE_TERMS = {"MOCK_IMPL", "ENABLE_GCC_WARNINGS", "ENABLE_GCC_WARNING", "DUMMY_TYPECHECK_INSTANCE", "DISABLE_GCC_WARNING", "DISABLE_GCC_WARNINGS"} in_function = False for lineno, line in enumerate(f): - if any(x in line for x in REGEXP_CONFUSE_TERMS): - continue - if not in_function: # find the start of a function m = re.match(r'^([a-zA-Z_][a-zA-Z_0-9]*),?\(', line) if m: func_name = m.group(1) + if func_name in REGEXP_CONFUSE_TERMS: + continue func_start = lineno in_function = True From e2512950b69d5f9f030673a15e8cf72b930538c7 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 12 Mar 2019 09:32:22 -0400 Subject: [PATCH 0599/2557] Improve failure message from check-best-practices --- doc/HACKING/HelpfulTools.md | 15 +++++++++++++++ scripts/maint/practracker/practracker.py | 9 ++++++++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/doc/HACKING/HelpfulTools.md b/doc/HACKING/HelpfulTools.md index d499238526..cba57e875d 100644 --- a/doc/HACKING/HelpfulTools.md +++ b/doc/HACKING/HelpfulTools.md @@ -371,3 +371,18 @@ source code. Here's how to use it: 6. See the Doxygen manual for more information; this summary just scratches the surface. + +Style and best-pratices checking +-------------------------------- + +We use scripts to check for various problems in the formatting and style +of our source code. The "check-spaces" test detects a bunch of violations +of our coding style on the local level. The "check-best-practices" test +looks for violations of some of our complexity guidelines. + +You can tell the tool about exceptions to the complexity guidelines via its +exceptions file (scripts/maint/practracker/exceptions.txt). But before you +do this, consider whether you shouldn't fix the underlying problem. Maybe +that file really _is_ too big. Maybe that function really _is_ doing too +much. (On the other hand, for stable release series, it is sometimes better +to leave things unrefactored.) diff --git a/scripts/maint/practracker/practracker.py b/scripts/maint/practracker/practracker.py index 541c5cbfb3..08b74c264f 100755 --- a/scripts/maint/practracker/practracker.py +++ b/scripts/maint/practracker/practracker.py @@ -133,7 +133,14 @@ def main(): # If new issues were found, try to give out some advice to the developer on how to resolve it. if (found_new_issues): - new_issues_str = "practracker FAILED as indicated by the problem lines above. Please use the exceptions file ({}) to find any previous state of these problems. If you are unable to fix the underlying best-practices issue right now then you need to either update the relevant exception line or add a new one.".format(exceptions_file) + new_issues_str = """\ +FAILURE: practracker found new problems in the code: see warnings above. + +Please fix the problems if you can, and update the exceptions file +({}) if you can't. + +See doc/HACKING/HelpfulTools.md for more information on using practracker.\ +""".format(exceptions_file) print(new_issues_str) sys.exit(found_new_issues) From 21c6e295ca74e2ab50a759123bca1fae36af36b3 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 12 Mar 2019 20:37:25 -0400 Subject: [PATCH 0600/2557] grandfather in two more functions --- scripts/maint/practracker/exceptions.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/maint/practracker/exceptions.txt b/scripts/maint/practracker/exceptions.txt index da354d8f83..3fa68aff47 100644 --- a/scripts/maint/practracker/exceptions.txt +++ b/scripts/maint/practracker/exceptions.txt @@ -166,7 +166,9 @@ problem function-size /src/feature/dirauth/bwauth.c:dirserv_read_measured_bandwi problem function-size /src/feature/dirauth/dsigs_parse.c:networkstatus_parse_detached_signatures() 196 problem function-size /src/feature/dirauth/voteflags.c:dirserv_compute_performance_thresholds() 172 problem file-size /src/feature/hs/hs_service.c 4171 +problem function-size /src/feature/hs/hs_cell.c:hs_cell_parse_introduce2() 154 problem function-size /src/feature/hs/hs_common.c:hs_get_responsible_hsdirs() 104 +problem function-size /src/feature/hs/hs_common.c:hs_get_extend_info_from_lspecs() 101 problem function-size /src/feature/hs/hs_client.c:send_introduce1() 104 problem function-size /src/feature/hs/hs_client.c:hs_config_client_authorization() 108 problem function-size /src/feature/hs/hs_config.c:config_generic_service() 149 From 3e38efdf165098062972300687975c3372cec70e Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Wed, 13 Mar 2019 12:59:44 +0200 Subject: [PATCH 0601/2557] practracker: Fix duplicate exceptions and handle them more "gracefully". --- scripts/maint/practracker/exceptions.txt | 1 - scripts/maint/practracker/problem.py | 6 ++++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/scripts/maint/practracker/exceptions.txt b/scripts/maint/practracker/exceptions.txt index 3fa68aff47..abe615f04a 100644 --- a/scripts/maint/practracker/exceptions.txt +++ b/scripts/maint/practracker/exceptions.txt @@ -173,7 +173,6 @@ problem function-size /src/feature/hs/hs_client.c:send_introduce1() 104 problem function-size /src/feature/hs/hs_client.c:hs_config_client_authorization() 108 problem function-size /src/feature/hs/hs_config.c:config_generic_service() 149 problem function-size /src/feature/hs/hs_cell.c:hs_cell_build_establish_intro() 115 -problem function-size /src/feature/hs/hs_cell.c:hs_cell_parse_introduce2() 147 problem file-size /src/feature/hs/hs_descriptor.c 3108 problem function-size /src/feature/hs/hs_descriptor.c:desc_encode_v3() 108 problem function-size /src/feature/hs/hs_descriptor.c:decrypt_desc_layer() 110 diff --git a/scripts/maint/practracker/problem.py b/scripts/maint/practracker/problem.py index 60ee3eb242..61420fb785 100644 --- a/scripts/maint/practracker/problem.py +++ b/scripts/maint/practracker/problem.py @@ -33,6 +33,12 @@ class ProblemVault(object): if problem is None: continue + # Fail if we see dup exceptions. There is really no reason to have dup exceptions. + if problem.key() in self.exceptions: + print("Duplicate exceptions lines found in exception file:\n\t{}\n\t{}\nAborting...".format(problem, self.exceptions[problem.key()]), + file=sys.stderr) + sys.exit(1) + self.exceptions[problem.key()] = problem #print "Registering exception: %s" % problem From 504aef6228d3a8460f43b9c9cf2cd4175ce4e1f7 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 13 Mar 2019 09:29:10 -0400 Subject: [PATCH 0602/2557] Changes file for practracker (29221) --- changes/bug29221 | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 changes/bug29221 diff --git a/changes/bug29221 b/changes/bug29221 new file mode 100644 index 0000000000..fbe08aa9a0 --- /dev/null +++ b/changes/bug29221 @@ -0,0 +1,5 @@ + o Minor features (development tools): + - Tor's test scripts now check for files and functions that seem + too long and complicated. Existing overlong functions and files are + accepted for now, but should eventually be refactored. Closes + ticket 29221. From a71999462b8d185c70169c4cba3efde6e55393ce Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Mon, 25 Feb 2019 14:59:28 +0200 Subject: [PATCH 0603/2557] Fix test prob distr parameters that caused warnings. They were causing the following warnings in circuitpadding/circuitpadding_sample_distribution: src/lib/math/prob_distr.c:1311:17: runtime error: division by zero SUMMARY: UndefinedBehaviorSanitizer: undefined-behavior src/lib/math/prob_distr.c:1311:17 in src/lib/math/prob_distr.c:1219:49: runtime error: division by zero SUMMARY: UndefinedBehaviorSanitizer: undefined-behavior src/lib/math/prob_distr.c:1219:49 in because the distributions were called with erroneous parameters (e.g. geometric distribution with p=0). We now defined these test probability distributions with more realistic parameters. As far as the circuitpadding_sample_distribution() test is concerned, it doesn't matter if the distributions return values outside of [0,10] since we already restrict the values into that interval using min=0 and max=10 (and RTT estimate is disabled). --- src/core/or/circuitpadding.c | 7 +++++-- src/core/or/circuitpadding.h | 11 ++++++++++- src/test/test_circuitpadding.c | 30 ++++++++++++++++-------------- 3 files changed, 31 insertions(+), 17 deletions(-) diff --git a/src/core/or/circuitpadding.c b/src/core/or/circuitpadding.c index b7e4b56846..9e87902778 100644 --- a/src/core/or/circuitpadding.c +++ b/src/core/or/circuitpadding.c @@ -389,11 +389,14 @@ circpad_distribution_sample_iat_delay(const circpad_state_t *state, double val = circpad_distribution_sample(state->iat_dist); /* These comparisons are safe, because the output is in the range * [0, 2**32), and double has a precision of 53 bits. */ + /* We want a positive sample value */ val = MAX(0, val); + /* Respect the maximum sample setting */ val = MIN(val, state->dist_max_sample_usec); - /* This addition is exact: val is at most 2**32-1, min_delay - * is at most 2**32-1, and doubles have a precision of 53 bits. */ + /* Now apply the shift: + * This addition is exact: val is at most 2**32-1, delay_shift is at most + * 2**32-1, and doubles have a precision of 53 bits. */ val += delay_shift; /* Clamp the distribution at infinite delay val */ diff --git a/src/core/or/circuitpadding.h b/src/core/or/circuitpadding.h index 74f9f35c98..fefdf2ba94 100644 --- a/src/core/or/circuitpadding.h +++ b/src/core/or/circuitpadding.h @@ -198,14 +198,23 @@ typedef enum { * These can be used instead of histograms for the inter-packet * timing distribution, or to specify a distribution on the number * of cells that can be sent while in a specific state of the state - * machine. */ + * machine. + * + * Each distribution takes up to two parameters which are described below. */ typedef enum { + /* No probability distribution is used */ CIRCPAD_DIST_NONE = 0, + /* Uniform distribution: param1 is lower bound and param2 is upper bound */ CIRCPAD_DIST_UNIFORM = 1, + /* Logistic distribution: param1 is Mu, param2 is sigma. */ CIRCPAD_DIST_LOGISTIC = 2, + /* Log-logistic distribution: param1 is Alpha, param2 is 1.0/Beta */ CIRCPAD_DIST_LOG_LOGISTIC = 3, + /* Geometric distribution: param1 is 'p' (success probability) */ CIRCPAD_DIST_GEOMETRIC = 4, + /* Weibull distribution: param1 is k, param2 is Lambda */ CIRCPAD_DIST_WEIBULL = 5, + /* Generalized Pareto distribution: param1 is sigma, param2 is xi */ CIRCPAD_DIST_PARETO = 6 } circpad_distribution_type_t; diff --git a/src/test/test_circuitpadding.c b/src/test/test_circuitpadding.c index 3570b179bd..d349a4d3c3 100644 --- a/src/test/test_circuitpadding.c +++ b/src/test/test_circuitpadding.c @@ -2096,6 +2096,7 @@ helper_circpad_circ_distribution_machine_setup(int min, int max) circpad_state_t *zero_st = &circ_client_machine.states[0]; zero_st->next_state[CIRCPAD_EVENT_NONPADDING_RECV] = 1; zero_st->iat_dist.type = CIRCPAD_DIST_UNIFORM; + /* param2 is upper bound, param1 is lower */ zero_st->iat_dist.param1 = min; zero_st->iat_dist.param2 = max; zero_st->dist_added_shift_usec = min; @@ -2104,48 +2105,50 @@ helper_circpad_circ_distribution_machine_setup(int min, int max) circpad_state_t *first_st = &circ_client_machine.states[1]; first_st->next_state[CIRCPAD_EVENT_NONPADDING_RECV] = 2; first_st->iat_dist.type = CIRCPAD_DIST_LOGISTIC; - first_st->iat_dist.param1 = min; - first_st->iat_dist.param2 = max; + /* param1 is Mu, param2 is sigma. */ + first_st->iat_dist.param1 = 9; + first_st->iat_dist.param2 = 3; first_st->dist_added_shift_usec = min; first_st->dist_max_sample_usec = max; circpad_state_t *second_st = &circ_client_machine.states[2]; second_st->next_state[CIRCPAD_EVENT_NONPADDING_RECV] = 3; second_st->iat_dist.type = CIRCPAD_DIST_LOG_LOGISTIC; - second_st->iat_dist.param1 = min; - second_st->iat_dist.param2 = max; + /* param1 is Alpha, param2 is 1.0/Beta */ + second_st->iat_dist.param1 = 1; + second_st->iat_dist.param2 = 0.5; second_st->dist_added_shift_usec = min; second_st->dist_max_sample_usec = max; circpad_state_t *third_st = &circ_client_machine.states[3]; third_st->next_state[CIRCPAD_EVENT_NONPADDING_RECV] = 4; third_st->iat_dist.type = CIRCPAD_DIST_GEOMETRIC; - third_st->iat_dist.param1 = min; - third_st->iat_dist.param2 = max; + /* param1 is 'p' (success probability) */ + third_st->iat_dist.param1 = 0.2; third_st->dist_added_shift_usec = min; third_st->dist_max_sample_usec = max; circpad_state_t *fourth_st = &circ_client_machine.states[4]; fourth_st->next_state[CIRCPAD_EVENT_NONPADDING_RECV] = 5; fourth_st->iat_dist.type = CIRCPAD_DIST_WEIBULL; - fourth_st->iat_dist.param1 = min; - fourth_st->iat_dist.param2 = max; + /* param1 is k, param2 is Lambda */ + fourth_st->iat_dist.param1 = 1.5; + fourth_st->iat_dist.param2 = 1; fourth_st->dist_added_shift_usec = min; fourth_st->dist_max_sample_usec = max; circpad_state_t *fifth_st = &circ_client_machine.states[5]; fifth_st->next_state[CIRCPAD_EVENT_NONPADDING_RECV] = 6; fifth_st->iat_dist.type = CIRCPAD_DIST_PARETO; - fifth_st->iat_dist.param1 = min; - fifth_st->iat_dist.param2 = max; + /* param1 is sigma, param2 is xi */ + fifth_st->iat_dist.param1 = 1; + fifth_st->iat_dist.param2 = 5; fifth_st->dist_added_shift_usec = min; fifth_st->dist_max_sample_usec = max; } /** Simple test that the padding delays sampled from a uniform distribution * actually faill within the uniform distribution range. */ -/* TODO: Upgrade this test so that each state tests a different prob - * distribution */ static void test_circuitpadding_sample_distribution(void *arg) { @@ -2159,8 +2162,7 @@ test_circuitpadding_sample_distribution(void *arg) MOCK(circpad_machine_schedule_padding, circpad_machine_schedule_padding_mock); - /* Initialize a machine with multiple probability distributions that should - * return values between 0 and 5 */ + /* Initialize a machine with multiple probability distributions */ circpad_machines_init(); helper_circpad_circ_distribution_machine_setup(0, 10); From 8b64ff7a3f2d9eb0837248a6c9b0e8b438374369 Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Mon, 25 Feb 2019 15:29:41 +0200 Subject: [PATCH 0604/2557] Silence unneeded clang warns that triggered in prob distr tests. See https://trac.torproject.org/projects/tor/ticket/29528#comment:3 and https://bugs.llvm.org/show_bug.cgi?id=19535 --- configure.ac | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/configure.ac b/configure.ac index 75bb0c7bd9..8f05637538 100644 --- a/configure.ac +++ b/configure.ac @@ -1196,6 +1196,11 @@ if test "$fragile_hardening" = "yes"; then AC_MSG_ERROR([The compiler supports -fsanitize=undefined, but for some reason I was not able to link when using it. Are you missing run-time support? With GCC you need libasan.*, and with Clang you need libclang_rt.ubsan*]) fi + TOR_TRY_COMPILE_WITH_CFLAGS([-fno-sanitize=float-divide-by-zero], also_link, CFLAGS_UBSAN="-fno-sanitize=float-divide-by-zero", true) + if test "$tor_cv_cflags__fno_sanitize_float_divide_by_zero" = "yes" && test "$tor_can_link__fno_sanitize_float_divide_by_zero" != "yes"; then + AC_MSG_ERROR([The compiler supports -fno-sanitize=float-divide-by-zero, but for some reason I was not able to link when using it. Are you missing run-time support? With GCC you need libasan.*, and with Clang you need libclang_rt.ubsan*]) + fi + TOR_CHECK_CFLAGS([-fno-omit-frame-pointer]) fi From 8d9f81bc9cdca3022ec3243145bf232fe4f28579 Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Tue, 5 Mar 2019 17:49:44 +0200 Subject: [PATCH 0605/2557] prob_distr: Implement type-safe downcasting functions. --- src/lib/math/prob_distr.c | 133 +++++++++++++------------------------- 1 file changed, 46 insertions(+), 87 deletions(-) diff --git a/src/lib/math/prob_distr.c b/src/lib/math/prob_distr.c index c952dadc06..2dff9e76e4 100644 --- a/src/lib/math/prob_distr.c +++ b/src/lib/math/prob_distr.c @@ -46,26 +46,27 @@ #include "lib/crypt_ops/crypto_rand.h" #include "lib/cc/ctassert.h" +#include "lib/log/util_bug.h" #include #include #include -/** Validators for downcasting macros below */ -#define validate_container_of(PTR, TYPE, FIELD) \ - (0 * sizeof((PTR) - &((TYPE *)(((char *)(PTR)) - \ - offsetof(TYPE, FIELD)))->FIELD)) -#define validate_const_container_of(PTR, TYPE, FIELD) \ - (0 * sizeof((PTR) - &((const TYPE *)(((const char *)(PTR)) - \ - offsetof(TYPE, FIELD)))->FIELD)) -/** Downcasting macro */ -#define container_of(PTR, TYPE, FIELD) \ - ((TYPE *)(((char *)(PTR)) - offsetof(TYPE, FIELD)) \ - + validate_container_of(PTR, TYPE, FIELD)) -/** Constified downcasting macro */ -#define const_container_of(PTR, TYPE, FIELD) \ - ((const TYPE *)(((const char *)(PTR)) - offsetof(TYPE, FIELD)) \ - + validate_const_container_of(PTR, TYPE, FIELD)) +/** Declare a function that downcasts from a generic dist struct to the actual + * subtype probablity distribution it represents. */ +#define DECLARE_PROB_DISTR_DOWNCAST_FN(name) \ + static inline \ + const struct name * \ + dist_to_const_##name(const struct dist *obj) { \ + tor_assert(obj->ops == &name##_ops); \ + return SUBTYPE_P(obj, struct name, base); \ + } +DECLARE_PROB_DISTR_DOWNCAST_FN(uniform) +DECLARE_PROB_DISTR_DOWNCAST_FN(geometric) +DECLARE_PROB_DISTR_DOWNCAST_FN(logistic) +DECLARE_PROB_DISTR_DOWNCAST_FN(log_logistic) +DECLARE_PROB_DISTR_DOWNCAST_FN(genpareto) +DECLARE_PROB_DISTR_DOWNCAST_FN(weibull) /** * Count number of one bits in 32-bit word. @@ -1360,8 +1361,7 @@ dist_isf(const struct dist *dist, double p) static double uniform_sample(const struct dist *dist) { - const struct uniform *U = const_container_of(dist, struct uniform, - base); + const struct uniform *U = dist_to_const_uniform(dist); double p0 = random_uniform_01(); return sample_uniform_interval(p0, U->a, U->b); @@ -1370,9 +1370,7 @@ uniform_sample(const struct dist *dist) static double uniform_cdf(const struct dist *dist, double x) { - const struct uniform *U = const_container_of(dist, struct uniform, - base); - + const struct uniform *U = dist_to_const_uniform(dist); if (x < U->a) return 0; else if (x < U->b) @@ -1384,8 +1382,7 @@ uniform_cdf(const struct dist *dist, double x) static double uniform_sf(const struct dist *dist, double x) { - const struct uniform *U = const_container_of(dist, struct uniform, - base); + const struct uniform *U = dist_to_const_uniform(dist); if (x > U->b) return 0; @@ -1398,8 +1395,7 @@ uniform_sf(const struct dist *dist, double x) static double uniform_icdf(const struct dist *dist, double p) { - const struct uniform *U = const_container_of(dist, struct uniform, - base); + const struct uniform *U = dist_to_const_uniform(dist); double w = U->b - U->a; return (p < 0.5 ? (U->a + w*p) : (U->b - w*(1 - p))); @@ -1408,8 +1404,7 @@ uniform_icdf(const struct dist *dist, double p) static double uniform_isf(const struct dist *dist, double p) { - const struct uniform *U = const_container_of(dist, struct uniform, - base); + const struct uniform *U = dist_to_const_uniform(dist); double w = U->b - U->a; return (p < 0.5 ? (U->b - w*p) : (U->a + w*(1 - p))); @@ -1429,8 +1424,7 @@ const struct dist_ops uniform_ops = { static double logistic_sample(const struct dist *dist) { - const struct logistic *L = const_container_of(dist, struct logistic, - base); + const struct logistic *L = dist_to_const_logistic(dist); uint32_t s = crypto_rand_u32(); double t = random_uniform_01(); double p0 = random_uniform_01(); @@ -1441,36 +1435,28 @@ logistic_sample(const struct dist *dist) static double logistic_cdf(const struct dist *dist, double x) { - const struct logistic *L = const_container_of(dist, struct logistic, - base); - + const struct logistic *L = dist_to_const_logistic(dist); return cdf_logistic(x, L->mu, L->sigma); } static double logistic_sf(const struct dist *dist, double x) { - const struct logistic *L = const_container_of(dist, struct logistic, - base); - + const struct logistic *L = dist_to_const_logistic(dist); return sf_logistic(x, L->mu, L->sigma); } static double logistic_icdf(const struct dist *dist, double p) { - const struct logistic *L = const_container_of(dist, struct logistic, - base); - + const struct logistic *L = dist_to_const_logistic(dist); return icdf_logistic(p, L->mu, L->sigma); } static double logistic_isf(const struct dist *dist, double p) { - const struct logistic *L = const_container_of(dist, struct logistic, - base); - + const struct logistic *L = dist_to_const_logistic(dist); return isf_logistic(p, L->mu, L->sigma); } @@ -1488,8 +1474,7 @@ const struct dist_ops logistic_ops = { static double log_logistic_sample(const struct dist *dist) { - const struct log_logistic *LL = const_container_of(dist, struct - log_logistic, base); + const struct log_logistic *LL = dist_to_const_log_logistic(dist); uint32_t s = crypto_rand_u32(); double p0 = random_uniform_01(); @@ -1499,36 +1484,28 @@ log_logistic_sample(const struct dist *dist) static double log_logistic_cdf(const struct dist *dist, double x) { - const struct log_logistic *LL = const_container_of(dist, - struct log_logistic, base); - + const struct log_logistic *LL = dist_to_const_log_logistic(dist); return cdf_log_logistic(x, LL->alpha, LL->beta); } static double log_logistic_sf(const struct dist *dist, double x) { - const struct log_logistic *LL = const_container_of(dist, - struct log_logistic, base); - + const struct log_logistic *LL = dist_to_const_log_logistic(dist); return sf_log_logistic(x, LL->alpha, LL->beta); } static double log_logistic_icdf(const struct dist *dist, double p) { - const struct log_logistic *LL = const_container_of(dist, - struct log_logistic, base); - + const struct log_logistic *LL = dist_to_const_log_logistic(dist); return icdf_log_logistic(p, LL->alpha, LL->beta); } static double log_logistic_isf(const struct dist *dist, double p) { - const struct log_logistic *LL = const_container_of(dist, - struct log_logistic, base); - + const struct log_logistic *LL = dist_to_const_log_logistic(dist); return isf_log_logistic(p, LL->alpha, LL->beta); } @@ -1546,8 +1523,7 @@ const struct dist_ops log_logistic_ops = { static double weibull_sample(const struct dist *dist) { - const struct weibull *W = const_container_of(dist, struct weibull, - base); + const struct weibull *W = dist_to_const_weibull(dist); uint32_t s = crypto_rand_u32(); double p0 = random_uniform_01(); @@ -1557,36 +1533,28 @@ weibull_sample(const struct dist *dist) static double weibull_cdf(const struct dist *dist, double x) { - const struct weibull *W = const_container_of(dist, struct weibull, - base); - + const struct weibull *W = dist_to_const_weibull(dist); return cdf_weibull(x, W->lambda, W->k); } static double weibull_sf(const struct dist *dist, double x) { - const struct weibull *W = const_container_of(dist, struct weibull, - base); - + const struct weibull *W = dist_to_const_weibull(dist); return sf_weibull(x, W->lambda, W->k); } static double weibull_icdf(const struct dist *dist, double p) { - const struct weibull *W = const_container_of(dist, struct weibull, - base); - + const struct weibull *W = dist_to_const_weibull(dist); return icdf_weibull(p, W->lambda, W->k); } static double weibull_isf(const struct dist *dist, double p) { - const struct weibull *W = const_container_of(dist, struct weibull, - base); - + const struct weibull *W = dist_to_const_weibull(dist); return isf_weibull(p, W->lambda, W->k); } @@ -1604,8 +1572,7 @@ const struct dist_ops weibull_ops = { static double genpareto_sample(const struct dist *dist) { - const struct genpareto *GP = const_container_of(dist, struct genpareto, - base); + const struct genpareto *GP = dist_to_const_genpareto(dist); uint32_t s = crypto_rand_u32(); double p0 = random_uniform_01(); @@ -1615,36 +1582,28 @@ genpareto_sample(const struct dist *dist) static double genpareto_cdf(const struct dist *dist, double x) { - const struct genpareto *GP = const_container_of(dist, struct genpareto, - base); - + const struct genpareto *GP = dist_to_const_genpareto(dist); return cdf_genpareto(x, GP->mu, GP->sigma, GP->xi); } static double genpareto_sf(const struct dist *dist, double x) { - const struct genpareto *GP = const_container_of(dist, struct genpareto, - base); - + const struct genpareto *GP = dist_to_const_genpareto(dist); return sf_genpareto(x, GP->mu, GP->sigma, GP->xi); } static double genpareto_icdf(const struct dist *dist, double p) { - const struct genpareto *GP = const_container_of(dist, struct genpareto, - base); - + const struct genpareto *GP = dist_to_const_genpareto(dist); return icdf_genpareto(p, GP->mu, GP->sigma, GP->xi); } static double genpareto_isf(const struct dist *dist, double p) { - const struct genpareto *GP = const_container_of(dist, struct genpareto, - base); - + const struct genpareto *GP = dist_to_const_genpareto(dist); return isf_genpareto(p, GP->mu, GP->sigma, GP->xi); } @@ -1662,7 +1621,7 @@ const struct dist_ops genpareto_ops = { static double geometric_sample(const struct dist *dist) { - const struct geometric *G = const_container_of(dist, struct geometric, base); + const struct geometric *G = dist_to_const_geometric(dist); uint32_t s = crypto_rand_u32(); double p0 = random_uniform_01(); @@ -1672,7 +1631,7 @@ geometric_sample(const struct dist *dist) static double geometric_cdf(const struct dist *dist, double x) { - const struct geometric *G = const_container_of(dist, struct geometric, base); + const struct geometric *G = dist_to_const_geometric(dist); if (x < 1) return 0; @@ -1683,7 +1642,7 @@ geometric_cdf(const struct dist *dist, double x) static double geometric_sf(const struct dist *dist, double x) { - const struct geometric *G = const_container_of(dist, struct geometric, base); + const struct geometric *G = dist_to_const_geometric(dist); if (x < 1) return 0; @@ -1694,7 +1653,7 @@ geometric_sf(const struct dist *dist, double x) static double geometric_icdf(const struct dist *dist, double p) { - const struct geometric *G = const_container_of(dist, struct geometric, base); + const struct geometric *G = dist_to_const_geometric(dist); return log1p(-p)/log1p(-G->p); } @@ -1702,7 +1661,7 @@ geometric_icdf(const struct dist *dist, double p) static double geometric_isf(const struct dist *dist, double p) { - const struct geometric *G = const_container_of(dist, struct geometric, base); + const struct geometric *G = dist_to_const_geometric(dist); return log(p)/log1p(-G->p); } From fe78ba855a50c1f83e449d63d32e754bc988595a Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Tue, 12 Mar 2019 19:04:27 +0200 Subject: [PATCH 0606/2557] prob_distr: Better document our public API. --- src/lib/math/prob_distr.c | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/lib/math/prob_distr.c b/src/lib/math/prob_distr.c index 2dff9e76e4..bfad06963d 100644 --- a/src/lib/math/prob_distr.c +++ b/src/lib/math/prob_distr.c @@ -1316,40 +1316,48 @@ sample_geometric(uint32_t s, double p0, double p) /** Public API for probability distributions: * - * For each probability distribution we define each public functions - * (sample/cdf/sf/icdf/isf) as part of its dist_ops structure. + * These are wrapper functions on top of the various probability distribution + * operations using the generic dist structure. + + * These are the functions that should be used by consumers of this API. */ +/** Returns the name of the distribution in dist. */ const char * dist_name(const struct dist *dist) { return dist->ops->name; } +/* Sample a value from dist and return it. */ double dist_sample(const struct dist *dist) { return dist->ops->sample(dist); } +/** Compute the CDF of dist at x. */ double dist_cdf(const struct dist *dist, double x) { return dist->ops->cdf(dist, x); } +/** Compute the SF (Survival function) of dist at x. */ double dist_sf(const struct dist *dist, double x) { return dist->ops->sf(dist, x); } +/** Compute the iCDF (Inverse CDF) of dist at x. */ double dist_icdf(const struct dist *dist, double p) { return dist->ops->icdf(dist, p); } +/** Compute the iSF (Inverse Survival function) of dist at x. */ double dist_isf(const struct dist *dist, double p) { @@ -1419,6 +1427,10 @@ const struct dist_ops uniform_ops = { .isf = uniform_isf, }; +/*******************************************************************/ + +/** Private functions for each probability distribution. */ + /** Functions for logistic distribution: */ static double From a55c89c47594afc47d1302fbebed9d104ec3d6ff Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Thu, 14 Mar 2019 02:15:32 +0200 Subject: [PATCH 0607/2557] practracker: Be compatible with python2 which is used by travis/jenkins. --- scripts/maint/practracker/practracker.py | 2 ++ scripts/maint/practracker/problem.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/scripts/maint/practracker/practracker.py b/scripts/maint/practracker/practracker.py index 08b74c264f..a6e6d0b607 100755 --- a/scripts/maint/practracker/practracker.py +++ b/scripts/maint/practracker/practracker.py @@ -18,6 +18,8 @@ the source code and then get saved in the repository for ever after: $ python3 ./scripts/maint/practracker/practracker.py . > ./scripts/maint/practracker/exceptions.txt """ +from __future__ import print_function + import os, sys import metrics diff --git a/scripts/maint/practracker/problem.py b/scripts/maint/practracker/problem.py index 61420fb785..ab3d55057e 100644 --- a/scripts/maint/practracker/problem.py +++ b/scripts/maint/practracker/problem.py @@ -7,6 +7,8 @@ problem is worse than a registered exception so that it only warns when things get worse. """ +from __future__ import print_function + import os.path import sys From 2d76945052c09a4f36519491fd414d1a314b27a7 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 14 Mar 2019 09:35:20 -0400 Subject: [PATCH 0608/2557] practracker: Create an exception for an as-yet-nonexistent file. We need a better way to do this. --- scripts/maint/practracker/exceptions.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/maint/practracker/exceptions.txt b/scripts/maint/practracker/exceptions.txt index abe615f04a..359daca1c1 100644 --- a/scripts/maint/practracker/exceptions.txt +++ b/scripts/maint/practracker/exceptions.txt @@ -101,6 +101,7 @@ problem function-size /src/app/main/main.c:dumpstats() 102 problem function-size /src/app/main/main.c:tor_init() 136 problem function-size /src/app/main/main.c:sandbox_init_filter() 291 problem function-size /src/app/main/main.c:run_tor_main_loop() 105 +problem include-count /src/app/main/shutdown.c 55 problem function-size /src/tools/tor-resolve.c:build_socks5_resolve_request() 104 problem function-size /src/tools/tor-resolve.c:do_resolve() 173 problem function-size /src/tools/tor-resolve.c:main() 112 From aee65084b421d0b5ff717cc9e01dd92f4e96be9e Mon Sep 17 00:00:00 2001 From: David Goulet Date: Tue, 29 Jan 2019 15:13:16 +0100 Subject: [PATCH 0609/2557] circ: Remove n_mux and p_mux members They are simply not used apart from assigning a pointer and asserting on the pointer depending on the cell direction. Closes #29196. Signed-off-by: David Goulet --- src/core/or/circuit_st.h | 6 ------ src/core/or/circuitlist.c | 4 ---- src/core/or/circuitmux.c | 29 +---------------------------- src/core/or/or_circuit_st.h | 5 ----- src/test/test_channel.c | 1 - src/test/test_circuitpadding.c | 1 - 6 files changed, 1 insertion(+), 45 deletions(-) diff --git a/src/core/or/circuit_st.h b/src/core/or/circuit_st.h index af343f082e..c0cd87b3c7 100644 --- a/src/core/or/circuit_st.h +++ b/src/core/or/circuit_st.h @@ -66,12 +66,6 @@ struct circuit_t { */ circid_t n_circ_id; - /** - * Circuit mux associated with n_chan to which this circuit is attached; - * NULL if we have no n_chan. - */ - circuitmux_t *n_mux; - /** Queue of cells waiting to be transmitted on n_chan */ cell_queue_t n_chan_cells; diff --git a/src/core/or/circuitlist.c b/src/core/or/circuitlist.c index 6b5f30e418..0d3665ce71 100644 --- a/src/core/or/circuitlist.c +++ b/src/core/or/circuitlist.c @@ -2433,13 +2433,9 @@ marked_circuit_free_cells(circuit_t *circ) return; } cell_queue_clear(&circ->n_chan_cells); - if (circ->n_mux) - circuitmux_clear_num_cells(circ->n_mux, circ); if (! CIRCUIT_IS_ORIGIN(circ)) { or_circuit_t *orcirc = TO_OR_CIRCUIT(circ); cell_queue_clear(&orcirc->p_chan_cells); - if (orcirc->p_mux) - circuitmux_clear_num_cells(orcirc->p_mux, circ); } } diff --git a/src/core/or/circuitmux.c b/src/core/or/circuitmux.c index 88f9ac7923..b2628bec3f 100644 --- a/src/core/or/circuitmux.c +++ b/src/core/or/circuitmux.c @@ -294,9 +294,6 @@ circuitmux_detach_all_circuits(circuitmux_t *cmux, smartlist_t *detached_out) circuitmux_make_circuit_inactive(cmux, circ); } - /* Clear n_mux */ - circ->n_mux = NULL; - if (detached_out) smartlist_add(detached_out, circ); } else if (circ->magic == OR_CIRCUIT_MAGIC) { @@ -309,12 +306,6 @@ circuitmux_detach_all_circuits(circuitmux_t *cmux, smartlist_t *detached_out) circuitmux_make_circuit_inactive(cmux, circ); } - /* - * It has a sensible p_chan and direction == CELL_DIRECTION_IN, - * so clear p_mux. - */ - TO_OR_CIRCUIT(circ)->p_mux = NULL; - if (detached_out) smartlist_add(detached_out, circ); } else { @@ -836,18 +827,14 @@ circuitmux_attach_circuit,(circuitmux_t *cmux, circuit_t *circ, */ log_info(LD_CIRC, "Circuit %u on channel %"PRIu64 " was already attached to " - "cmux %p (trying to attach to %p)", + "(trying to attach to %p)", (unsigned)circ_id, (channel_id), - ((direction == CELL_DIRECTION_OUT) ? - circ->n_mux : TO_OR_CIRCUIT(circ)->p_mux), cmux); /* * The mux pointer on this circuit and the direction in result should * match; otherwise assert. */ - if (direction == CELL_DIRECTION_OUT) tor_assert(circ->n_mux == cmux); - else tor_assert(TO_OR_CIRCUIT(circ)->p_mux == cmux); tor_assert(hashent->muxinfo.direction == direction); /* @@ -872,13 +859,6 @@ circuitmux_attach_circuit,(circuitmux_t *cmux, circuit_t *circ, "Attaching circuit %u on channel %"PRIu64 " to cmux %p", (unsigned)circ_id, (channel_id), cmux); - /* - * Assert that the circuit doesn't already have a mux for this - * direction. - */ - if (direction == CELL_DIRECTION_OUT) tor_assert(circ->n_mux == NULL); - else tor_assert(TO_OR_CIRCUIT(circ)->p_mux == NULL); - /* Insert it in the map */ hashent = tor_malloc_zero(sizeof(*hashent)); hashent->chan_id = channel_id; @@ -902,10 +882,6 @@ circuitmux_attach_circuit,(circuitmux_t *cmux, circuit_t *circ, HT_INSERT(chanid_circid_muxinfo_map, cmux->chanid_circid_map, hashent); - /* Set the circuit's mux for this direction */ - if (direction == CELL_DIRECTION_OUT) circ->n_mux = cmux; - else TO_OR_CIRCUIT(circ)->p_mux = cmux; - /* Update counters */ ++(cmux->n_circuits); if (cell_count > 0) { @@ -993,9 +969,6 @@ circuitmux_detach_circuit,(circuitmux_t *cmux, circuit_t *circ)) /* Consistency check: the direction must match the direction searched */ tor_assert(last_searched_direction == hashent->muxinfo.direction); - /* Clear the circuit's mux for this direction */ - if (last_searched_direction == CELL_DIRECTION_OUT) circ->n_mux = NULL; - else TO_OR_CIRCUIT(circ)->p_mux = NULL; /* Now remove it from the map */ HT_REMOVE(chanid_circid_muxinfo_map, cmux->chanid_circid_map, hashent); diff --git a/src/core/or/or_circuit_st.h b/src/core/or/or_circuit_st.h index 6b6feb9d89..062e4ac854 100644 --- a/src/core/or/or_circuit_st.h +++ b/src/core/or/or_circuit_st.h @@ -33,11 +33,6 @@ struct or_circuit_t { cell_queue_t p_chan_cells; /** The channel that is previous in this circuit. */ channel_t *p_chan; - /** - * Circuit mux associated with p_chan to which this circuit is attached; - * NULL if we have no p_chan. - */ - circuitmux_t *p_mux; /** Linked list of Exit streams associated with this circuit. */ edge_connection_t *n_streams; /** Linked list of Exit streams associated with this circuit that are diff --git a/src/test/test_channel.c b/src/test/test_channel.c index e55b9b0750..4c2bbc86b4 100644 --- a/src/test/test_channel.c +++ b/src/test/test_channel.c @@ -598,7 +598,6 @@ test_channel_outbound_cell(void *arg) circuit_set_n_circid_chan(TO_CIRCUIT(circ), 42, chan); tt_int_op(channel_num_circuits(chan), OP_EQ, 1); /* Test the cmux state. */ - tt_ptr_op(TO_CIRCUIT(circ)->n_mux, OP_EQ, chan->cmux); tt_int_op(circuitmux_is_circuit_attached(chan->cmux, TO_CIRCUIT(circ)), OP_EQ, 1); diff --git a/src/test/test_circuitpadding.c b/src/test/test_circuitpadding.c index a2381023ca..aaf7875a13 100644 --- a/src/test/test_circuitpadding.c +++ b/src/test/test_circuitpadding.c @@ -115,7 +115,6 @@ new_fake_orcirc(channel_t *nchan, channel_t *pchan) //circ->n_chan = nchan; circ->n_circ_id = get_unique_circ_id_by_chan(nchan); - circ->n_mux = NULL; /* ?? */ cell_queue_init(&(circ->n_chan_cells)); circ->n_hop = NULL; circ->streams_blocked_on_n_chan = 0; From 0aaeec19e72c428321981194a68b154d1d67fd84 Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Fri, 15 Mar 2019 17:13:06 +0200 Subject: [PATCH 0610/2557] Satisfy practracker broken by #29665 and #28656. --- scripts/maint/practracker/exceptions.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/maint/practracker/exceptions.txt b/scripts/maint/practracker/exceptions.txt index 359daca1c1..fee22ec7fc 100644 --- a/scripts/maint/practracker/exceptions.txt +++ b/scripts/maint/practracker/exceptions.txt @@ -26,7 +26,7 @@ problem function-size /src/core/or/channeltls.c:channel_tls_process_versions_cel problem function-size /src/core/or/channeltls.c:channel_tls_process_netinfo_cell() 214 problem function-size /src/core/or/channeltls.c:channel_tls_process_certs_cell() 246 problem function-size /src/core/or/channeltls.c:channel_tls_process_authenticate_cell() 202 -problem file-size /src/core/or/circuituse.c 3146 +problem file-size /src/core/or/circuituse.c 3150 problem function-size /src/core/or/circuituse.c:circuit_is_acceptable() 129 problem function-size /src/core/or/circuituse.c:circuit_expire_building() 394 problem function-size /src/core/or/circuituse.c:circuit_log_ancient_one_hop_circuits() 126 @@ -220,7 +220,7 @@ problem function-size /src/feature/nodelist/routerlist.c:router_add_to_routerlis problem function-size /src/feature/nodelist/routerlist.c:routerlist_remove_old_routers() 121 problem function-size /src/feature/nodelist/routerlist.c:update_consensus_router_descriptor_downloads() 136 problem function-size /src/feature/nodelist/routerlist.c:update_extrainfo_downloads() 103 -problem function-size /src/feature/nodelist/nodelist.c:compute_frac_paths_available() 187 +problem function-size /src/feature/nodelist/nodelist.c:compute_frac_paths_available() 193 problem function-size /src/feature/nodelist/node_select.c:router_pick_directory_server_impl() 123 problem function-size /src/feature/nodelist/node_select.c:compute_weighted_bandwidths() 205 problem function-size /src/feature/nodelist/node_select.c:router_pick_trusteddirserver_impl() 114 From 78a8827fd4d4c814e63a8670cf88453f1586f4c5 Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Mon, 25 Feb 2019 15:29:41 +0200 Subject: [PATCH 0611/2557] Silence unneeded clang warns that triggered in prob distr tests. See https://trac.torproject.org/projects/tor/ticket/29528#comment:3 and https://bugs.llvm.org/show_bug.cgi?id=19535 --- configure.ac | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/configure.ac b/configure.ac index 91a0498440..0f7eb61f25 100644 --- a/configure.ac +++ b/configure.ac @@ -1198,6 +1198,11 @@ if test "$fragile_hardening" = "yes"; then AC_MSG_ERROR([The compiler supports -fsanitize=undefined, but for some reason I was not able to link when using it. Are you missing run-time support? With GCC you need libasan.*, and with Clang you need libclang_rt.ubsan*]) fi + TOR_TRY_COMPILE_WITH_CFLAGS([-fno-sanitize=float-divide-by-zero], also_link, CFLAGS_UBSAN="-fno-sanitize=float-divide-by-zero", true) + if test "$tor_cv_cflags__fno_sanitize_float_divide_by_zero" = "yes" && test "$tor_can_link__fno_sanitize_float_divide_by_zero" != "yes"; then + AC_MSG_ERROR([The compiler supports -fno-sanitize=float-divide-by-zero, but for some reason I was not able to link when using it. Are you missing run-time support? With GCC you need libasan.*, and with Clang you need libclang_rt.ubsan*]) + fi + TOR_CHECK_CFLAGS([-fno-omit-frame-pointer]) fi From cfcf6de1a83ebb192374836a8c35a317fc2894e1 Mon Sep 17 00:00:00 2001 From: teor Date: Wed, 20 Mar 2019 18:07:00 +1000 Subject: [PATCH 0612/2557] Changes file for bug 29527 The previous commit introduces these changes: Stop warning about undefined behavior in the probability distribution tests. Float division by zero may technically be undefined behaviour in C, but it's well-defined in IEEE 754. Partial backport of 29298. Closes ticket 29527; bugfix on 0.4.0.1-alpha. --- changes/bug29527 | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 changes/bug29527 diff --git a/changes/bug29527 b/changes/bug29527 new file mode 100644 index 0000000000..6f36a9e1a0 --- /dev/null +++ b/changes/bug29527 @@ -0,0 +1,5 @@ + o Minor features (circuit padding): + - Stop warning about undefined behavior in the probability distribution + tests. Float division by zero may technically be undefined behaviour in + C, but it's well-defined in IEEE 754. Partial backport of 29298. + Closes ticket 29527; bugfix on 0.4.0.1-alpha. From 0bc9ed9d38cb29783d000b6e22677ae16727976b Mon Sep 17 00:00:00 2001 From: rl1987 Date: Wed, 20 Mar 2019 18:54:11 +0200 Subject: [PATCH 0613/2557] Move casts to separate C file to prevent compiler from optimising them away --- src/test/include.am | 2 ++ src/test/ptr_helpers.c | 50 ++++++++++++++++++++++++++++++++++++++++ src/test/ptr_helpers.h | 18 +++++++++++++++ src/test/test_ptr_slow.c | 13 ++++++----- 4 files changed, 77 insertions(+), 6 deletions(-) create mode 100644 src/test/ptr_helpers.c create mode 100644 src/test/ptr_helpers.h diff --git a/src/test/include.am b/src/test/include.am index e6cebe1d1a..700107d6ce 100644 --- a/src/test/include.am +++ b/src/test/include.am @@ -211,6 +211,7 @@ src_test_test_slow_SOURCES += \ src/test/test_crypto_slow.c \ src/test/test_process_slow.c \ src/test/test_prob_distr.c \ + src/test/ptr_helpers.c \ src/test/test_ptr_slow.c \ src/test/testing_common.c \ src/test/testing_rsakeys.c \ @@ -315,6 +316,7 @@ noinst_HEADERS+= \ src/test/log_test_helpers.h \ src/test/rend_test_helpers.h \ src/test/test.h \ + src/test/ptr_helpers.h \ src/test/test_helpers.h \ src/test/test_dir_common.h \ src/test/test_connection.h \ diff --git a/src/test/ptr_helpers.c b/src/test/ptr_helpers.c new file mode 100644 index 0000000000..296238feeb --- /dev/null +++ b/src/test/ptr_helpers.c @@ -0,0 +1,50 @@ +/* Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "ptr_helpers.h" + +/** + * Cast (inptr_t value) to a void pointer. + */ +void * +cast_intptr_to_voidstar(intptr_t x) +{ + void *r = (void *)x; + + return r; +} + +/** + * Cast x (void pointer) to inptr_t value. + */ +intptr_t +cast_voidstar_to_intptr(void *x) +{ + intptr_t r = (intptr_t)x; + + return r; +} + +/** + * Cast x (uinptr_t value) to void pointer. + */ +void * +cast_uintptr_to_voidstar(uintptr_t x) +{ + void *r = (void *)x; + + return r; +} + +/** + * Cast x (void pointer) to uinptr_t value. + */ +uintptr_t +cast_voidstar_to_uintptr(void *x) +{ + uintptr_t r = (uintptr_t)x; + + return r; +} diff --git a/src/test/ptr_helpers.h b/src/test/ptr_helpers.h new file mode 100644 index 0000000000..fe2c8c9705 --- /dev/null +++ b/src/test/ptr_helpers.h @@ -0,0 +1,18 @@ +/* Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include + +void * +cast_intptr_to_voidstar(intptr_t x); + +intptr_t +cast_voidstar_to_intptr(void *x); + +void * +cast_uintptr_to_voidstar(uintptr_t x); + +uintptr_t +cast_voidstar_to_uintptr(void *x); diff --git a/src/test/test_ptr_slow.c b/src/test/test_ptr_slow.c index f064a3e7c2..07481fb1ec 100644 --- a/src/test/test_ptr_slow.c +++ b/src/test/test_ptr_slow.c @@ -6,6 +6,7 @@ #include "orconfig.h" #include "core/or/or.h" #include "test/test.h" +#include "test/ptr_helpers.h" #include #include @@ -15,9 +16,9 @@ static void assert_int_voidptr_roundtrip(int a) { intptr_t ap = (intptr_t)a; - void *b = (void *)ap; - intptr_t c = (intptr_t)b; - void *d = (void *)c; + void *b = cast_intptr_to_voidstar(ap); + intptr_t c = cast_voidstar_to_intptr(b); + void *d = cast_intptr_to_voidstar(c); tt_assert(ap == c); tt_assert(b == d); @@ -45,9 +46,9 @@ static void assert_uint_voidptr_roundtrip(unsigned int a) { uintptr_t ap = (uintptr_t)a; - void *b = (void *)ap; - uintptr_t c = (uintptr_t)b; - void *d = (void *)c; + void *b = cast_uintptr_to_voidstar(ap); + uintptr_t c = cast_voidstar_to_uintptr(b); + void *d = cast_uintptr_to_voidstar(c); tt_assert(ap == c); tt_assert(b == d); From 280109473fe30d1e67f8db09ce279cc1f714a682 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Wed, 20 Mar 2019 19:00:03 +0200 Subject: [PATCH 0614/2557] Check more values of int --- src/test/test_ptr_slow.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/test/test_ptr_slow.c b/src/test/test_ptr_slow.c index 07481fb1ec..5b366b7d38 100644 --- a/src/test/test_ptr_slow.c +++ b/src/test/test_ptr_slow.c @@ -33,7 +33,11 @@ test_int_voidstar_interop(void *arg) int a; (void)arg; - for (a = 0; a <= 1024; a++) { + for (a = -1024; a <= 1024; a++) { + assert_int_voidptr_roundtrip(a); + } + + for (a = INT_MIN; a <= INT_MIN+1024; a++) { assert_int_voidptr_roundtrip(a); } From 72e0dc0822deb8587b3cae2edf584ece13cb6bb8 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Wed, 20 Mar 2019 19:06:40 +0200 Subject: [PATCH 0615/2557] Check roundtrip for each bit of {unsigned} int values --- src/test/test_ptr_slow.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/test/test_ptr_slow.c b/src/test/test_ptr_slow.c index 5b366b7d38..0142fd1c8a 100644 --- a/src/test/test_ptr_slow.c +++ b/src/test/test_ptr_slow.c @@ -44,6 +44,12 @@ test_int_voidstar_interop(void *arg) for (a = INT_MAX-1024; a < INT_MAX; a++) { assert_int_voidptr_roundtrip(a); } + + a = 1; + for (unsigned long i = 0; i < sizeof(int) * 8; i++) { + assert_int_voidptr_roundtrip(a); + a = (a << 1); + } } static void @@ -74,6 +80,12 @@ test_uint_voidstar_interop(void *arg) for (a = UINT_MAX-1024; a < UINT_MAX; a++) { assert_uint_voidptr_roundtrip(a); } + + a = 1; + for (unsigned long i = 0; i < sizeof(int) * 8; i++) { + assert_uint_voidptr_roundtrip(a); + a = (a << 1); + } } struct testcase_t slow_ptr_tests[] = { From bc64fb4e33ff98ca7271ee369fef24b3cf693574 Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Wed, 13 Mar 2019 14:54:01 +0200 Subject: [PATCH 0616/2557] circpad: Don't pad if Tor is in dormant mode. This is something we should think about harder, but we probably want dormant mode to be more powerful than padding in case a client has been inactive for a day or so. After all, there are probably no circuits open at this point and dormant mode will not allow the client to open more circuits. Furthermore, padding should not block dormant mode from being activated, since dormant mode relies on SocksPort activity, and circuit padding does not mess with that. --- src/core/or/circuitpadding.c | 9 ++++++++- src/feature/hibernate/hibernate.h | 1 + src/test/test_circuitpadding.c | 7 +++++++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/src/core/or/circuitpadding.c b/src/core/or/circuitpadding.c index 2895d49e18..599d88f493 100644 --- a/src/core/or/circuitpadding.c +++ b/src/core/or/circuitpadding.c @@ -48,6 +48,7 @@ #include "core/or/circuitpadding.h" #include "core/or/circuitlist.h" #include "core/or/circuituse.h" +#include "core/mainloop/netstatus.h" #include "core/or/relay.h" #include "feature/stats/rephist.h" #include "feature/nodelist/networkstatus.h" @@ -965,7 +966,7 @@ circpad_send_padding_cell_for_callback(circpad_machine_state_t *mi) mi->padding_scheduled_at_usec = 0; circpad_statenum_t state = mi->current_state; - // Make sure circuit didn't close on us + /* Make sure circuit didn't close on us */ if (mi->on_circ->marked_for_close) { log_fn(LOG_INFO,LD_CIRC, "Padding callback on a circuit marked for close. Ignoring."); @@ -1157,6 +1158,12 @@ circpad_machine_schedule_padding,(circpad_machine_state_t *mi)) struct timeval timeout; tor_assert(mi); + /* Don't schedule padding if we are currently in dormant mode. */ + if (!is_participating_on_network()) { + log_info(LD_CIRC, "Not scheduling padding because we are dormant."); + return CIRCPAD_STATE_UNCHANGED; + } + // Don't pad in end (but also don't cancel any previously // scheduled padding either). if (mi->current_state == CIRCPAD_STATE_END) { diff --git a/src/feature/hibernate/hibernate.h b/src/feature/hibernate/hibernate.h index 3309ef0ce3..2e245f6ab1 100644 --- a/src/feature/hibernate/hibernate.h +++ b/src/feature/hibernate/hibernate.h @@ -32,6 +32,7 @@ int getinfo_helper_accounting(control_connection_t *conn, const char **errmsg); uint64_t get_accounting_max_total(void); void accounting_free_all(void); +bool accounting_tor_is_dormant(void); #ifdef HIBERNATE_PRIVATE /** Possible values of hibernate_state */ diff --git a/src/test/test_circuitpadding.c b/src/test/test_circuitpadding.c index 1976e54bb9..ba58cd0d97 100644 --- a/src/test/test_circuitpadding.c +++ b/src/test/test_circuitpadding.c @@ -17,6 +17,7 @@ #include "core/or/circuitlist.h" #include "core/or/circuitbuild.h" #include "core/or/circuitpadding.h" +#include "core/mainloop/netstatus.h" #include "core/crypto/relay_crypto.h" #include "core/or/protover.h" #include "feature/nodelist/nodelist.h" @@ -1021,6 +1022,9 @@ test_circuitpadding_tokens(void *arg) monotime_coarse_set_mock_time_nsec(1*TOR_NSEC_PER_USEC); curr_mocked_time = 1*TOR_NSEC_PER_USEC; + /* This is needed so that we are not considered to be dormant */ + note_user_activity(20); + timers_initialize(); helper_create_basic_machine(); @@ -1750,6 +1754,9 @@ test_circuitpadding_conditions(void *arg) monotime_coarse_set_mock_time_nsec(1*TOR_NSEC_PER_USEC); curr_mocked_time = 1*TOR_NSEC_PER_USEC; + /* This is needed so that we are not considered to be dormant */ + note_user_activity(20); + timers_initialize(); helper_create_conditional_machines(); From ebc7556dd0af1b7cc3e12ecc04d1db567d348b2e Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 21 Mar 2019 09:36:19 -0400 Subject: [PATCH 0617/2557] Bump version to 0.4.0.3-alpha --- configure.ac | 4 ++-- contrib/win32build/tor-mingw.nsi.in | 2 +- src/win32/orconfig.h | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/configure.ac b/configure.ac index 91a0498440..c5e1b6d564 100644 --- a/configure.ac +++ b/configure.ac @@ -4,7 +4,7 @@ dnl Copyright (c) 2007-2019, The Tor Project, Inc. dnl See LICENSE for licensing information AC_PREREQ([2.63]) -AC_INIT([tor],[0.4.0.2-alpha-dev]) +AC_INIT([tor],[0.4.0.3-alpha]) AC_CONFIG_SRCDIR([src/app/main/tor_main.c]) AC_CONFIG_MACRO_DIR([m4]) @@ -14,7 +14,7 @@ AC_CONFIG_MACRO_DIR([m4]) # version number changes. Tor uses it to make sure that it # only shuts down for missing "required protocols" when those protocols # are listed as required by a consensus after this date. -AC_DEFINE(APPROX_RELEASE_DATE, ["2019-02-21"], # for 0.4.0.2-alpha-dev +AC_DEFINE(APPROX_RELEASE_DATE, ["2019-03-21"], # for 0.4.0.3-alpha [Approximate date when this software was released. (Updated when the version changes.)]) # "foreign" means we don't follow GNU package layout standards diff --git a/contrib/win32build/tor-mingw.nsi.in b/contrib/win32build/tor-mingw.nsi.in index c937ec2f75..2a01308f72 100644 --- a/contrib/win32build/tor-mingw.nsi.in +++ b/contrib/win32build/tor-mingw.nsi.in @@ -8,7 +8,7 @@ !include "LogicLib.nsh" !include "FileFunc.nsh" !insertmacro GetParameters -!define VERSION "0.4.0.2-alpha-dev" +!define VERSION "0.4.0.3-alpha" !define INSTALLER "tor-${VERSION}-win32.exe" !define WEBSITE "https://www.torproject.org/" !define LICENSE "LICENSE" diff --git a/src/win32/orconfig.h b/src/win32/orconfig.h index 017ed223e6..a667099e93 100644 --- a/src/win32/orconfig.h +++ b/src/win32/orconfig.h @@ -218,7 +218,7 @@ #define USING_TWOS_COMPLEMENT /* Version number of package */ -#define VERSION "0.4.0.2-alpha-dev" +#define VERSION "0.4.0.3-alpha" From 846d379b50b4f4790a9fe2ec88746748e0fab2b7 Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Wed, 13 Mar 2019 15:15:03 +0200 Subject: [PATCH 0618/2557] circpad/prob_distr: Use crypto_fast_rng() instead of the old RNG. --- changes/bug28636 | 8 +++++++ src/core/or/circuitpadding.c | 9 ++++---- src/lib/crypt_ops/crypto_rand.h | 3 +++ src/lib/crypt_ops/crypto_rand_numeric.c | 30 ++++++++++++++++++++++++- src/lib/math/prob_distr.c | 16 ++++++------- 5 files changed, 53 insertions(+), 13 deletions(-) create mode 100644 changes/bug28636 diff --git a/changes/bug28636 b/changes/bug28636 new file mode 100644 index 0000000000..240655cbea --- /dev/null +++ b/changes/bug28636 @@ -0,0 +1,8 @@ + o Minor bugfixes (circuit padding): + - The circuit padding subsystem does not schedule padding if dormant mode + is enabled. Fixes bug 28636; bugfix on 0.4.0.1-alpha. + + o Minor feature (circuit padding): + - We now use a fast RNG when scheduling circuit padding. Part of ticket + 28636. + diff --git a/src/core/or/circuitpadding.c b/src/core/or/circuitpadding.c index 599d88f493..4514111a96 100644 --- a/src/core/or/circuitpadding.c +++ b/src/core/or/circuitpadding.c @@ -449,7 +449,8 @@ circpad_machine_sample_delay(circpad_machine_state_t *mi) histogram_total_tokens = state->histogram_total_tokens; } - bin_choice = crypto_rand_uint64(histogram_total_tokens); + bin_choice = crypto_fast_rng_get_uint64(get_thread_fast_rng(), + histogram_total_tokens); /* Skip all the initial zero bins */ while (!histogram[curr_bin]) { @@ -498,12 +499,12 @@ circpad_machine_sample_delay(circpad_machine_state_t *mi) bin_end = circpad_histogram_bin_to_usec(mi, curr_bin+1); /* Bin edges are monotonically increasing so this is a bug. Handle it. */ - if (BUG(bin_start > bin_end)) { + if (BUG(bin_start >= bin_end)) { return bin_start; } - /* Sample randomly from within the bin width */ - return (circpad_delay_t)crypto_rand_uint64_range(bin_start, bin_end); + return (circpad_delay_t)crypto_fast_rng_uint64_range(get_thread_fast_rng(), + bin_start, bin_end); } /** diff --git a/src/lib/crypt_ops/crypto_rand.h b/src/lib/crypt_ops/crypto_rand.h index 6f09aedf6a..c51d6a4480 100644 --- a/src/lib/crypt_ops/crypto_rand.h +++ b/src/lib/crypt_ops/crypto_rand.h @@ -66,6 +66,9 @@ void crypto_fast_rng_free_(crypto_fast_rng_t *); unsigned crypto_fast_rng_get_uint(crypto_fast_rng_t *rng, unsigned limit); uint64_t crypto_fast_rng_get_uint64(crypto_fast_rng_t *rng, uint64_t limit); +uint32_t crypto_fast_rng_get_u32(crypto_fast_rng_t *rng); +uint64_t crypto_fast_rng_uint64_range(crypto_fast_rng_t *rng, + uint64_t min, uint64_t max); double crypto_fast_rng_get_double(crypto_fast_rng_t *rng); /** diff --git a/src/lib/crypt_ops/crypto_rand_numeric.c b/src/lib/crypt_ops/crypto_rand_numeric.c index d02c5cdcfa..ffbfa2d56c 100644 --- a/src/lib/crypt_ops/crypto_rand_numeric.c +++ b/src/lib/crypt_ops/crypto_rand_numeric.c @@ -155,7 +155,34 @@ crypto_fast_rng_get_uint64(crypto_fast_rng_t *rng, uint64_t limit) } /** - * As crypto_rand_, but extract the result from a crypto_fast_rng_t. + * As crypto_rand_u32, but extract the result from a crypto_fast_rng_t. + */ +uint32_t +crypto_fast_rng_get_u32(crypto_fast_rng_t *rng) +{ + uint32_t val; + crypto_fast_rng_getbytes(rng, (void*)&val, sizeof(val)); + return val; +} + +/** + * As crypto_rand_uint64_range(), but extract the result from a + * crypto_fast_rng_t. + */ +uint64_t +crypto_fast_rng_uint64_range(crypto_fast_rng_t *rng, + uint64_t min, uint64_t max) +{ + /* Handle corrupted input */ + if (BUG(min >= max)) { + return min; + } + + return min + crypto_fast_rng_get_uint64(rng, max - min); +} + +/** + * As crypto_rand_get_double() but extract the result from a crypto_fast_rng_t. */ double crypto_fast_rng_get_double(crypto_fast_rng_t *rng) @@ -164,3 +191,4 @@ crypto_fast_rng_get_double(crypto_fast_rng_t *rng) crypto_fast_rng_getbytes(rng, (void*)&u, sizeof(u)); return ((double)u) / UINT_MAX_AS_DOUBLE; } + diff --git a/src/lib/math/prob_distr.c b/src/lib/math/prob_distr.c index bfad06963d..d44dc28265 100644 --- a/src/lib/math/prob_distr.c +++ b/src/lib/math/prob_distr.c @@ -459,7 +459,7 @@ random_uniform_01(void) * system is broken. */ z = 0; - while ((x = crypto_rand_u32()) == 0) { + while ((x = crypto_fast_rng_get_u32(get_thread_fast_rng())) == 0) { if (z >= 1088) /* Your bit sampler is broken. Go home. */ return 0; @@ -473,8 +473,8 @@ random_uniform_01(void) * occur only with measure zero in the uniform distribution on * [0, 1]. */ - hi = crypto_rand_u32() | UINT32_C(0x80000000); - lo = crypto_rand_u32() | UINT32_C(0x00000001); + hi = crypto_fast_rng_get_u32(get_thread_fast_rng()) | UINT32_C(0x80000000); + lo = crypto_fast_rng_get_u32(get_thread_fast_rng()) | UINT32_C(0x00000001); /* Round to nearest scaled significand in [2^63, 2^64]. */ s = hi*(double)4294967296 + lo; @@ -1437,7 +1437,7 @@ static double logistic_sample(const struct dist *dist) { const struct logistic *L = dist_to_const_logistic(dist); - uint32_t s = crypto_rand_u32(); + uint32_t s = crypto_fast_rng_get_u32(get_thread_fast_rng()); double t = random_uniform_01(); double p0 = random_uniform_01(); @@ -1487,7 +1487,7 @@ static double log_logistic_sample(const struct dist *dist) { const struct log_logistic *LL = dist_to_const_log_logistic(dist); - uint32_t s = crypto_rand_u32(); + uint32_t s = crypto_fast_rng_get_u32(get_thread_fast_rng()); double p0 = random_uniform_01(); return sample_log_logistic_scaleshape(s, p0, LL->alpha, LL->beta); @@ -1536,7 +1536,7 @@ static double weibull_sample(const struct dist *dist) { const struct weibull *W = dist_to_const_weibull(dist); - uint32_t s = crypto_rand_u32(); + uint32_t s = crypto_fast_rng_get_u32(get_thread_fast_rng()); double p0 = random_uniform_01(); return sample_weibull(s, p0, W->lambda, W->k); @@ -1585,7 +1585,7 @@ static double genpareto_sample(const struct dist *dist) { const struct genpareto *GP = dist_to_const_genpareto(dist); - uint32_t s = crypto_rand_u32(); + uint32_t s = crypto_fast_rng_get_u32(get_thread_fast_rng()); double p0 = random_uniform_01(); return sample_genpareto_locscale(s, p0, GP->mu, GP->sigma, GP->xi); @@ -1634,7 +1634,7 @@ static double geometric_sample(const struct dist *dist) { const struct geometric *G = dist_to_const_geometric(dist); - uint32_t s = crypto_rand_u32(); + uint32_t s = crypto_fast_rng_get_u32(get_thread_fast_rng()); double p0 = random_uniform_01(); return sample_geometric(s, p0, G->p); From 57291602532bfe55060668293130b0aac1fd702e Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Wed, 13 Mar 2019 15:24:04 +0200 Subject: [PATCH 0619/2557] circpad: Rename circpad_machine_state_t to circpad_machine_runtime_t. The name of circpad_machine_state_t was very confusing since it was conflicting with circpad_state_t and circpad_circuit_state_t. Right now here is the current meaning of these structs: circpad_state_t -> A state of the state machine. circpad_machine_runtime_t -> The current mutable runtime info of the state machine. circpad_circuit_state_t -> Circuit conditions based on which we should apply a machine to the circuit --- src/core/or/circuit_st.h | 8 ++--- src/core/or/circuitpadding.c | 64 +++++++++++++++++----------------- src/core/or/circuitpadding.h | 44 +++++++++++------------ src/test/test_circuitpadding.c | 22 ++++++------ 4 files changed, 69 insertions(+), 69 deletions(-) diff --git a/src/core/or/circuit_st.h b/src/core/or/circuit_st.h index af343f082e..c972b99318 100644 --- a/src/core/or/circuit_st.h +++ b/src/core/or/circuit_st.h @@ -13,7 +13,7 @@ struct hs_token_t; struct circpad_machine_spec_t; -struct circpad_machine_state_t; +struct circpad_machine_runtime_t; /** Number of padding state machines on a circuit. */ #define CIRCPAD_MAX_MACHINES (2) @@ -193,8 +193,8 @@ struct circuit_t { * and we can have up to CIRCPAD_MAX_MACHINES such machines. */ const struct circpad_machine_spec_t *padding_machine[CIRCPAD_MAX_MACHINES]; - /** Adaptive Padding machine info for above machines. This is the - * per-circuit mutable information, such as the current state and + /** Adaptive Padding machine runtime info for above machines. This is + * the per-circuit mutable information, such as the current state and * histogram token counts. Some of it is optional (aka NULL). * If a machine is being shut down, these indexes can be NULL * without the corresponding padding_machine being NULL, while we @@ -202,7 +202,7 @@ struct circuit_t { * * Each element of this array corresponds to a different padding machine, * and we can have up to CIRCPAD_MAX_MACHINES such machines. */ - struct circpad_machine_state_t *padding_info[CIRCPAD_MAX_MACHINES]; + struct circpad_machine_runtime_t *padding_info[CIRCPAD_MAX_MACHINES]; }; #endif diff --git a/src/core/or/circuitpadding.c b/src/core/or/circuitpadding.c index 4514111a96..b921f1993f 100644 --- a/src/core/or/circuitpadding.c +++ b/src/core/or/circuitpadding.c @@ -23,7 +23,7 @@ * As specified by prop#254, clients can negotiate padding with relays by using * PADDING_NEGOTIATE cells. After successful padding negotiation, padding * machines are assigned to the circuit in their mutable form as a - * circpad_machine_state_t. + * circpad_machine_runtime_t. * * Each state of a padding state machine can be either: * - A histogram that specifies inter-arrival padding delays. @@ -188,11 +188,11 @@ circpad_circuit_free_all_machineinfos(circuit_t *circ) /** * Allocate a new mutable machineinfo structure. */ -STATIC circpad_machine_state_t * +STATIC circpad_machine_runtime_t * circpad_circuit_machineinfo_new(circuit_t *on_circ, int machine_index) { - circpad_machine_state_t *mi = - tor_malloc_zero(sizeof(circpad_machine_state_t)); + circpad_machine_runtime_t *mi = + tor_malloc_zero(sizeof(circpad_machine_runtime_t)); mi->machine_index = machine_index; mi->on_circ = on_circ; @@ -207,7 +207,7 @@ circpad_circuit_machineinfo_new(circuit_t *on_circ, int machine_index) * invalid state. */ STATIC const circpad_state_t * -circpad_machine_current_state(const circpad_machine_state_t *mi) +circpad_machine_current_state(const circpad_machine_runtime_t *mi) { const circpad_machine_spec_t *machine = CIRCPAD_GET_MACHINE(mi); @@ -234,7 +234,7 @@ circpad_machine_current_state(const circpad_machine_state_t *mi) * CIRCPAD_DELAY_INFINITE is returned. */ STATIC circpad_delay_t -circpad_histogram_bin_to_usec(const circpad_machine_state_t *mi, +circpad_histogram_bin_to_usec(const circpad_machine_runtime_t *mi, circpad_hist_index_t bin) { const circpad_state_t *state = circpad_machine_current_state(mi); @@ -265,7 +265,7 @@ circpad_histogram_bin_to_usec(const circpad_machine_state_t *mi, * (The upper bound is included in the bin.) */ STATIC circpad_delay_t -histogram_get_bin_upper_bound(const circpad_machine_state_t *mi, +histogram_get_bin_upper_bound(const circpad_machine_runtime_t *mi, circpad_hist_index_t bin) { return circpad_histogram_bin_to_usec(mi, bin+1) - 1; @@ -273,7 +273,7 @@ histogram_get_bin_upper_bound(const circpad_machine_state_t *mi, /** Return the midpoint of the histogram bin bin_index. */ static circpad_delay_t -circpad_get_histogram_bin_midpoint(const circpad_machine_state_t *mi, +circpad_get_histogram_bin_midpoint(const circpad_machine_runtime_t *mi, int bin_index) { circpad_delay_t left_bound = circpad_histogram_bin_to_usec(mi, bin_index); @@ -291,7 +291,7 @@ circpad_get_histogram_bin_midpoint(const circpad_machine_state_t *mi, * the highest non-infinity bin, that bin index will be returned. */ STATIC circpad_hist_index_t -circpad_histogram_usec_to_bin(const circpad_machine_state_t *mi, +circpad_histogram_usec_to_bin(const circpad_machine_runtime_t *mi, circpad_delay_t usec) { const circpad_state_t *state = circpad_machine_current_state(mi); @@ -331,7 +331,7 @@ circpad_histogram_usec_to_bin(const circpad_machine_state_t *mi, * Called after a state transition, or if the bins are empty. */ STATIC void -circpad_machine_setup_tokens(circpad_machine_state_t *mi) +circpad_machine_setup_tokens(circpad_machine_runtime_t *mi) { const circpad_state_t *state = circpad_machine_current_state(mi); @@ -363,7 +363,7 @@ circpad_machine_setup_tokens(circpad_machine_state_t *mi) * Choose a length for this state (in cells), if specified. */ static void -circpad_choose_state_length(circpad_machine_state_t *mi) +circpad_choose_state_length(circpad_machine_runtime_t *mi) { const circpad_state_t *state = circpad_machine_current_state(mi); double length; @@ -413,7 +413,7 @@ circpad_distribution_sample_iat_delay(const circpad_state_t *state, * that bin's [start,end) time range. */ STATIC circpad_delay_t -circpad_machine_sample_delay(circpad_machine_state_t *mi) +circpad_machine_sample_delay(circpad_machine_runtime_t *mi) { const circpad_state_t *state = circpad_machine_current_state(mi); const circpad_hist_token_t *histogram = NULL; @@ -601,7 +601,7 @@ circpad_distribution_sample(circpad_distribution_t dist) * greater than the target, and that has tokens remaining. */ static circpad_hist_index_t -circpad_machine_first_higher_index(const circpad_machine_state_t *mi, +circpad_machine_first_higher_index(const circpad_machine_runtime_t *mi, circpad_delay_t target_bin_usec) { circpad_hist_index_t bin = circpad_histogram_usec_to_bin(mi, @@ -623,7 +623,7 @@ circpad_machine_first_higher_index(const circpad_machine_state_t *mi, * target_bin_usec, and that still has tokens remaining. */ static circpad_hist_index_t -circpad_machine_first_lower_index(const circpad_machine_state_t *mi, +circpad_machine_first_lower_index(const circpad_machine_runtime_t *mi, circpad_delay_t target_bin_usec) { circpad_hist_index_t bin = circpad_histogram_usec_to_bin(mi, @@ -644,7 +644,7 @@ circpad_machine_first_lower_index(const circpad_machine_state_t *mi, * greater than the target. */ STATIC void -circpad_machine_remove_higher_token(circpad_machine_state_t *mi, +circpad_machine_remove_higher_token(circpad_machine_runtime_t *mi, circpad_delay_t target_bin_usec) { /* We need to remove the token from the first bin @@ -665,7 +665,7 @@ circpad_machine_remove_higher_token(circpad_machine_state_t *mi, * lower than the target. */ STATIC void -circpad_machine_remove_lower_token(circpad_machine_state_t *mi, +circpad_machine_remove_lower_token(circpad_machine_runtime_t *mi, circpad_delay_t target_bin_usec) { circpad_hist_index_t bin = circpad_machine_first_lower_index(mi, @@ -694,7 +694,7 @@ circpad_machine_remove_lower_token(circpad_machine_state_t *mi, * If it is false, use bin index distance only. */ STATIC void -circpad_machine_remove_closest_token(circpad_machine_state_t *mi, +circpad_machine_remove_closest_token(circpad_machine_runtime_t *mi, circpad_delay_t target_bin_usec, bool use_usec) { @@ -776,7 +776,7 @@ circpad_machine_remove_closest_token(circpad_machine_state_t *mi, * If it is empty, do nothing. */ static void -circpad_machine_remove_exact(circpad_machine_state_t *mi, +circpad_machine_remove_exact(circpad_machine_runtime_t *mi, circpad_delay_t target_bin_usec) { circpad_hist_index_t bin = circpad_histogram_usec_to_bin(mi, @@ -793,7 +793,7 @@ circpad_machine_remove_exact(circpad_machine_state_t *mi, * otherwise returns 0. */ static circpad_decision_t -check_machine_token_supply(circpad_machine_state_t *mi) +check_machine_token_supply(circpad_machine_runtime_t *mi) { uint32_t histogram_total_tokens = 0; @@ -833,7 +833,7 @@ check_machine_token_supply(circpad_machine_state_t *mi) * Returns 1 if we transition states, 0 otherwise. */ STATIC circpad_decision_t -circpad_machine_remove_token(circpad_machine_state_t *mi) +circpad_machine_remove_token(circpad_machine_runtime_t *mi) { const circpad_state_t *state = NULL; circpad_time_t current_time; @@ -960,7 +960,7 @@ circpad_send_command_to_hop,(origin_circuit_t *circ, uint8_t hopnum, * CIRCPAD_STATE_CHANGED. Otherwise return CIRCPAD_STATE_UNCHANGED. */ circpad_decision_t -circpad_send_padding_cell_for_callback(circpad_machine_state_t *mi) +circpad_send_padding_cell_for_callback(circpad_machine_runtime_t *mi) { circuit_t *circ = mi->on_circ; int machine_idx = mi->machine_index; @@ -1046,7 +1046,7 @@ circpad_send_padding_cell_for_callback(circpad_machine_state_t *mi) /** * Tor-timer compatible callback that tells us to send a padding cell. * - * Timers are associated with circpad_machine_state_t's. When the machineinfo + * Timers are associated with circpad_machine_runtime_t's. When the machineinfo * is freed on a circuit, the timers are cancelled. Since the lifetime * of machineinfo is always longer than the timers, handles are not * needed. @@ -1055,7 +1055,7 @@ static void circpad_send_padding_callback(tor_timer_t *timer, void *args, const struct monotime_t *time) { - circpad_machine_state_t *mi = ((circpad_machine_state_t*)args); + circpad_machine_runtime_t *mi = ((circpad_machine_runtime_t*)args); (void)timer; (void)time; if (mi && mi->on_circ) { @@ -1105,7 +1105,7 @@ circpad_new_consensus_params(const networkstatus_t *ns) * Returns 1 if limits are set and we've hit them. Otherwise returns 0. */ STATIC bool -circpad_machine_reached_padding_limit(circpad_machine_state_t *mi) +circpad_machine_reached_padding_limit(circpad_machine_runtime_t *mi) { const circpad_machine_spec_t *machine = CIRCPAD_GET_MACHINE(mi); @@ -1153,7 +1153,7 @@ circpad_machine_reached_padding_limit(circpad_machine_state_t *mi) * 0 otherwise. */ MOCK_IMPL(circpad_decision_t, -circpad_machine_schedule_padding,(circpad_machine_state_t *mi)) +circpad_machine_schedule_padding,(circpad_machine_runtime_t *mi)) { circpad_delay_t in_usec = 0; struct timeval timeout; @@ -1249,7 +1249,7 @@ circpad_machine_schedule_padding,(circpad_machine_state_t *mi)) * not access it. */ static void -circpad_machine_spec_transitioned_to_end(circpad_machine_state_t *mi) +circpad_machine_spec_transitioned_to_end(circpad_machine_runtime_t *mi) { const circpad_machine_spec_t *machine = CIRCPAD_GET_MACHINE(mi); @@ -1299,7 +1299,7 @@ circpad_machine_spec_transitioned_to_end(circpad_machine_state_t *mi) * Returns 1 if we transition states, 0 otherwise. */ MOCK_IMPL(circpad_decision_t, -circpad_machine_spec_transition,(circpad_machine_state_t *mi, +circpad_machine_spec_transition,(circpad_machine_runtime_t *mi, circpad_event_t event)) { const circpad_state_t *state = @@ -1380,7 +1380,7 @@ circpad_machine_spec_transition,(circpad_machine_state_t *mi, */ static void circpad_estimate_circ_rtt_on_received(circuit_t *circ, - circpad_machine_state_t *mi) + circpad_machine_runtime_t *mi) { /* Origin circuits don't estimate RTT. They could do it easily enough, * but they have no reason to use it in any delay calculations. */ @@ -1427,7 +1427,7 @@ circpad_estimate_circ_rtt_on_received(circuit_t *circ, */ static void circpad_estimate_circ_rtt_on_send(circuit_t *circ, - circpad_machine_state_t *mi) + circpad_machine_runtime_t *mi) { /* Origin circuits don't estimate RTT. They could do it easily enough, * but they have no reason to use it in any delay calculations. */ @@ -1572,7 +1572,7 @@ circpad_cell_event_padding_received(circuit_t *on_circ) * Return 1 if we decide to transition, 0 otherwise. */ circpad_decision_t -circpad_internal_event_infinity(circpad_machine_state_t *mi) +circpad_internal_event_infinity(circpad_machine_runtime_t *mi) { return circpad_machine_spec_transition(mi, CIRCPAD_EVENT_INFINITY); } @@ -1586,7 +1586,7 @@ circpad_internal_event_infinity(circpad_machine_state_t *mi) * Return 1 if we decide to transition, 0 otherwise. */ circpad_decision_t -circpad_internal_event_bins_empty(circpad_machine_state_t *mi) +circpad_internal_event_bins_empty(circpad_machine_runtime_t *mi) { if (circpad_machine_spec_transition(mi, CIRCPAD_EVENT_BINS_EMPTY) == CIRCPAD_STATE_CHANGED) { @@ -1605,7 +1605,7 @@ circpad_internal_event_bins_empty(circpad_machine_state_t *mi) * Return 1 if we decide to transition, 0 otherwise. */ circpad_decision_t -circpad_internal_event_state_length_up(circpad_machine_state_t *mi) +circpad_internal_event_state_length_up(circpad_machine_runtime_t *mi) { return circpad_machine_spec_transition(mi, CIRCPAD_EVENT_LENGTH_COUNT); } diff --git a/src/core/or/circuitpadding.h b/src/core/or/circuitpadding.h index 54039fa59a..7f6696380e 100644 --- a/src/core/or/circuitpadding.h +++ b/src/core/or/circuitpadding.h @@ -88,7 +88,7 @@ typedef uint32_t circpad_delay_t; /** * Macro to clarify when we're checking the infinity bin. * - * Works with either circpad_state_t or circpad_machine_state_t + * Works with either circpad_state_t or circpad_machine_runtime_t */ #define CIRCPAD_INFINITY_BIN(mi) ((mi)->histogram_len-1) @@ -245,7 +245,7 @@ typedef uint16_t circpad_statenum_t; * A state of a padding state machine. The information here are immutable and * represent the initial form of the state; it does not get updated as things * happen. The mutable information that gets updated in runtime are carried in - * a circpad_machine_state_t. + * a circpad_machine_runtime_t. * * This struct describes the histograms and parameters of a single * state in the adaptive padding machine. Instances of this struct @@ -467,7 +467,7 @@ typedef struct circpad_state_t { * * XXX: Play with layout to minimize space on x64 Linux (most common relay). */ -typedef struct circpad_machine_state_t { +typedef struct circpad_machine_runtime_t { /** The callback pointer for the padding callbacks. * * These timers stick around the machineinfo until the machineinfo's circuit @@ -551,7 +551,7 @@ typedef struct circpad_machine_state_t { * CIRCPAD_MAX_MACHINES define). */ unsigned machine_index : 1; -} circpad_machine_state_t; +} circpad_machine_runtime_t; /** Helper macro to get an actual state machine from a machineinfo */ #define CIRCPAD_GET_MACHINE(machineinfo) \ @@ -629,11 +629,11 @@ void circpad_cell_event_padding_received(struct circuit_t *on_circ); /** Internal events are events the machines send to themselves */ circpad_decision_t -circpad_internal_event_infinity(circpad_machine_state_t *mi); +circpad_internal_event_infinity(circpad_machine_runtime_t *mi); circpad_decision_t -circpad_internal_event_bins_empty(circpad_machine_state_t *); +circpad_internal_event_bins_empty(circpad_machine_runtime_t *); circpad_decision_t circpad_internal_event_state_length_up( - circpad_machine_state_t *); + circpad_machine_runtime_t *); /** Machine creation events are events that cause us to set up or * tear down padding state machines. */ @@ -676,47 +676,47 @@ bool circpad_padding_negotiated(struct circuit_t *circ, uint8_t response); MOCK_DECL(circpad_decision_t, -circpad_machine_schedule_padding,(circpad_machine_state_t *)); +circpad_machine_schedule_padding,(circpad_machine_runtime_t *)); MOCK_DECL(circpad_decision_t, -circpad_machine_spec_transition, (circpad_machine_state_t *mi, +circpad_machine_spec_transition, (circpad_machine_runtime_t *mi, circpad_event_t event)); circpad_decision_t circpad_send_padding_cell_for_callback( - circpad_machine_state_t *mi); + circpad_machine_runtime_t *mi); #ifdef CIRCUITPADDING_PRIVATE STATIC circpad_delay_t -circpad_machine_sample_delay(circpad_machine_state_t *mi); +circpad_machine_sample_delay(circpad_machine_runtime_t *mi); STATIC bool -circpad_machine_reached_padding_limit(circpad_machine_state_t *mi); +circpad_machine_reached_padding_limit(circpad_machine_runtime_t *mi); STATIC -circpad_decision_t circpad_machine_remove_token(circpad_machine_state_t *mi); +circpad_decision_t circpad_machine_remove_token(circpad_machine_runtime_t *mi); STATIC circpad_delay_t -circpad_histogram_bin_to_usec(const circpad_machine_state_t *mi, +circpad_histogram_bin_to_usec(const circpad_machine_runtime_t *mi, circpad_hist_index_t bin); STATIC const circpad_state_t * -circpad_machine_current_state(const circpad_machine_state_t *mi); +circpad_machine_current_state(const circpad_machine_runtime_t *mi); STATIC circpad_hist_index_t circpad_histogram_usec_to_bin( - const circpad_machine_state_t *mi, + const circpad_machine_runtime_t *mi, circpad_delay_t us); -STATIC circpad_machine_state_t *circpad_circuit_machineinfo_new( +STATIC circpad_machine_runtime_t *circpad_circuit_machineinfo_new( struct circuit_t *on_circ, int machine_index); -STATIC void circpad_machine_remove_higher_token(circpad_machine_state_t *mi, +STATIC void circpad_machine_remove_higher_token(circpad_machine_runtime_t *mi, circpad_delay_t target_bin_us); -STATIC void circpad_machine_remove_lower_token(circpad_machine_state_t *mi, +STATIC void circpad_machine_remove_lower_token(circpad_machine_runtime_t *mi, circpad_delay_t target_bin_us); -STATIC void circpad_machine_remove_closest_token(circpad_machine_state_t *mi, +STATIC void circpad_machine_remove_closest_token(circpad_machine_runtime_t *mi, circpad_delay_t target_bin_us, bool use_usec); -STATIC void circpad_machine_setup_tokens(circpad_machine_state_t *mi); +STATIC void circpad_machine_setup_tokens(circpad_machine_runtime_t *mi); MOCK_DECL(STATIC signed_error_t, circpad_send_command_to_hop,(struct origin_circuit_t *circ, uint8_t hopnum, @@ -724,7 +724,7 @@ circpad_send_command_to_hop,(struct origin_circuit_t *circ, uint8_t hopnum, ssize_t payload_len)); STATIC circpad_delay_t -histogram_get_bin_upper_bound(const circpad_machine_state_t *mi, +histogram_get_bin_upper_bound(const circpad_machine_runtime_t *mi, circpad_hist_index_t bin); #ifdef TOR_UNIT_TESTS diff --git a/src/test/test_circuitpadding.c b/src/test/test_circuitpadding.c index ba58cd0d97..d02f40f21c 100644 --- a/src/test/test_circuitpadding.c +++ b/src/test/test_circuitpadding.c @@ -493,7 +493,7 @@ helper_create_machine_with_big_histogram(circpad_removal_t removal_strategy) } static circpad_decision_t -circpad_machine_schedule_padding_mock(circpad_machine_state_t *mi) +circpad_machine_schedule_padding_mock(circpad_machine_runtime_t *mi) { (void)mi; return 0; @@ -509,7 +509,7 @@ mock_monotime_absolute_usec(void) static void test_circuitpadding_token_removal_higher(void *arg) { - circpad_machine_state_t *mi; + circpad_machine_runtime_t *mi; (void)arg; /* Mock it up */ @@ -614,7 +614,7 @@ test_circuitpadding_token_removal_higher(void *arg) static void test_circuitpadding_token_removal_lower(void *arg) { - circpad_machine_state_t *mi; + circpad_machine_runtime_t *mi; (void)arg; /* Mock it up */ @@ -712,7 +712,7 @@ test_circuitpadding_token_removal_lower(void *arg) static void test_circuitpadding_closest_token_removal(void *arg) { - circpad_machine_state_t *mi; + circpad_machine_runtime_t *mi; (void)arg; /* Mock it up */ @@ -818,7 +818,7 @@ test_circuitpadding_closest_token_removal(void *arg) static void test_circuitpadding_closest_token_removal_usec(void *arg) { - circpad_machine_state_t *mi; + circpad_machine_runtime_t *mi; (void)arg; /* Mock it up */ @@ -929,7 +929,7 @@ test_circuitpadding_closest_token_removal_usec(void *arg) static void test_circuitpadding_token_removal_exact(void *arg) { - circpad_machine_state_t *mi; + circpad_machine_runtime_t *mi; (void)arg; /* Mock it up */ @@ -990,7 +990,7 @@ void test_circuitpadding_tokens(void *arg) { const circpad_state_t *state; - circpad_machine_state_t *mi; + circpad_machine_runtime_t *mi; (void)arg; /** Test plan: @@ -2157,7 +2157,7 @@ helper_circpad_circ_distribution_machine_setup(int min, int max) static void test_circuitpadding_sample_distribution(void *arg) { - circpad_machine_state_t *mi; + circpad_machine_runtime_t *mi; int n_samples; int n_states; @@ -2202,7 +2202,7 @@ test_circuitpadding_sample_distribution(void *arg) } static circpad_decision_t -circpad_machine_spec_transition_mock(circpad_machine_state_t *mi, +circpad_machine_spec_transition_mock(circpad_machine_runtime_t *mi, circpad_event_t event) { (void) mi; @@ -2217,7 +2217,7 @@ test_circuitpadding_machine_rate_limiting(void *arg) { (void) arg; bool retval; - circpad_machine_state_t *mi; + circpad_machine_runtime_t *mi; int i; /* Ignore machine transitions for the purposes of this function, we only @@ -2285,7 +2285,7 @@ test_circuitpadding_global_rate_limiting(void *arg) { (void) arg; bool retval; - circpad_machine_state_t *mi; + circpad_machine_runtime_t *mi; int i; /* Ignore machine transitions for the purposes of this function, we only From 13b28063f9f2a3d2a587eabac358452a42277052 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Fri, 22 Mar 2019 12:57:58 +0200 Subject: [PATCH 0620/2557] Compile time check for being able to fit {unsigned} int inside void pointer --- configure.ac | 1 + src/test/test_ptr_slow.c | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/configure.ac b/configure.ac index 6036cdffe5..8ac2d71e4f 100644 --- a/configure.ac +++ b/configure.ac @@ -1588,6 +1588,7 @@ AC_CHECK_MEMBERS([struct timeval.tv_sec], , , AC_CHECK_SIZEOF(char) AC_CHECK_SIZEOF(short) AC_CHECK_SIZEOF(int) +AC_CHECK_SIZEOF(unsigned int) AC_CHECK_SIZEOF(long) AC_CHECK_SIZEOF(long long) AC_CHECK_SIZEOF(__int64) diff --git a/src/test/test_ptr_slow.c b/src/test/test_ptr_slow.c index 0142fd1c8a..37560d4359 100644 --- a/src/test/test_ptr_slow.c +++ b/src/test/test_ptr_slow.c @@ -11,6 +11,15 @@ #include #include +#if SIZEOF_INT > SIZEOF_VOID_P +#error "sizeof(int) > sizeof(void *) - Tor cannot be built on this platform!" +#endif + +#if SIZEOF_UNSIGNED_INT > SIZEOF_VOID_P +#error "sizeof(unsigned int) > sizeof(void *) - Tor cannot be built on this \ +platform!" +#endif + /** Assert that a can be cast to void * and back. */ static void assert_int_voidptr_roundtrip(int a) From 7b30f8dc8c93ac215a1e7330fee1baa5f1437a98 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Fri, 22 Mar 2019 13:04:06 +0200 Subject: [PATCH 0621/2557] Write missing function comments --- src/test/test_ptr_slow.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/test/test_ptr_slow.c b/src/test/test_ptr_slow.c index 37560d4359..2d4fc9d4dd 100644 --- a/src/test/test_ptr_slow.c +++ b/src/test/test_ptr_slow.c @@ -36,6 +36,7 @@ assert_int_voidptr_roundtrip(int a) return; } +/** Test for possibility of casting `int` to `void *` and back. */ static void test_int_voidstar_interop(void *arg) { @@ -61,6 +62,7 @@ test_int_voidstar_interop(void *arg) } } +/** Assert that a can be cast to void * and back. */ static void assert_uint_voidptr_roundtrip(unsigned int a) { @@ -76,6 +78,7 @@ assert_uint_voidptr_roundtrip(unsigned int a) return; } +/** Test for possibility of casting `int` to `void *` and back. */ static void test_uint_voidstar_interop(void *arg) { From a5df9402b611f9fcfcbd22e486ae0ae16246ecd0 Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Thu, 21 Mar 2019 16:49:56 +0200 Subject: [PATCH 0622/2557] prob-distr: Decrease false positive rate of stochastic tests. --- changes/bug29693 | 3 +++ src/test/test_prob_distr.c | 15 +++++++++------ 2 files changed, 12 insertions(+), 6 deletions(-) create mode 100644 changes/bug29693 diff --git a/changes/bug29693 b/changes/bug29693 new file mode 100644 index 0000000000..33ce051c40 --- /dev/null +++ b/changes/bug29693 @@ -0,0 +1,3 @@ + o Minor bugfixes (unit tests): + - Decrease the false positive rate of stochastic probability distribution + tests. Fixes bug 29693; bugfix on 0.4.0.1-alpha. \ No newline at end of file diff --git a/src/test/test_prob_distr.c b/src/test/test_prob_distr.c index 42cc6d70f6..37cfdae7d9 100644 --- a/src/test/test_prob_distr.c +++ b/src/test/test_prob_distr.c @@ -878,11 +878,14 @@ test_uniform_interval(void *arg) * is higher: 1 - Binom(0; n, alpha) = 1 - (1 - alpha)^n. For n = 10, * this is about 10%, and for n = 100 it is well over 50%. * - * We can drive it down by running each test twice, and accepting it if - * it passes at least once; in that case, it is as if we used Binom(2; - * 2, alpha) = alpha^2 as the false positive rate for each test, and - * for n = 10 tests, it would be 0.1%, and for n = 100 tests, still - * only 1%. + * Given that these tests will run with every CI job, we want to drive down the + * false positive rate. We can drive it down by running each test four times, + * and accepting it if it passes at least once; in that case, it is as if we + * used Binom(4; 2, alpha) = alpha^4 as the false positive rate for each test, + * and for n = 10 tests, it would be 9.99999959506e-08. If each CI build has 14 + * jobs, then the chance of a CI build failing is 1.39999903326e-06, which + * means that a CI build will break with probability 50% after about 495106 + * builds. * * The critical value for a chi^2 distribution with 100 degrees of * freedom and false positive rate alpha = 1% was taken from: @@ -895,7 +898,7 @@ test_uniform_interval(void *arg) static const size_t NSAMPLES = 100000; /* Number of chances we give to the test to succeed. */ -static const unsigned NTRIALS = 2; +static const unsigned NTRIALS = 4; /* Number of times we want the test to pass per NTRIALS. */ static const unsigned NPASSES_MIN = 1; From 307c156fbe2f70a697afbf5340e40655173125a6 Mon Sep 17 00:00:00 2001 From: Taylor Yu Date: Fri, 22 Mar 2019 17:45:06 -0500 Subject: [PATCH 0623/2557] Set file encoding in practracker.py Explicitly set the file encoding to UTF-8 in practracker.py, to avoid problems in some CI environments. Fixes bug 29789; bug not in any released Tor. --- scripts/maint/practracker/practracker.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/scripts/maint/practracker/practracker.py b/scripts/maint/practracker/practracker.py index a6e6d0b607..3274bd15f7 100755 --- a/scripts/maint/practracker/practracker.py +++ b/scripts/maint/practracker/practracker.py @@ -46,6 +46,13 @@ TOR_TOPDIR = None ####################################################### +if sys.version_info[0] <= 2: + def open_file(fname): + return open(fname, 'r') +else: + def open_file(fname): + return open(fname, 'r', encoding='utf-8') + def consider_file_size(fname, f): """Consider file size issues for 'f' and return True if a new issue was found""" file_size = metrics.get_file_len(f) @@ -85,7 +92,7 @@ def consider_all_metrics(files_list): """Consider metrics for all files, and return True if new issues were found""" found_new_issues = False for fname in files_list: - with open(fname, 'r') as f: + with open_file(fname) as f: found_new_issues |= consider_metrics_for_file(fname, f) return found_new_issues From 8bc3ac6a84a11adb728f4ce49f23414c27fe84cd Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Sat, 23 Mar 2019 08:44:36 -0400 Subject: [PATCH 0624/2557] Bump to 0.4.0.3-alpha-dev --- configure.ac | 4 ++-- contrib/win32build/tor-mingw.nsi.in | 2 +- src/win32/orconfig.h | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/configure.ac b/configure.ac index c5e1b6d564..17a4d9680e 100644 --- a/configure.ac +++ b/configure.ac @@ -4,7 +4,7 @@ dnl Copyright (c) 2007-2019, The Tor Project, Inc. dnl See LICENSE for licensing information AC_PREREQ([2.63]) -AC_INIT([tor],[0.4.0.3-alpha]) +AC_INIT([tor],[0.4.0.3-alpha-dev]) AC_CONFIG_SRCDIR([src/app/main/tor_main.c]) AC_CONFIG_MACRO_DIR([m4]) @@ -14,7 +14,7 @@ AC_CONFIG_MACRO_DIR([m4]) # version number changes. Tor uses it to make sure that it # only shuts down for missing "required protocols" when those protocols # are listed as required by a consensus after this date. -AC_DEFINE(APPROX_RELEASE_DATE, ["2019-03-21"], # for 0.4.0.3-alpha +AC_DEFINE(APPROX_RELEASE_DATE, ["2019-03-23"], # for 0.4.0.3-alpha-dev [Approximate date when this software was released. (Updated when the version changes.)]) # "foreign" means we don't follow GNU package layout standards diff --git a/contrib/win32build/tor-mingw.nsi.in b/contrib/win32build/tor-mingw.nsi.in index 2a01308f72..0d0d4f493e 100644 --- a/contrib/win32build/tor-mingw.nsi.in +++ b/contrib/win32build/tor-mingw.nsi.in @@ -8,7 +8,7 @@ !include "LogicLib.nsh" !include "FileFunc.nsh" !insertmacro GetParameters -!define VERSION "0.4.0.3-alpha" +!define VERSION "0.4.0.3-alpha-dev" !define INSTALLER "tor-${VERSION}-win32.exe" !define WEBSITE "https://www.torproject.org/" !define LICENSE "LICENSE" diff --git a/src/win32/orconfig.h b/src/win32/orconfig.h index a667099e93..d3441610b6 100644 --- a/src/win32/orconfig.h +++ b/src/win32/orconfig.h @@ -218,7 +218,7 @@ #define USING_TWOS_COMPLEMENT /* Version number of package */ -#define VERSION "0.4.0.3-alpha" +#define VERSION "0.4.0.3-alpha-dev" From 9d0ce0afcce9bbf969d11ba081f829aef5071e9c Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Sat, 23 Mar 2019 08:46:29 -0400 Subject: [PATCH 0625/2557] forward-port changelog for 0.4.0.3-alpha --- ChangeLog | 84 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/ChangeLog b/ChangeLog index dd2a98469d..9b512c6aed 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,87 @@ +Changes in version 0.4.0.3-alpha - 2019-03-22 + Tor 0.4.0.3-alpha is the third in its series; it fixes several small + bugs from earlier versions. + + o Minor features (address selection): + - Treat the subnet 100.64.0.0/10 as public for some purposes; + private for others. This subnet is the RFC 6598 (Carrier Grade + NAT) IP range, and is deployed by many ISPs as an alternative to + RFC 1918 that does not break existing internal networks. Tor now + blocks SOCKS and control ports on these addresses and warns users + if client ports or ExtORPorts are listening on a RFC 6598 address. + Closes ticket 28525. Patch by Neel Chauhan. + + o Minor features (geoip): + - Update geoip and geoip6 to the March 4 2019 Maxmind GeoLite2 + Country database. Closes ticket 29666. + + o Minor bugfixes (circuitpadding): + - Inspect the circuit-level cell queue before sending padding, to + avoid sending padding when too much data is queued. Fixes bug + 29204; bugfix on 0.4.0.1-alpha. + + o Minor bugfixes (logging): + - Correct a misleading error message when IPv4Only or IPv6Only is + used but the resolved address can not be interpreted as an address + of the specified IP version. Fixes bug 13221; bugfix on + 0.2.3.9-alpha. Patch from Kris Katterjohn. + - Log the correct port number for listening sockets when "auto" is + used to let Tor pick the port number. Previously, port 0 was + logged instead of the actual port number. Fixes bug 29144; bugfix + on 0.3.5.1-alpha. Patch from Kris Katterjohn. + - Stop logging a BUG() warning when Tor is waiting for exit + descriptors. Fixes bug 28656; bugfix on 0.3.5.1-alpha. + + o Minor bugfixes (memory management): + - Refactor the shared random state's memory management so that it + actually takes ownership of the shared random value pointers. + Fixes bug 29706; bugfix on 0.2.9.1-alpha. + + o Minor bugfixes (memory management, testing): + - Stop leaking parts of the shared random state in the shared-random + unit tests. Fixes bug 29599; bugfix on 0.2.9.1-alpha. + + o Minor bugfixes (pluggable transports): + - Fix an assertion failure crash bug when a pluggable transport is + terminated during the bootstrap phase. Fixes bug 29562; bugfix + on 0.4.0.1-alpha. + + o Minor bugfixes (Rust, protover): + - Add a missing "Padding" value to the Rust implementation of + protover. Fixes bug 29631; bugfix on 0.4.0.1-alpha. + + o Minor bugfixes (single onion services): + - Allow connections to single onion services to remain idle without + being disconnected. Previously, relays acting as rendezvous points + for single onion services were mistakenly closing idle rendezvous + circuits after 60 seconds, thinking that they were unused + directory-fetching circuits that had served their purpose. Fixes + bug 29665; bugfix on 0.2.1.26. + + o Minor bugfixes (stats): + - When ExtraInfoStatistics is 0, stop including PaddingStatistics in + relay and bridge extra-info documents. Fixes bug 29017; bugfix + on 0.3.1.1-alpha. + + o Minor bugfixes (testing): + - Downgrade some LOG_ERR messages in the address/* tests to + warnings. The LOG_ERR messages were occurring when we had no + configured network. We were failing the unit tests, because we + backported 28668 to 0.3.5.8, but did not backport 29530. Fixes bug + 29530; bugfix on 0.3.5.8. + - Fix our gcov wrapper script to look for object files at the + correct locations. Fixes bug 29435; bugfix on 0.3.5.1-alpha. + - Decrease the false positive rate of stochastic probability + distribution tests. Fixes bug 29693; bugfix on 0.4.0.1-alpha. + + o Minor bugfixes (Windows, CI): + - Skip the Appveyor 32-bit Windows Server 2016 job, and 64-bit + Windows Server 2012 R2 job. The remaining 2 jobs still provide + coverage of 64/32-bit, and Windows Server 2016/2012 R2. Also set + fast_finish, so failed jobs terminate the build immediately. Fixes + bug 29601; bugfix on 0.3.5.4-alpha. + + Changes in version 0.3.5.8 - 2019-02-21 Tor 0.3.5.8 backports serveral fixes from later releases, including fixes for an annoying SOCKS-parsing bug that affected users in earlier 0.3.5.x From 9529d99982eb64998e12db7704fa977bf02f541c Mon Sep 17 00:00:00 2001 From: rl1987 Date: Sun, 24 Mar 2019 09:25:30 +0200 Subject: [PATCH 0626/2557] Tweak changes file --- changes/ticket29537 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/changes/ticket29537 b/changes/ticket29537 index 048e13c65f..afe2308205 100644 --- a/changes/ticket29537 +++ b/changes/ticket29537 @@ -1,3 +1,3 @@ o Testing: - - Check that all valid values of `int` and `unsigned int` can be - represented by `void *`. Resolves issue 29537. + - Check that representative subsets of values of `int` and `unsigned int` + can be represented by `void *`. Resolves issue 29537. From f09205ef5384175f5d222f3137e42ae44486a30b Mon Sep 17 00:00:00 2001 From: rl1987 Date: Sun, 24 Mar 2019 10:10:52 +0200 Subject: [PATCH 0627/2557] Refactor test_utils_general() to fix Coverity warnings --- changes/bug29823 | 5 ++ src/test/test_shared_random.c | 134 +++++++++++++++++++--------------- 2 files changed, 82 insertions(+), 57 deletions(-) create mode 100644 changes/bug29823 diff --git a/changes/bug29823 b/changes/bug29823 new file mode 100644 index 0000000000..d856cf1fef --- /dev/null +++ b/changes/bug29823 @@ -0,0 +1,5 @@ + o Minor bugfixes (unit tests): + - Split test_utils_general() to several smaller test functions in + test_utils_general(). This makes it easier to perform resource + deallocation on assert failure and fixes Coverity warnings CID 1444117 + and CID 1444118. Fixes bug 29823; bugfix on 0.2.9.1-alpha. diff --git a/src/test/test_shared_random.c b/src/test/test_shared_random.c index 5fa7e80d07..480799383b 100644 --- a/src/test/test_shared_random.c +++ b/src/test/test_shared_random.c @@ -1081,70 +1081,85 @@ test_sr_get_majority_srv_from_votes(void *arg) smartlist_free(votes); } -/* Test utils that don't depend on authority state */ +/* Testing sr_srv_dup(). */ static void -test_utils_general(void *arg) +test_sr_svr_dup(void *arg) { - (void) arg; + (void)arg; - /* Testing sr_srv_dup(). */ - { - sr_srv_t *srv = NULL, *dup_srv = NULL; - const char *srv_value = - "1BDB7C3E973936E4D13A49F37C859B3DC69C429334CF9412E3FEF6399C52D47A"; - srv = tor_malloc_zero(sizeof(*srv)); - srv->num_reveals = 42; - memcpy(srv->value, srv_value, sizeof(srv->value)); - dup_srv = sr_srv_dup(srv); - tt_assert(dup_srv); - tt_u64_op(dup_srv->num_reveals, OP_EQ, srv->num_reveals); - tt_mem_op(dup_srv->value, OP_EQ, srv->value, sizeof(srv->value)); - tor_free(srv); - tor_free(dup_srv); - } + sr_srv_t *srv = NULL, *dup_srv = NULL; + const char *srv_value = + "1BDB7C3E973936E4D13A49F37C859B3DC69C429334CF9412E3FEF6399C52D47A"; + srv = tor_malloc_zero(sizeof(*srv)); + srv->num_reveals = 42; + memcpy(srv->value, srv_value, sizeof(srv->value)); + dup_srv = sr_srv_dup(srv); + tt_assert(dup_srv); + tt_u64_op(dup_srv->num_reveals, OP_EQ, srv->num_reveals); + tt_mem_op(dup_srv->value, OP_EQ, srv->value, sizeof(srv->value)); - /* Testing commitments_are_the_same(). Currently, the check is to test the - * value of the encoded commit so let's make sure that actually works. */ - { - /* Payload of 57 bytes that is the length of sr_commit_t->encoded_commit. - * 56 bytes of payload and a NUL terminated byte at the end ('\x00') - * which comes down to SR_COMMIT_BASE64_LEN + 1. */ - const char *payload = - "\x5d\xb9\x60\xb6\xcc\x51\x68\x52\x31\xd9\x88\x88\x71\x71\xe0\x30" - "\x59\x55\x7f\xcd\x61\xc0\x4b\x05\xb8\xcd\xc1\x48\xe9\xcd\x16\x1f" - "\x70\x15\x0c\xfc\xd3\x1a\x75\xd0\x93\x6c\xc4\xe0\x5c\xbe\xe2\x18" - "\xc7\xaf\x72\xb6\x7c\x9b\x52\x00"; - sr_commit_t commit1, commit2; - memcpy(commit1.encoded_commit, payload, sizeof(commit1.encoded_commit)); - memcpy(commit2.encoded_commit, payload, sizeof(commit2.encoded_commit)); - tt_int_op(commitments_are_the_same(&commit1, &commit2), OP_EQ, 1); - /* Let's corrupt one of them. */ - memset(commit1.encoded_commit, 'A', sizeof(commit1.encoded_commit)); - tt_int_op(commitments_are_the_same(&commit1, &commit2), OP_EQ, 0); - } + done: + tor_free(srv); + tor_free(dup_srv); +} - /* Testing commit_is_authoritative(). */ - { - crypto_pk_t *k = crypto_pk_new(); - char digest[DIGEST_LEN]; - sr_commit_t commit; +/* Testing commitments_are_the_same(). Currently, the check is to test the + * value of the encoded commit so let's make sure that actually works. */ +static void +test_commitments_are_the_same(void *arg) +{ + (void)arg; - tt_assert(!crypto_pk_generate_key(k)); + /* Payload of 57 bytes that is the length of sr_commit_t->encoded_commit. + * 56 bytes of payload and a NUL terminated byte at the end ('\x00') + * which comes down to SR_COMMIT_BASE64_LEN + 1. */ + const char *payload = + "\x5d\xb9\x60\xb6\xcc\x51\x68\x52\x31\xd9\x88\x88\x71\x71\xe0\x30" + "\x59\x55\x7f\xcd\x61\xc0\x4b\x05\xb8\xcd\xc1\x48\xe9\xcd\x16\x1f" + "\x70\x15\x0c\xfc\xd3\x1a\x75\xd0\x93\x6c\xc4\xe0\x5c\xbe\xe2\x18" + "\xc7\xaf\x72\xb6\x7c\x9b\x52\x00"; + sr_commit_t commit1, commit2; + memcpy(commit1.encoded_commit, payload, sizeof(commit1.encoded_commit)); + memcpy(commit2.encoded_commit, payload, sizeof(commit2.encoded_commit)); + tt_int_op(commitments_are_the_same(&commit1, &commit2), OP_EQ, 1); + /* Let's corrupt one of them. */ + memset(commit1.encoded_commit, 'A', sizeof(commit1.encoded_commit)); + tt_int_op(commitments_are_the_same(&commit1, &commit2), OP_EQ, 0); - tt_int_op(0, OP_EQ, crypto_pk_get_digest(k, digest)); - memcpy(commit.rsa_identity, digest, sizeof(commit.rsa_identity)); - tt_int_op(commit_is_authoritative(&commit, digest), OP_EQ, 1); - /* Change the pubkey. */ - memset(commit.rsa_identity, 0, sizeof(commit.rsa_identity)); - tt_int_op(commit_is_authoritative(&commit, digest), OP_EQ, 0); - crypto_pk_free(k); - } + done: + return; +} - /* Testing get_phase_str(). */ - { - tt_str_op(get_phase_str(SR_PHASE_REVEAL), OP_EQ, "reveal"); - tt_str_op(get_phase_str(SR_PHASE_COMMIT), OP_EQ, "commit"); - } +/* Testing commit_is_authoritative(). */ +static void +test_commit_is_authoritative(void *arg) +{ + (void)arg; + + crypto_pk_t *k = crypto_pk_new(); + char digest[DIGEST_LEN]; + sr_commit_t commit; + + tt_assert(!crypto_pk_generate_key(k)); + + tt_int_op(0, OP_EQ, crypto_pk_get_digest(k, digest)); + memcpy(commit.rsa_identity, digest, sizeof(commit.rsa_identity)); + tt_int_op(commit_is_authoritative(&commit, digest), OP_EQ, 1); + /* Change the pubkey. */ + memset(commit.rsa_identity, 0, sizeof(commit.rsa_identity)); + tt_int_op(commit_is_authoritative(&commit, digest), OP_EQ, 0); + + done: + crypto_pk_free(k); +} + +static void +test_get_phase_str(void *arg) +{ + (void)arg; + + tt_str_op(get_phase_str(SR_PHASE_REVEAL), OP_EQ, "reveal"); + tt_str_op(get_phase_str(SR_PHASE_COMMIT), OP_EQ, "commit"); done: return; @@ -1649,7 +1664,12 @@ struct testcase_t sr_tests[] = { { "sr_compute_srv", test_sr_compute_srv, TT_FORK, NULL, NULL }, { "sr_get_majority_srv_from_votes", test_sr_get_majority_srv_from_votes, TT_FORK, NULL, NULL }, - { "utils_general", test_utils_general, TT_FORK, NULL, NULL }, + { "sr_svr_dup", test_sr_svr_dup, TT_FORK, NULL, NULL }, + { "commitments_are_the_same", test_commitments_are_the_same, TT_FORK, NULL, + NULL }, + { "commit_is_authoritative", test_commit_is_authoritative, TT_FORK, NULL, + NULL }, + { "get_phase_str", test_get_phase_str, TT_FORK, NULL, NULL }, { "utils_auth", test_utils_auth, TT_FORK, NULL, NULL }, { "state_transition", test_state_transition, TT_FORK, NULL, NULL }, { "state_update", test_state_update, TT_FORK, From 4be522b2e6eb634b141d7876234acf40a931cab6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Mon, 25 Mar 2019 03:00:53 +0100 Subject: [PATCH 0628/2557] Pass NULL to lpApplicationName in CreateProcessA(). When NULL is given to lpApplicationName we enable Windows' "magical" path interpretation logic, which makes Tor 0.4.x behave in the same way as previous Tor versions did when it comes to executing binaries in different system paths. For more information about this have a look at the CreateProcessA() documentation on MSDN -- especially the string interpretation example is useful to understand this issue. This bug was introduced in commit bfb94dd2ca8. See: https://bugs.torproject.org/29874 --- changes/bug29874 | 4 ++++ src/lib/process/process_win32.c | 7 +------ 2 files changed, 5 insertions(+), 6 deletions(-) create mode 100644 changes/bug29874 diff --git a/changes/bug29874 b/changes/bug29874 new file mode 100644 index 0000000000..8534753b51 --- /dev/null +++ b/changes/bug29874 @@ -0,0 +1,4 @@ + o Minor bugfixes (pluggable transports): + - Restore old behaviour when it comes to discovering the path of a given + Pluggable Transport exe-file. Fixes bug 29874; bugfix on 0.4.0.1-alpha. + diff --git a/src/lib/process/process_win32.c b/src/lib/process/process_win32.c index 21d0b23476..ddbe76bfd9 100644 --- a/src/lib/process/process_win32.c +++ b/src/lib/process/process_win32.c @@ -152,11 +152,6 @@ process_win32_exec(process_t *process) HANDLE stdin_pipe_read = NULL; HANDLE stdin_pipe_write = NULL; BOOL ret = FALSE; - const char *filename = process_get_command(process); - - /* Not much we can do if we haven't been told what to start. */ - if (BUG(filename == NULL)) - return PROCESS_STATUS_ERROR; /* Setup our security attributes. */ SECURITY_ATTRIBUTES security_attributes; @@ -211,7 +206,7 @@ process_win32_exec(process_t *process) char *joined_argv = tor_join_win_cmdline((const char **)argv); /* Create the child process */ - ret = CreateProcessA(filename, + ret = CreateProcessA(NULL, joined_argv, NULL, NULL, From 5d2f5e482e9985ad00f517ac3725b2336fbb930b Mon Sep 17 00:00:00 2001 From: Taylor Yu Date: Wed, 20 Mar 2019 20:51:12 -0500 Subject: [PATCH 0629/2557] Correctly report PT vs proxy during bootstrap Previously, or_connection_t did not record whether or not the connection uses a pluggable transport. Instead, it stored the underlying proxy protocol of the pluggable transport in proxy_type. This made bootstrap reporting treat pluggable transport connections as plain proxy connections. Store a separate bit indicating whether a pluggable transport is in use, and decode this during bootstrap reporting. Fixes bug 28925; bugfix on 0.4.0.1-alpha. --- changes/bug28925 | 4 ++++ src/core/mainloop/connection.c | 16 +++++++++++----- src/core/mainloop/connection.h | 2 +- src/core/or/connection_or.c | 16 +++++++++++++--- src/core/or/or_connection_st.h | 2 ++ 5 files changed, 31 insertions(+), 9 deletions(-) create mode 100644 changes/bug28925 diff --git a/changes/bug28925 b/changes/bug28925 new file mode 100644 index 0000000000..a867443885 --- /dev/null +++ b/changes/bug28925 @@ -0,0 +1,4 @@ + o Minor bugfixes (bootstrap reporting): + - During bootstrap reporting, correctly distinguish pluggable + transports from plain proxies. Fixes bug 28925; bugfix on + 0.4.0.1-alpha. diff --git a/src/core/mainloop/connection.c b/src/core/mainloop/connection.c index 41be3833ab..c8b19344bd 100644 --- a/src/core/mainloop/connection.c +++ b/src/core/mainloop/connection.c @@ -5361,17 +5361,20 @@ assert_connection_ok(connection_t *conn, time_t now) } /** Fills addr and port with the details of the global - * proxy server we are using. - * conn contains the connection we are using the proxy for. + * proxy server we are using. Store a 1 to the int pointed to by + * is_put_out if the connection is using a pluggable + * transport; store 0 otherwise. conn contains the connection + * we are using the proxy for. * * Return 0 on success, -1 on failure. */ int get_proxy_addrport(tor_addr_t *addr, uint16_t *port, int *proxy_type, - const connection_t *conn) + int *is_pt_out, const connection_t *conn) { const or_options_t *options = get_options(); + *is_pt_out = 0; /* Client Transport Plugins can use another proxy, but that should be hidden * from the rest of tor (as the plugin is responsible for dealing with the * proxy), check it first, then check the rest of the proxy types to allow @@ -5387,6 +5390,7 @@ get_proxy_addrport(tor_addr_t *addr, uint16_t *port, int *proxy_type, tor_addr_copy(addr, &transport->addr); *port = transport->port; *proxy_type = transport->socks_version; + *is_pt_out = 1; return 0; } @@ -5423,11 +5427,13 @@ log_failed_proxy_connection(connection_t *conn) { tor_addr_t proxy_addr; uint16_t proxy_port; - int proxy_type; + int proxy_type, is_pt; - if (get_proxy_addrport(&proxy_addr, &proxy_port, &proxy_type, conn) != 0) + if (get_proxy_addrport(&proxy_addr, &proxy_port, &proxy_type, &is_pt, + conn) != 0) return; /* if we have no proxy set up, leave this function. */ + (void)is_pt; log_warn(LD_NET, "The connection to the %s proxy server at %s just failed. " "Make sure that the proxy server is up and running.", diff --git a/src/core/mainloop/connection.h b/src/core/mainloop/connection.h index f4f0e839ae..411f13a8b8 100644 --- a/src/core/mainloop/connection.h +++ b/src/core/mainloop/connection.h @@ -187,7 +187,7 @@ int connection_proxy_connect(connection_t *conn, int type); int connection_read_proxy_handshake(connection_t *conn); void log_failed_proxy_connection(connection_t *conn); int get_proxy_addrport(tor_addr_t *addr, uint16_t *port, int *proxy_type, - const connection_t *conn); + int *is_pt_out, const connection_t *conn); int retry_all_listeners(smartlist_t *new_conns, int close_all_noncontrol); diff --git a/src/core/or/connection_or.c b/src/core/or/connection_or.c index 55047da167..debf482cb3 100644 --- a/src/core/or/connection_or.c +++ b/src/core/or/connection_or.c @@ -437,7 +437,15 @@ connection_or_state_publish(const or_connection_t *conn, uint8_t state) msg.type = ORCONN_MSGTYPE_STATE; msg.u.state.gid = conn->base_.global_identifier; - msg.u.state.proxy_type = conn->proxy_type; + if (conn->is_pt) { + /* Do extra decoding because conn->proxy_type indicates the proxy + * protocol that tor uses to talk with the transport plugin, + * instead of PROXY_PLUGGABLE. */ + tor_assert_nonfatal(conn->proxy_type != PROXY_NONE); + msg.u.state.proxy_type = PROXY_PLUGGABLE; + } else { + msg.u.state.proxy_type = conn->proxy_type; + } msg.u.state.state = state; if (conn->chan) { msg.u.state.chan = TLS_CHAN_TO_BASE(conn->chan)->global_identifier; @@ -1472,7 +1480,7 @@ connection_or_connect, (const tor_addr_t *_addr, uint16_t port, int r; tor_addr_t proxy_addr; uint16_t proxy_port; - int proxy_type; + int proxy_type, is_pt = 0; tor_assert(_addr); tor_assert(id_digest); @@ -1516,13 +1524,15 @@ connection_or_connect, (const tor_addr_t *_addr, uint16_t port, conn->is_outgoing = 1; /* If we are using a proxy server, find it and use it. */ - r = get_proxy_addrport(&proxy_addr, &proxy_port, &proxy_type, TO_CONN(conn)); + r = get_proxy_addrport(&proxy_addr, &proxy_port, &proxy_type, &is_pt, + TO_CONN(conn)); if (r == 0) { conn->proxy_type = proxy_type; if (proxy_type != PROXY_NONE) { tor_addr_copy(&addr, &proxy_addr); port = proxy_port; conn->base_.proxy_state = PROXY_INFANT; + conn->is_pt = is_pt; } connection_or_change_state(conn, OR_CONN_STATE_CONNECTING); connection_or_event_status(conn, OR_CONN_EVENT_LAUNCHED, 0); diff --git a/src/core/or/or_connection_st.h b/src/core/or/or_connection_st.h index d5db5e8694..a5ce844bff 100644 --- a/src/core/or/or_connection_st.h +++ b/src/core/or/or_connection_st.h @@ -67,6 +67,8 @@ struct or_connection_t { * geoip cache and handled by the DoS mitigation subsystem. We use this to * insure we have a coherent count of concurrent connection. */ unsigned int tracked_for_dos_mitigation : 1; + /** True iff this connection is using a pluggable transport */ + unsigned int is_pt : 1; uint16_t link_proto; /**< What protocol version are we using? 0 for * "none negotiated yet." */ From 68260e85b5fd4f5e0bb87a4c58b187a8309b0cb5 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Mon, 25 Mar 2019 10:17:30 +0200 Subject: [PATCH 0630/2557] Move sizeof check to torint.h --- src/lib/cc/torint.h | 9 +++++++++ src/test/test_ptr_slow.c | 9 --------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/lib/cc/torint.h b/src/lib/cc/torint.h index c9b2d329f2..9a66aada18 100644 --- a/src/lib/cc/torint.h +++ b/src/lib/cc/torint.h @@ -125,4 +125,13 @@ typedef int32_t ssize_t; /** Any size_t larger than this amount is likely to be an underflow. */ #define SIZE_T_CEILING ((size_t)(SSIZE_MAX-16)) +#if SIZEOF_INT > SIZEOF_VOID_P +#error "sizeof(int) > sizeof(void *) - Tor cannot be built on this platform!" +#endif + +#if SIZEOF_UNSIGNED_INT > SIZEOF_VOID_P +#error "sizeof(unsigned int) > sizeof(void *) - Tor cannot be built on this \ +platform!" +#endif + #endif /* !defined(TOR_TORINT_H) */ diff --git a/src/test/test_ptr_slow.c b/src/test/test_ptr_slow.c index 2d4fc9d4dd..76bdbf1891 100644 --- a/src/test/test_ptr_slow.c +++ b/src/test/test_ptr_slow.c @@ -11,15 +11,6 @@ #include #include -#if SIZEOF_INT > SIZEOF_VOID_P -#error "sizeof(int) > sizeof(void *) - Tor cannot be built on this platform!" -#endif - -#if SIZEOF_UNSIGNED_INT > SIZEOF_VOID_P -#error "sizeof(unsigned int) > sizeof(void *) - Tor cannot be built on this \ -platform!" -#endif - /** Assert that a can be cast to void * and back. */ static void assert_int_voidptr_roundtrip(int a) From a20a2025a5738cf5bcff52d36d88d3279af133e1 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 25 Mar 2019 09:08:04 -0400 Subject: [PATCH 0631/2557] practracker: sort filenames and directories. This helps ensure that we'll get output in a stable order. Closes ticket 29882. --- scripts/maint/practracker/util.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/maint/practracker/util.py b/scripts/maint/practracker/util.py index 63de72d5a3..b0ca73b997 100644 --- a/scripts/maint/practracker/util.py +++ b/scripts/maint/practracker/util.py @@ -11,6 +11,8 @@ def get_tor_c_files(tor_topdir): files_list = [] for root, directories, filenames in os.walk(tor_topdir): + directories.sort() + filenames.sort() for filename in filenames: # We only care about .c files if not filename.endswith(".c"): @@ -24,4 +26,3 @@ def get_tor_c_files(tor_topdir): files_list.append(full_path) return files_list - From 135b51c9d3436b3afcf323a193c755deafa53f7d Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 25 Mar 2019 09:28:24 -0400 Subject: [PATCH 0632/2557] practracker: allow comments in exceptions file Also, distinguish between empty lines (which we should ignore) and incorrect lines (which we should warn about). --- scripts/maint/practracker/problem.py | 31 +++++++++++++++++++++------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/scripts/maint/practracker/problem.py b/scripts/maint/practracker/problem.py index ab3d55057e..d5ebedd17f 100644 --- a/scripts/maint/practracker/problem.py +++ b/scripts/maint/practracker/problem.py @@ -10,6 +10,7 @@ get worse. from __future__ import print_function import os.path +import re import sys class ProblemVault(object): @@ -30,8 +31,15 @@ class ProblemVault(object): def register_exceptions(self, exception_file): # Register exceptions - for line in exception_file: - problem = get_old_problem_from_exception_str(line) + for lineno, line in enumerate(exception_file, 1): + try: + problem = get_old_problem_from_exception_str(line) + except ValueError as v: + print("Exception file line {} not recognized: {}" + .format(lineno,v), + file=sys.stderr) + continue + if problem is None: continue @@ -122,11 +130,20 @@ class FunctionSizeProblem(Problem): def __init__(self, problem_location, metric_value): super(FunctionSizeProblem, self).__init__("function-size", problem_location, metric_value) +comment_re = re.compile(r'#.*$') + def get_old_problem_from_exception_str(exception_str): - try: - _, problem_type, problem_location, metric_value = exception_str.split(" ") - except ValueError: + orig_str = exception_str + exception_str = comment_re.sub("", exception_str) + fields = exception_str.split() + if len(fields) == 0: + # empty line or comment return None + elif len(fields) == 4: + # valid line + _, problem_type, problem_location, metric_value = fields + else: + raise ValueError("Misformatted line {!r}".format(orig_str)) if problem_type == "file-size": return FileSizeProblem(problem_location, metric_value) @@ -135,6 +152,4 @@ def get_old_problem_from_exception_str(exception_str): elif problem_type == "function-size": return FunctionSizeProblem(problem_location, metric_value) else: -# print("Unknown exception line '{}'".format(exception_str)) - return None - + raise ValueError("Unknown exception type {!r}".format(orig_str)) From a49f506e05364eb0fd14d02e3cd482941942928e Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 25 Mar 2019 12:11:59 -0400 Subject: [PATCH 0633/2557] Split all controller events code into a new control_events.c Also, split the formatting code shared by control.c and control_events.c into controller_fmt.c. --- scripts/maint/practracker/exceptions.txt | 18 +- src/app/config/config.c | 1 + src/app/config/statefile.c | 2 +- src/app/main/main.c | 1 + src/core/include.am | 4 + src/core/mainloop/connection.c | 1 + src/core/mainloop/mainloop.c | 1 + src/core/or/circuitbuild.c | 2 +- src/core/or/circuitlist.c | 2 +- src/core/or/circuitstats.c | 2 +- src/core/or/circuituse.c | 2 +- src/core/or/command.c | 2 +- src/core/or/connection_edge.c | 2 +- src/core/or/connection_or.c | 2 +- src/core/or/relay.c | 2 +- src/core/proto/proto_socks.c | 2 +- src/feature/client/addressmap.c | 2 +- src/feature/client/dnsserv.c | 2 +- src/feature/client/entrynodes.c | 2 +- src/feature/client/transports.c | 2 +- src/feature/control/btrack_orconn_cevent.c | 2 +- src/feature/control/control.c | 2526 +------------------- src/feature/control/control.h | 317 +-- src/feature/control/control_bootstrap.c | 2 +- src/feature/control/control_events.c | 2280 ++++++++++++++++++ src/feature/control/control_events.h | 343 +++ src/feature/control/control_fmt.c | 289 +++ src/feature/control/control_fmt.h | 28 + src/feature/dirclient/dirclient.c | 2 +- src/feature/hibernate/hibernate.c | 2 +- src/feature/hs/hs_control.c | 2 +- src/feature/nodelist/dirlist.c | 2 +- src/feature/nodelist/networkstatus.c | 2 +- src/feature/nodelist/nodelist.c | 2 +- src/feature/nodelist/routerlist.c | 2 +- src/feature/relay/dns.c | 2 +- src/feature/relay/ext_orport.c | 3 +- src/feature/relay/router.c | 2 +- src/feature/relay/selftest.c | 2 +- src/feature/rend/rendclient.c | 2 +- src/feature/rend/rendcommon.c | 2 +- src/feature/rend/rendservice.c | 2 +- src/feature/stats/geoip_stats.c | 2 +- src/test/test_controller_events.c | 3 +- src/test/test_extorport.c | 2 +- src/test/test_hs.c | 3 +- src/test/test_hs_control.c | 3 +- src/test/test_pt.c | 4 +- src/test/test_util.c | 1 + src/test/testing_common.c | 1 + 50 files changed, 3012 insertions(+), 2877 deletions(-) create mode 100644 src/feature/control/control_events.c create mode 100644 src/feature/control/control_events.h create mode 100644 src/feature/control/control_fmt.c create mode 100644 src/feature/control/control_fmt.h diff --git a/scripts/maint/practracker/exceptions.txt b/scripts/maint/practracker/exceptions.txt index fee22ec7fc..15f16fdb4e 100644 --- a/scripts/maint/practracker/exceptions.txt +++ b/scripts/maint/practracker/exceptions.txt @@ -59,13 +59,13 @@ problem function-size /src/core/or/relay.c:connection_edge_process_relay_cell_no problem function-size /src/core/or/relay.c:connection_edge_process_relay_cell() 520 problem function-size /src/core/or/relay.c:connection_edge_package_raw_inbuf() 130 problem function-size /src/core/or/relay.c:circuit_resume_edge_reading_helper() 148 -problem file-size /src/core/mainloop/mainloop.c 3050 -problem include-count /src/core/mainloop/mainloop.c 65 +problem file-size /src/core/mainloop/mainloop.c 3051 +problem include-count /src/core/mainloop/mainloop.c 66 problem function-size /src/core/mainloop/mainloop.c:conn_close_if_marked() 108 problem function-size /src/core/mainloop/mainloop.c:run_connection_housekeeping() 123 problem function-size /src/core/mainloop/mainloop.c:CALLBACK() 116 -problem file-size /src/core/mainloop/connection.c 5547 -problem include-count /src/core/mainloop/connection.c 60 +problem file-size /src/core/mainloop/connection.c 5548 +problem include-count /src/core/mainloop/connection.c 61 problem function-size /src/core/mainloop/connection.c:connection_free_minimal() 184 problem function-size /src/core/mainloop/connection.c:connection_listener_new() 328 problem function-size /src/core/mainloop/connection.c:connection_handle_listener_read() 161 @@ -79,8 +79,8 @@ problem function-size /src/core/mainloop/connection.c:connection_handle_write_im problem function-size /src/core/mainloop/connection.c:assert_connection_ok() 143 problem function-size /src/app/config/confparse.c:config_assign_value() 205 problem function-size /src/app/config/confparse.c:config_get_assigned_option() 129 -problem file-size /src/app/config/config.c 8488 -problem include-count /src/app/config/config.c 84 +problem file-size /src/app/config/config.c 8489 +problem include-count /src/app/config/config.c 85 problem function-size /src/app/config/config.c:options_act_reversible() 296 problem function-size /src/app/config/config.c:options_act() 588 problem function-size /src/app/config/config.c:resolve_my_address() 192 @@ -96,7 +96,7 @@ problem function-size /src/app/config/config.c:parse_port_config() 452 problem function-size /src/app/config/config.c:parse_ports() 170 problem function-size /src/app/config/config.c:getinfo_helper_config() 116 problem function-size /src/app/main/ntmain.c:nt_service_install() 125 -problem include-count /src/app/main/main.c 83 +problem include-count /src/app/main/main.c 84 problem function-size /src/app/main/main.c:dumpstats() 102 problem function-size /src/app/main/main.c:tor_init() 136 problem function-size /src/app/main/main.c:sandbox_init_filter() 291 @@ -110,7 +110,7 @@ problem function-size /src/feature/keymgt/loadkey.c:ed_key_init_from_file() 333 problem function-size /src/feature/dircommon/consdiff.c:gen_ed_diff() 204 problem function-size /src/feature/dircommon/consdiff.c:apply_ed_diff() 159 problem file-size /src/feature/control/control.c 7592 -problem include-count /src/feature/control/control.c 83 +problem include-count /src/feature/control/control.c 90 problem function-size /src/feature/control/control.c:handle_control_authenticate() 188 problem function-size /src/feature/control/control.c:getinfo_helper_misc() 109 problem function-size /src/feature/control/control.c:getinfo_helper_dir() 304 @@ -122,7 +122,7 @@ problem function-size /src/feature/control/control.c:handle_control_hspost() 117 problem function-size /src/feature/control/control.c:handle_control_add_onion() 293 problem function-size /src/feature/control/control.c:add_onion_helper_keyarg() 125 problem function-size /src/feature/control/control.c:connection_control_process_inbuf() 239 -problem function-size /src/feature/control/control.c:control_event_stream_status() 119 +problem function-size /src/feature/control/control_events.c:control_event_stream_status() 119 problem function-size /src/feature/stats/rephist.c:rep_hist_load_mtbf_data() 185 problem function-size /src/feature/stats/rephist.c:rep_hist_format_exit_stats() 148 problem function-size /src/feature/dircache/consdiffmgr.c:consdiffmgr_cleanup() 115 diff --git a/src/app/config/config.c b/src/app/config/config.c index 7476f78175..1966910a81 100644 --- a/src/app/config/config.c +++ b/src/app/config/config.c @@ -86,6 +86,7 @@ #include "feature/client/entrynodes.h" #include "feature/client/transports.h" #include "feature/control/control.h" +#include "feature/control/control_events.h" #include "feature/dirauth/bwauth.h" #include "feature/dirauth/guardfraction.h" #include "feature/dircache/consdiffmgr.h" diff --git a/src/app/config/statefile.c b/src/app/config/statefile.c index 9681f6f8b3..fdfd68b244 100644 --- a/src/app/config/statefile.c +++ b/src/app/config/statefile.c @@ -36,7 +36,7 @@ #include "core/mainloop/mainloop.h" #include "core/mainloop/netstatus.h" #include "core/mainloop/connection.h" -#include "feature/control/control.h" +#include "feature/control/control_events.h" #include "feature/client/entrynodes.h" #include "feature/hibernate/hibernate.h" #include "feature/stats/rephist.h" diff --git a/src/app/main/main.c b/src/app/main/main.c index ec15109f6c..56978f9bab 100644 --- a/src/app/main/main.c +++ b/src/app/main/main.c @@ -43,6 +43,7 @@ #include "feature/client/entrynodes.h" #include "feature/client/transports.h" #include "feature/control/control.h" +#include "feature/control/control_events.h" #include "feature/dirauth/bwauth.h" #include "feature/dirauth/keypin.h" #include "feature/dirauth/process_descs.h" diff --git a/src/core/include.am b/src/core/include.am index ae47c75e09..58fcb784fd 100644 --- a/src/core/include.am +++ b/src/core/include.am @@ -71,6 +71,8 @@ LIBTOR_APP_A_SOURCES = \ src/feature/control/btrack_orconn_maps.c \ src/feature/control/control.c \ src/feature/control/control_bootstrap.c \ + src/feature/control/control_events.c \ + src/feature/control/control_fmt.c \ src/feature/control/fmt_serverstatus.c \ src/feature/control/getinfo_geoip.c \ src/feature/dirauth/keypin.c \ @@ -288,6 +290,8 @@ noinst_HEADERS += \ src/feature/control/btrack_sys.h \ src/feature/control/control.h \ src/feature/control/control_connection_st.h \ + src/feature/control/control_events.h \ + src/feature/control/control_fmt.h \ src/feature/control/fmt_serverstatus.h \ src/feature/control/getinfo_geoip.h \ src/feature/dirauth/authmode.h \ diff --git a/src/core/mainloop/connection.c b/src/core/mainloop/connection.c index 65ccd3a94e..40e548a135 100644 --- a/src/core/mainloop/connection.c +++ b/src/core/mainloop/connection.c @@ -88,6 +88,7 @@ #include "feature/client/entrynodes.h" #include "feature/client/transports.h" #include "feature/control/control.h" +#include "feature/control/control_events.h" #include "feature/dirauth/authmode.h" #include "feature/dircache/dirserv.h" #include "feature/dircommon/directory.h" diff --git a/src/core/mainloop/mainloop.c b/src/core/mainloop/mainloop.c index 18e87fa87a..c9f2b0d896 100644 --- a/src/core/mainloop/mainloop.c +++ b/src/core/mainloop/mainloop.c @@ -73,6 +73,7 @@ #include "feature/client/entrynodes.h" #include "feature/client/transports.h" #include "feature/control/control.h" +#include "feature/control/control_events.h" #include "feature/dirauth/authmode.h" #include "feature/dirauth/reachability.h" #include "feature/dircache/consdiffmgr.h" diff --git a/src/core/or/circuitbuild.c b/src/core/or/circuitbuild.c index 3ec1e01f11..f8e87bf026 100644 --- a/src/core/or/circuitbuild.c +++ b/src/core/or/circuitbuild.c @@ -55,7 +55,7 @@ #include "feature/client/circpathbias.h" #include "feature/client/entrynodes.h" #include "feature/client/transports.h" -#include "feature/control/control.h" +#include "feature/control/control_events.h" #include "feature/dircommon/directory.h" #include "feature/nodelist/describe.h" #include "feature/nodelist/microdesc.h" diff --git a/src/core/or/circuitlist.c b/src/core/or/circuitlist.c index 0d3665ce71..afbde06434 100644 --- a/src/core/or/circuitlist.c +++ b/src/core/or/circuitlist.c @@ -67,7 +67,7 @@ #include "app/config/config.h" #include "core/or/connection_edge.h" #include "core/or/connection_or.h" -#include "feature/control/control.h" +#include "feature/control/control_events.h" #include "lib/crypt_ops/crypto_rand.h" #include "lib/crypt_ops/crypto_util.h" #include "lib/crypt_ops/crypto_dh.h" diff --git a/src/core/or/circuitstats.c b/src/core/or/circuitstats.c index c6ea2fff97..e5a3bac30b 100644 --- a/src/core/or/circuitstats.c +++ b/src/core/or/circuitstats.c @@ -30,7 +30,7 @@ #include "core/or/circuitstats.h" #include "app/config/config.h" #include "app/config/confparse.h" -#include "feature/control/control.h" +#include "feature/control/control_events.h" #include "lib/crypt_ops/crypto_rand.h" #include "core/mainloop/mainloop.h" #include "feature/nodelist/networkstatus.h" diff --git a/src/core/or/circuituse.c b/src/core/or/circuituse.c index fd782c0cd1..2d0e75e53b 100644 --- a/src/core/or/circuituse.c +++ b/src/core/or/circuituse.c @@ -42,7 +42,7 @@ #include "feature/client/bridges.h" #include "feature/client/circpathbias.h" #include "feature/client/entrynodes.h" -#include "feature/control/control.h" +#include "feature/control/control_events.h" #include "feature/dircommon/directory.h" #include "feature/hs/hs_circuit.h" #include "feature/hs/hs_client.h" diff --git a/src/core/or/command.c b/src/core/or/command.c index 5fb6640c22..77e5447ce3 100644 --- a/src/core/or/command.c +++ b/src/core/or/command.c @@ -49,7 +49,7 @@ #include "core/or/dos.h" #include "core/or/onion.h" #include "core/or/relay.h" -#include "feature/control/control.h" +#include "feature/control/control_events.h" #include "feature/hibernate/hibernate.h" #include "feature/nodelist/describe.h" #include "feature/nodelist/nodelist.h" diff --git a/src/core/or/connection_edge.c b/src/core/or/connection_edge.c index cc240bdc98..071a8c91ed 100644 --- a/src/core/or/connection_edge.c +++ b/src/core/or/connection_edge.c @@ -78,7 +78,7 @@ #include "feature/client/addressmap.h" #include "feature/client/circpathbias.h" #include "feature/client/dnsserv.h" -#include "feature/control/control.h" +#include "feature/control/control_events.h" #include "feature/dircache/dirserv.h" #include "feature/dircommon/directory.h" #include "feature/hibernate/hibernate.h" diff --git a/src/core/or/connection_or.c b/src/core/or/connection_or.c index 55047da167..841ba8fa32 100644 --- a/src/core/or/connection_or.c +++ b/src/core/or/connection_or.c @@ -39,7 +39,7 @@ #include "app/config/config.h" #include "core/mainloop/connection.h" #include "core/or/connection_or.h" -#include "feature/control/control.h" +#include "feature/control/control_events.h" #include "lib/crypt_ops/crypto_rand.h" #include "lib/crypt_ops/crypto_util.h" #include "feature/dirauth/reachability.h" diff --git a/src/core/or/relay.c b/src/core/or/relay.c index 7f7fa2fe1f..a166904a5f 100644 --- a/src/core/or/relay.c +++ b/src/core/or/relay.c @@ -61,7 +61,7 @@ #include "core/mainloop/connection.h" #include "core/or/connection_edge.h" #include "core/or/connection_or.h" -#include "feature/control/control.h" +#include "feature/control/control_events.h" #include "lib/crypt_ops/crypto_rand.h" #include "lib/crypt_ops/crypto_util.h" #include "feature/dircommon/directory.h" diff --git a/src/core/proto/proto_socks.c b/src/core/proto/proto_socks.c index ac0c9e911b..b657a7b758 100644 --- a/src/core/proto/proto_socks.c +++ b/src/core/proto/proto_socks.c @@ -8,7 +8,7 @@ #include "feature/client/addressmap.h" #include "lib/buf/buffers.h" #include "core/mainloop/connection.h" -#include "feature/control/control.h" +#include "feature/control/control_events.h" #include "app/config/config.h" #include "lib/crypt_ops/crypto_util.h" #include "feature/relay/ext_orport.h" diff --git a/src/feature/client/addressmap.c b/src/feature/client/addressmap.c index bbe786a6a2..c5a27ce8c6 100644 --- a/src/feature/client/addressmap.c +++ b/src/feature/client/addressmap.c @@ -22,7 +22,7 @@ #include "core/or/circuituse.h" #include "app/config/config.h" #include "core/or/connection_edge.h" -#include "feature/control/control.h" +#include "feature/control/control_events.h" #include "feature/relay/dns.h" #include "feature/nodelist/nodelist.h" #include "feature/nodelist/routerset.h" diff --git a/src/feature/client/dnsserv.c b/src/feature/client/dnsserv.c index 44e0caaafa..7fb3fff6c1 100644 --- a/src/feature/client/dnsserv.c +++ b/src/feature/client/dnsserv.c @@ -26,7 +26,7 @@ #include "app/config/config.h" #include "core/mainloop/connection.h" #include "core/or/connection_edge.h" -#include "feature/control/control.h" +#include "feature/control/control_events.h" #include "core/mainloop/mainloop.h" #include "core/mainloop/netstatus.h" #include "core/or/policies.h" diff --git a/src/feature/client/entrynodes.c b/src/feature/client/entrynodes.c index e543289ce0..e59f8b34e0 100644 --- a/src/feature/client/entrynodes.c +++ b/src/feature/client/entrynodes.c @@ -128,7 +128,7 @@ #include "feature/client/circpathbias.h" #include "feature/client/entrynodes.h" #include "feature/client/transports.h" -#include "feature/control/control.h" +#include "feature/control/control_events.h" #include "feature/dircommon/directory.h" #include "feature/nodelist/describe.h" #include "feature/nodelist/microdesc.h" diff --git a/src/feature/client/transports.c b/src/feature/client/transports.c index 6fb357b466..97bfc8ae30 100644 --- a/src/feature/client/transports.c +++ b/src/feature/client/transports.c @@ -100,7 +100,7 @@ #include "app/config/statefile.h" #include "core/or/connection_or.h" #include "feature/relay/ext_orport.h" -#include "feature/control/control.h" +#include "feature/control/control_events.h" #include "lib/encoding/confline.h" #include "lib/encoding/kvline.h" diff --git a/src/feature/control/btrack_orconn_cevent.c b/src/feature/control/btrack_orconn_cevent.c index ee142f2873..535aa8f614 100644 --- a/src/feature/control/btrack_orconn_cevent.c +++ b/src/feature/control/btrack_orconn_cevent.c @@ -20,7 +20,7 @@ #include "core/or/orconn_event.h" #include "feature/control/btrack_orconn.h" #include "feature/control/btrack_orconn_cevent.h" -#include "feature/control/control.h" +#include "feature/control/control_events.h" /** * Have we completed our first OR connection? diff --git a/src/feature/control/control.c b/src/feature/control/control.c index f4bb0d38a8..d56ff41b0e 100644 --- a/src/feature/control/control.c +++ b/src/feature/control/control.c @@ -1,4 +1,3 @@ - /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ @@ -33,7 +32,9 @@ * stack. **/ +#define CONTROL_MODULE_PRIVATE #define CONTROL_PRIVATE +#define CONTROL_EVENTS_PRIVATE #define OCIRC_EVENT_PRIVATE #include "core/or/or.h" @@ -62,6 +63,8 @@ #include "feature/client/dnsserv.h" #include "feature/client/entrynodes.h" #include "feature/control/control.h" +#include "feature/control/control_events.h" +#include "feature/control/control_fmt.h" #include "feature/control/fmt_serverstatus.h" #include "feature/control/getinfo_geoip.h" #include "feature/dircache/dirserv.h" @@ -134,30 +137,6 @@ * finished authentication and is accepting commands. */ #define STATE_IS_OPEN(s) ((s) == CONTROL_CONN_STATE_OPEN) -/** Bitfield: The bit 1<<e is set if any open control - * connection is interested in events of type e. We use this - * so that we can decide to skip generating event messages that nobody - * has interest in without having to walk over the global connection - * list to find out. - **/ -typedef uint64_t event_mask_t; - -/** An event mask of all the events that any controller is interested in - * receiving. */ -static event_mask_t global_event_mask = 0; - -/** True iff we have disabled log messages from being sent to the controller */ -static int disable_log_messages = 0; - -/** Macro: true if any control connection is interested in events of type - * e. */ -#define EVENT_IS_INTERESTING(e) \ - (!! (global_event_mask & EVENT_MASK_(e))) - -/** Macro: true if any event from the bitfield 'e' is interesting. */ -#define ANY_EVENT_IS_INTERESTING(e) \ - (!! (global_event_mask & (e))) - /** If we're using cookie-type authentication, how long should our cookies be? */ #define AUTHENTICATION_COOKIE_LEN 32 @@ -181,20 +160,7 @@ static uint8_t *authentication_cookie = NULL; */ static smartlist_t *detached_onion_services = NULL; -static void connection_printf_to_buf(control_connection_t *conn, - const char *format, ...) - CHECK_PRINTF(2,3); -static void send_control_event_impl(uint16_t event, - const char *format, va_list ap) - CHECK_PRINTF(2,0); -static int control_event_status(int type, int severity, const char *format, - va_list args) - CHECK_PRINTF(3,0); - static void send_control_done(control_connection_t *conn); -static void send_control_event(uint16_t event, - const char *format, ...) - CHECK_PRINTF(2,3); static int handle_control_setconf(control_connection_t *conn, uint32_t len, char *body); static int handle_control_resetconf(control_connection_t *conn, uint32_t len, @@ -247,18 +213,8 @@ static int handle_control_add_onion(control_connection_t *conn, uint32_t len, const char *body); static int handle_control_del_onion(control_connection_t *conn, uint32_t len, const char *body); -static int write_stream_target_to_buf(entry_connection_t *conn, char *buf, - size_t len); -static void orconn_target_get_name(char *buf, size_t len, - or_connection_t *conn); - -static int get_cached_network_liveness(void); -static void set_cached_network_liveness(int liveness); - -static void flush_queued_events_cb(mainloop_event_t *event, void *arg); static char * download_status_to_string(const download_status_t *dl); -static void control_get_bytes_rw_last_sec(uint64_t *r, uint64_t *w); /** Convert a connection_t* to an control_connection_t*; assert if the cast is * invalid. */ @@ -269,211 +225,6 @@ TO_CONTROL_CONN(connection_t *c) return DOWNCAST(control_connection_t, c); } -/** Given a control event code for a message event, return the corresponding - * log severity. */ -static inline int -event_to_log_severity(int event) -{ - switch (event) { - case EVENT_DEBUG_MSG: return LOG_DEBUG; - case EVENT_INFO_MSG: return LOG_INFO; - case EVENT_NOTICE_MSG: return LOG_NOTICE; - case EVENT_WARN_MSG: return LOG_WARN; - case EVENT_ERR_MSG: return LOG_ERR; - default: return -1; - } -} - -/** Given a log severity, return the corresponding control event code. */ -static inline int -log_severity_to_event(int severity) -{ - switch (severity) { - case LOG_DEBUG: return EVENT_DEBUG_MSG; - case LOG_INFO: return EVENT_INFO_MSG; - case LOG_NOTICE: return EVENT_NOTICE_MSG; - case LOG_WARN: return EVENT_WARN_MSG; - case LOG_ERR: return EVENT_ERR_MSG; - default: return -1; - } -} - -/** Helper: clear bandwidth counters of all origin circuits. */ -static void -clear_circ_bw_fields(void) -{ - origin_circuit_t *ocirc; - SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) { - if (!CIRCUIT_IS_ORIGIN(circ)) - continue; - ocirc = TO_ORIGIN_CIRCUIT(circ); - ocirc->n_written_circ_bw = ocirc->n_read_circ_bw = 0; - ocirc->n_overhead_written_circ_bw = ocirc->n_overhead_read_circ_bw = 0; - ocirc->n_delivered_written_circ_bw = ocirc->n_delivered_read_circ_bw = 0; - } - SMARTLIST_FOREACH_END(circ); -} - -/** Set global_event_mask* to the bitwise OR of each live control - * connection's event_mask field. */ -void -control_update_global_event_mask(void) -{ - smartlist_t *conns = get_connection_array(); - event_mask_t old_mask, new_mask; - old_mask = global_event_mask; - int any_old_per_sec_events = control_any_per_second_event_enabled(); - - global_event_mask = 0; - SMARTLIST_FOREACH(conns, connection_t *, _conn, - { - if (_conn->type == CONN_TYPE_CONTROL && - STATE_IS_OPEN(_conn->state)) { - control_connection_t *conn = TO_CONTROL_CONN(_conn); - global_event_mask |= conn->event_mask; - } - }); - - new_mask = global_event_mask; - - /* Handle the aftermath. Set up the log callback to tell us only what - * we want to hear...*/ - control_adjust_event_log_severity(); - - /* Macro: true if ev was false before and is true now. */ -#define NEWLY_ENABLED(ev) \ - (! (old_mask & (ev)) && (new_mask & (ev))) - - /* ...then, if we've started logging stream or circ bw, clear the - * appropriate fields. */ - if (NEWLY_ENABLED(EVENT_STREAM_BANDWIDTH_USED)) { - SMARTLIST_FOREACH(conns, connection_t *, conn, - { - if (conn->type == CONN_TYPE_AP) { - edge_connection_t *edge_conn = TO_EDGE_CONN(conn); - edge_conn->n_written = edge_conn->n_read = 0; - } - }); - } - if (NEWLY_ENABLED(EVENT_CIRC_BANDWIDTH_USED)) { - clear_circ_bw_fields(); - } - if (NEWLY_ENABLED(EVENT_BANDWIDTH_USED)) { - uint64_t r, w; - control_get_bytes_rw_last_sec(&r, &w); - } - if (any_old_per_sec_events != control_any_per_second_event_enabled()) { - rescan_periodic_events(get_options()); - } - -#undef NEWLY_ENABLED -} - -/** Adjust the log severities that result in control_event_logmsg being called - * to match the severity of log messages that any controllers are interested - * in. */ -void -control_adjust_event_log_severity(void) -{ - int i; - int min_log_event=EVENT_ERR_MSG, max_log_event=EVENT_DEBUG_MSG; - - for (i = EVENT_DEBUG_MSG; i <= EVENT_ERR_MSG; ++i) { - if (EVENT_IS_INTERESTING(i)) { - min_log_event = i; - break; - } - } - for (i = EVENT_ERR_MSG; i >= EVENT_DEBUG_MSG; --i) { - if (EVENT_IS_INTERESTING(i)) { - max_log_event = i; - break; - } - } - if (EVENT_IS_INTERESTING(EVENT_STATUS_GENERAL)) { - if (min_log_event > EVENT_NOTICE_MSG) - min_log_event = EVENT_NOTICE_MSG; - if (max_log_event < EVENT_ERR_MSG) - max_log_event = EVENT_ERR_MSG; - } - if (min_log_event <= max_log_event) - change_callback_log_severity(event_to_log_severity(min_log_event), - event_to_log_severity(max_log_event), - control_event_logmsg); - else - change_callback_log_severity(LOG_ERR, LOG_ERR, - control_event_logmsg); -} - -/** Return true iff the event with code c is being sent to any current - * control connection. This is useful if the amount of work needed to prepare - * to call the appropriate control_event_...() function is high. - */ -int -control_event_is_interesting(int event) -{ - return EVENT_IS_INTERESTING(event); -} - -/** Return true if any event that needs to fire once a second is enabled. */ -int -control_any_per_second_event_enabled(void) -{ - return ANY_EVENT_IS_INTERESTING( - EVENT_MASK_(EVENT_BANDWIDTH_USED) | - EVENT_MASK_(EVENT_CELL_STATS) | - EVENT_MASK_(EVENT_CIRC_BANDWIDTH_USED) | - EVENT_MASK_(EVENT_CONN_BW) | - EVENT_MASK_(EVENT_STREAM_BANDWIDTH_USED) - ); -} - -/* The value of 'get_bytes_read()' the previous time that - * control_get_bytes_rw_last_sec() as called. */ -static uint64_t stats_prev_n_read = 0; -/* The value of 'get_bytes_written()' the previous time that - * control_get_bytes_rw_last_sec() as called. */ -static uint64_t stats_prev_n_written = 0; - -/** - * Set n_read and n_written to the total number of bytes read - * and written by Tor since the last call to this function. - * - * Call this only from the main thread. - */ -static void -control_get_bytes_rw_last_sec(uint64_t *n_read, - uint64_t *n_written) -{ - const uint64_t stats_n_bytes_read = get_bytes_read(); - const uint64_t stats_n_bytes_written = get_bytes_written(); - - *n_read = stats_n_bytes_read - stats_prev_n_read; - *n_written = stats_n_bytes_written - stats_prev_n_written; - stats_prev_n_read = stats_n_bytes_read; - stats_prev_n_written = stats_n_bytes_written; -} - -/** - * Run all the controller events (if any) that are scheduled to trigger once - * per second. - */ -void -control_per_second_events(void) -{ - if (!control_any_per_second_event_enabled()) - return; - - uint64_t bytes_read, bytes_written; - control_get_bytes_rw_last_sec(&bytes_read, &bytes_written); - control_event_bandwidth_used((uint32_t)bytes_read,(uint32_t)bytes_written); - - control_event_stream_bandwidth_used(); - control_event_conn_bandwidth_used(); - control_event_circ_bandwidth_used(); - control_event_circuit_cell_stats(); -} - /** Append a NUL-terminated string s to the end of * conn-\>outbuf. */ @@ -484,107 +235,6 @@ connection_write_str_to_buf(const char *s, control_connection_t *conn) connection_buf_add(s, len, TO_CONN(conn)); } -/** Given a len-character string in data, made of lines - * terminated by CRLF, allocate a new string in *out, and copy the - * contents of data into *out, adding a period before any period - * that appears at the start of a line, and adding a period-CRLF line at - * the end. Replace all LF characters sequences with CRLF. Return the number - * of bytes in *out. - */ -STATIC size_t -write_escaped_data(const char *data, size_t len, char **out) -{ - tor_assert(len < SIZE_MAX - 9); - size_t sz_out = len+8+1; - char *outp; - const char *start = data, *end; - size_t i; - int start_of_line; - for (i=0; i < len; ++i) { - if (data[i] == '\n') { - sz_out += 2; /* Maybe add a CR; maybe add a dot. */ - if (sz_out >= SIZE_T_CEILING) { - log_warn(LD_BUG, "Input to write_escaped_data was too long"); - *out = tor_strdup(".\r\n"); - return 3; - } - } - } - *out = outp = tor_malloc(sz_out); - end = data+len; - start_of_line = 1; - while (data < end) { - if (*data == '\n') { - if (data > start && data[-1] != '\r') - *outp++ = '\r'; - start_of_line = 1; - } else if (*data == '.') { - if (start_of_line) { - start_of_line = 0; - *outp++ = '.'; - } - } else { - start_of_line = 0; - } - *outp++ = *data++; - } - if (outp < *out+2 || fast_memcmp(outp-2, "\r\n", 2)) { - *outp++ = '\r'; - *outp++ = '\n'; - } - *outp++ = '.'; - *outp++ = '\r'; - *outp++ = '\n'; - *outp = '\0'; /* NUL-terminate just in case. */ - tor_assert(outp >= *out); - tor_assert((size_t)(outp - *out) <= sz_out); - return outp - *out; -} - -/** Given a len-character string in data, made of lines - * terminated by CRLF, allocate a new string in *out, and copy - * the contents of data into *out, removing any period - * that appears at the start of a line, and replacing all CRLF sequences - * with LF. Return the number of - * bytes in *out. */ -STATIC size_t -read_escaped_data(const char *data, size_t len, char **out) -{ - char *outp; - const char *next; - const char *end; - - *out = outp = tor_malloc(len+1); - - end = data+len; - - while (data < end) { - /* we're at the start of a line. */ - if (*data == '.') - ++data; - next = memchr(data, '\n', end-data); - if (next) { - size_t n_to_copy = next-data; - /* Don't copy a CR that precedes this LF. */ - if (n_to_copy && *(next-1) == '\r') - --n_to_copy; - memcpy(outp, data, n_to_copy); - outp += n_to_copy; - data = next+1; /* This will point at the start of the next line, - * or the end of the string, or a period. */ - } else { - memcpy(outp, data, end-data); - outp += (end-data); - *outp = '\0'; - return outp - *out; - } - *outp++ = '\n'; - } - - *outp = '\0'; - return outp - *out; -} - /** If the first in_len_max characters in start contain a * double-quoted string with escaped characters, return the length of that * string (as encoded, including quotes). Otherwise return -1. */ @@ -716,29 +366,6 @@ control_connection_add_local_fd(tor_socket_t sock, unsigned flags) return 0; } -/** Acts like sprintf, but writes its formatted string to the end of - * conn-\>outbuf. */ -static void -connection_printf_to_buf(control_connection_t *conn, const char *format, ...) -{ - va_list ap; - char *buf = NULL; - int len; - - va_start(ap,format); - len = tor_vasprintf(&buf, format, ap); - va_end(ap); - - if (len < 0) { - log_err(LD_BUG, "Unable to format string for controller."); - tor_assert(0); - } - - connection_buf_add(buf, (size_t)len, TO_CONN(conn)); - - tor_free(buf); -} - /** Write all of the open control ports to ControlPortWriteToFile */ void control_ports_write_to_file(void) @@ -790,256 +417,6 @@ send_control_done(control_connection_t *conn) connection_write_str_to_buf("250 OK\r\n", conn); } -/** Represents an event that's queued to be sent to one or more - * controllers. */ -typedef struct queued_event_s { - uint16_t event; - char *msg; -} queued_event_t; - -/** Pointer to int. If this is greater than 0, we don't allow new events to be - * queued. */ -static tor_threadlocal_t block_event_queue_flag; - -/** Holds a smartlist of queued_event_t objects that may need to be sent - * to one or more controllers */ -static smartlist_t *queued_control_events = NULL; - -/** True if the flush_queued_events_event is pending. */ -static int flush_queued_event_pending = 0; - -/** Lock to protect the above fields. */ -static tor_mutex_t *queued_control_events_lock = NULL; - -/** An event that should fire in order to flush the contents of - * queued_control_events. */ -static mainloop_event_t *flush_queued_events_event = NULL; - -void -control_initialize_event_queue(void) -{ - if (queued_control_events == NULL) { - queued_control_events = smartlist_new(); - } - - if (flush_queued_events_event == NULL) { - struct event_base *b = tor_libevent_get_base(); - if (b) { - flush_queued_events_event = - mainloop_event_new(flush_queued_events_cb, NULL); - tor_assert(flush_queued_events_event); - } - } - - if (queued_control_events_lock == NULL) { - queued_control_events_lock = tor_mutex_new(); - tor_threadlocal_init(&block_event_queue_flag); - } -} - -static int * -get_block_event_queue(void) -{ - int *val = tor_threadlocal_get(&block_event_queue_flag); - if (PREDICT_UNLIKELY(val == NULL)) { - val = tor_malloc_zero(sizeof(int)); - tor_threadlocal_set(&block_event_queue_flag, val); - } - return val; -} - -/** Helper: inserts an event on the list of events queued to be sent to - * one or more controllers, and schedules the events to be flushed if needed. - * - * This function takes ownership of msg, and may free it. - * - * We queue these events rather than send them immediately in order to break - * the dependency in our callgraph from code that generates events for the - * controller, and the network layer at large. Otherwise, nearly every - * interesting part of Tor would potentially call every other interesting part - * of Tor. - */ -MOCK_IMPL(STATIC void, -queue_control_event_string,(uint16_t event, char *msg)) -{ - /* This is redundant with checks done elsewhere, but it's a last-ditch - * attempt to avoid queueing something we shouldn't have to queue. */ - if (PREDICT_UNLIKELY( ! EVENT_IS_INTERESTING(event) )) { - tor_free(msg); - return; - } - - int *block_event_queue = get_block_event_queue(); - if (*block_event_queue) { - tor_free(msg); - return; - } - - queued_event_t *ev = tor_malloc(sizeof(*ev)); - ev->event = event; - ev->msg = msg; - - /* No queueing an event while queueing an event */ - ++*block_event_queue; - - tor_mutex_acquire(queued_control_events_lock); - tor_assert(queued_control_events); - smartlist_add(queued_control_events, ev); - - int activate_event = 0; - if (! flush_queued_event_pending && in_main_thread()) { - activate_event = 1; - flush_queued_event_pending = 1; - } - - tor_mutex_release(queued_control_events_lock); - - --*block_event_queue; - - /* We just put an event on the queue; mark the queue to be - * flushed. We only do this from the main thread for now; otherwise, - * we'd need to incur locking overhead in Libevent or use a socket. - */ - if (activate_event) { - tor_assert(flush_queued_events_event); - mainloop_event_activate(flush_queued_events_event); - } -} - -#define queued_event_free(ev) \ - FREE_AND_NULL(queued_event_t, queued_event_free_, (ev)) - -/** Release all storage held by ev. */ -static void -queued_event_free_(queued_event_t *ev) -{ - if (ev == NULL) - return; - - tor_free(ev->msg); - tor_free(ev); -} - -/** Send every queued event to every controller that's interested in it, - * and remove the events from the queue. If force is true, - * then make all controllers send their data out immediately, since we - * may be about to shut down. */ -static void -queued_events_flush_all(int force) -{ - /* Make sure that we get all the pending log events, if there are any. */ - flush_pending_log_callbacks(); - - if (PREDICT_UNLIKELY(queued_control_events == NULL)) { - return; - } - smartlist_t *all_conns = get_connection_array(); - smartlist_t *controllers = smartlist_new(); - smartlist_t *queued_events; - - int *block_event_queue = get_block_event_queue(); - ++*block_event_queue; - - tor_mutex_acquire(queued_control_events_lock); - /* No queueing an event while flushing events. */ - flush_queued_event_pending = 0; - queued_events = queued_control_events; - queued_control_events = smartlist_new(); - tor_mutex_release(queued_control_events_lock); - - /* Gather all the controllers that will care... */ - SMARTLIST_FOREACH_BEGIN(all_conns, connection_t *, conn) { - if (conn->type == CONN_TYPE_CONTROL && - !conn->marked_for_close && - conn->state == CONTROL_CONN_STATE_OPEN) { - control_connection_t *control_conn = TO_CONTROL_CONN(conn); - - smartlist_add(controllers, control_conn); - } - } SMARTLIST_FOREACH_END(conn); - - SMARTLIST_FOREACH_BEGIN(queued_events, queued_event_t *, ev) { - const event_mask_t bit = ((event_mask_t)1) << ev->event; - const size_t msg_len = strlen(ev->msg); - SMARTLIST_FOREACH_BEGIN(controllers, control_connection_t *, - control_conn) { - if (control_conn->event_mask & bit) { - connection_buf_add(ev->msg, msg_len, TO_CONN(control_conn)); - } - } SMARTLIST_FOREACH_END(control_conn); - - queued_event_free(ev); - } SMARTLIST_FOREACH_END(ev); - - if (force) { - SMARTLIST_FOREACH_BEGIN(controllers, control_connection_t *, - control_conn) { - connection_flush(TO_CONN(control_conn)); - } SMARTLIST_FOREACH_END(control_conn); - } - - smartlist_free(queued_events); - smartlist_free(controllers); - - --*block_event_queue; -} - -/** Libevent callback: Flushes pending events to controllers that are - * interested in them. */ -static void -flush_queued_events_cb(mainloop_event_t *event, void *arg) -{ - (void) event; - (void) arg; - queued_events_flush_all(0); -} - -/** Send an event to all v1 controllers that are listening for code - * event. The event's body is given by msg. - * - * The EXTENDED_FORMAT and NONEXTENDED_FORMAT flags behave similarly with - * respect to the EXTENDED_EVENTS feature. */ -MOCK_IMPL(STATIC void, -send_control_event_string,(uint16_t event, - const char *msg)) -{ - tor_assert(event >= EVENT_MIN_ && event <= EVENT_MAX_); - queue_control_event_string(event, tor_strdup(msg)); -} - -/** Helper for send_control_event and control_event_status: - * Send an event to all v1 controllers that are listening for code - * event. The event's body is created by the printf-style format in - * format, and other arguments as provided. */ -static void -send_control_event_impl(uint16_t event, - const char *format, va_list ap) -{ - char *buf = NULL; - int len; - - len = tor_vasprintf(&buf, format, ap); - if (len < 0) { - log_warn(LD_BUG, "Unable to format event for controller."); - return; - } - - queue_control_event_string(event, buf); -} - -/** Send an event to all v1 controllers that are listening for code - * event. The event's body is created by the printf-style format in - * format, and other arguments as provided. */ -static void -send_control_event(uint16_t event, - const char *format, ...) -{ - va_list ap; - va_start(ap, format); - send_control_event_impl(event, format, ap); - va_end(ap); -} - /** Given a text circuit id, return the corresponding circuit. */ static origin_circuit_t * get_circ(const char *id) @@ -2805,104 +2182,6 @@ getinfo_helper_downloads(control_connection_t *control_conn, } } -/** Allocate and return a description of circ's current status, - * including its path (if any). */ -static char * -circuit_describe_status_for_controller(origin_circuit_t *circ) -{ - char *rv; - smartlist_t *descparts = smartlist_new(); - - { - char *vpath = circuit_list_path_for_controller(circ); - if (*vpath) { - smartlist_add(descparts, vpath); - } else { - tor_free(vpath); /* empty path; don't put an extra space in the result */ - } - } - - { - cpath_build_state_t *build_state = circ->build_state; - smartlist_t *flaglist = smartlist_new(); - char *flaglist_joined; - - if (build_state->onehop_tunnel) - smartlist_add(flaglist, (void *)"ONEHOP_TUNNEL"); - if (build_state->is_internal) - smartlist_add(flaglist, (void *)"IS_INTERNAL"); - if (build_state->need_capacity) - smartlist_add(flaglist, (void *)"NEED_CAPACITY"); - if (build_state->need_uptime) - smartlist_add(flaglist, (void *)"NEED_UPTIME"); - - /* Only emit a BUILD_FLAGS argument if it will have a non-empty value. */ - if (smartlist_len(flaglist)) { - flaglist_joined = smartlist_join_strings(flaglist, ",", 0, NULL); - - smartlist_add_asprintf(descparts, "BUILD_FLAGS=%s", flaglist_joined); - - tor_free(flaglist_joined); - } - - smartlist_free(flaglist); - } - - smartlist_add_asprintf(descparts, "PURPOSE=%s", - circuit_purpose_to_controller_string(circ->base_.purpose)); - - { - const char *hs_state = - circuit_purpose_to_controller_hs_state_string(circ->base_.purpose); - - if (hs_state != NULL) { - smartlist_add_asprintf(descparts, "HS_STATE=%s", hs_state); - } - } - - if (circ->rend_data != NULL || circ->hs_ident != NULL) { - char addr[HS_SERVICE_ADDR_LEN_BASE32 + 1]; - const char *onion_address; - if (circ->rend_data) { - onion_address = rend_data_get_address(circ->rend_data); - } else { - hs_build_address(&circ->hs_ident->identity_pk, HS_VERSION_THREE, addr); - onion_address = addr; - } - smartlist_add_asprintf(descparts, "REND_QUERY=%s", onion_address); - } - - { - char tbuf[ISO_TIME_USEC_LEN+1]; - format_iso_time_nospace_usec(tbuf, &circ->base_.timestamp_created); - - smartlist_add_asprintf(descparts, "TIME_CREATED=%s", tbuf); - } - - // Show username and/or password if available. - if (circ->socks_username_len > 0) { - char* socks_username_escaped = esc_for_log_len(circ->socks_username, - (size_t) circ->socks_username_len); - smartlist_add_asprintf(descparts, "SOCKS_USERNAME=%s", - socks_username_escaped); - tor_free(socks_username_escaped); - } - if (circ->socks_password_len > 0) { - char* socks_password_escaped = esc_for_log_len(circ->socks_password, - (size_t) circ->socks_password_len); - smartlist_add_asprintf(descparts, "SOCKS_PASSWORD=%s", - socks_password_escaped); - tor_free(socks_password_escaped); - } - - rv = smartlist_join_strings(descparts, " ", 0, NULL); - - SMARTLIST_FOREACH(descparts, char *, cp, tor_free(cp)); - smartlist_free(descparts); - - return rv; -} - /** Implementation helper for GETINFO: knows how to generate summaries of the * current states of things we send events about. */ static int @@ -5647,1294 +4926,23 @@ connection_control_process_inbuf(control_connection_t *conn) conn->incoming_cmd_cur_len = 0; goto again; } - -/** Something major has happened to circuit circ: tell any - * interested control connections. */ -int -control_event_circuit_status(origin_circuit_t *circ, circuit_status_event_t tp, - int reason_code) -{ - const char *status; - char reasons[64] = ""; - - if (!EVENT_IS_INTERESTING(EVENT_CIRCUIT_STATUS)) - return 0; - tor_assert(circ); - - switch (tp) - { - case CIRC_EVENT_LAUNCHED: status = "LAUNCHED"; break; - case CIRC_EVENT_BUILT: status = "BUILT"; break; - case CIRC_EVENT_EXTENDED: status = "EXTENDED"; break; - case CIRC_EVENT_FAILED: status = "FAILED"; break; - case CIRC_EVENT_CLOSED: status = "CLOSED"; break; - default: - log_warn(LD_BUG, "Unrecognized status code %d", (int)tp); - tor_fragile_assert(); - return 0; - } - - if (tp == CIRC_EVENT_FAILED || tp == CIRC_EVENT_CLOSED) { - const char *reason_str = circuit_end_reason_to_control_string(reason_code); - char unk_reason_buf[16]; - if (!reason_str) { - tor_snprintf(unk_reason_buf, 16, "UNKNOWN_%d", reason_code); - reason_str = unk_reason_buf; - } - if (reason_code > 0 && reason_code & END_CIRC_REASON_FLAG_REMOTE) { - tor_snprintf(reasons, sizeof(reasons), - " REASON=DESTROYED REMOTE_REASON=%s", reason_str); - } else { - tor_snprintf(reasons, sizeof(reasons), - " REASON=%s", reason_str); - } - } - - { - char *circdesc = circuit_describe_status_for_controller(circ); - const char *sp = strlen(circdesc) ? " " : ""; - send_control_event(EVENT_CIRCUIT_STATUS, - "650 CIRC %lu %s%s%s%s\r\n", - (unsigned long)circ->global_identifier, - status, sp, - circdesc, - reasons); - tor_free(circdesc); - } - - return 0; -} - -/** Something minor has happened to circuit circ: tell any - * interested control connections. */ -static int -control_event_circuit_status_minor(origin_circuit_t *circ, - circuit_status_minor_event_t e, - int purpose, const struct timeval *tv) -{ - const char *event_desc; - char event_tail[160] = ""; - if (!EVENT_IS_INTERESTING(EVENT_CIRCUIT_STATUS_MINOR)) - return 0; - tor_assert(circ); - - switch (e) - { - case CIRC_MINOR_EVENT_PURPOSE_CHANGED: - event_desc = "PURPOSE_CHANGED"; - - { - /* event_tail can currently be up to 68 chars long */ - const char *hs_state_str = - circuit_purpose_to_controller_hs_state_string(purpose); - tor_snprintf(event_tail, sizeof(event_tail), - " OLD_PURPOSE=%s%s%s", - circuit_purpose_to_controller_string(purpose), - (hs_state_str != NULL) ? " OLD_HS_STATE=" : "", - (hs_state_str != NULL) ? hs_state_str : ""); - } - - break; - case CIRC_MINOR_EVENT_CANNIBALIZED: - event_desc = "CANNIBALIZED"; - - { - /* event_tail can currently be up to 130 chars long */ - const char *hs_state_str = - circuit_purpose_to_controller_hs_state_string(purpose); - const struct timeval *old_timestamp_began = tv; - char tbuf[ISO_TIME_USEC_LEN+1]; - format_iso_time_nospace_usec(tbuf, old_timestamp_began); - - tor_snprintf(event_tail, sizeof(event_tail), - " OLD_PURPOSE=%s%s%s OLD_TIME_CREATED=%s", - circuit_purpose_to_controller_string(purpose), - (hs_state_str != NULL) ? " OLD_HS_STATE=" : "", - (hs_state_str != NULL) ? hs_state_str : "", - tbuf); - } - - break; - default: - log_warn(LD_BUG, "Unrecognized status code %d", (int)e); - tor_fragile_assert(); - return 0; - } - - { - char *circdesc = circuit_describe_status_for_controller(circ); - const char *sp = strlen(circdesc) ? " " : ""; - send_control_event(EVENT_CIRCUIT_STATUS_MINOR, - "650 CIRC_MINOR %lu %s%s%s%s\r\n", - (unsigned long)circ->global_identifier, - event_desc, sp, - circdesc, - event_tail); - tor_free(circdesc); - } - - return 0; -} - -/** - * circ has changed its purpose from old_purpose: tell any - * interested controllers. - */ -int -control_event_circuit_purpose_changed(origin_circuit_t *circ, - int old_purpose) -{ - return control_event_circuit_status_minor(circ, - CIRC_MINOR_EVENT_PURPOSE_CHANGED, - old_purpose, - NULL); -} - -/** - * circ has changed its purpose from old_purpose, and its - * created-time from old_tv_created: tell any interested controllers. - */ -int -control_event_circuit_cannibalized(origin_circuit_t *circ, - int old_purpose, - const struct timeval *old_tv_created) -{ - return control_event_circuit_status_minor(circ, - CIRC_MINOR_EVENT_CANNIBALIZED, - old_purpose, - old_tv_created); -} - -/** Given an AP connection conn and a len-character buffer - * buf, determine the address:port combination requested on - * conn, and write it to buf. Return 0 on success, -1 on - * failure. */ -static int -write_stream_target_to_buf(entry_connection_t *conn, char *buf, size_t len) -{ - char buf2[256]; - if (conn->chosen_exit_name) - if (tor_snprintf(buf2, sizeof(buf2), ".%s.exit", conn->chosen_exit_name)<0) - return -1; - if (!conn->socks_request) - return -1; - if (tor_snprintf(buf, len, "%s%s%s:%d", - conn->socks_request->address, - conn->chosen_exit_name ? buf2 : "", - !conn->chosen_exit_name && connection_edge_is_rendezvous_stream( - ENTRY_TO_EDGE_CONN(conn)) ? ".onion" : "", - conn->socks_request->port)<0) - return -1; - return 0; -} - -/** Something has happened to the stream associated with AP connection - * conn: tell any interested control connections. */ -int -control_event_stream_status(entry_connection_t *conn, stream_status_event_t tp, - int reason_code) -{ - char reason_buf[64]; - char addrport_buf[64]; - const char *status; - circuit_t *circ; - origin_circuit_t *origin_circ = NULL; - char buf[256]; - const char *purpose = ""; - tor_assert(conn->socks_request); - - if (!EVENT_IS_INTERESTING(EVENT_STREAM_STATUS)) - return 0; - - if (tp == STREAM_EVENT_CLOSED && - (reason_code & END_STREAM_REASON_FLAG_ALREADY_SENT_CLOSED)) - return 0; - - write_stream_target_to_buf(conn, buf, sizeof(buf)); - - reason_buf[0] = '\0'; - switch (tp) - { - case STREAM_EVENT_SENT_CONNECT: status = "SENTCONNECT"; break; - case STREAM_EVENT_SENT_RESOLVE: status = "SENTRESOLVE"; break; - case STREAM_EVENT_SUCCEEDED: status = "SUCCEEDED"; break; - case STREAM_EVENT_FAILED: status = "FAILED"; break; - case STREAM_EVENT_CLOSED: status = "CLOSED"; break; - case STREAM_EVENT_NEW: status = "NEW"; break; - case STREAM_EVENT_NEW_RESOLVE: status = "NEWRESOLVE"; break; - case STREAM_EVENT_FAILED_RETRIABLE: status = "DETACHED"; break; - case STREAM_EVENT_REMAP: status = "REMAP"; break; - default: - log_warn(LD_BUG, "Unrecognized status code %d", (int)tp); - return 0; - } - if (reason_code && (tp == STREAM_EVENT_FAILED || - tp == STREAM_EVENT_CLOSED || - tp == STREAM_EVENT_FAILED_RETRIABLE)) { - const char *reason_str = stream_end_reason_to_control_string(reason_code); - char *r = NULL; - if (!reason_str) { - tor_asprintf(&r, " UNKNOWN_%d", reason_code); - reason_str = r; - } - if (reason_code & END_STREAM_REASON_FLAG_REMOTE) - tor_snprintf(reason_buf, sizeof(reason_buf), - " REASON=END REMOTE_REASON=%s", reason_str); - else - tor_snprintf(reason_buf, sizeof(reason_buf), - " REASON=%s", reason_str); - tor_free(r); - } else if (reason_code && tp == STREAM_EVENT_REMAP) { - switch (reason_code) { - case REMAP_STREAM_SOURCE_CACHE: - strlcpy(reason_buf, " SOURCE=CACHE", sizeof(reason_buf)); - break; - case REMAP_STREAM_SOURCE_EXIT: - strlcpy(reason_buf, " SOURCE=EXIT", sizeof(reason_buf)); - break; - default: - tor_snprintf(reason_buf, sizeof(reason_buf), " REASON=UNKNOWN_%d", - reason_code); - /* XXX do we want SOURCE=UNKNOWN_%d above instead? -RD */ - break; - } - } - - if (tp == STREAM_EVENT_NEW || tp == STREAM_EVENT_NEW_RESOLVE) { - /* - * When the control conn is an AF_UNIX socket and we have no address, - * it gets set to "(Tor_internal)"; see dnsserv_launch_request() in - * dnsserv.c. - */ - if (strcmp(ENTRY_TO_CONN(conn)->address, "(Tor_internal)") != 0) { - tor_snprintf(addrport_buf,sizeof(addrport_buf), " SOURCE_ADDR=%s:%d", - ENTRY_TO_CONN(conn)->address, ENTRY_TO_CONN(conn)->port); - } else { - /* - * else leave it blank so control on AF_UNIX doesn't need to make - * something up. - */ - addrport_buf[0] = '\0'; - } - } else { - addrport_buf[0] = '\0'; - } - - if (tp == STREAM_EVENT_NEW_RESOLVE) { - purpose = " PURPOSE=DNS_REQUEST"; - } else if (tp == STREAM_EVENT_NEW) { - if (conn->use_begindir) { - connection_t *linked = ENTRY_TO_CONN(conn)->linked_conn; - int linked_dir_purpose = -1; - if (linked && linked->type == CONN_TYPE_DIR) - linked_dir_purpose = linked->purpose; - if (DIR_PURPOSE_IS_UPLOAD(linked_dir_purpose)) - purpose = " PURPOSE=DIR_UPLOAD"; - else - purpose = " PURPOSE=DIR_FETCH"; - } else - purpose = " PURPOSE=USER"; - } - - circ = circuit_get_by_edge_conn(ENTRY_TO_EDGE_CONN(conn)); - if (circ && CIRCUIT_IS_ORIGIN(circ)) - origin_circ = TO_ORIGIN_CIRCUIT(circ); - send_control_event(EVENT_STREAM_STATUS, - "650 STREAM %"PRIu64" %s %lu %s%s%s%s\r\n", - (ENTRY_TO_CONN(conn)->global_identifier), - status, - origin_circ? - (unsigned long)origin_circ->global_identifier : 0ul, - buf, reason_buf, addrport_buf, purpose); - - /* XXX need to specify its intended exit, etc? */ - - return 0; -} - -/** Figure out the best name for the target router of an OR connection - * conn, and write it into the len-character buffer - * name. */ -static void -orconn_target_get_name(char *name, size_t len, or_connection_t *conn) -{ - const node_t *node = node_get_by_id(conn->identity_digest); - if (node) { - tor_assert(len > MAX_VERBOSE_NICKNAME_LEN); - node_get_verbose_nickname(node, name); - } else if (! tor_digest_is_zero(conn->identity_digest)) { - name[0] = '$'; - base16_encode(name+1, len-1, conn->identity_digest, - DIGEST_LEN); - } else { - tor_snprintf(name, len, "%s:%d", - conn->base_.address, conn->base_.port); - } -} - -/** Called when the status of an OR connection conn changes: tell any - * interested control connections. tp is the new status for the - * connection. If conn has just closed or failed, then reason - * may be the reason why. - */ -int -control_event_or_conn_status(or_connection_t *conn, or_conn_status_event_t tp, - int reason) -{ - int ncircs = 0; - const char *status; - char name[128]; - char ncircs_buf[32] = {0}; /* > 8 + log10(2^32)=10 + 2 */ - - if (!EVENT_IS_INTERESTING(EVENT_OR_CONN_STATUS)) - return 0; - - switch (tp) - { - case OR_CONN_EVENT_LAUNCHED: status = "LAUNCHED"; break; - case OR_CONN_EVENT_CONNECTED: status = "CONNECTED"; break; - case OR_CONN_EVENT_FAILED: status = "FAILED"; break; - case OR_CONN_EVENT_CLOSED: status = "CLOSED"; break; - case OR_CONN_EVENT_NEW: status = "NEW"; break; - default: - log_warn(LD_BUG, "Unrecognized status code %d", (int)tp); - return 0; - } - if (conn->chan) { - ncircs = circuit_count_pending_on_channel(TLS_CHAN_TO_BASE(conn->chan)); - } else { - ncircs = 0; - } - ncircs += connection_or_get_num_circuits(conn); - if (ncircs && (tp == OR_CONN_EVENT_FAILED || tp == OR_CONN_EVENT_CLOSED)) { - tor_snprintf(ncircs_buf, sizeof(ncircs_buf), " NCIRCS=%d", ncircs); - } - - orconn_target_get_name(name, sizeof(name), conn); - send_control_event(EVENT_OR_CONN_STATUS, - "650 ORCONN %s %s%s%s%s ID=%"PRIu64"\r\n", - name, status, - reason ? " REASON=" : "", - orconn_end_reason_to_control_string(reason), - ncircs_buf, - (conn->base_.global_identifier)); - - return 0; -} - -/** - * Print out STREAM_BW event for a single conn - */ -int -control_event_stream_bandwidth(edge_connection_t *edge_conn) -{ - struct timeval now; - char tbuf[ISO_TIME_USEC_LEN+1]; - if (EVENT_IS_INTERESTING(EVENT_STREAM_BANDWIDTH_USED)) { - if (!edge_conn->n_read && !edge_conn->n_written) - return 0; - - tor_gettimeofday(&now); - format_iso_time_nospace_usec(tbuf, &now); - send_control_event(EVENT_STREAM_BANDWIDTH_USED, - "650 STREAM_BW %"PRIu64" %lu %lu %s\r\n", - (edge_conn->base_.global_identifier), - (unsigned long)edge_conn->n_read, - (unsigned long)edge_conn->n_written, - tbuf); - - edge_conn->n_written = edge_conn->n_read = 0; - } - - return 0; -} - -/** A second or more has elapsed: tell any interested control - * connections how much bandwidth streams have used. */ -int -control_event_stream_bandwidth_used(void) -{ - if (EVENT_IS_INTERESTING(EVENT_STREAM_BANDWIDTH_USED)) { - smartlist_t *conns = get_connection_array(); - edge_connection_t *edge_conn; - struct timeval now; - char tbuf[ISO_TIME_USEC_LEN+1]; - - SMARTLIST_FOREACH_BEGIN(conns, connection_t *, conn) - { - if (conn->type != CONN_TYPE_AP) - continue; - edge_conn = TO_EDGE_CONN(conn); - if (!edge_conn->n_read && !edge_conn->n_written) - continue; - - tor_gettimeofday(&now); - format_iso_time_nospace_usec(tbuf, &now); - send_control_event(EVENT_STREAM_BANDWIDTH_USED, - "650 STREAM_BW %"PRIu64" %lu %lu %s\r\n", - (edge_conn->base_.global_identifier), - (unsigned long)edge_conn->n_read, - (unsigned long)edge_conn->n_written, - tbuf); - - edge_conn->n_written = edge_conn->n_read = 0; - } - SMARTLIST_FOREACH_END(conn); - } - - return 0; -} - -/** A second or more has elapsed: tell any interested control connections - * how much bandwidth origin circuits have used. */ -int -control_event_circ_bandwidth_used(void) -{ - if (!EVENT_IS_INTERESTING(EVENT_CIRC_BANDWIDTH_USED)) - return 0; - - SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) { - if (!CIRCUIT_IS_ORIGIN(circ)) - continue; - - control_event_circ_bandwidth_used_for_circ(TO_ORIGIN_CIRCUIT(circ)); - } - SMARTLIST_FOREACH_END(circ); - - return 0; -} - -/** - * Emit a CIRC_BW event line for a specific circuit. - * - * This function sets the values it emits to 0, and does not emit - * an event if there is no new data to report since the last call. - * - * Therefore, it may be called at any frequency. - */ -int -control_event_circ_bandwidth_used_for_circ(origin_circuit_t *ocirc) -{ - struct timeval now; - char tbuf[ISO_TIME_USEC_LEN+1]; - - tor_assert(ocirc); - - if (!EVENT_IS_INTERESTING(EVENT_CIRC_BANDWIDTH_USED)) - return 0; - - /* n_read_circ_bw and n_written_circ_bw are always updated - * when there is any new cell on a circuit, and set to 0 after - * the event, below. - * - * Therefore, checking them is sufficient to determine if there - * is new data to report. */ - if (!ocirc->n_read_circ_bw && !ocirc->n_written_circ_bw) - return 0; - - tor_gettimeofday(&now); - format_iso_time_nospace_usec(tbuf, &now); - send_control_event(EVENT_CIRC_BANDWIDTH_USED, - "650 CIRC_BW ID=%d READ=%lu WRITTEN=%lu TIME=%s " - "DELIVERED_READ=%lu OVERHEAD_READ=%lu " - "DELIVERED_WRITTEN=%lu OVERHEAD_WRITTEN=%lu\r\n", - ocirc->global_identifier, - (unsigned long)ocirc->n_read_circ_bw, - (unsigned long)ocirc->n_written_circ_bw, - tbuf, - (unsigned long)ocirc->n_delivered_read_circ_bw, - (unsigned long)ocirc->n_overhead_read_circ_bw, - (unsigned long)ocirc->n_delivered_written_circ_bw, - (unsigned long)ocirc->n_overhead_written_circ_bw); - ocirc->n_written_circ_bw = ocirc->n_read_circ_bw = 0; - ocirc->n_overhead_written_circ_bw = ocirc->n_overhead_read_circ_bw = 0; - ocirc->n_delivered_written_circ_bw = ocirc->n_delivered_read_circ_bw = 0; - - return 0; -} - -/** Print out CONN_BW event for a single OR/DIR/EXIT conn and reset - * bandwidth counters. */ -int -control_event_conn_bandwidth(connection_t *conn) -{ - const char *conn_type_str; - if (!get_options()->TestingEnableConnBwEvent || - !EVENT_IS_INTERESTING(EVENT_CONN_BW)) - return 0; - if (!conn->n_read_conn_bw && !conn->n_written_conn_bw) - return 0; - switch (conn->type) { - case CONN_TYPE_OR: - conn_type_str = "OR"; - break; - case CONN_TYPE_DIR: - conn_type_str = "DIR"; - break; - case CONN_TYPE_EXIT: - conn_type_str = "EXIT"; - break; - default: - return 0; - } - send_control_event(EVENT_CONN_BW, - "650 CONN_BW ID=%"PRIu64" TYPE=%s " - "READ=%lu WRITTEN=%lu\r\n", - (conn->global_identifier), - conn_type_str, - (unsigned long)conn->n_read_conn_bw, - (unsigned long)conn->n_written_conn_bw); - conn->n_written_conn_bw = conn->n_read_conn_bw = 0; - return 0; -} - -/** A second or more has elapsed: tell any interested control - * connections how much bandwidth connections have used. */ -int -control_event_conn_bandwidth_used(void) -{ - if (get_options()->TestingEnableConnBwEvent && - EVENT_IS_INTERESTING(EVENT_CONN_BW)) { - SMARTLIST_FOREACH(get_connection_array(), connection_t *, conn, - control_event_conn_bandwidth(conn)); - } - return 0; -} - -/** Helper: iterate over cell statistics of circ and sum up added - * cells, removed cells, and waiting times by cell command and direction. - * Store results in cell_stats. Free cell statistics of the - * circuit afterwards. */ -void -sum_up_cell_stats_by_command(circuit_t *circ, cell_stats_t *cell_stats) -{ - memset(cell_stats, 0, sizeof(cell_stats_t)); - SMARTLIST_FOREACH_BEGIN(circ->testing_cell_stats, - const testing_cell_stats_entry_t *, ent) { - tor_assert(ent->command <= CELL_COMMAND_MAX_); - if (!ent->removed && !ent->exitward) { - cell_stats->added_cells_appward[ent->command] += 1; - } else if (!ent->removed && ent->exitward) { - cell_stats->added_cells_exitward[ent->command] += 1; - } else if (!ent->exitward) { - cell_stats->removed_cells_appward[ent->command] += 1; - cell_stats->total_time_appward[ent->command] += ent->waiting_time * 10; - } else { - cell_stats->removed_cells_exitward[ent->command] += 1; - cell_stats->total_time_exitward[ent->command] += ent->waiting_time * 10; - } - } SMARTLIST_FOREACH_END(ent); - circuit_clear_testing_cell_stats(circ); -} - -/** Helper: append a cell statistics string to event_parts, - * prefixed with key=. Statistics consist of comma-separated - * key:value pairs with lower-case command strings as keys and cell - * numbers or total waiting times as values. A key:value pair is included - * if the entry in include_if_non_zero is not zero, but with - * the (possibly zero) entry from number_to_include. Both - * arrays are expected to have a length of CELL_COMMAND_MAX_ + 1. If no - * entry in include_if_non_zero is positive, no string will - * be added to event_parts. */ -void -append_cell_stats_by_command(smartlist_t *event_parts, const char *key, - const uint64_t *include_if_non_zero, - const uint64_t *number_to_include) -{ - smartlist_t *key_value_strings = smartlist_new(); - int i; - for (i = 0; i <= CELL_COMMAND_MAX_; i++) { - if (include_if_non_zero[i] > 0) { - smartlist_add_asprintf(key_value_strings, "%s:%"PRIu64, - cell_command_to_string(i), - (number_to_include[i])); - } - } - if (smartlist_len(key_value_strings) > 0) { - char *joined = smartlist_join_strings(key_value_strings, ",", 0, NULL); - smartlist_add_asprintf(event_parts, "%s=%s", key, joined); - SMARTLIST_FOREACH(key_value_strings, char *, cp, tor_free(cp)); - tor_free(joined); - } - smartlist_free(key_value_strings); -} - -/** Helper: format cell_stats for circ for inclusion in a - * CELL_STATS event and write result string to event_string. */ -void -format_cell_stats(char **event_string, circuit_t *circ, - cell_stats_t *cell_stats) -{ - smartlist_t *event_parts = smartlist_new(); - if (CIRCUIT_IS_ORIGIN(circ)) { - origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ); - smartlist_add_asprintf(event_parts, "ID=%lu", - (unsigned long)ocirc->global_identifier); - } else if (TO_OR_CIRCUIT(circ)->p_chan) { - or_circuit_t *or_circ = TO_OR_CIRCUIT(circ); - smartlist_add_asprintf(event_parts, "InboundQueue=%lu", - (unsigned long)or_circ->p_circ_id); - smartlist_add_asprintf(event_parts, "InboundConn=%"PRIu64, - (or_circ->p_chan->global_identifier)); - append_cell_stats_by_command(event_parts, "InboundAdded", - cell_stats->added_cells_appward, - cell_stats->added_cells_appward); - append_cell_stats_by_command(event_parts, "InboundRemoved", - cell_stats->removed_cells_appward, - cell_stats->removed_cells_appward); - append_cell_stats_by_command(event_parts, "InboundTime", - cell_stats->removed_cells_appward, - cell_stats->total_time_appward); - } - if (circ->n_chan) { - smartlist_add_asprintf(event_parts, "OutboundQueue=%lu", - (unsigned long)circ->n_circ_id); - smartlist_add_asprintf(event_parts, "OutboundConn=%"PRIu64, - (circ->n_chan->global_identifier)); - append_cell_stats_by_command(event_parts, "OutboundAdded", - cell_stats->added_cells_exitward, - cell_stats->added_cells_exitward); - append_cell_stats_by_command(event_parts, "OutboundRemoved", - cell_stats->removed_cells_exitward, - cell_stats->removed_cells_exitward); - append_cell_stats_by_command(event_parts, "OutboundTime", - cell_stats->removed_cells_exitward, - cell_stats->total_time_exitward); - } - *event_string = smartlist_join_strings(event_parts, " ", 0, NULL); - SMARTLIST_FOREACH(event_parts, char *, cp, tor_free(cp)); - smartlist_free(event_parts); -} - -/** A second or more has elapsed: tell any interested control connection - * how many cells have been processed for a given circuit. */ -int -control_event_circuit_cell_stats(void) -{ - cell_stats_t *cell_stats; - char *event_string; - if (!get_options()->TestingEnableCellStatsEvent || - !EVENT_IS_INTERESTING(EVENT_CELL_STATS)) - return 0; - cell_stats = tor_malloc(sizeof(cell_stats_t)); - SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) { - if (!circ->testing_cell_stats) - continue; - sum_up_cell_stats_by_command(circ, cell_stats); - format_cell_stats(&event_string, circ, cell_stats); - send_control_event(EVENT_CELL_STATS, - "650 CELL_STATS %s\r\n", event_string); - tor_free(event_string); - } - SMARTLIST_FOREACH_END(circ); - tor_free(cell_stats); - return 0; -} - -/* about 5 minutes worth. */ -#define N_BW_EVENTS_TO_CACHE 300 -/* Index into cached_bw_events to next write. */ -static int next_measurement_idx = 0; -/* number of entries set in n_measurements */ -static int n_measurements = 0; -static struct cached_bw_event_s { - uint32_t n_read; - uint32_t n_written; -} cached_bw_events[N_BW_EVENTS_TO_CACHE]; - -/** A second or more has elapsed: tell any interested control - * connections how much bandwidth we used. */ -int -control_event_bandwidth_used(uint32_t n_read, uint32_t n_written) -{ - cached_bw_events[next_measurement_idx].n_read = n_read; - cached_bw_events[next_measurement_idx].n_written = n_written; - if (++next_measurement_idx == N_BW_EVENTS_TO_CACHE) - next_measurement_idx = 0; - if (n_measurements < N_BW_EVENTS_TO_CACHE) - ++n_measurements; - - if (EVENT_IS_INTERESTING(EVENT_BANDWIDTH_USED)) { - send_control_event(EVENT_BANDWIDTH_USED, - "650 BW %lu %lu\r\n", - (unsigned long)n_read, - (unsigned long)n_written); - } - - return 0; -} - -STATIC char * -get_bw_samples(void) -{ - int i; - int idx = (next_measurement_idx + N_BW_EVENTS_TO_CACHE - n_measurements) - % N_BW_EVENTS_TO_CACHE; - tor_assert(0 <= idx && idx < N_BW_EVENTS_TO_CACHE); - - smartlist_t *elements = smartlist_new(); - - for (i = 0; i < n_measurements; ++i) { - tor_assert(0 <= idx && idx < N_BW_EVENTS_TO_CACHE); - const struct cached_bw_event_s *bwe = &cached_bw_events[idx]; - - smartlist_add_asprintf(elements, "%u,%u", - (unsigned)bwe->n_read, - (unsigned)bwe->n_written); - - idx = (idx + 1) % N_BW_EVENTS_TO_CACHE; - } - - char *result = smartlist_join_strings(elements, " ", 0, NULL); - - SMARTLIST_FOREACH(elements, char *, cp, tor_free(cp)); - smartlist_free(elements); - - return result; -} - -/** Called when we are sending a log message to the controllers: suspend - * sending further log messages to the controllers until we're done. Used by - * CONN_LOG_PROTECT. */ -void -disable_control_logging(void) -{ - ++disable_log_messages; -} - -/** We're done sending a log message to the controllers: re-enable controller - * logging. Used by CONN_LOG_PROTECT. */ -void -enable_control_logging(void) -{ - if (--disable_log_messages < 0) - tor_assert(0); -} - -/** We got a log message: tell any interested control connections. */ -void -control_event_logmsg(int severity, uint32_t domain, const char *msg) -{ - int event; - - /* Don't even think of trying to add stuff to a buffer from a cpuworker - * thread. (See #25987 for plan to fix.) */ - if (! in_main_thread()) - return; - - if (disable_log_messages) - return; - - if (domain == LD_BUG && EVENT_IS_INTERESTING(EVENT_STATUS_GENERAL) && - severity <= LOG_NOTICE) { - char *esc = esc_for_log(msg); - ++disable_log_messages; - control_event_general_status(severity, "BUG REASON=%s", esc); - --disable_log_messages; - tor_free(esc); - } - - event = log_severity_to_event(severity); - if (event >= 0 && EVENT_IS_INTERESTING(event)) { - char *b = NULL; - const char *s; - if (strchr(msg, '\n')) { - char *cp; - b = tor_strdup(msg); - for (cp = b; *cp; ++cp) - if (*cp == '\r' || *cp == '\n') - *cp = ' '; - } - switch (severity) { - case LOG_DEBUG: s = "DEBUG"; break; - case LOG_INFO: s = "INFO"; break; - case LOG_NOTICE: s = "NOTICE"; break; - case LOG_WARN: s = "WARN"; break; - case LOG_ERR: s = "ERR"; break; - default: s = "UnknownLogSeverity"; break; - } - ++disable_log_messages; - send_control_event(event, "650 %s %s\r\n", s, b?b:msg); - if (severity == LOG_ERR) { - /* Force a flush, since we may be about to die horribly */ - queued_events_flush_all(1); - } - --disable_log_messages; - tor_free(b); - } -} - -/** - * Logging callback: called when there is a queued pending log callback. - */ -void -control_event_logmsg_pending(void) -{ - if (! in_main_thread()) { - /* We can't handle this case yet, since we're using a - * mainloop_event_t to invoke queued_events_flush_all. We ought to - * use a different mechanism instead: see #25987. - **/ - return; - } - tor_assert(flush_queued_events_event); - mainloop_event_activate(flush_queued_events_event); -} - -/** Called whenever we receive new router descriptors: tell any - * interested control connections. routers is a list of - * routerinfo_t's. - */ -int -control_event_descriptors_changed(smartlist_t *routers) -{ - char *msg; - - if (!EVENT_IS_INTERESTING(EVENT_NEW_DESC)) - return 0; - - { - smartlist_t *names = smartlist_new(); - char *ids; - SMARTLIST_FOREACH(routers, routerinfo_t *, ri, { - char *b = tor_malloc(MAX_VERBOSE_NICKNAME_LEN+1); - router_get_verbose_nickname(b, ri); - smartlist_add(names, b); - }); - ids = smartlist_join_strings(names, " ", 0, NULL); - tor_asprintf(&msg, "650 NEWDESC %s\r\n", ids); - send_control_event_string(EVENT_NEW_DESC, msg); - tor_free(ids); - tor_free(msg); - SMARTLIST_FOREACH(names, char *, cp, tor_free(cp)); - smartlist_free(names); - } - return 0; -} - -/** Called when an address mapping on from from changes to to. - * expires values less than 3 are special; see connection_edge.c. If - * error is non-NULL, it is an error code describing the failure - * mode of the mapping. - */ -int -control_event_address_mapped(const char *from, const char *to, time_t expires, - const char *error, const int cached) -{ - if (!EVENT_IS_INTERESTING(EVENT_ADDRMAP)) - return 0; - - if (expires < 3 || expires == TIME_MAX) - send_control_event(EVENT_ADDRMAP, - "650 ADDRMAP %s %s NEVER %s%s" - "CACHED=\"%s\"\r\n", - from, to, error?error:"", error?" ":"", - cached?"YES":"NO"); - else { - char buf[ISO_TIME_LEN+1]; - char buf2[ISO_TIME_LEN+1]; - format_local_iso_time(buf,expires); - format_iso_time(buf2,expires); - send_control_event(EVENT_ADDRMAP, - "650 ADDRMAP %s %s \"%s\"" - " %s%sEXPIRES=\"%s\" CACHED=\"%s\"\r\n", - from, to, buf, - error?error:"", error?" ":"", - buf2, cached?"YES":"NO"); - } - - return 0; -} - /** Cached liveness for network liveness events and GETINFO */ static int network_is_live = 0; -static int +int get_cached_network_liveness(void) { return network_is_live; } -static void +void set_cached_network_liveness(int liveness) { network_is_live = liveness; } -/** The network liveness has changed; this is called from circuitstats.c - * whenever we receive a cell, or when timeout expires and we assume the - * network is down. */ -int -control_event_network_liveness_update(int liveness) -{ - if (liveness > 0) { - if (get_cached_network_liveness() <= 0) { - /* Update cached liveness */ - set_cached_network_liveness(1); - log_debug(LD_CONTROL, "Sending NETWORK_LIVENESS UP"); - send_control_event_string(EVENT_NETWORK_LIVENESS, - "650 NETWORK_LIVENESS UP\r\n"); - } - /* else was already live, no-op */ - } else { - if (get_cached_network_liveness() > 0) { - /* Update cached liveness */ - set_cached_network_liveness(0); - log_debug(LD_CONTROL, "Sending NETWORK_LIVENESS DOWN"); - send_control_event_string(EVENT_NETWORK_LIVENESS, - "650 NETWORK_LIVENESS DOWN\r\n"); - } - /* else was already dead, no-op */ - } - - return 0; -} - -/** Helper function for NS-style events. Constructs and sends an event - * of type event with string event_string out of the set of - * networkstatuses statuses. Currently it is used for NS events - * and NEWCONSENSUS events. */ -static int -control_event_networkstatus_changed_helper(smartlist_t *statuses, - uint16_t event, - const char *event_string) -{ - smartlist_t *strs; - char *s, *esc = NULL; - if (!EVENT_IS_INTERESTING(event) || !smartlist_len(statuses)) - return 0; - - strs = smartlist_new(); - smartlist_add_strdup(strs, "650+"); - smartlist_add_strdup(strs, event_string); - smartlist_add_strdup(strs, "\r\n"); - SMARTLIST_FOREACH(statuses, const routerstatus_t *, rs, - { - s = networkstatus_getinfo_helper_single(rs); - if (!s) continue; - smartlist_add(strs, s); - }); - - s = smartlist_join_strings(strs, "", 0, NULL); - write_escaped_data(s, strlen(s), &esc); - SMARTLIST_FOREACH(strs, char *, cp, tor_free(cp)); - smartlist_free(strs); - tor_free(s); - send_control_event_string(event, esc); - send_control_event_string(event, - "650 OK\r\n"); - - tor_free(esc); - return 0; -} - -/** Called when the routerstatus_ts statuses have changed: sends - * an NS event to any controller that cares. */ -int -control_event_networkstatus_changed(smartlist_t *statuses) -{ - return control_event_networkstatus_changed_helper(statuses, EVENT_NS, "NS"); -} - -/** Called when we get a new consensus networkstatus. Sends a NEWCONSENSUS - * event consisting of an NS-style line for each relay in the consensus. */ -int -control_event_newconsensus(const networkstatus_t *consensus) -{ - if (!control_event_is_interesting(EVENT_NEWCONSENSUS)) - return 0; - return control_event_networkstatus_changed_helper( - consensus->routerstatus_list, EVENT_NEWCONSENSUS, "NEWCONSENSUS"); -} - -/** Called when we compute a new circuitbuildtimeout */ -int -control_event_buildtimeout_set(buildtimeout_set_event_t type, - const char *args) -{ - const char *type_string = NULL; - - if (!control_event_is_interesting(EVENT_BUILDTIMEOUT_SET)) - return 0; - - switch (type) { - case BUILDTIMEOUT_SET_EVENT_COMPUTED: - type_string = "COMPUTED"; - break; - case BUILDTIMEOUT_SET_EVENT_RESET: - type_string = "RESET"; - break; - case BUILDTIMEOUT_SET_EVENT_SUSPENDED: - type_string = "SUSPENDED"; - break; - case BUILDTIMEOUT_SET_EVENT_DISCARD: - type_string = "DISCARD"; - break; - case BUILDTIMEOUT_SET_EVENT_RESUME: - type_string = "RESUME"; - break; - default: - type_string = "UNKNOWN"; - break; - } - - send_control_event(EVENT_BUILDTIMEOUT_SET, - "650 BUILDTIMEOUT_SET %s %s\r\n", - type_string, args); - - return 0; -} - -/** Called when a signal has been processed from signal_callback */ -int -control_event_signal(uintptr_t signal_num) -{ - const char *signal_string = NULL; - - if (!control_event_is_interesting(EVENT_GOT_SIGNAL)) - return 0; - - switch (signal_num) { - case SIGHUP: - signal_string = "RELOAD"; - break; - case SIGUSR1: - signal_string = "DUMP"; - break; - case SIGUSR2: - signal_string = "DEBUG"; - break; - case SIGNEWNYM: - signal_string = "NEWNYM"; - break; - case SIGCLEARDNSCACHE: - signal_string = "CLEARDNSCACHE"; - break; - case SIGHEARTBEAT: - signal_string = "HEARTBEAT"; - break; - default: - log_warn(LD_BUG, "Unrecognized signal %lu in control_event_signal", - (unsigned long)signal_num); - return -1; - } - - send_control_event(EVENT_GOT_SIGNAL, "650 SIGNAL %s\r\n", - signal_string); - return 0; -} - -/** Called when a single local_routerstatus_t has changed: Sends an NS event - * to any controller that cares. */ -int -control_event_networkstatus_changed_single(const routerstatus_t *rs) -{ - smartlist_t *statuses; - int r; - - if (!EVENT_IS_INTERESTING(EVENT_NS)) - return 0; - - statuses = smartlist_new(); - smartlist_add(statuses, (void*)rs); - r = control_event_networkstatus_changed(statuses); - smartlist_free(statuses); - return r; -} - -/** Our own router descriptor has changed; tell any controllers that care. - */ -int -control_event_my_descriptor_changed(void) -{ - send_control_event(EVENT_DESCCHANGED, "650 DESCCHANGED\r\n"); - return 0; -} - -/** Helper: sends a status event where type is one of - * EVENT_STATUS_{GENERAL,CLIENT,SERVER}, where severity is one of - * LOG_{NOTICE,WARN,ERR}, and where format is a printf-style format - * string corresponding to args. */ -static int -control_event_status(int type, int severity, const char *format, va_list args) -{ - char *user_buf = NULL; - char format_buf[160]; - const char *status, *sev; - - switch (type) { - case EVENT_STATUS_GENERAL: - status = "STATUS_GENERAL"; - break; - case EVENT_STATUS_CLIENT: - status = "STATUS_CLIENT"; - break; - case EVENT_STATUS_SERVER: - status = "STATUS_SERVER"; - break; - default: - log_warn(LD_BUG, "Unrecognized status type %d", type); - return -1; - } - switch (severity) { - case LOG_NOTICE: - sev = "NOTICE"; - break; - case LOG_WARN: - sev = "WARN"; - break; - case LOG_ERR: - sev = "ERR"; - break; - default: - log_warn(LD_BUG, "Unrecognized status severity %d", severity); - return -1; - } - if (tor_snprintf(format_buf, sizeof(format_buf), "650 %s %s", - status, sev)<0) { - log_warn(LD_BUG, "Format string too long."); - return -1; - } - tor_vasprintf(&user_buf, format, args); - - send_control_event(type, "%s %s\r\n", format_buf, user_buf); - tor_free(user_buf); - return 0; -} - -#define CONTROL_EVENT_STATUS_BODY(event, sev) \ - int r; \ - do { \ - va_list ap; \ - if (!EVENT_IS_INTERESTING(event)) \ - return 0; \ - \ - va_start(ap, format); \ - r = control_event_status((event), (sev), format, ap); \ - va_end(ap); \ - } while (0) - -/** Format and send an EVENT_STATUS_GENERAL event whose main text is obtained - * by formatting the arguments using the printf-style format. */ -int -control_event_general_status(int severity, const char *format, ...) -{ - CONTROL_EVENT_STATUS_BODY(EVENT_STATUS_GENERAL, severity); - return r; -} - -/** Format and send an EVENT_STATUS_GENERAL LOG_ERR event, and flush it to the - * controller(s) immediately. */ -int -control_event_general_error(const char *format, ...) -{ - CONTROL_EVENT_STATUS_BODY(EVENT_STATUS_GENERAL, LOG_ERR); - /* Force a flush, since we may be about to die horribly */ - queued_events_flush_all(1); - return r; -} - -/** Format and send an EVENT_STATUS_CLIENT event whose main text is obtained - * by formatting the arguments using the printf-style format. */ -int -control_event_client_status(int severity, const char *format, ...) -{ - CONTROL_EVENT_STATUS_BODY(EVENT_STATUS_CLIENT, severity); - return r; -} - -/** Format and send an EVENT_STATUS_CLIENT LOG_ERR event, and flush it to the - * controller(s) immediately. */ -int -control_event_client_error(const char *format, ...) -{ - CONTROL_EVENT_STATUS_BODY(EVENT_STATUS_CLIENT, LOG_ERR); - /* Force a flush, since we may be about to die horribly */ - queued_events_flush_all(1); - return r; -} - -/** Format and send an EVENT_STATUS_SERVER event whose main text is obtained - * by formatting the arguments using the printf-style format. */ -int -control_event_server_status(int severity, const char *format, ...) -{ - CONTROL_EVENT_STATUS_BODY(EVENT_STATUS_SERVER, severity); - return r; -} - -/** Format and send an EVENT_STATUS_SERVER LOG_ERR event, and flush it to the - * controller(s) immediately. */ -int -control_event_server_error(const char *format, ...) -{ - CONTROL_EVENT_STATUS_BODY(EVENT_STATUS_SERVER, LOG_ERR); - /* Force a flush, since we may be about to die horribly */ - queued_events_flush_all(1); - return r; -} - -/** Called when the status of an entry guard with the given nickname - * and identity digest has changed to status: tells any - * controllers that care. */ -int -control_event_guard(const char *nickname, const char *digest, - const char *status) -{ - char hbuf[HEX_DIGEST_LEN+1]; - base16_encode(hbuf, sizeof(hbuf), digest, DIGEST_LEN); - if (!EVENT_IS_INTERESTING(EVENT_GUARD)) - return 0; - - { - char buf[MAX_VERBOSE_NICKNAME_LEN+1]; - const node_t *node = node_get_by_id(digest); - if (node) { - node_get_verbose_nickname(node, buf); - } else { - tor_snprintf(buf, sizeof(buf), "$%s~%s", hbuf, nickname); - } - send_control_event(EVENT_GUARD, - "650 GUARD ENTRY %s %s\r\n", buf, status); - } - return 0; -} - -/** Called when a configuration option changes. This is generally triggered - * by SETCONF requests and RELOAD/SIGHUP signals. The elements is - * a smartlist_t containing (key, value, ...) pairs in sequence. - * value can be NULL. */ -int -control_event_conf_changed(const smartlist_t *elements) -{ - int i; - char *result; - smartlist_t *lines; - if (!EVENT_IS_INTERESTING(EVENT_CONF_CHANGED) || - smartlist_len(elements) == 0) { - return 0; - } - lines = smartlist_new(); - for (i = 0; i < smartlist_len(elements); i += 2) { - char *k = smartlist_get(elements, i); - char *v = smartlist_get(elements, i+1); - if (v == NULL) { - smartlist_add_asprintf(lines, "650-%s", k); - } else { - smartlist_add_asprintf(lines, "650-%s=%s", k, v); - } - } - result = smartlist_join_strings(lines, "\r\n", 0, NULL); - send_control_event(EVENT_CONF_CHANGED, - "650-CONF_CHANGED\r\n%s\r\n650 OK\r\n", result); - tor_free(result); - SMARTLIST_FOREACH(lines, char *, cp, tor_free(cp)); - smartlist_free(lines); - return 0; -} - /** Helper: Return a newly allocated string containing a path to the * file where we store our authentication cookie. */ char * @@ -7040,75 +5048,6 @@ monitor_owning_controller_process(const char *process_spec) } } -/** We just generated a new summary of which countries we've seen clients - * from recently. Send a copy to the controller in case it wants to - * display it for the user. */ -void -control_event_clients_seen(const char *controller_str) -{ - send_control_event(EVENT_CLIENTS_SEEN, - "650 CLIENTS_SEEN %s\r\n", controller_str); -} - -/** A new pluggable transport called transport_name was - * launched on addr:port. mode is either - * "server" or "client" depending on the mode of the pluggable - * transport. - * "650" SP "TRANSPORT_LAUNCHED" SP Mode SP Name SP Address SP Port - */ -void -control_event_transport_launched(const char *mode, const char *transport_name, - tor_addr_t *addr, uint16_t port) -{ - send_control_event(EVENT_TRANSPORT_LAUNCHED, - "650 TRANSPORT_LAUNCHED %s %s %s %u\r\n", - mode, transport_name, fmt_addr(addr), port); -} - -/** A pluggable transport called pt_name has emitted a log message - * found in message at severity log level. */ -void -control_event_pt_log(const char *log) -{ - send_control_event(EVENT_PT_LOG, - "650 PT_LOG %s\r\n", - log); -} - -/** A pluggable transport has emitted a STATUS message found in - * status. */ -void -control_event_pt_status(const char *status) -{ - send_control_event(EVENT_PT_STATUS, - "650 PT_STATUS %s\r\n", - status); -} - -/** Convert rendezvous auth type to string for HS_DESC control events - */ -const char * -rend_auth_type_to_string(rend_auth_type_t auth_type) -{ - const char *str; - - switch (auth_type) { - case REND_NO_AUTH: - str = "NO_AUTH"; - break; - case REND_BASIC_AUTH: - str = "BASIC_AUTH"; - break; - case REND_STEALTH_AUTH: - str = "STEALTH_AUTH"; - break; - default: - str = "UNKNOWN"; - } - - return str; -} - /** Return a longname the node whose identity is id_digest. If * node_get_by_id() returns NULL, base 16 encoding of id_digest is * returned instead. @@ -7124,434 +5063,11 @@ node_describe_longname_by_id,(const char *id_digest)) return longname; } -/** Return either the onion address if the given pointer is a non empty - * string else the unknown string. */ -static const char * -rend_hsaddress_str_or_unknown(const char *onion_address) -{ - static const char *str_unknown = "UNKNOWN"; - const char *str_ret = str_unknown; - - /* No valid pointer, unknown it is. */ - if (!onion_address) { - goto end; - } - /* Empty onion address thus we don't know, unknown it is. */ - if (onion_address[0] == '\0') { - goto end; - } - /* All checks are good so return the given onion address. */ - str_ret = onion_address; - - end: - return str_ret; -} - -/** send HS_DESC requested event. - * - * rend_query is used to fetch requested onion address and auth type. - * hs_dir is the description of contacting hs directory. - * desc_id_base32 is the ID of requested hs descriptor. - * hsdir_index is the HSDir fetch index value for v3, an hex string. - */ -void -control_event_hs_descriptor_requested(const char *onion_address, - rend_auth_type_t auth_type, - const char *id_digest, - const char *desc_id, - const char *hsdir_index) -{ - char *hsdir_index_field = NULL; - - if (BUG(!id_digest || !desc_id)) { - return; - } - - if (hsdir_index) { - tor_asprintf(&hsdir_index_field, " HSDIR_INDEX=%s", hsdir_index); - } - - send_control_event(EVENT_HS_DESC, - "650 HS_DESC REQUESTED %s %s %s %s%s\r\n", - rend_hsaddress_str_or_unknown(onion_address), - rend_auth_type_to_string(auth_type), - node_describe_longname_by_id(id_digest), - desc_id, - hsdir_index_field ? hsdir_index_field : ""); - tor_free(hsdir_index_field); -} - -/** For an HS descriptor query rend_data, using the - * onion_address and HSDir fingerprint hsdir_fp, find out - * which descriptor ID in the query is the right one. - * - * Return a pointer of the binary descriptor ID found in the query's object - * or NULL if not found. */ -static const char * -get_desc_id_from_query(const rend_data_t *rend_data, const char *hsdir_fp) -{ - int replica; - const char *desc_id = NULL; - const rend_data_v2_t *rend_data_v2 = TO_REND_DATA_V2(rend_data); - - /* Possible if the fetch was done using a descriptor ID. This means that - * the HSFETCH command was used. */ - if (!tor_digest_is_zero(rend_data_v2->desc_id_fetch)) { - desc_id = rend_data_v2->desc_id_fetch; - goto end; - } - - /* Without a directory fingerprint at this stage, we can't do much. */ - if (hsdir_fp == NULL) { - goto end; - } - - /* OK, we have an onion address so now let's find which descriptor ID - * is the one associated with the HSDir fingerprint. */ - for (replica = 0; replica < REND_NUMBER_OF_NON_CONSECUTIVE_REPLICAS; - replica++) { - const char *digest = rend_data_get_desc_id(rend_data, replica, NULL); - - SMARTLIST_FOREACH_BEGIN(rend_data->hsdirs_fp, char *, fingerprint) { - if (tor_memcmp(fingerprint, hsdir_fp, DIGEST_LEN) == 0) { - /* Found it! This descriptor ID is the right one. */ - desc_id = digest; - goto end; - } - } SMARTLIST_FOREACH_END(fingerprint); - } - - end: - return desc_id; -} - -/** send HS_DESC CREATED event when a local service generates a descriptor. - * - * onion_address is service address. - * desc_id is the descriptor ID. - * replica is the the descriptor replica number. If it is negative, it - * is ignored. - */ -void -control_event_hs_descriptor_created(const char *onion_address, - const char *desc_id, - int replica) -{ - char *replica_field = NULL; - - if (BUG(!onion_address || !desc_id)) { - return; - } - - if (replica >= 0) { - tor_asprintf(&replica_field, " REPLICA=%d", replica); - } - - send_control_event(EVENT_HS_DESC, - "650 HS_DESC CREATED %s UNKNOWN UNKNOWN %s%s\r\n", - onion_address, desc_id, - replica_field ? replica_field : ""); - tor_free(replica_field); -} - -/** send HS_DESC upload event. - * - * onion_address is service address. - * hs_dir is the description of contacting hs directory. - * desc_id is the ID of requested hs descriptor. - */ -void -control_event_hs_descriptor_upload(const char *onion_address, - const char *id_digest, - const char *desc_id, - const char *hsdir_index) -{ - char *hsdir_index_field = NULL; - - if (BUG(!onion_address || !id_digest || !desc_id)) { - return; - } - - if (hsdir_index) { - tor_asprintf(&hsdir_index_field, " HSDIR_INDEX=%s", hsdir_index); - } - - send_control_event(EVENT_HS_DESC, - "650 HS_DESC UPLOAD %s UNKNOWN %s %s%s\r\n", - onion_address, - node_describe_longname_by_id(id_digest), - desc_id, - hsdir_index_field ? hsdir_index_field : ""); - tor_free(hsdir_index_field); -} - -/** send HS_DESC event after got response from hs directory. - * - * NOTE: this is an internal function used by following functions: - * control_event_hsv2_descriptor_received - * control_event_hsv2_descriptor_failed - * control_event_hsv3_descriptor_failed - * - * So do not call this function directly. - */ -static void -event_hs_descriptor_receive_end(const char *action, - const char *onion_address, - const char *desc_id, - rend_auth_type_t auth_type, - const char *hsdir_id_digest, - const char *reason) -{ - char *reason_field = NULL; - - if (BUG(!action || !onion_address)) { - return; - } - - if (reason) { - tor_asprintf(&reason_field, " REASON=%s", reason); - } - - send_control_event(EVENT_HS_DESC, - "650 HS_DESC %s %s %s %s%s%s\r\n", - action, - rend_hsaddress_str_or_unknown(onion_address), - rend_auth_type_to_string(auth_type), - hsdir_id_digest ? - node_describe_longname_by_id(hsdir_id_digest) : - "UNKNOWN", - desc_id ? desc_id : "", - reason_field ? reason_field : ""); - - tor_free(reason_field); -} - -/** send HS_DESC event after got response from hs directory. - * - * NOTE: this is an internal function used by following functions: - * control_event_hs_descriptor_uploaded - * control_event_hs_descriptor_upload_failed - * - * So do not call this function directly. - */ -void -control_event_hs_descriptor_upload_end(const char *action, - const char *onion_address, - const char *id_digest, - const char *reason) -{ - char *reason_field = NULL; - - if (BUG(!action || !id_digest)) { - return; - } - - if (reason) { - tor_asprintf(&reason_field, " REASON=%s", reason); - } - - send_control_event(EVENT_HS_DESC, - "650 HS_DESC %s %s UNKNOWN %s%s\r\n", - action, - rend_hsaddress_str_or_unknown(onion_address), - node_describe_longname_by_id(id_digest), - reason_field ? reason_field : ""); - - tor_free(reason_field); -} - -/** send HS_DESC RECEIVED event - * - * called when we successfully received a hidden service descriptor. - */ -void -control_event_hsv2_descriptor_received(const char *onion_address, - const rend_data_t *rend_data, - const char *hsdir_id_digest) -{ - char *desc_id_field = NULL; - const char *desc_id; - - if (BUG(!rend_data || !hsdir_id_digest || !onion_address)) { - return; - } - - desc_id = get_desc_id_from_query(rend_data, hsdir_id_digest); - if (desc_id != NULL) { - char desc_id_base32[REND_DESC_ID_V2_LEN_BASE32 + 1]; - /* Set the descriptor ID digest to base32 so we can send it. */ - base32_encode(desc_id_base32, sizeof(desc_id_base32), desc_id, - DIGEST_LEN); - /* Extra whitespace is needed before the value. */ - tor_asprintf(&desc_id_field, " %s", desc_id_base32); - } - - event_hs_descriptor_receive_end("RECEIVED", onion_address, desc_id_field, - TO_REND_DATA_V2(rend_data)->auth_type, - hsdir_id_digest, NULL); - tor_free(desc_id_field); -} - -/* Send HS_DESC RECEIVED event - * - * Called when we successfully received a hidden service descriptor. */ -void -control_event_hsv3_descriptor_received(const char *onion_address, - const char *desc_id, - const char *hsdir_id_digest) -{ - char *desc_id_field = NULL; - - if (BUG(!onion_address || !desc_id || !hsdir_id_digest)) { - return; - } - - /* Because DescriptorID is an optional positional value, we need to add a - * whitespace before in order to not be next to the HsDir value. */ - tor_asprintf(&desc_id_field, " %s", desc_id); - - event_hs_descriptor_receive_end("RECEIVED", onion_address, desc_id_field, - REND_NO_AUTH, hsdir_id_digest, NULL); - tor_free(desc_id_field); -} - -/** send HS_DESC UPLOADED event - * - * called when we successfully uploaded a hidden service descriptor. - */ -void -control_event_hs_descriptor_uploaded(const char *id_digest, - const char *onion_address) -{ - if (BUG(!id_digest)) { - return; - } - - control_event_hs_descriptor_upload_end("UPLOADED", onion_address, - id_digest, NULL); -} - -/** Send HS_DESC event to inform controller that query rend_data - * failed to retrieve hidden service descriptor from directory identified by - * id_digest. If NULL, "UNKNOWN" is used. If reason is not NULL, - * add it to REASON= field. - */ -void -control_event_hsv2_descriptor_failed(const rend_data_t *rend_data, - const char *hsdir_id_digest, - const char *reason) -{ - char *desc_id_field = NULL; - const char *desc_id; - - if (BUG(!rend_data)) { - return; - } - - desc_id = get_desc_id_from_query(rend_data, hsdir_id_digest); - if (desc_id != NULL) { - char desc_id_base32[REND_DESC_ID_V2_LEN_BASE32 + 1]; - /* Set the descriptor ID digest to base32 so we can send it. */ - base32_encode(desc_id_base32, sizeof(desc_id_base32), desc_id, - DIGEST_LEN); - /* Extra whitespace is needed before the value. */ - tor_asprintf(&desc_id_field, " %s", desc_id_base32); - } - - event_hs_descriptor_receive_end("FAILED", rend_data_get_address(rend_data), - desc_id_field, - TO_REND_DATA_V2(rend_data)->auth_type, - hsdir_id_digest, reason); - tor_free(desc_id_field); -} - -/** Send HS_DESC event to inform controller that the query to - * onion_address failed to retrieve hidden service descriptor - * desc_id from directory identified by hsdir_id_digest. If - * NULL, "UNKNOWN" is used. If reason is not NULL, add it to REASON= - * field. */ -void -control_event_hsv3_descriptor_failed(const char *onion_address, - const char *desc_id, - const char *hsdir_id_digest, - const char *reason) -{ - char *desc_id_field = NULL; - - if (BUG(!onion_address || !desc_id || !reason)) { - return; - } - - /* Because DescriptorID is an optional positional value, we need to add a - * whitespace before in order to not be next to the HsDir value. */ - tor_asprintf(&desc_id_field, " %s", desc_id); - - event_hs_descriptor_receive_end("FAILED", onion_address, desc_id_field, - REND_NO_AUTH, hsdir_id_digest, reason); - tor_free(desc_id_field); -} - -/** Send HS_DESC_CONTENT event after completion of a successful fetch - * from hs directory. If hsdir_id_digest is NULL, it is replaced - * by "UNKNOWN". If content is NULL, it is replaced by an empty - * string. The onion_address or desc_id set to NULL will - * not trigger the control event. */ -void -control_event_hs_descriptor_content(const char *onion_address, - const char *desc_id, - const char *hsdir_id_digest, - const char *content) -{ - static const char *event_name = "HS_DESC_CONTENT"; - char *esc_content = NULL; - - if (!onion_address || !desc_id) { - log_warn(LD_BUG, "Called with onion_address==%p, desc_id==%p, ", - onion_address, desc_id); - return; - } - - if (content == NULL) { - /* Point it to empty content so it can still be escaped. */ - content = ""; - } - write_escaped_data(content, strlen(content), &esc_content); - - send_control_event(EVENT_HS_DESC_CONTENT, - "650+%s %s %s %s\r\n%s650 OK\r\n", - event_name, - rend_hsaddress_str_or_unknown(onion_address), - desc_id, - hsdir_id_digest ? - node_describe_longname_by_id(hsdir_id_digest) : - "UNKNOWN", - esc_content); - tor_free(esc_content); -} - -/** Send HS_DESC event to inform controller upload of hidden service - * descriptor identified by id_digest failed. If reason - * is not NULL, add it to REASON= field. - */ -void -control_event_hs_descriptor_upload_failed(const char *id_digest, - const char *onion_address, - const char *reason) -{ - if (BUG(!id_digest)) { - return; - } - control_event_hs_descriptor_upload_end("FAILED", onion_address, - id_digest, reason); -} - /** Free any leftover allocated memory of the control.c subsystem. */ void control_free_all(void) { - smartlist_t *queued_events = NULL; - - stats_prev_n_read = stats_prev_n_written = 0; + control_events_free_all(); if (authentication_cookie) /* Free the auth cookie */ tor_free(authentication_cookie); @@ -7559,34 +5075,6 @@ control_free_all(void) SMARTLIST_FOREACH(detached_onion_services, char *, cp, tor_free(cp)); smartlist_free(detached_onion_services); } - - if (queued_control_events_lock) { - tor_mutex_acquire(queued_control_events_lock); - flush_queued_event_pending = 0; - queued_events = queued_control_events; - queued_control_events = NULL; - tor_mutex_release(queued_control_events_lock); - } - if (queued_events) { - SMARTLIST_FOREACH(queued_events, queued_event_t *, ev, - queued_event_free(ev)); - smartlist_free(queued_events); - } - if (flush_queued_events_event) { - mainloop_event_free(flush_queued_events_event); - flush_queued_events_event = NULL; - } control_event_bootstrap_reset(); authentication_cookie_is_set = 0; - global_event_mask = 0; - disable_log_messages = 0; } - -#ifdef TOR_UNIT_TESTS -/* For testing: change the value of global_event_mask */ -void -control_testing_set_global_event_mask(uint64_t mask) -{ - global_event_mask = mask; -} -#endif /* defined(TOR_UNIT_TESTS) */ diff --git a/src/feature/control/control.h b/src/feature/control/control.h index b2ab4c1997..f9742d555e 100644 --- a/src/feature/control/control.h +++ b/src/feature/control/control.h @@ -12,84 +12,6 @@ #ifndef TOR_CONTROL_H #define TOR_CONTROL_H -#include "core/or/ocirc_event.h" - -/** Used to indicate the type of a CIRC_MINOR event passed to the controller. - * The various types are defined in control-spec.txt . */ -typedef enum circuit_status_minor_event_t { - CIRC_MINOR_EVENT_PURPOSE_CHANGED, - CIRC_MINOR_EVENT_CANNIBALIZED, -} circuit_status_minor_event_t; - -#include "core/or/orconn_event.h" - -/** Used to indicate the type of a stream event passed to the controller. - * The various types are defined in control-spec.txt */ -typedef enum stream_status_event_t { - STREAM_EVENT_SENT_CONNECT = 0, - STREAM_EVENT_SENT_RESOLVE = 1, - STREAM_EVENT_SUCCEEDED = 2, - STREAM_EVENT_FAILED = 3, - STREAM_EVENT_CLOSED = 4, - STREAM_EVENT_NEW = 5, - STREAM_EVENT_NEW_RESOLVE = 6, - STREAM_EVENT_FAILED_RETRIABLE = 7, - STREAM_EVENT_REMAP = 8 -} stream_status_event_t; - -/** Used to indicate the type of a buildtime event */ -typedef enum buildtimeout_set_event_t { - BUILDTIMEOUT_SET_EVENT_COMPUTED = 0, - BUILDTIMEOUT_SET_EVENT_RESET = 1, - BUILDTIMEOUT_SET_EVENT_SUSPENDED = 2, - BUILDTIMEOUT_SET_EVENT_DISCARD = 3, - BUILDTIMEOUT_SET_EVENT_RESUME = 4 -} buildtimeout_set_event_t; - -/** Enum describing various stages of bootstrapping, for use with controller - * bootstrap status events. The values range from 0 to 100. */ -typedef enum { - BOOTSTRAP_STATUS_UNDEF=-1, - BOOTSTRAP_STATUS_STARTING=0, - - /* Initial connection to any relay */ - - BOOTSTRAP_STATUS_CONN_PT=1, - BOOTSTRAP_STATUS_CONN_DONE_PT=2, - BOOTSTRAP_STATUS_CONN_PROXY=3, - BOOTSTRAP_STATUS_CONN_DONE_PROXY=4, - BOOTSTRAP_STATUS_CONN=5, - BOOTSTRAP_STATUS_CONN_DONE=10, - BOOTSTRAP_STATUS_HANDSHAKE=14, - BOOTSTRAP_STATUS_HANDSHAKE_DONE=15, - - /* Loading directory info */ - - BOOTSTRAP_STATUS_ONEHOP_CREATE=20, - BOOTSTRAP_STATUS_REQUESTING_STATUS=25, - BOOTSTRAP_STATUS_LOADING_STATUS=30, - BOOTSTRAP_STATUS_LOADING_KEYS=40, - BOOTSTRAP_STATUS_REQUESTING_DESCRIPTORS=45, - BOOTSTRAP_STATUS_LOADING_DESCRIPTORS=50, - BOOTSTRAP_STATUS_ENOUGH_DIRINFO=75, - - /* Connecting to a relay for AP circuits */ - - BOOTSTRAP_STATUS_AP_CONN_PT=76, - BOOTSTRAP_STATUS_AP_CONN_DONE_PT=77, - BOOTSTRAP_STATUS_AP_CONN_PROXY=78, - BOOTSTRAP_STATUS_AP_CONN_DONE_PROXY=79, - BOOTSTRAP_STATUS_AP_CONN=80, - BOOTSTRAP_STATUS_AP_CONN_DONE=85, - BOOTSTRAP_STATUS_AP_HANDSHAKE=89, - BOOTSTRAP_STATUS_AP_HANDSHAKE_DONE=90, - - /* Creating AP circuits */ - - BOOTSTRAP_STATUS_CIRCUIT_CREATE=95, - BOOTSTRAP_STATUS_DONE=100 -} bootstrap_status_t; - control_connection_t *TO_CONTROL_CONN(connection_t *); #define CONTROL_CONN_STATE_MIN_ 1 @@ -100,18 +22,6 @@ control_connection_t *TO_CONTROL_CONN(connection_t *); #define CONTROL_CONN_STATE_NEEDAUTH 2 #define CONTROL_CONN_STATE_MAX_ 2 -/** Reason for remapping an AP connection's address: we have a cached - * answer. */ -#define REMAP_STREAM_SOURCE_CACHE 1 -/** Reason for remapping an AP connection's address: the exit node told us an - * answer. */ -#define REMAP_STREAM_SOURCE_EXIT 2 - -void control_initialize_event_queue(void); - -void control_update_global_event_mask(void); -void control_adjust_event_log_severity(void); - void control_ports_write_to_file(void); /** Log information about the connection conn, protecting it as with @@ -132,65 +42,6 @@ void connection_control_closed(control_connection_t *conn); int connection_control_process_inbuf(control_connection_t *conn); -#define EVENT_NS 0x000F -int control_event_is_interesting(int event); - -void control_per_second_events(void); -int control_any_per_second_event_enabled(void); - -int control_event_circuit_status(origin_circuit_t *circ, - circuit_status_event_t e, int reason); -int control_event_circuit_purpose_changed(origin_circuit_t *circ, - int old_purpose); -int control_event_circuit_cannibalized(origin_circuit_t *circ, - int old_purpose, - const struct timeval *old_tv_created); -int control_event_stream_status(entry_connection_t *conn, - stream_status_event_t e, - int reason); -int control_event_or_conn_status(or_connection_t *conn, - or_conn_status_event_t e, int reason); -int control_event_bandwidth_used(uint32_t n_read, uint32_t n_written); -int control_event_stream_bandwidth(edge_connection_t *edge_conn); -int control_event_stream_bandwidth_used(void); -int control_event_circ_bandwidth_used(void); -int control_event_circ_bandwidth_used_for_circ(origin_circuit_t *ocirc); -int control_event_conn_bandwidth(connection_t *conn); -int control_event_conn_bandwidth_used(void); -int control_event_circuit_cell_stats(void); -void control_event_logmsg(int severity, uint32_t domain, const char *msg); -void control_event_logmsg_pending(void); -int control_event_descriptors_changed(smartlist_t *routers); -int control_event_address_mapped(const char *from, const char *to, - time_t expires, const char *error, - const int cached); -int control_event_my_descriptor_changed(void); -int control_event_network_liveness_update(int liveness); -int control_event_networkstatus_changed(smartlist_t *statuses); - -int control_event_newconsensus(const networkstatus_t *consensus); -int control_event_networkstatus_changed_single(const routerstatus_t *rs); -int control_event_general_status(int severity, const char *format, ...) - CHECK_PRINTF(2,3); -int control_event_client_status(int severity, const char *format, ...) - CHECK_PRINTF(2,3); -int control_event_server_status(int severity, const char *format, ...) - CHECK_PRINTF(2,3); - -int control_event_general_error(const char *format, ...) - CHECK_PRINTF(1,2); -int control_event_client_error(const char *format, ...) - CHECK_PRINTF(1,2); -int control_event_server_error(const char *format, ...) - CHECK_PRINTF(1,2); - -int control_event_guard(const char *nickname, const char *digest, - const char *status); -int control_event_conf_changed(const smartlist_t *elements); -int control_event_buildtimeout_set(buildtimeout_set_event_t type, - const char *args); -int control_event_signal(uintptr_t signal); - int init_control_cookie_authentication(int enabled); char *get_controller_cookie_file_name(void); struct config_line_t; @@ -200,176 +51,13 @@ void enable_control_logging(void); void monitor_owning_controller_process(const char *process_spec); -void control_event_bootstrap(bootstrap_status_t status, int progress); -MOCK_DECL(void, control_event_bootstrap_prob_or,(const char *warn, - int reason, - or_connection_t *or_conn)); -void control_event_boot_dir(bootstrap_status_t status, int progress); -void control_event_boot_first_orconn(void); -void control_event_bootstrap_problem(const char *warn, const char *reason, - const connection_t *conn, int dowarn); -char *control_event_boot_last_msg(void); -void control_event_bootstrap_reset(void); - -void control_event_clients_seen(const char *controller_str); -void control_event_transport_launched(const char *mode, - const char *transport_name, - tor_addr_t *addr, uint16_t port); -void control_event_pt_log(const char *log); -void control_event_pt_status(const char *status); const char *rend_auth_type_to_string(rend_auth_type_t auth_type); MOCK_DECL(const char *, node_describe_longname_by_id,(const char *id_digest)); -void control_event_hs_descriptor_requested(const char *onion_address, - rend_auth_type_t auth_type, - const char *id_digest, - const char *desc_id, - const char *hsdir_index); -void control_event_hs_descriptor_created(const char *onion_address, - const char *desc_id, - int replica); -void control_event_hs_descriptor_upload(const char *onion_address, - const char *desc_id, - const char *hs_dir, - const char *hsdir_index); -void control_event_hs_descriptor_upload_end(const char *action, - const char *onion_address, - const char *hs_dir, - const char *reason); -void control_event_hs_descriptor_uploaded(const char *hs_dir, - const char *onion_address); -/* Hidden service v2 HS_DESC specific. */ -void control_event_hsv2_descriptor_failed(const rend_data_t *rend_data, - const char *id_digest, - const char *reason); -void control_event_hsv2_descriptor_received(const char *onion_address, - const rend_data_t *rend_data, - const char *id_digest); -/* Hidden service v3 HS_DESC specific. */ -void control_event_hsv3_descriptor_failed(const char *onion_address, - const char *desc_id, - const char *hsdir_id_digest, - const char *reason); -void control_event_hsv3_descriptor_received(const char *onion_address, - const char *desc_id, - const char *hsdir_id_digest); -void control_event_hs_descriptor_upload_failed(const char *hs_dir, - const char *onion_address, - const char *reason); -void control_event_hs_descriptor_content(const char *onion_address, - const char *desc_id, - const char *hsdir_fp, - const char *content); void control_free_all(void); #ifdef CONTROL_PRIVATE #include "lib/crypt_ops/crypto_ed25519.h" -/* Recognized asynchronous event types. It's okay to expand this list - * because it is used both as a list of v0 event types, and as indices - * into the bitfield to determine which controllers want which events. - */ -/* This bitfield has no event zero 0x0000 */ -#define EVENT_MIN_ 0x0001 -#define EVENT_CIRCUIT_STATUS 0x0001 -#define EVENT_STREAM_STATUS 0x0002 -#define EVENT_OR_CONN_STATUS 0x0003 -#define EVENT_BANDWIDTH_USED 0x0004 -#define EVENT_CIRCUIT_STATUS_MINOR 0x0005 -#define EVENT_NEW_DESC 0x0006 -#define EVENT_DEBUG_MSG 0x0007 -#define EVENT_INFO_MSG 0x0008 -#define EVENT_NOTICE_MSG 0x0009 -#define EVENT_WARN_MSG 0x000A -#define EVENT_ERR_MSG 0x000B -#define EVENT_ADDRMAP 0x000C -/* There was an AUTHDIR_NEWDESCS event, but it no longer exists. We - can reclaim 0x000D. */ -#define EVENT_DESCCHANGED 0x000E -/* Exposed above */ -// #define EVENT_NS 0x000F -#define EVENT_STATUS_CLIENT 0x0010 -#define EVENT_STATUS_SERVER 0x0011 -#define EVENT_STATUS_GENERAL 0x0012 -#define EVENT_GUARD 0x0013 -#define EVENT_STREAM_BANDWIDTH_USED 0x0014 -#define EVENT_CLIENTS_SEEN 0x0015 -#define EVENT_NEWCONSENSUS 0x0016 -#define EVENT_BUILDTIMEOUT_SET 0x0017 -#define EVENT_GOT_SIGNAL 0x0018 -#define EVENT_CONF_CHANGED 0x0019 -#define EVENT_CONN_BW 0x001A -#define EVENT_CELL_STATS 0x001B -/* UNUSED : 0x001C */ -#define EVENT_CIRC_BANDWIDTH_USED 0x001D -#define EVENT_TRANSPORT_LAUNCHED 0x0020 -#define EVENT_HS_DESC 0x0021 -#define EVENT_HS_DESC_CONTENT 0x0022 -#define EVENT_NETWORK_LIVENESS 0x0023 -#define EVENT_PT_LOG 0x0024 -#define EVENT_PT_STATUS 0x0025 -#define EVENT_MAX_ 0x0025 - -/* sizeof(control_connection_t.event_mask) in bits, currently a uint64_t */ -#define EVENT_CAPACITY_ 0x0040 - -/* If EVENT_MAX_ ever hits 0x0040, we need to make the mask into a - * different structure, as it can only handle a maximum left shift of 1<<63. */ - -#if EVENT_MAX_ >= EVENT_CAPACITY_ -#error control_connection_t.event_mask has an event greater than its capacity -#endif - -#define EVENT_MASK_(e) (((uint64_t)1)<<(e)) - -#define EVENT_MASK_NONE_ ((uint64_t)0x0) - -#define EVENT_MASK_ABOVE_MIN_ ((~((uint64_t)0x0)) << EVENT_MIN_) -#define EVENT_MASK_BELOW_MAX_ ((~((uint64_t)0x0)) \ - >> (EVENT_CAPACITY_ - EVENT_MAX_ \ - - EVENT_MIN_)) - -#define EVENT_MASK_ALL_ (EVENT_MASK_ABOVE_MIN_ \ - & EVENT_MASK_BELOW_MAX_) - -/* Used only by control.c and test.c */ -STATIC size_t write_escaped_data(const char *data, size_t len, char **out); -STATIC size_t read_escaped_data(const char *data, size_t len, char **out); - -#ifdef TOR_UNIT_TESTS -MOCK_DECL(STATIC void, - send_control_event_string,(uint16_t event, const char *msg)); - -MOCK_DECL(STATIC void, - queue_control_event_string,(uint16_t event, char *msg)); - -void control_testing_set_global_event_mask(uint64_t mask); -#endif /* defined(TOR_UNIT_TESTS) */ - -/** Helper structure: temporarily stores cell statistics for a circuit. */ -typedef struct cell_stats_t { - /** Number of cells added in app-ward direction by command. */ - uint64_t added_cells_appward[CELL_COMMAND_MAX_ + 1]; - /** Number of cells added in exit-ward direction by command. */ - uint64_t added_cells_exitward[CELL_COMMAND_MAX_ + 1]; - /** Number of cells removed in app-ward direction by command. */ - uint64_t removed_cells_appward[CELL_COMMAND_MAX_ + 1]; - /** Number of cells removed in exit-ward direction by command. */ - uint64_t removed_cells_exitward[CELL_COMMAND_MAX_ + 1]; - /** Total waiting time of cells in app-ward direction by command. */ - uint64_t total_time_appward[CELL_COMMAND_MAX_ + 1]; - /** Total waiting time of cells in exit-ward direction by command. */ - uint64_t total_time_exitward[CELL_COMMAND_MAX_ + 1]; -} cell_stats_t; -void sum_up_cell_stats_by_command(circuit_t *circ, - cell_stats_t *cell_stats); -void append_cell_stats_by_command(smartlist_t *event_parts, - const char *key, - const uint64_t *include_if_non_zero, - const uint64_t *number_to_include); -void format_cell_stats(char **event_string, circuit_t *circ, - cell_stats_t *cell_stats); -STATIC char *get_bw_samples(void); - /* ADD_ONION secret key to create an ephemeral service. The command supports * multiple versions so this union stores the key and passes it to the HS * subsystem depending on the requested version. */ @@ -428,4 +116,9 @@ STATIC int getinfo_helper_current_time( #endif /* defined(CONTROL_PRIVATE) */ +#ifdef CONTROL_MODULE_PRIVATE +int get_cached_network_liveness(void); +void set_cached_network_liveness(int liveness); +#endif /* defined(CONTROL_MODULE_PRIVATE) */ + #endif /* !defined(TOR_CONTROL_H) */ diff --git a/src/feature/control/control_bootstrap.c b/src/feature/control/control_bootstrap.c index 8153d7595a..098e24682e 100644 --- a/src/feature/control/control_bootstrap.c +++ b/src/feature/control/control_bootstrap.c @@ -14,7 +14,7 @@ #include "core/or/connection_st.h" #include "core/or/or_connection_st.h" #include "core/or/reasons.h" -#include "feature/control/control.h" +#include "feature/control/control_events.h" #include "feature/hibernate/hibernate.h" #include "lib/malloc/malloc.h" diff --git a/src/feature/control/control_events.c b/src/feature/control/control_events.c new file mode 100644 index 0000000000..07e1a9d94a --- /dev/null +++ b/src/feature/control/control_events.c @@ -0,0 +1,2280 @@ +/* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file control_events.c + * \brief Implement the event-reporting part of the controller API. + **/ + +#define CONTROL_MODULE_PRIVATE +#define CONTROL_EVENTS_PRIVATE +#define OCIRC_EVENT_PRIVATE + +#include "core/or/or.h" +#include "app/config/config.h" +#include "core/mainloop/connection.h" +#include "core/mainloop/mainloop.h" +#include "core/or/channeltls.h" +#include "core/or/circuitlist.h" +#include "core/or/command.h" +#include "core/or/connection_edge.h" +#include "core/or/connection_or.h" +#include "core/or/reasons.h" +#include "feature/control/control.h" +#include "feature/control/control_events.h" +#include "feature/control/control_fmt.h" +#include "feature/dircommon/directory.h" +#include "feature/nodelist/networkstatus.h" +#include "feature/nodelist/nodelist.h" +#include "feature/nodelist/routerinfo.h" + +#include "feature/control/control_connection_st.h" +#include "core/or/entry_connection_st.h" +#include "feature/nodelist/networkstatus_st.h" +#include "core/or/or_connection_st.h" +#include "core/or/or_circuit_st.h" +#include "core/or/origin_circuit_st.h" + +#include "lib/evloop/compat_libevent.h" + +static void flush_queued_events_cb(mainloop_event_t *event, void *arg); +static void control_get_bytes_rw_last_sec(uint64_t *r, uint64_t *w); + +/** Yield true iff s is the state of a control_connection_t that has + * finished authentication and is accepting commands. */ +#define STATE_IS_OPEN(s) ((s) == CONTROL_CONN_STATE_OPEN) + +/** An event mask of all the events that any controller is interested in + * receiving. */ +static event_mask_t global_event_mask = 0; + +/** True iff we have disabled log messages from being sent to the controller */ +static int disable_log_messages = 0; + +/** Macro: true if any control connection is interested in events of type + * e. */ +#define EVENT_IS_INTERESTING(e) \ + (!! (global_event_mask & EVENT_MASK_(e))) + +/** Macro: true if any event from the bitfield 'e' is interesting. */ +#define ANY_EVENT_IS_INTERESTING(e) \ + (!! (global_event_mask & (e))) + +static void send_control_event_impl(uint16_t event, + const char *format, va_list ap) + CHECK_PRINTF(2,0); +static int control_event_status(int type, int severity, const char *format, + va_list args) + CHECK_PRINTF(3,0); + +static void send_control_event(uint16_t event, + const char *format, ...) + CHECK_PRINTF(2,3); + +/** Given a log severity, return the corresponding control event code. */ +static inline int +log_severity_to_event(int severity) +{ + switch (severity) { + case LOG_DEBUG: return EVENT_DEBUG_MSG; + case LOG_INFO: return EVENT_INFO_MSG; + case LOG_NOTICE: return EVENT_NOTICE_MSG; + case LOG_WARN: return EVENT_WARN_MSG; + case LOG_ERR: return EVENT_ERR_MSG; + default: return -1; + } +} + +/** Helper: clear bandwidth counters of all origin circuits. */ +static void +clear_circ_bw_fields(void) +{ + origin_circuit_t *ocirc; + SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) { + if (!CIRCUIT_IS_ORIGIN(circ)) + continue; + ocirc = TO_ORIGIN_CIRCUIT(circ); + ocirc->n_written_circ_bw = ocirc->n_read_circ_bw = 0; + ocirc->n_overhead_written_circ_bw = ocirc->n_overhead_read_circ_bw = 0; + ocirc->n_delivered_written_circ_bw = ocirc->n_delivered_read_circ_bw = 0; + } + SMARTLIST_FOREACH_END(circ); +} + +/** Set global_event_mask* to the bitwise OR of each live control + * connection's event_mask field. */ +void +control_update_global_event_mask(void) +{ + smartlist_t *conns = get_connection_array(); + event_mask_t old_mask, new_mask; + old_mask = global_event_mask; + int any_old_per_sec_events = control_any_per_second_event_enabled(); + + global_event_mask = 0; + SMARTLIST_FOREACH(conns, connection_t *, _conn, + { + if (_conn->type == CONN_TYPE_CONTROL && + STATE_IS_OPEN(_conn->state)) { + control_connection_t *conn = TO_CONTROL_CONN(_conn); + global_event_mask |= conn->event_mask; + } + }); + + new_mask = global_event_mask; + + /* Handle the aftermath. Set up the log callback to tell us only what + * we want to hear...*/ + control_adjust_event_log_severity(); + + /* Macro: true if ev was false before and is true now. */ +#define NEWLY_ENABLED(ev) \ + (! (old_mask & (ev)) && (new_mask & (ev))) + + /* ...then, if we've started logging stream or circ bw, clear the + * appropriate fields. */ + if (NEWLY_ENABLED(EVENT_STREAM_BANDWIDTH_USED)) { + SMARTLIST_FOREACH(conns, connection_t *, conn, + { + if (conn->type == CONN_TYPE_AP) { + edge_connection_t *edge_conn = TO_EDGE_CONN(conn); + edge_conn->n_written = edge_conn->n_read = 0; + } + }); + } + if (NEWLY_ENABLED(EVENT_CIRC_BANDWIDTH_USED)) { + clear_circ_bw_fields(); + } + if (NEWLY_ENABLED(EVENT_BANDWIDTH_USED)) { + uint64_t r, w; + control_get_bytes_rw_last_sec(&r, &w); + } + if (any_old_per_sec_events != control_any_per_second_event_enabled()) { + rescan_periodic_events(get_options()); + } + +#undef NEWLY_ENABLED +} + +/** Given a control event code for a message event, return the corresponding + * log severity. */ +static inline int +event_to_log_severity(int event) +{ + switch (event) { + case EVENT_DEBUG_MSG: return LOG_DEBUG; + case EVENT_INFO_MSG: return LOG_INFO; + case EVENT_NOTICE_MSG: return LOG_NOTICE; + case EVENT_WARN_MSG: return LOG_WARN; + case EVENT_ERR_MSG: return LOG_ERR; + default: return -1; + } +} + +/** Adjust the log severities that result in control_event_logmsg being called + * to match the severity of log messages that any controllers are interested + * in. */ +void +control_adjust_event_log_severity(void) +{ + int i; + int min_log_event=EVENT_ERR_MSG, max_log_event=EVENT_DEBUG_MSG; + + for (i = EVENT_DEBUG_MSG; i <= EVENT_ERR_MSG; ++i) { + if (EVENT_IS_INTERESTING(i)) { + min_log_event = i; + break; + } + } + for (i = EVENT_ERR_MSG; i >= EVENT_DEBUG_MSG; --i) { + if (EVENT_IS_INTERESTING(i)) { + max_log_event = i; + break; + } + } + if (EVENT_IS_INTERESTING(EVENT_STATUS_GENERAL)) { + if (min_log_event > EVENT_NOTICE_MSG) + min_log_event = EVENT_NOTICE_MSG; + if (max_log_event < EVENT_ERR_MSG) + max_log_event = EVENT_ERR_MSG; + } + if (min_log_event <= max_log_event) + change_callback_log_severity(event_to_log_severity(min_log_event), + event_to_log_severity(max_log_event), + control_event_logmsg); + else + change_callback_log_severity(LOG_ERR, LOG_ERR, + control_event_logmsg); +} + +/** Return true iff the event with code c is being sent to any current + * control connection. This is useful if the amount of work needed to prepare + * to call the appropriate control_event_...() function is high. + */ +int +control_event_is_interesting(int event) +{ + return EVENT_IS_INTERESTING(event); +} + +/** Return true if any event that needs to fire once a second is enabled. */ +int +control_any_per_second_event_enabled(void) +{ + return ANY_EVENT_IS_INTERESTING( + EVENT_MASK_(EVENT_BANDWIDTH_USED) | + EVENT_MASK_(EVENT_CELL_STATS) | + EVENT_MASK_(EVENT_CIRC_BANDWIDTH_USED) | + EVENT_MASK_(EVENT_CONN_BW) | + EVENT_MASK_(EVENT_STREAM_BANDWIDTH_USED) + ); +} + +/* The value of 'get_bytes_read()' the previous time that + * control_get_bytes_rw_last_sec() as called. */ +static uint64_t stats_prev_n_read = 0; +/* The value of 'get_bytes_written()' the previous time that + * control_get_bytes_rw_last_sec() as called. */ +static uint64_t stats_prev_n_written = 0; + +/** + * Set n_read and n_written to the total number of bytes read + * and written by Tor since the last call to this function. + * + * Call this only from the main thread. + */ +static void +control_get_bytes_rw_last_sec(uint64_t *n_read, + uint64_t *n_written) +{ + const uint64_t stats_n_bytes_read = get_bytes_read(); + const uint64_t stats_n_bytes_written = get_bytes_written(); + + *n_read = stats_n_bytes_read - stats_prev_n_read; + *n_written = stats_n_bytes_written - stats_prev_n_written; + stats_prev_n_read = stats_n_bytes_read; + stats_prev_n_written = stats_n_bytes_written; +} + +/** + * Run all the controller events (if any) that are scheduled to trigger once + * per second. + */ +void +control_per_second_events(void) +{ + if (!control_any_per_second_event_enabled()) + return; + + uint64_t bytes_read, bytes_written; + control_get_bytes_rw_last_sec(&bytes_read, &bytes_written); + control_event_bandwidth_used((uint32_t)bytes_read,(uint32_t)bytes_written); + + control_event_stream_bandwidth_used(); + control_event_conn_bandwidth_used(); + control_event_circ_bandwidth_used(); + control_event_circuit_cell_stats(); +} + +/** Represents an event that's queued to be sent to one or more + * controllers. */ +typedef struct queued_event_s { + uint16_t event; + char *msg; +} queued_event_t; + +/** Pointer to int. If this is greater than 0, we don't allow new events to be + * queued. */ +static tor_threadlocal_t block_event_queue_flag; + +/** Holds a smartlist of queued_event_t objects that may need to be sent + * to one or more controllers */ +static smartlist_t *queued_control_events = NULL; + +/** True if the flush_queued_events_event is pending. */ +static int flush_queued_event_pending = 0; + +/** Lock to protect the above fields. */ +static tor_mutex_t *queued_control_events_lock = NULL; + +/** An event that should fire in order to flush the contents of + * queued_control_events. */ +static mainloop_event_t *flush_queued_events_event = NULL; + +void +control_initialize_event_queue(void) +{ + if (queued_control_events == NULL) { + queued_control_events = smartlist_new(); + } + + if (flush_queued_events_event == NULL) { + struct event_base *b = tor_libevent_get_base(); + if (b) { + flush_queued_events_event = + mainloop_event_new(flush_queued_events_cb, NULL); + tor_assert(flush_queued_events_event); + } + } + + if (queued_control_events_lock == NULL) { + queued_control_events_lock = tor_mutex_new(); + tor_threadlocal_init(&block_event_queue_flag); + } +} + +static int * +get_block_event_queue(void) +{ + int *val = tor_threadlocal_get(&block_event_queue_flag); + if (PREDICT_UNLIKELY(val == NULL)) { + val = tor_malloc_zero(sizeof(int)); + tor_threadlocal_set(&block_event_queue_flag, val); + } + return val; +} + +/** Helper: inserts an event on the list of events queued to be sent to + * one or more controllers, and schedules the events to be flushed if needed. + * + * This function takes ownership of msg, and may free it. + * + * We queue these events rather than send them immediately in order to break + * the dependency in our callgraph from code that generates events for the + * controller, and the network layer at large. Otherwise, nearly every + * interesting part of Tor would potentially call every other interesting part + * of Tor. + */ +MOCK_IMPL(STATIC void, +queue_control_event_string,(uint16_t event, char *msg)) +{ + /* This is redundant with checks done elsewhere, but it's a last-ditch + * attempt to avoid queueing something we shouldn't have to queue. */ + if (PREDICT_UNLIKELY( ! EVENT_IS_INTERESTING(event) )) { + tor_free(msg); + return; + } + + int *block_event_queue = get_block_event_queue(); + if (*block_event_queue) { + tor_free(msg); + return; + } + + queued_event_t *ev = tor_malloc(sizeof(*ev)); + ev->event = event; + ev->msg = msg; + + /* No queueing an event while queueing an event */ + ++*block_event_queue; + + tor_mutex_acquire(queued_control_events_lock); + tor_assert(queued_control_events); + smartlist_add(queued_control_events, ev); + + int activate_event = 0; + if (! flush_queued_event_pending && in_main_thread()) { + activate_event = 1; + flush_queued_event_pending = 1; + } + + tor_mutex_release(queued_control_events_lock); + + --*block_event_queue; + + /* We just put an event on the queue; mark the queue to be + * flushed. We only do this from the main thread for now; otherwise, + * we'd need to incur locking overhead in Libevent or use a socket. + */ + if (activate_event) { + tor_assert(flush_queued_events_event); + mainloop_event_activate(flush_queued_events_event); + } +} + +#define queued_event_free(ev) \ + FREE_AND_NULL(queued_event_t, queued_event_free_, (ev)) + +/** Release all storage held by ev. */ +static void +queued_event_free_(queued_event_t *ev) +{ + if (ev == NULL) + return; + + tor_free(ev->msg); + tor_free(ev); +} + +/** Send every queued event to every controller that's interested in it, + * and remove the events from the queue. If force is true, + * then make all controllers send their data out immediately, since we + * may be about to shut down. */ +static void +queued_events_flush_all(int force) +{ + /* Make sure that we get all the pending log events, if there are any. */ + flush_pending_log_callbacks(); + + if (PREDICT_UNLIKELY(queued_control_events == NULL)) { + return; + } + smartlist_t *all_conns = get_connection_array(); + smartlist_t *controllers = smartlist_new(); + smartlist_t *queued_events; + + int *block_event_queue = get_block_event_queue(); + ++*block_event_queue; + + tor_mutex_acquire(queued_control_events_lock); + /* No queueing an event while flushing events. */ + flush_queued_event_pending = 0; + queued_events = queued_control_events; + queued_control_events = smartlist_new(); + tor_mutex_release(queued_control_events_lock); + + /* Gather all the controllers that will care... */ + SMARTLIST_FOREACH_BEGIN(all_conns, connection_t *, conn) { + if (conn->type == CONN_TYPE_CONTROL && + !conn->marked_for_close && + conn->state == CONTROL_CONN_STATE_OPEN) { + control_connection_t *control_conn = TO_CONTROL_CONN(conn); + + smartlist_add(controllers, control_conn); + } + } SMARTLIST_FOREACH_END(conn); + + SMARTLIST_FOREACH_BEGIN(queued_events, queued_event_t *, ev) { + const event_mask_t bit = ((event_mask_t)1) << ev->event; + const size_t msg_len = strlen(ev->msg); + SMARTLIST_FOREACH_BEGIN(controllers, control_connection_t *, + control_conn) { + if (control_conn->event_mask & bit) { + connection_buf_add(ev->msg, msg_len, TO_CONN(control_conn)); + } + } SMARTLIST_FOREACH_END(control_conn); + + queued_event_free(ev); + } SMARTLIST_FOREACH_END(ev); + + if (force) { + SMARTLIST_FOREACH_BEGIN(controllers, control_connection_t *, + control_conn) { + connection_flush(TO_CONN(control_conn)); + } SMARTLIST_FOREACH_END(control_conn); + } + + smartlist_free(queued_events); + smartlist_free(controllers); + + --*block_event_queue; +} + +/** Libevent callback: Flushes pending events to controllers that are + * interested in them. */ +static void +flush_queued_events_cb(mainloop_event_t *event, void *arg) +{ + (void) event; + (void) arg; + queued_events_flush_all(0); +} + +/** Send an event to all v1 controllers that are listening for code + * event. The event's body is given by msg. + * + * The EXTENDED_FORMAT and NONEXTENDED_FORMAT flags behave similarly with + * respect to the EXTENDED_EVENTS feature. */ +MOCK_IMPL(STATIC void, +send_control_event_string,(uint16_t event, + const char *msg)) +{ + tor_assert(event >= EVENT_MIN_ && event <= EVENT_MAX_); + queue_control_event_string(event, tor_strdup(msg)); +} + +/** Helper for send_control_event and control_event_status: + * Send an event to all v1 controllers that are listening for code + * event. The event's body is created by the printf-style format in + * format, and other arguments as provided. */ +static void +send_control_event_impl(uint16_t event, + const char *format, va_list ap) +{ + char *buf = NULL; + int len; + + len = tor_vasprintf(&buf, format, ap); + if (len < 0) { + log_warn(LD_BUG, "Unable to format event for controller."); + return; + } + + queue_control_event_string(event, buf); +} + +/** Send an event to all v1 controllers that are listening for code + * event. The event's body is created by the printf-style format in + * format, and other arguments as provided. */ +static void +send_control_event(uint16_t event, + const char *format, ...) +{ + va_list ap; + va_start(ap, format); + send_control_event_impl(event, format, ap); + va_end(ap); +} + +/** Something major has happened to circuit circ: tell any + * interested control connections. */ +int +control_event_circuit_status(origin_circuit_t *circ, circuit_status_event_t tp, + int reason_code) +{ + const char *status; + char reasons[64] = ""; + + if (!EVENT_IS_INTERESTING(EVENT_CIRCUIT_STATUS)) + return 0; + tor_assert(circ); + + switch (tp) + { + case CIRC_EVENT_LAUNCHED: status = "LAUNCHED"; break; + case CIRC_EVENT_BUILT: status = "BUILT"; break; + case CIRC_EVENT_EXTENDED: status = "EXTENDED"; break; + case CIRC_EVENT_FAILED: status = "FAILED"; break; + case CIRC_EVENT_CLOSED: status = "CLOSED"; break; + default: + log_warn(LD_BUG, "Unrecognized status code %d", (int)tp); + tor_fragile_assert(); + return 0; + } + + if (tp == CIRC_EVENT_FAILED || tp == CIRC_EVENT_CLOSED) { + const char *reason_str = circuit_end_reason_to_control_string(reason_code); + char unk_reason_buf[16]; + if (!reason_str) { + tor_snprintf(unk_reason_buf, 16, "UNKNOWN_%d", reason_code); + reason_str = unk_reason_buf; + } + if (reason_code > 0 && reason_code & END_CIRC_REASON_FLAG_REMOTE) { + tor_snprintf(reasons, sizeof(reasons), + " REASON=DESTROYED REMOTE_REASON=%s", reason_str); + } else { + tor_snprintf(reasons, sizeof(reasons), + " REASON=%s", reason_str); + } + } + + { + char *circdesc = circuit_describe_status_for_controller(circ); + const char *sp = strlen(circdesc) ? " " : ""; + send_control_event(EVENT_CIRCUIT_STATUS, + "650 CIRC %lu %s%s%s%s\r\n", + (unsigned long)circ->global_identifier, + status, sp, + circdesc, + reasons); + tor_free(circdesc); + } + + return 0; +} + +/** Something minor has happened to circuit circ: tell any + * interested control connections. */ +static int +control_event_circuit_status_minor(origin_circuit_t *circ, + circuit_status_minor_event_t e, + int purpose, const struct timeval *tv) +{ + const char *event_desc; + char event_tail[160] = ""; + if (!EVENT_IS_INTERESTING(EVENT_CIRCUIT_STATUS_MINOR)) + return 0; + tor_assert(circ); + + switch (e) + { + case CIRC_MINOR_EVENT_PURPOSE_CHANGED: + event_desc = "PURPOSE_CHANGED"; + + { + /* event_tail can currently be up to 68 chars long */ + const char *hs_state_str = + circuit_purpose_to_controller_hs_state_string(purpose); + tor_snprintf(event_tail, sizeof(event_tail), + " OLD_PURPOSE=%s%s%s", + circuit_purpose_to_controller_string(purpose), + (hs_state_str != NULL) ? " OLD_HS_STATE=" : "", + (hs_state_str != NULL) ? hs_state_str : ""); + } + + break; + case CIRC_MINOR_EVENT_CANNIBALIZED: + event_desc = "CANNIBALIZED"; + + { + /* event_tail can currently be up to 130 chars long */ + const char *hs_state_str = + circuit_purpose_to_controller_hs_state_string(purpose); + const struct timeval *old_timestamp_began = tv; + char tbuf[ISO_TIME_USEC_LEN+1]; + format_iso_time_nospace_usec(tbuf, old_timestamp_began); + + tor_snprintf(event_tail, sizeof(event_tail), + " OLD_PURPOSE=%s%s%s OLD_TIME_CREATED=%s", + circuit_purpose_to_controller_string(purpose), + (hs_state_str != NULL) ? " OLD_HS_STATE=" : "", + (hs_state_str != NULL) ? hs_state_str : "", + tbuf); + } + + break; + default: + log_warn(LD_BUG, "Unrecognized status code %d", (int)e); + tor_fragile_assert(); + return 0; + } + + { + char *circdesc = circuit_describe_status_for_controller(circ); + const char *sp = strlen(circdesc) ? " " : ""; + send_control_event(EVENT_CIRCUIT_STATUS_MINOR, + "650 CIRC_MINOR %lu %s%s%s%s\r\n", + (unsigned long)circ->global_identifier, + event_desc, sp, + circdesc, + event_tail); + tor_free(circdesc); + } + + return 0; +} + +/** + * circ has changed its purpose from old_purpose: tell any + * interested controllers. + */ +int +control_event_circuit_purpose_changed(origin_circuit_t *circ, + int old_purpose) +{ + return control_event_circuit_status_minor(circ, + CIRC_MINOR_EVENT_PURPOSE_CHANGED, + old_purpose, + NULL); +} + +/** + * circ has changed its purpose from old_purpose, and its + * created-time from old_tv_created: tell any interested controllers. + */ +int +control_event_circuit_cannibalized(origin_circuit_t *circ, + int old_purpose, + const struct timeval *old_tv_created) +{ + return control_event_circuit_status_minor(circ, + CIRC_MINOR_EVENT_CANNIBALIZED, + old_purpose, + old_tv_created); +} + +/** Something has happened to the stream associated with AP connection + * conn: tell any interested control connections. */ +int +control_event_stream_status(entry_connection_t *conn, stream_status_event_t tp, + int reason_code) +{ + char reason_buf[64]; + char addrport_buf[64]; + const char *status; + circuit_t *circ; + origin_circuit_t *origin_circ = NULL; + char buf[256]; + const char *purpose = ""; + tor_assert(conn->socks_request); + + if (!EVENT_IS_INTERESTING(EVENT_STREAM_STATUS)) + return 0; + + if (tp == STREAM_EVENT_CLOSED && + (reason_code & END_STREAM_REASON_FLAG_ALREADY_SENT_CLOSED)) + return 0; + + write_stream_target_to_buf(conn, buf, sizeof(buf)); + + reason_buf[0] = '\0'; + switch (tp) + { + case STREAM_EVENT_SENT_CONNECT: status = "SENTCONNECT"; break; + case STREAM_EVENT_SENT_RESOLVE: status = "SENTRESOLVE"; break; + case STREAM_EVENT_SUCCEEDED: status = "SUCCEEDED"; break; + case STREAM_EVENT_FAILED: status = "FAILED"; break; + case STREAM_EVENT_CLOSED: status = "CLOSED"; break; + case STREAM_EVENT_NEW: status = "NEW"; break; + case STREAM_EVENT_NEW_RESOLVE: status = "NEWRESOLVE"; break; + case STREAM_EVENT_FAILED_RETRIABLE: status = "DETACHED"; break; + case STREAM_EVENT_REMAP: status = "REMAP"; break; + default: + log_warn(LD_BUG, "Unrecognized status code %d", (int)tp); + return 0; + } + if (reason_code && (tp == STREAM_EVENT_FAILED || + tp == STREAM_EVENT_CLOSED || + tp == STREAM_EVENT_FAILED_RETRIABLE)) { + const char *reason_str = stream_end_reason_to_control_string(reason_code); + char *r = NULL; + if (!reason_str) { + tor_asprintf(&r, " UNKNOWN_%d", reason_code); + reason_str = r; + } + if (reason_code & END_STREAM_REASON_FLAG_REMOTE) + tor_snprintf(reason_buf, sizeof(reason_buf), + " REASON=END REMOTE_REASON=%s", reason_str); + else + tor_snprintf(reason_buf, sizeof(reason_buf), + " REASON=%s", reason_str); + tor_free(r); + } else if (reason_code && tp == STREAM_EVENT_REMAP) { + switch (reason_code) { + case REMAP_STREAM_SOURCE_CACHE: + strlcpy(reason_buf, " SOURCE=CACHE", sizeof(reason_buf)); + break; + case REMAP_STREAM_SOURCE_EXIT: + strlcpy(reason_buf, " SOURCE=EXIT", sizeof(reason_buf)); + break; + default: + tor_snprintf(reason_buf, sizeof(reason_buf), " REASON=UNKNOWN_%d", + reason_code); + /* XXX do we want SOURCE=UNKNOWN_%d above instead? -RD */ + break; + } + } + + if (tp == STREAM_EVENT_NEW || tp == STREAM_EVENT_NEW_RESOLVE) { + /* + * When the control conn is an AF_UNIX socket and we have no address, + * it gets set to "(Tor_internal)"; see dnsserv_launch_request() in + * dnsserv.c. + */ + if (strcmp(ENTRY_TO_CONN(conn)->address, "(Tor_internal)") != 0) { + tor_snprintf(addrport_buf,sizeof(addrport_buf), " SOURCE_ADDR=%s:%d", + ENTRY_TO_CONN(conn)->address, ENTRY_TO_CONN(conn)->port); + } else { + /* + * else leave it blank so control on AF_UNIX doesn't need to make + * something up. + */ + addrport_buf[0] = '\0'; + } + } else { + addrport_buf[0] = '\0'; + } + + if (tp == STREAM_EVENT_NEW_RESOLVE) { + purpose = " PURPOSE=DNS_REQUEST"; + } else if (tp == STREAM_EVENT_NEW) { + if (conn->use_begindir) { + connection_t *linked = ENTRY_TO_CONN(conn)->linked_conn; + int linked_dir_purpose = -1; + if (linked && linked->type == CONN_TYPE_DIR) + linked_dir_purpose = linked->purpose; + if (DIR_PURPOSE_IS_UPLOAD(linked_dir_purpose)) + purpose = " PURPOSE=DIR_UPLOAD"; + else + purpose = " PURPOSE=DIR_FETCH"; + } else + purpose = " PURPOSE=USER"; + } + + circ = circuit_get_by_edge_conn(ENTRY_TO_EDGE_CONN(conn)); + if (circ && CIRCUIT_IS_ORIGIN(circ)) + origin_circ = TO_ORIGIN_CIRCUIT(circ); + send_control_event(EVENT_STREAM_STATUS, + "650 STREAM %"PRIu64" %s %lu %s%s%s%s\r\n", + (ENTRY_TO_CONN(conn)->global_identifier), + status, + origin_circ? + (unsigned long)origin_circ->global_identifier : 0ul, + buf, reason_buf, addrport_buf, purpose); + + /* XXX need to specify its intended exit, etc? */ + + return 0; +} + +/** Called when the status of an OR connection conn changes: tell any + * interested control connections. tp is the new status for the + * connection. If conn has just closed or failed, then reason + * may be the reason why. + */ +int +control_event_or_conn_status(or_connection_t *conn, or_conn_status_event_t tp, + int reason) +{ + int ncircs = 0; + const char *status; + char name[128]; + char ncircs_buf[32] = {0}; /* > 8 + log10(2^32)=10 + 2 */ + + if (!EVENT_IS_INTERESTING(EVENT_OR_CONN_STATUS)) + return 0; + + switch (tp) + { + case OR_CONN_EVENT_LAUNCHED: status = "LAUNCHED"; break; + case OR_CONN_EVENT_CONNECTED: status = "CONNECTED"; break; + case OR_CONN_EVENT_FAILED: status = "FAILED"; break; + case OR_CONN_EVENT_CLOSED: status = "CLOSED"; break; + case OR_CONN_EVENT_NEW: status = "NEW"; break; + default: + log_warn(LD_BUG, "Unrecognized status code %d", (int)tp); + return 0; + } + if (conn->chan) { + ncircs = circuit_count_pending_on_channel(TLS_CHAN_TO_BASE(conn->chan)); + } else { + ncircs = 0; + } + ncircs += connection_or_get_num_circuits(conn); + if (ncircs && (tp == OR_CONN_EVENT_FAILED || tp == OR_CONN_EVENT_CLOSED)) { + tor_snprintf(ncircs_buf, sizeof(ncircs_buf), " NCIRCS=%d", ncircs); + } + + orconn_target_get_name(name, sizeof(name), conn); + send_control_event(EVENT_OR_CONN_STATUS, + "650 ORCONN %s %s%s%s%s ID=%"PRIu64"\r\n", + name, status, + reason ? " REASON=" : "", + orconn_end_reason_to_control_string(reason), + ncircs_buf, + (conn->base_.global_identifier)); + + return 0; +} + +/** + * Print out STREAM_BW event for a single conn + */ +int +control_event_stream_bandwidth(edge_connection_t *edge_conn) +{ + struct timeval now; + char tbuf[ISO_TIME_USEC_LEN+1]; + if (EVENT_IS_INTERESTING(EVENT_STREAM_BANDWIDTH_USED)) { + if (!edge_conn->n_read && !edge_conn->n_written) + return 0; + + tor_gettimeofday(&now); + format_iso_time_nospace_usec(tbuf, &now); + send_control_event(EVENT_STREAM_BANDWIDTH_USED, + "650 STREAM_BW %"PRIu64" %lu %lu %s\r\n", + (edge_conn->base_.global_identifier), + (unsigned long)edge_conn->n_read, + (unsigned long)edge_conn->n_written, + tbuf); + + edge_conn->n_written = edge_conn->n_read = 0; + } + + return 0; +} + +/** A second or more has elapsed: tell any interested control + * connections how much bandwidth streams have used. */ +int +control_event_stream_bandwidth_used(void) +{ + if (EVENT_IS_INTERESTING(EVENT_STREAM_BANDWIDTH_USED)) { + smartlist_t *conns = get_connection_array(); + edge_connection_t *edge_conn; + struct timeval now; + char tbuf[ISO_TIME_USEC_LEN+1]; + + SMARTLIST_FOREACH_BEGIN(conns, connection_t *, conn) + { + if (conn->type != CONN_TYPE_AP) + continue; + edge_conn = TO_EDGE_CONN(conn); + if (!edge_conn->n_read && !edge_conn->n_written) + continue; + + tor_gettimeofday(&now); + format_iso_time_nospace_usec(tbuf, &now); + send_control_event(EVENT_STREAM_BANDWIDTH_USED, + "650 STREAM_BW %"PRIu64" %lu %lu %s\r\n", + (edge_conn->base_.global_identifier), + (unsigned long)edge_conn->n_read, + (unsigned long)edge_conn->n_written, + tbuf); + + edge_conn->n_written = edge_conn->n_read = 0; + } + SMARTLIST_FOREACH_END(conn); + } + + return 0; +} + +/** A second or more has elapsed: tell any interested control connections + * how much bandwidth origin circuits have used. */ +int +control_event_circ_bandwidth_used(void) +{ + if (!EVENT_IS_INTERESTING(EVENT_CIRC_BANDWIDTH_USED)) + return 0; + + SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) { + if (!CIRCUIT_IS_ORIGIN(circ)) + continue; + + control_event_circ_bandwidth_used_for_circ(TO_ORIGIN_CIRCUIT(circ)); + } + SMARTLIST_FOREACH_END(circ); + + return 0; +} + +/** + * Emit a CIRC_BW event line for a specific circuit. + * + * This function sets the values it emits to 0, and does not emit + * an event if there is no new data to report since the last call. + * + * Therefore, it may be called at any frequency. + */ +int +control_event_circ_bandwidth_used_for_circ(origin_circuit_t *ocirc) +{ + struct timeval now; + char tbuf[ISO_TIME_USEC_LEN+1]; + + tor_assert(ocirc); + + if (!EVENT_IS_INTERESTING(EVENT_CIRC_BANDWIDTH_USED)) + return 0; + + /* n_read_circ_bw and n_written_circ_bw are always updated + * when there is any new cell on a circuit, and set to 0 after + * the event, below. + * + * Therefore, checking them is sufficient to determine if there + * is new data to report. */ + if (!ocirc->n_read_circ_bw && !ocirc->n_written_circ_bw) + return 0; + + tor_gettimeofday(&now); + format_iso_time_nospace_usec(tbuf, &now); + send_control_event(EVENT_CIRC_BANDWIDTH_USED, + "650 CIRC_BW ID=%d READ=%lu WRITTEN=%lu TIME=%s " + "DELIVERED_READ=%lu OVERHEAD_READ=%lu " + "DELIVERED_WRITTEN=%lu OVERHEAD_WRITTEN=%lu\r\n", + ocirc->global_identifier, + (unsigned long)ocirc->n_read_circ_bw, + (unsigned long)ocirc->n_written_circ_bw, + tbuf, + (unsigned long)ocirc->n_delivered_read_circ_bw, + (unsigned long)ocirc->n_overhead_read_circ_bw, + (unsigned long)ocirc->n_delivered_written_circ_bw, + (unsigned long)ocirc->n_overhead_written_circ_bw); + ocirc->n_written_circ_bw = ocirc->n_read_circ_bw = 0; + ocirc->n_overhead_written_circ_bw = ocirc->n_overhead_read_circ_bw = 0; + ocirc->n_delivered_written_circ_bw = ocirc->n_delivered_read_circ_bw = 0; + + return 0; +} + +/** Print out CONN_BW event for a single OR/DIR/EXIT conn and reset + * bandwidth counters. */ +int +control_event_conn_bandwidth(connection_t *conn) +{ + const char *conn_type_str; + if (!get_options()->TestingEnableConnBwEvent || + !EVENT_IS_INTERESTING(EVENT_CONN_BW)) + return 0; + if (!conn->n_read_conn_bw && !conn->n_written_conn_bw) + return 0; + switch (conn->type) { + case CONN_TYPE_OR: + conn_type_str = "OR"; + break; + case CONN_TYPE_DIR: + conn_type_str = "DIR"; + break; + case CONN_TYPE_EXIT: + conn_type_str = "EXIT"; + break; + default: + return 0; + } + send_control_event(EVENT_CONN_BW, + "650 CONN_BW ID=%"PRIu64" TYPE=%s " + "READ=%lu WRITTEN=%lu\r\n", + (conn->global_identifier), + conn_type_str, + (unsigned long)conn->n_read_conn_bw, + (unsigned long)conn->n_written_conn_bw); + conn->n_written_conn_bw = conn->n_read_conn_bw = 0; + return 0; +} + +/** A second or more has elapsed: tell any interested control + * connections how much bandwidth connections have used. */ +int +control_event_conn_bandwidth_used(void) +{ + if (get_options()->TestingEnableConnBwEvent && + EVENT_IS_INTERESTING(EVENT_CONN_BW)) { + SMARTLIST_FOREACH(get_connection_array(), connection_t *, conn, + control_event_conn_bandwidth(conn)); + } + return 0; +} + +/** Helper: iterate over cell statistics of circ and sum up added + * cells, removed cells, and waiting times by cell command and direction. + * Store results in cell_stats. Free cell statistics of the + * circuit afterwards. */ +void +sum_up_cell_stats_by_command(circuit_t *circ, cell_stats_t *cell_stats) +{ + memset(cell_stats, 0, sizeof(cell_stats_t)); + SMARTLIST_FOREACH_BEGIN(circ->testing_cell_stats, + const testing_cell_stats_entry_t *, ent) { + tor_assert(ent->command <= CELL_COMMAND_MAX_); + if (!ent->removed && !ent->exitward) { + cell_stats->added_cells_appward[ent->command] += 1; + } else if (!ent->removed && ent->exitward) { + cell_stats->added_cells_exitward[ent->command] += 1; + } else if (!ent->exitward) { + cell_stats->removed_cells_appward[ent->command] += 1; + cell_stats->total_time_appward[ent->command] += ent->waiting_time * 10; + } else { + cell_stats->removed_cells_exitward[ent->command] += 1; + cell_stats->total_time_exitward[ent->command] += ent->waiting_time * 10; + } + } SMARTLIST_FOREACH_END(ent); + circuit_clear_testing_cell_stats(circ); +} + +/** Helper: append a cell statistics string to event_parts, + * prefixed with key=. Statistics consist of comma-separated + * key:value pairs with lower-case command strings as keys and cell + * numbers or total waiting times as values. A key:value pair is included + * if the entry in include_if_non_zero is not zero, but with + * the (possibly zero) entry from number_to_include. Both + * arrays are expected to have a length of CELL_COMMAND_MAX_ + 1. If no + * entry in include_if_non_zero is positive, no string will + * be added to event_parts. */ +void +append_cell_stats_by_command(smartlist_t *event_parts, const char *key, + const uint64_t *include_if_non_zero, + const uint64_t *number_to_include) +{ + smartlist_t *key_value_strings = smartlist_new(); + int i; + for (i = 0; i <= CELL_COMMAND_MAX_; i++) { + if (include_if_non_zero[i] > 0) { + smartlist_add_asprintf(key_value_strings, "%s:%"PRIu64, + cell_command_to_string(i), + (number_to_include[i])); + } + } + if (smartlist_len(key_value_strings) > 0) { + char *joined = smartlist_join_strings(key_value_strings, ",", 0, NULL); + smartlist_add_asprintf(event_parts, "%s=%s", key, joined); + SMARTLIST_FOREACH(key_value_strings, char *, cp, tor_free(cp)); + tor_free(joined); + } + smartlist_free(key_value_strings); +} + +/** Helper: format cell_stats for circ for inclusion in a + * CELL_STATS event and write result string to event_string. */ +void +format_cell_stats(char **event_string, circuit_t *circ, + cell_stats_t *cell_stats) +{ + smartlist_t *event_parts = smartlist_new(); + if (CIRCUIT_IS_ORIGIN(circ)) { + origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ); + smartlist_add_asprintf(event_parts, "ID=%lu", + (unsigned long)ocirc->global_identifier); + } else if (TO_OR_CIRCUIT(circ)->p_chan) { + or_circuit_t *or_circ = TO_OR_CIRCUIT(circ); + smartlist_add_asprintf(event_parts, "InboundQueue=%lu", + (unsigned long)or_circ->p_circ_id); + smartlist_add_asprintf(event_parts, "InboundConn=%"PRIu64, + (or_circ->p_chan->global_identifier)); + append_cell_stats_by_command(event_parts, "InboundAdded", + cell_stats->added_cells_appward, + cell_stats->added_cells_appward); + append_cell_stats_by_command(event_parts, "InboundRemoved", + cell_stats->removed_cells_appward, + cell_stats->removed_cells_appward); + append_cell_stats_by_command(event_parts, "InboundTime", + cell_stats->removed_cells_appward, + cell_stats->total_time_appward); + } + if (circ->n_chan) { + smartlist_add_asprintf(event_parts, "OutboundQueue=%lu", + (unsigned long)circ->n_circ_id); + smartlist_add_asprintf(event_parts, "OutboundConn=%"PRIu64, + (circ->n_chan->global_identifier)); + append_cell_stats_by_command(event_parts, "OutboundAdded", + cell_stats->added_cells_exitward, + cell_stats->added_cells_exitward); + append_cell_stats_by_command(event_parts, "OutboundRemoved", + cell_stats->removed_cells_exitward, + cell_stats->removed_cells_exitward); + append_cell_stats_by_command(event_parts, "OutboundTime", + cell_stats->removed_cells_exitward, + cell_stats->total_time_exitward); + } + *event_string = smartlist_join_strings(event_parts, " ", 0, NULL); + SMARTLIST_FOREACH(event_parts, char *, cp, tor_free(cp)); + smartlist_free(event_parts); +} + +/** A second or more has elapsed: tell any interested control connection + * how many cells have been processed for a given circuit. */ +int +control_event_circuit_cell_stats(void) +{ + cell_stats_t *cell_stats; + char *event_string; + if (!get_options()->TestingEnableCellStatsEvent || + !EVENT_IS_INTERESTING(EVENT_CELL_STATS)) + return 0; + cell_stats = tor_malloc(sizeof(cell_stats_t)); + SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) { + if (!circ->testing_cell_stats) + continue; + sum_up_cell_stats_by_command(circ, cell_stats); + format_cell_stats(&event_string, circ, cell_stats); + send_control_event(EVENT_CELL_STATS, + "650 CELL_STATS %s\r\n", event_string); + tor_free(event_string); + } + SMARTLIST_FOREACH_END(circ); + tor_free(cell_stats); + return 0; +} + +/* about 5 minutes worth. */ +#define N_BW_EVENTS_TO_CACHE 300 +/* Index into cached_bw_events to next write. */ +static int next_measurement_idx = 0; +/* number of entries set in n_measurements */ +static int n_measurements = 0; +static struct cached_bw_event_s { + uint32_t n_read; + uint32_t n_written; +} cached_bw_events[N_BW_EVENTS_TO_CACHE]; + +/** A second or more has elapsed: tell any interested control + * connections how much bandwidth we used. */ +int +control_event_bandwidth_used(uint32_t n_read, uint32_t n_written) +{ + cached_bw_events[next_measurement_idx].n_read = n_read; + cached_bw_events[next_measurement_idx].n_written = n_written; + if (++next_measurement_idx == N_BW_EVENTS_TO_CACHE) + next_measurement_idx = 0; + if (n_measurements < N_BW_EVENTS_TO_CACHE) + ++n_measurements; + + if (EVENT_IS_INTERESTING(EVENT_BANDWIDTH_USED)) { + send_control_event(EVENT_BANDWIDTH_USED, + "650 BW %lu %lu\r\n", + (unsigned long)n_read, + (unsigned long)n_written); + } + + return 0; +} + +char * +get_bw_samples(void) +{ + int i; + int idx = (next_measurement_idx + N_BW_EVENTS_TO_CACHE - n_measurements) + % N_BW_EVENTS_TO_CACHE; + tor_assert(0 <= idx && idx < N_BW_EVENTS_TO_CACHE); + + smartlist_t *elements = smartlist_new(); + + for (i = 0; i < n_measurements; ++i) { + tor_assert(0 <= idx && idx < N_BW_EVENTS_TO_CACHE); + const struct cached_bw_event_s *bwe = &cached_bw_events[idx]; + + smartlist_add_asprintf(elements, "%u,%u", + (unsigned)bwe->n_read, + (unsigned)bwe->n_written); + + idx = (idx + 1) % N_BW_EVENTS_TO_CACHE; + } + + char *result = smartlist_join_strings(elements, " ", 0, NULL); + + SMARTLIST_FOREACH(elements, char *, cp, tor_free(cp)); + smartlist_free(elements); + + return result; +} + +/** Called when we are sending a log message to the controllers: suspend + * sending further log messages to the controllers until we're done. Used by + * CONN_LOG_PROTECT. */ +void +disable_control_logging(void) +{ + ++disable_log_messages; +} + +/** We're done sending a log message to the controllers: re-enable controller + * logging. Used by CONN_LOG_PROTECT. */ +void +enable_control_logging(void) +{ + if (--disable_log_messages < 0) + tor_assert(0); +} + +/** We got a log message: tell any interested control connections. */ +void +control_event_logmsg(int severity, uint32_t domain, const char *msg) +{ + int event; + + /* Don't even think of trying to add stuff to a buffer from a cpuworker + * thread. (See #25987 for plan to fix.) */ + if (! in_main_thread()) + return; + + if (disable_log_messages) + return; + + if (domain == LD_BUG && EVENT_IS_INTERESTING(EVENT_STATUS_GENERAL) && + severity <= LOG_NOTICE) { + char *esc = esc_for_log(msg); + ++disable_log_messages; + control_event_general_status(severity, "BUG REASON=%s", esc); + --disable_log_messages; + tor_free(esc); + } + + event = log_severity_to_event(severity); + if (event >= 0 && EVENT_IS_INTERESTING(event)) { + char *b = NULL; + const char *s; + if (strchr(msg, '\n')) { + char *cp; + b = tor_strdup(msg); + for (cp = b; *cp; ++cp) + if (*cp == '\r' || *cp == '\n') + *cp = ' '; + } + switch (severity) { + case LOG_DEBUG: s = "DEBUG"; break; + case LOG_INFO: s = "INFO"; break; + case LOG_NOTICE: s = "NOTICE"; break; + case LOG_WARN: s = "WARN"; break; + case LOG_ERR: s = "ERR"; break; + default: s = "UnknownLogSeverity"; break; + } + ++disable_log_messages; + send_control_event(event, "650 %s %s\r\n", s, b?b:msg); + if (severity == LOG_ERR) { + /* Force a flush, since we may be about to die horribly */ + queued_events_flush_all(1); + } + --disable_log_messages; + tor_free(b); + } +} + +/** + * Logging callback: called when there is a queued pending log callback. + */ +void +control_event_logmsg_pending(void) +{ + if (! in_main_thread()) { + /* We can't handle this case yet, since we're using a + * mainloop_event_t to invoke queued_events_flush_all. We ought to + * use a different mechanism instead: see #25987. + **/ + return; + } + tor_assert(flush_queued_events_event); + mainloop_event_activate(flush_queued_events_event); +} + +/** Called whenever we receive new router descriptors: tell any + * interested control connections. routers is a list of + * routerinfo_t's. + */ +int +control_event_descriptors_changed(smartlist_t *routers) +{ + char *msg; + + if (!EVENT_IS_INTERESTING(EVENT_NEW_DESC)) + return 0; + + { + smartlist_t *names = smartlist_new(); + char *ids; + SMARTLIST_FOREACH(routers, routerinfo_t *, ri, { + char *b = tor_malloc(MAX_VERBOSE_NICKNAME_LEN+1); + router_get_verbose_nickname(b, ri); + smartlist_add(names, b); + }); + ids = smartlist_join_strings(names, " ", 0, NULL); + tor_asprintf(&msg, "650 NEWDESC %s\r\n", ids); + send_control_event_string(EVENT_NEW_DESC, msg); + tor_free(ids); + tor_free(msg); + SMARTLIST_FOREACH(names, char *, cp, tor_free(cp)); + smartlist_free(names); + } + return 0; +} + +/** Called when an address mapping on from from changes to to. + * expires values less than 3 are special; see connection_edge.c. If + * error is non-NULL, it is an error code describing the failure + * mode of the mapping. + */ +int +control_event_address_mapped(const char *from, const char *to, time_t expires, + const char *error, const int cached) +{ + if (!EVENT_IS_INTERESTING(EVENT_ADDRMAP)) + return 0; + + if (expires < 3 || expires == TIME_MAX) + send_control_event(EVENT_ADDRMAP, + "650 ADDRMAP %s %s NEVER %s%s" + "CACHED=\"%s\"\r\n", + from, to, error?error:"", error?" ":"", + cached?"YES":"NO"); + else { + char buf[ISO_TIME_LEN+1]; + char buf2[ISO_TIME_LEN+1]; + format_local_iso_time(buf,expires); + format_iso_time(buf2,expires); + send_control_event(EVENT_ADDRMAP, + "650 ADDRMAP %s %s \"%s\"" + " %s%sEXPIRES=\"%s\" CACHED=\"%s\"\r\n", + from, to, buf, + error?error:"", error?" ":"", + buf2, cached?"YES":"NO"); + } + + return 0; +} +/** The network liveness has changed; this is called from circuitstats.c + * whenever we receive a cell, or when timeout expires and we assume the + * network is down. */ +int +control_event_network_liveness_update(int liveness) +{ + if (liveness > 0) { + if (get_cached_network_liveness() <= 0) { + /* Update cached liveness */ + set_cached_network_liveness(1); + log_debug(LD_CONTROL, "Sending NETWORK_LIVENESS UP"); + send_control_event_string(EVENT_NETWORK_LIVENESS, + "650 NETWORK_LIVENESS UP\r\n"); + } + /* else was already live, no-op */ + } else { + if (get_cached_network_liveness() > 0) { + /* Update cached liveness */ + set_cached_network_liveness(0); + log_debug(LD_CONTROL, "Sending NETWORK_LIVENESS DOWN"); + send_control_event_string(EVENT_NETWORK_LIVENESS, + "650 NETWORK_LIVENESS DOWN\r\n"); + } + /* else was already dead, no-op */ + } + + return 0; +} + +/** Helper function for NS-style events. Constructs and sends an event + * of type event with string event_string out of the set of + * networkstatuses statuses. Currently it is used for NS events + * and NEWCONSENSUS events. */ +static int +control_event_networkstatus_changed_helper(smartlist_t *statuses, + uint16_t event, + const char *event_string) +{ + smartlist_t *strs; + char *s, *esc = NULL; + if (!EVENT_IS_INTERESTING(event) || !smartlist_len(statuses)) + return 0; + + strs = smartlist_new(); + smartlist_add_strdup(strs, "650+"); + smartlist_add_strdup(strs, event_string); + smartlist_add_strdup(strs, "\r\n"); + SMARTLIST_FOREACH(statuses, const routerstatus_t *, rs, + { + s = networkstatus_getinfo_helper_single(rs); + if (!s) continue; + smartlist_add(strs, s); + }); + + s = smartlist_join_strings(strs, "", 0, NULL); + write_escaped_data(s, strlen(s), &esc); + SMARTLIST_FOREACH(strs, char *, cp, tor_free(cp)); + smartlist_free(strs); + tor_free(s); + send_control_event_string(event, esc); + send_control_event_string(event, + "650 OK\r\n"); + + tor_free(esc); + return 0; +} + +/** Called when the routerstatus_ts statuses have changed: sends + * an NS event to any controller that cares. */ +int +control_event_networkstatus_changed(smartlist_t *statuses) +{ + return control_event_networkstatus_changed_helper(statuses, EVENT_NS, "NS"); +} + +/** Called when we get a new consensus networkstatus. Sends a NEWCONSENSUS + * event consisting of an NS-style line for each relay in the consensus. */ +int +control_event_newconsensus(const networkstatus_t *consensus) +{ + if (!control_event_is_interesting(EVENT_NEWCONSENSUS)) + return 0; + return control_event_networkstatus_changed_helper( + consensus->routerstatus_list, EVENT_NEWCONSENSUS, "NEWCONSENSUS"); +} + +/** Called when we compute a new circuitbuildtimeout */ +int +control_event_buildtimeout_set(buildtimeout_set_event_t type, + const char *args) +{ + const char *type_string = NULL; + + if (!control_event_is_interesting(EVENT_BUILDTIMEOUT_SET)) + return 0; + + switch (type) { + case BUILDTIMEOUT_SET_EVENT_COMPUTED: + type_string = "COMPUTED"; + break; + case BUILDTIMEOUT_SET_EVENT_RESET: + type_string = "RESET"; + break; + case BUILDTIMEOUT_SET_EVENT_SUSPENDED: + type_string = "SUSPENDED"; + break; + case BUILDTIMEOUT_SET_EVENT_DISCARD: + type_string = "DISCARD"; + break; + case BUILDTIMEOUT_SET_EVENT_RESUME: + type_string = "RESUME"; + break; + default: + type_string = "UNKNOWN"; + break; + } + + send_control_event(EVENT_BUILDTIMEOUT_SET, + "650 BUILDTIMEOUT_SET %s %s\r\n", + type_string, args); + + return 0; +} + +/** Called when a signal has been processed from signal_callback */ +int +control_event_signal(uintptr_t signal_num) +{ + const char *signal_string = NULL; + + if (!control_event_is_interesting(EVENT_GOT_SIGNAL)) + return 0; + + switch (signal_num) { + case SIGHUP: + signal_string = "RELOAD"; + break; + case SIGUSR1: + signal_string = "DUMP"; + break; + case SIGUSR2: + signal_string = "DEBUG"; + break; + case SIGNEWNYM: + signal_string = "NEWNYM"; + break; + case SIGCLEARDNSCACHE: + signal_string = "CLEARDNSCACHE"; + break; + case SIGHEARTBEAT: + signal_string = "HEARTBEAT"; + break; + default: + log_warn(LD_BUG, "Unrecognized signal %lu in control_event_signal", + (unsigned long)signal_num); + return -1; + } + + send_control_event(EVENT_GOT_SIGNAL, "650 SIGNAL %s\r\n", + signal_string); + return 0; +} + +/** Called when a single local_routerstatus_t has changed: Sends an NS event + * to any controller that cares. */ +int +control_event_networkstatus_changed_single(const routerstatus_t *rs) +{ + smartlist_t *statuses; + int r; + + if (!EVENT_IS_INTERESTING(EVENT_NS)) + return 0; + + statuses = smartlist_new(); + smartlist_add(statuses, (void*)rs); + r = control_event_networkstatus_changed(statuses); + smartlist_free(statuses); + return r; +} + +/** Our own router descriptor has changed; tell any controllers that care. + */ +int +control_event_my_descriptor_changed(void) +{ + send_control_event(EVENT_DESCCHANGED, "650 DESCCHANGED\r\n"); + return 0; +} + +/** Helper: sends a status event where type is one of + * EVENT_STATUS_{GENERAL,CLIENT,SERVER}, where severity is one of + * LOG_{NOTICE,WARN,ERR}, and where format is a printf-style format + * string corresponding to args. */ +static int +control_event_status(int type, int severity, const char *format, va_list args) +{ + char *user_buf = NULL; + char format_buf[160]; + const char *status, *sev; + + switch (type) { + case EVENT_STATUS_GENERAL: + status = "STATUS_GENERAL"; + break; + case EVENT_STATUS_CLIENT: + status = "STATUS_CLIENT"; + break; + case EVENT_STATUS_SERVER: + status = "STATUS_SERVER"; + break; + default: + log_warn(LD_BUG, "Unrecognized status type %d", type); + return -1; + } + switch (severity) { + case LOG_NOTICE: + sev = "NOTICE"; + break; + case LOG_WARN: + sev = "WARN"; + break; + case LOG_ERR: + sev = "ERR"; + break; + default: + log_warn(LD_BUG, "Unrecognized status severity %d", severity); + return -1; + } + if (tor_snprintf(format_buf, sizeof(format_buf), "650 %s %s", + status, sev)<0) { + log_warn(LD_BUG, "Format string too long."); + return -1; + } + tor_vasprintf(&user_buf, format, args); + + send_control_event(type, "%s %s\r\n", format_buf, user_buf); + tor_free(user_buf); + return 0; +} + +#define CONTROL_EVENT_STATUS_BODY(event, sev) \ + int r; \ + do { \ + va_list ap; \ + if (!EVENT_IS_INTERESTING(event)) \ + return 0; \ + \ + va_start(ap, format); \ + r = control_event_status((event), (sev), format, ap); \ + va_end(ap); \ + } while (0) + +/** Format and send an EVENT_STATUS_GENERAL event whose main text is obtained + * by formatting the arguments using the printf-style format. */ +int +control_event_general_status(int severity, const char *format, ...) +{ + CONTROL_EVENT_STATUS_BODY(EVENT_STATUS_GENERAL, severity); + return r; +} + +/** Format and send an EVENT_STATUS_GENERAL LOG_ERR event, and flush it to the + * controller(s) immediately. */ +int +control_event_general_error(const char *format, ...) +{ + CONTROL_EVENT_STATUS_BODY(EVENT_STATUS_GENERAL, LOG_ERR); + /* Force a flush, since we may be about to die horribly */ + queued_events_flush_all(1); + return r; +} + +/** Format and send an EVENT_STATUS_CLIENT event whose main text is obtained + * by formatting the arguments using the printf-style format. */ +int +control_event_client_status(int severity, const char *format, ...) +{ + CONTROL_EVENT_STATUS_BODY(EVENT_STATUS_CLIENT, severity); + return r; +} + +/** Format and send an EVENT_STATUS_CLIENT LOG_ERR event, and flush it to the + * controller(s) immediately. */ +int +control_event_client_error(const char *format, ...) +{ + CONTROL_EVENT_STATUS_BODY(EVENT_STATUS_CLIENT, LOG_ERR); + /* Force a flush, since we may be about to die horribly */ + queued_events_flush_all(1); + return r; +} + +/** Format and send an EVENT_STATUS_SERVER event whose main text is obtained + * by formatting the arguments using the printf-style format. */ +int +control_event_server_status(int severity, const char *format, ...) +{ + CONTROL_EVENT_STATUS_BODY(EVENT_STATUS_SERVER, severity); + return r; +} + +/** Format and send an EVENT_STATUS_SERVER LOG_ERR event, and flush it to the + * controller(s) immediately. */ +int +control_event_server_error(const char *format, ...) +{ + CONTROL_EVENT_STATUS_BODY(EVENT_STATUS_SERVER, LOG_ERR); + /* Force a flush, since we may be about to die horribly */ + queued_events_flush_all(1); + return r; +} + +/** Called when the status of an entry guard with the given nickname + * and identity digest has changed to status: tells any + * controllers that care. */ +int +control_event_guard(const char *nickname, const char *digest, + const char *status) +{ + char hbuf[HEX_DIGEST_LEN+1]; + base16_encode(hbuf, sizeof(hbuf), digest, DIGEST_LEN); + if (!EVENT_IS_INTERESTING(EVENT_GUARD)) + return 0; + + { + char buf[MAX_VERBOSE_NICKNAME_LEN+1]; + const node_t *node = node_get_by_id(digest); + if (node) { + node_get_verbose_nickname(node, buf); + } else { + tor_snprintf(buf, sizeof(buf), "$%s~%s", hbuf, nickname); + } + send_control_event(EVENT_GUARD, + "650 GUARD ENTRY %s %s\r\n", buf, status); + } + return 0; +} + +/** Called when a configuration option changes. This is generally triggered + * by SETCONF requests and RELOAD/SIGHUP signals. The elements is + * a smartlist_t containing (key, value, ...) pairs in sequence. + * value can be NULL. */ +int +control_event_conf_changed(const smartlist_t *elements) +{ + int i; + char *result; + smartlist_t *lines; + if (!EVENT_IS_INTERESTING(EVENT_CONF_CHANGED) || + smartlist_len(elements) == 0) { + return 0; + } + lines = smartlist_new(); + for (i = 0; i < smartlist_len(elements); i += 2) { + char *k = smartlist_get(elements, i); + char *v = smartlist_get(elements, i+1); + if (v == NULL) { + smartlist_add_asprintf(lines, "650-%s", k); + } else { + smartlist_add_asprintf(lines, "650-%s=%s", k, v); + } + } + result = smartlist_join_strings(lines, "\r\n", 0, NULL); + send_control_event(EVENT_CONF_CHANGED, + "650-CONF_CHANGED\r\n%s\r\n650 OK\r\n", result); + tor_free(result); + SMARTLIST_FOREACH(lines, char *, cp, tor_free(cp)); + smartlist_free(lines); + return 0; +} + +/** We just generated a new summary of which countries we've seen clients + * from recently. Send a copy to the controller in case it wants to + * display it for the user. */ +void +control_event_clients_seen(const char *controller_str) +{ + send_control_event(EVENT_CLIENTS_SEEN, + "650 CLIENTS_SEEN %s\r\n", controller_str); +} + +/** A new pluggable transport called transport_name was + * launched on addr:port. mode is either + * "server" or "client" depending on the mode of the pluggable + * transport. + * "650" SP "TRANSPORT_LAUNCHED" SP Mode SP Name SP Address SP Port + */ +void +control_event_transport_launched(const char *mode, const char *transport_name, + tor_addr_t *addr, uint16_t port) +{ + send_control_event(EVENT_TRANSPORT_LAUNCHED, + "650 TRANSPORT_LAUNCHED %s %s %s %u\r\n", + mode, transport_name, fmt_addr(addr), port); +} + +/** A pluggable transport called pt_name has emitted a log message + * found in message at severity log level. */ +void +control_event_pt_log(const char *log) +{ + send_control_event(EVENT_PT_LOG, + "650 PT_LOG %s\r\n", + log); +} + +/** A pluggable transport has emitted a STATUS message found in + * status. */ +void +control_event_pt_status(const char *status) +{ + send_control_event(EVENT_PT_STATUS, + "650 PT_STATUS %s\r\n", + status); +} + +/** Convert rendezvous auth type to string for HS_DESC control events + */ +const char * +rend_auth_type_to_string(rend_auth_type_t auth_type) +{ + const char *str; + + switch (auth_type) { + case REND_NO_AUTH: + str = "NO_AUTH"; + break; + case REND_BASIC_AUTH: + str = "BASIC_AUTH"; + break; + case REND_STEALTH_AUTH: + str = "STEALTH_AUTH"; + break; + default: + str = "UNKNOWN"; + } + + return str; +} + +/** Return either the onion address if the given pointer is a non empty + * string else the unknown string. */ +static const char * +rend_hsaddress_str_or_unknown(const char *onion_address) +{ + static const char *str_unknown = "UNKNOWN"; + const char *str_ret = str_unknown; + + /* No valid pointer, unknown it is. */ + if (!onion_address) { + goto end; + } + /* Empty onion address thus we don't know, unknown it is. */ + if (onion_address[0] == '\0') { + goto end; + } + /* All checks are good so return the given onion address. */ + str_ret = onion_address; + + end: + return str_ret; +} + +/** send HS_DESC requested event. + * + * rend_query is used to fetch requested onion address and auth type. + * hs_dir is the description of contacting hs directory. + * desc_id_base32 is the ID of requested hs descriptor. + * hsdir_index is the HSDir fetch index value for v3, an hex string. + */ +void +control_event_hs_descriptor_requested(const char *onion_address, + rend_auth_type_t auth_type, + const char *id_digest, + const char *desc_id, + const char *hsdir_index) +{ + char *hsdir_index_field = NULL; + + if (BUG(!id_digest || !desc_id)) { + return; + } + + if (hsdir_index) { + tor_asprintf(&hsdir_index_field, " HSDIR_INDEX=%s", hsdir_index); + } + + send_control_event(EVENT_HS_DESC, + "650 HS_DESC REQUESTED %s %s %s %s%s\r\n", + rend_hsaddress_str_or_unknown(onion_address), + rend_auth_type_to_string(auth_type), + node_describe_longname_by_id(id_digest), + desc_id, + hsdir_index_field ? hsdir_index_field : ""); + tor_free(hsdir_index_field); +} + +/** send HS_DESC CREATED event when a local service generates a descriptor. + * + * onion_address is service address. + * desc_id is the descriptor ID. + * replica is the the descriptor replica number. If it is negative, it + * is ignored. + */ +void +control_event_hs_descriptor_created(const char *onion_address, + const char *desc_id, + int replica) +{ + char *replica_field = NULL; + + if (BUG(!onion_address || !desc_id)) { + return; + } + + if (replica >= 0) { + tor_asprintf(&replica_field, " REPLICA=%d", replica); + } + + send_control_event(EVENT_HS_DESC, + "650 HS_DESC CREATED %s UNKNOWN UNKNOWN %s%s\r\n", + onion_address, desc_id, + replica_field ? replica_field : ""); + tor_free(replica_field); +} + +/** send HS_DESC upload event. + * + * onion_address is service address. + * hs_dir is the description of contacting hs directory. + * desc_id is the ID of requested hs descriptor. + */ +void +control_event_hs_descriptor_upload(const char *onion_address, + const char *id_digest, + const char *desc_id, + const char *hsdir_index) +{ + char *hsdir_index_field = NULL; + + if (BUG(!onion_address || !id_digest || !desc_id)) { + return; + } + + if (hsdir_index) { + tor_asprintf(&hsdir_index_field, " HSDIR_INDEX=%s", hsdir_index); + } + + send_control_event(EVENT_HS_DESC, + "650 HS_DESC UPLOAD %s UNKNOWN %s %s%s\r\n", + onion_address, + node_describe_longname_by_id(id_digest), + desc_id, + hsdir_index_field ? hsdir_index_field : ""); + tor_free(hsdir_index_field); +} + +/** send HS_DESC event after got response from hs directory. + * + * NOTE: this is an internal function used by following functions: + * control_event_hsv2_descriptor_received + * control_event_hsv2_descriptor_failed + * control_event_hsv3_descriptor_failed + * + * So do not call this function directly. + */ +static void +event_hs_descriptor_receive_end(const char *action, + const char *onion_address, + const char *desc_id, + rend_auth_type_t auth_type, + const char *hsdir_id_digest, + const char *reason) +{ + char *reason_field = NULL; + + if (BUG(!action || !onion_address)) { + return; + } + + if (reason) { + tor_asprintf(&reason_field, " REASON=%s", reason); + } + + send_control_event(EVENT_HS_DESC, + "650 HS_DESC %s %s %s %s%s%s\r\n", + action, + rend_hsaddress_str_or_unknown(onion_address), + rend_auth_type_to_string(auth_type), + hsdir_id_digest ? + node_describe_longname_by_id(hsdir_id_digest) : + "UNKNOWN", + desc_id ? desc_id : "", + reason_field ? reason_field : ""); + + tor_free(reason_field); +} + +/** send HS_DESC event after got response from hs directory. + * + * NOTE: this is an internal function used by following functions: + * control_event_hs_descriptor_uploaded + * control_event_hs_descriptor_upload_failed + * + * So do not call this function directly. + */ +void +control_event_hs_descriptor_upload_end(const char *action, + const char *onion_address, + const char *id_digest, + const char *reason) +{ + char *reason_field = NULL; + + if (BUG(!action || !id_digest)) { + return; + } + + if (reason) { + tor_asprintf(&reason_field, " REASON=%s", reason); + } + + send_control_event(EVENT_HS_DESC, + "650 HS_DESC %s %s UNKNOWN %s%s\r\n", + action, + rend_hsaddress_str_or_unknown(onion_address), + node_describe_longname_by_id(id_digest), + reason_field ? reason_field : ""); + + tor_free(reason_field); +} + +/** For an HS descriptor query rend_data, using the + * onion_address and HSDir fingerprint hsdir_fp, find out + * which descriptor ID in the query is the right one. + * + * Return a pointer of the binary descriptor ID found in the query's object + * or NULL if not found. */ +static const char * +get_desc_id_from_query(const rend_data_t *rend_data, const char *hsdir_fp) +{ + int replica; + const char *desc_id = NULL; + const rend_data_v2_t *rend_data_v2 = TO_REND_DATA_V2(rend_data); + + /* Possible if the fetch was done using a descriptor ID. This means that + * the HSFETCH command was used. */ + if (!tor_digest_is_zero(rend_data_v2->desc_id_fetch)) { + desc_id = rend_data_v2->desc_id_fetch; + goto end; + } + + /* Without a directory fingerprint at this stage, we can't do much. */ + if (hsdir_fp == NULL) { + goto end; + } + + /* OK, we have an onion address so now let's find which descriptor ID + * is the one associated with the HSDir fingerprint. */ + for (replica = 0; replica < REND_NUMBER_OF_NON_CONSECUTIVE_REPLICAS; + replica++) { + const char *digest = rend_data_get_desc_id(rend_data, replica, NULL); + + SMARTLIST_FOREACH_BEGIN(rend_data->hsdirs_fp, char *, fingerprint) { + if (tor_memcmp(fingerprint, hsdir_fp, DIGEST_LEN) == 0) { + /* Found it! This descriptor ID is the right one. */ + desc_id = digest; + goto end; + } + } SMARTLIST_FOREACH_END(fingerprint); + } + + end: + return desc_id; +} + +/** send HS_DESC RECEIVED event + * + * called when we successfully received a hidden service descriptor. + */ +void +control_event_hsv2_descriptor_received(const char *onion_address, + const rend_data_t *rend_data, + const char *hsdir_id_digest) +{ + char *desc_id_field = NULL; + const char *desc_id; + + if (BUG(!rend_data || !hsdir_id_digest || !onion_address)) { + return; + } + + desc_id = get_desc_id_from_query(rend_data, hsdir_id_digest); + if (desc_id != NULL) { + char desc_id_base32[REND_DESC_ID_V2_LEN_BASE32 + 1]; + /* Set the descriptor ID digest to base32 so we can send it. */ + base32_encode(desc_id_base32, sizeof(desc_id_base32), desc_id, + DIGEST_LEN); + /* Extra whitespace is needed before the value. */ + tor_asprintf(&desc_id_field, " %s", desc_id_base32); + } + + event_hs_descriptor_receive_end("RECEIVED", onion_address, desc_id_field, + TO_REND_DATA_V2(rend_data)->auth_type, + hsdir_id_digest, NULL); + tor_free(desc_id_field); +} + +/* Send HS_DESC RECEIVED event + * + * Called when we successfully received a hidden service descriptor. */ +void +control_event_hsv3_descriptor_received(const char *onion_address, + const char *desc_id, + const char *hsdir_id_digest) +{ + char *desc_id_field = NULL; + + if (BUG(!onion_address || !desc_id || !hsdir_id_digest)) { + return; + } + + /* Because DescriptorID is an optional positional value, we need to add a + * whitespace before in order to not be next to the HsDir value. */ + tor_asprintf(&desc_id_field, " %s", desc_id); + + event_hs_descriptor_receive_end("RECEIVED", onion_address, desc_id_field, + REND_NO_AUTH, hsdir_id_digest, NULL); + tor_free(desc_id_field); +} + +/** send HS_DESC UPLOADED event + * + * called when we successfully uploaded a hidden service descriptor. + */ +void +control_event_hs_descriptor_uploaded(const char *id_digest, + const char *onion_address) +{ + if (BUG(!id_digest)) { + return; + } + + control_event_hs_descriptor_upload_end("UPLOADED", onion_address, + id_digest, NULL); +} + +/** Send HS_DESC event to inform controller that query rend_data + * failed to retrieve hidden service descriptor from directory identified by + * id_digest. If NULL, "UNKNOWN" is used. If reason is not NULL, + * add it to REASON= field. + */ +void +control_event_hsv2_descriptor_failed(const rend_data_t *rend_data, + const char *hsdir_id_digest, + const char *reason) +{ + char *desc_id_field = NULL; + const char *desc_id; + + if (BUG(!rend_data)) { + return; + } + + desc_id = get_desc_id_from_query(rend_data, hsdir_id_digest); + if (desc_id != NULL) { + char desc_id_base32[REND_DESC_ID_V2_LEN_BASE32 + 1]; + /* Set the descriptor ID digest to base32 so we can send it. */ + base32_encode(desc_id_base32, sizeof(desc_id_base32), desc_id, + DIGEST_LEN); + /* Extra whitespace is needed before the value. */ + tor_asprintf(&desc_id_field, " %s", desc_id_base32); + } + + event_hs_descriptor_receive_end("FAILED", rend_data_get_address(rend_data), + desc_id_field, + TO_REND_DATA_V2(rend_data)->auth_type, + hsdir_id_digest, reason); + tor_free(desc_id_field); +} + +/** Send HS_DESC event to inform controller that the query to + * onion_address failed to retrieve hidden service descriptor + * desc_id from directory identified by hsdir_id_digest. If + * NULL, "UNKNOWN" is used. If reason is not NULL, add it to REASON= + * field. */ +void +control_event_hsv3_descriptor_failed(const char *onion_address, + const char *desc_id, + const char *hsdir_id_digest, + const char *reason) +{ + char *desc_id_field = NULL; + + if (BUG(!onion_address || !desc_id || !reason)) { + return; + } + + /* Because DescriptorID is an optional positional value, we need to add a + * whitespace before in order to not be next to the HsDir value. */ + tor_asprintf(&desc_id_field, " %s", desc_id); + + event_hs_descriptor_receive_end("FAILED", onion_address, desc_id_field, + REND_NO_AUTH, hsdir_id_digest, reason); + tor_free(desc_id_field); +} + +/** Send HS_DESC_CONTENT event after completion of a successful fetch + * from hs directory. If hsdir_id_digest is NULL, it is replaced + * by "UNKNOWN". If content is NULL, it is replaced by an empty + * string. The onion_address or desc_id set to NULL will + * not trigger the control event. */ +void +control_event_hs_descriptor_content(const char *onion_address, + const char *desc_id, + const char *hsdir_id_digest, + const char *content) +{ + static const char *event_name = "HS_DESC_CONTENT"; + char *esc_content = NULL; + + if (!onion_address || !desc_id) { + log_warn(LD_BUG, "Called with onion_address==%p, desc_id==%p, ", + onion_address, desc_id); + return; + } + + if (content == NULL) { + /* Point it to empty content so it can still be escaped. */ + content = ""; + } + write_escaped_data(content, strlen(content), &esc_content); + + send_control_event(EVENT_HS_DESC_CONTENT, + "650+%s %s %s %s\r\n%s650 OK\r\n", + event_name, + rend_hsaddress_str_or_unknown(onion_address), + desc_id, + hsdir_id_digest ? + node_describe_longname_by_id(hsdir_id_digest) : + "UNKNOWN", + esc_content); + tor_free(esc_content); +} + +/** Send HS_DESC event to inform controller upload of hidden service + * descriptor identified by id_digest failed. If reason + * is not NULL, add it to REASON= field. + */ +void +control_event_hs_descriptor_upload_failed(const char *id_digest, + const char *onion_address, + const char *reason) +{ + if (BUG(!id_digest)) { + return; + } + control_event_hs_descriptor_upload_end("FAILED", onion_address, + id_digest, reason); +} + +void +control_events_free_all(void) +{ + smartlist_t *queued_events = NULL; + + stats_prev_n_read = stats_prev_n_written = 0; + + if (queued_control_events_lock) { + tor_mutex_acquire(queued_control_events_lock); + flush_queued_event_pending = 0; + queued_events = queued_control_events; + queued_control_events = NULL; + tor_mutex_release(queued_control_events_lock); + } + if (queued_events) { + SMARTLIST_FOREACH(queued_events, queued_event_t *, ev, + queued_event_free(ev)); + smartlist_free(queued_events); + } + if (flush_queued_events_event) { + mainloop_event_free(flush_queued_events_event); + flush_queued_events_event = NULL; + } + global_event_mask = 0; + disable_log_messages = 0; +} + +#ifdef TOR_UNIT_TESTS +/* For testing: change the value of global_event_mask */ +void +control_testing_set_global_event_mask(uint64_t mask) +{ + global_event_mask = mask; +} +#endif /* defined(TOR_UNIT_TESTS) */ diff --git a/src/feature/control/control_events.h b/src/feature/control/control_events.h new file mode 100644 index 0000000000..9e79346714 --- /dev/null +++ b/src/feature/control/control_events.h @@ -0,0 +1,343 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file control_events.h + * \brief Header file for control_events.c. + **/ + +#ifndef TOR_CONTROL_EVENTS_H +#define TOR_CONTROL_EVENTS_H + +#include "core/or/ocirc_event.h" + +/** Used to indicate the type of a CIRC_MINOR event passed to the controller. + * The various types are defined in control-spec.txt . */ +typedef enum circuit_status_minor_event_t { + CIRC_MINOR_EVENT_PURPOSE_CHANGED, + CIRC_MINOR_EVENT_CANNIBALIZED, +} circuit_status_minor_event_t; + +#include "core/or/orconn_event.h" + +/** Used to indicate the type of a stream event passed to the controller. + * The various types are defined in control-spec.txt */ +typedef enum stream_status_event_t { + STREAM_EVENT_SENT_CONNECT = 0, + STREAM_EVENT_SENT_RESOLVE = 1, + STREAM_EVENT_SUCCEEDED = 2, + STREAM_EVENT_FAILED = 3, + STREAM_EVENT_CLOSED = 4, + STREAM_EVENT_NEW = 5, + STREAM_EVENT_NEW_RESOLVE = 6, + STREAM_EVENT_FAILED_RETRIABLE = 7, + STREAM_EVENT_REMAP = 8 +} stream_status_event_t; + +/** Used to indicate the type of a buildtime event */ +typedef enum buildtimeout_set_event_t { + BUILDTIMEOUT_SET_EVENT_COMPUTED = 0, + BUILDTIMEOUT_SET_EVENT_RESET = 1, + BUILDTIMEOUT_SET_EVENT_SUSPENDED = 2, + BUILDTIMEOUT_SET_EVENT_DISCARD = 3, + BUILDTIMEOUT_SET_EVENT_RESUME = 4 +} buildtimeout_set_event_t; + +/** Enum describing various stages of bootstrapping, for use with controller + * bootstrap status events. The values range from 0 to 100. */ +typedef enum { + BOOTSTRAP_STATUS_UNDEF=-1, + BOOTSTRAP_STATUS_STARTING=0, + + /* Initial connection to any relay */ + + BOOTSTRAP_STATUS_CONN_PT=1, + BOOTSTRAP_STATUS_CONN_DONE_PT=2, + BOOTSTRAP_STATUS_CONN_PROXY=3, + BOOTSTRAP_STATUS_CONN_DONE_PROXY=4, + BOOTSTRAP_STATUS_CONN=5, + BOOTSTRAP_STATUS_CONN_DONE=10, + BOOTSTRAP_STATUS_HANDSHAKE=14, + BOOTSTRAP_STATUS_HANDSHAKE_DONE=15, + + /* Loading directory info */ + + BOOTSTRAP_STATUS_ONEHOP_CREATE=20, + BOOTSTRAP_STATUS_REQUESTING_STATUS=25, + BOOTSTRAP_STATUS_LOADING_STATUS=30, + BOOTSTRAP_STATUS_LOADING_KEYS=40, + BOOTSTRAP_STATUS_REQUESTING_DESCRIPTORS=45, + BOOTSTRAP_STATUS_LOADING_DESCRIPTORS=50, + BOOTSTRAP_STATUS_ENOUGH_DIRINFO=75, + + /* Connecting to a relay for AP circuits */ + + BOOTSTRAP_STATUS_AP_CONN_PT=76, + BOOTSTRAP_STATUS_AP_CONN_DONE_PT=77, + BOOTSTRAP_STATUS_AP_CONN_PROXY=78, + BOOTSTRAP_STATUS_AP_CONN_DONE_PROXY=79, + BOOTSTRAP_STATUS_AP_CONN=80, + BOOTSTRAP_STATUS_AP_CONN_DONE=85, + BOOTSTRAP_STATUS_AP_HANDSHAKE=89, + BOOTSTRAP_STATUS_AP_HANDSHAKE_DONE=90, + + /* Creating AP circuits */ + + BOOTSTRAP_STATUS_CIRCUIT_CREATE=95, + BOOTSTRAP_STATUS_DONE=100 +} bootstrap_status_t; + +/** Reason for remapping an AP connection's address: we have a cached + * answer. */ +#define REMAP_STREAM_SOURCE_CACHE 1 +/** Reason for remapping an AP connection's address: the exit node told us an + * answer. */ +#define REMAP_STREAM_SOURCE_EXIT 2 + +void control_initialize_event_queue(void); + +void control_update_global_event_mask(void); +void control_adjust_event_log_severity(void); + +#define EVENT_NS 0x000F +int control_event_is_interesting(int event); + +void control_per_second_events(void); +int control_any_per_second_event_enabled(void); + +int control_event_circuit_status(origin_circuit_t *circ, + circuit_status_event_t e, int reason); +int control_event_circuit_purpose_changed(origin_circuit_t *circ, + int old_purpose); +int control_event_circuit_cannibalized(origin_circuit_t *circ, + int old_purpose, + const struct timeval *old_tv_created); +int control_event_stream_status(entry_connection_t *conn, + stream_status_event_t e, + int reason); +int control_event_or_conn_status(or_connection_t *conn, + or_conn_status_event_t e, int reason); +int control_event_bandwidth_used(uint32_t n_read, uint32_t n_written); +int control_event_stream_bandwidth(edge_connection_t *edge_conn); +int control_event_stream_bandwidth_used(void); +int control_event_circ_bandwidth_used(void); +int control_event_circ_bandwidth_used_for_circ(origin_circuit_t *ocirc); +int control_event_conn_bandwidth(connection_t *conn); +int control_event_conn_bandwidth_used(void); +int control_event_circuit_cell_stats(void); +void control_event_logmsg(int severity, uint32_t domain, const char *msg); +void control_event_logmsg_pending(void); +int control_event_descriptors_changed(smartlist_t *routers); +int control_event_address_mapped(const char *from, const char *to, + time_t expires, const char *error, + const int cached); +int control_event_my_descriptor_changed(void); +int control_event_network_liveness_update(int liveness); +int control_event_networkstatus_changed(smartlist_t *statuses); + +int control_event_newconsensus(const networkstatus_t *consensus); +int control_event_networkstatus_changed_single(const routerstatus_t *rs); +int control_event_general_status(int severity, const char *format, ...) + CHECK_PRINTF(2,3); +int control_event_client_status(int severity, const char *format, ...) + CHECK_PRINTF(2,3); +int control_event_server_status(int severity, const char *format, ...) + CHECK_PRINTF(2,3); + +int control_event_general_error(const char *format, ...) + CHECK_PRINTF(1,2); +int control_event_client_error(const char *format, ...) + CHECK_PRINTF(1,2); +int control_event_server_error(const char *format, ...) + CHECK_PRINTF(1,2); + +int control_event_guard(const char *nickname, const char *digest, + const char *status); +int control_event_conf_changed(const smartlist_t *elements); +int control_event_buildtimeout_set(buildtimeout_set_event_t type, + const char *args); +int control_event_signal(uintptr_t signal); + +void control_event_bootstrap(bootstrap_status_t status, int progress); +MOCK_DECL(void, control_event_bootstrap_prob_or,(const char *warn, + int reason, + or_connection_t *or_conn)); +void control_event_boot_dir(bootstrap_status_t status, int progress); +void control_event_boot_first_orconn(void); +void control_event_bootstrap_problem(const char *warn, const char *reason, + const connection_t *conn, int dowarn); +char *control_event_boot_last_msg(void); +void control_event_bootstrap_reset(void); + +void control_event_clients_seen(const char *controller_str); +void control_event_transport_launched(const char *mode, + const char *transport_name, + tor_addr_t *addr, uint16_t port); +void control_event_pt_log(const char *log); +void control_event_pt_status(const char *status); + +void control_event_hs_descriptor_requested(const char *onion_address, + rend_auth_type_t auth_type, + const char *id_digest, + const char *desc_id, + const char *hsdir_index); +void control_event_hs_descriptor_created(const char *onion_address, + const char *desc_id, + int replica); +void control_event_hs_descriptor_upload(const char *onion_address, + const char *desc_id, + const char *hs_dir, + const char *hsdir_index); +void control_event_hs_descriptor_upload_end(const char *action, + const char *onion_address, + const char *hs_dir, + const char *reason); +void control_event_hs_descriptor_uploaded(const char *hs_dir, + const char *onion_address); +/* Hidden service v2 HS_DESC specific. */ +void control_event_hsv2_descriptor_failed(const rend_data_t *rend_data, + const char *id_digest, + const char *reason); +void control_event_hsv2_descriptor_received(const char *onion_address, + const rend_data_t *rend_data, + const char *id_digest); +/* Hidden service v3 HS_DESC specific. */ +void control_event_hsv3_descriptor_failed(const char *onion_address, + const char *desc_id, + const char *hsdir_id_digest, + const char *reason); +void control_event_hsv3_descriptor_received(const char *onion_address, + const char *desc_id, + const char *hsdir_id_digest); +void control_event_hs_descriptor_upload_failed(const char *hs_dir, + const char *onion_address, + const char *reason); +void control_event_hs_descriptor_content(const char *onion_address, + const char *desc_id, + const char *hsdir_fp, + const char *content); + +void control_events_free_all(void); + +#ifdef CONTROL_MODULE_PRIVATE +char *get_bw_samples(void); +#endif /* defined(CONTROL_MODULE_PRIVATE) */ + +#ifdef CONTROL_EVENTS_PRIVATE +/** Bitfield: The bit 1<<e is set if any open control + * connection is interested in events of type e. We use this + * so that we can decide to skip generating event messages that nobody + * has interest in without having to walk over the global connection + * list to find out. + **/ +typedef uint64_t event_mask_t; + +/* Recognized asynchronous event types. It's okay to expand this list + * because it is used both as a list of v0 event types, and as indices + * into the bitfield to determine which controllers want which events. + */ +/* This bitfield has no event zero 0x0000 */ +#define EVENT_MIN_ 0x0001 +#define EVENT_CIRCUIT_STATUS 0x0001 +#define EVENT_STREAM_STATUS 0x0002 +#define EVENT_OR_CONN_STATUS 0x0003 +#define EVENT_BANDWIDTH_USED 0x0004 +#define EVENT_CIRCUIT_STATUS_MINOR 0x0005 +#define EVENT_NEW_DESC 0x0006 +#define EVENT_DEBUG_MSG 0x0007 +#define EVENT_INFO_MSG 0x0008 +#define EVENT_NOTICE_MSG 0x0009 +#define EVENT_WARN_MSG 0x000A +#define EVENT_ERR_MSG 0x000B +#define EVENT_ADDRMAP 0x000C +/* There was an AUTHDIR_NEWDESCS event, but it no longer exists. We + can reclaim 0x000D. */ +#define EVENT_DESCCHANGED 0x000E +/* Exposed above */ +// #define EVENT_NS 0x000F +#define EVENT_STATUS_CLIENT 0x0010 +#define EVENT_STATUS_SERVER 0x0011 +#define EVENT_STATUS_GENERAL 0x0012 +#define EVENT_GUARD 0x0013 +#define EVENT_STREAM_BANDWIDTH_USED 0x0014 +#define EVENT_CLIENTS_SEEN 0x0015 +#define EVENT_NEWCONSENSUS 0x0016 +#define EVENT_BUILDTIMEOUT_SET 0x0017 +#define EVENT_GOT_SIGNAL 0x0018 +#define EVENT_CONF_CHANGED 0x0019 +#define EVENT_CONN_BW 0x001A +#define EVENT_CELL_STATS 0x001B +/* UNUSED : 0x001C */ +#define EVENT_CIRC_BANDWIDTH_USED 0x001D +#define EVENT_TRANSPORT_LAUNCHED 0x0020 +#define EVENT_HS_DESC 0x0021 +#define EVENT_HS_DESC_CONTENT 0x0022 +#define EVENT_NETWORK_LIVENESS 0x0023 +#define EVENT_PT_LOG 0x0024 +#define EVENT_PT_STATUS 0x0025 +#define EVENT_MAX_ 0x0025 + +/* sizeof(control_connection_t.event_mask) in bits, currently a uint64_t */ +#define EVENT_CAPACITY_ 0x0040 + +/* If EVENT_MAX_ ever hits 0x0040, we need to make the mask into a + * different structure, as it can only handle a maximum left shift of 1<<63. */ + +#if EVENT_MAX_ >= EVENT_CAPACITY_ +#error control_connection_t.event_mask has an event greater than its capacity +#endif + +#define EVENT_MASK_(e) (((uint64_t)1)<<(e)) + +#define EVENT_MASK_NONE_ ((uint64_t)0x0) + +#define EVENT_MASK_ABOVE_MIN_ ((~((uint64_t)0x0)) << EVENT_MIN_) +#define EVENT_MASK_BELOW_MAX_ ((~((uint64_t)0x0)) \ + >> (EVENT_CAPACITY_ - EVENT_MAX_ \ + - EVENT_MIN_)) + +#define EVENT_MASK_ALL_ (EVENT_MASK_ABOVE_MIN_ \ + & EVENT_MASK_BELOW_MAX_) + +/** Helper structure: temporarily stores cell statistics for a circuit. */ +typedef struct cell_stats_t { + /** Number of cells added in app-ward direction by command. */ + uint64_t added_cells_appward[CELL_COMMAND_MAX_ + 1]; + /** Number of cells added in exit-ward direction by command. */ + uint64_t added_cells_exitward[CELL_COMMAND_MAX_ + 1]; + /** Number of cells removed in app-ward direction by command. */ + uint64_t removed_cells_appward[CELL_COMMAND_MAX_ + 1]; + /** Number of cells removed in exit-ward direction by command. */ + uint64_t removed_cells_exitward[CELL_COMMAND_MAX_ + 1]; + /** Total waiting time of cells in app-ward direction by command. */ + uint64_t total_time_appward[CELL_COMMAND_MAX_ + 1]; + /** Total waiting time of cells in exit-ward direction by command. */ + uint64_t total_time_exitward[CELL_COMMAND_MAX_ + 1]; +} cell_stats_t; + +void sum_up_cell_stats_by_command(circuit_t *circ, + cell_stats_t *cell_stats); +void append_cell_stats_by_command(smartlist_t *event_parts, + const char *key, + const uint64_t *include_if_non_zero, + const uint64_t *number_to_include); +void format_cell_stats(char **event_string, circuit_t *circ, + cell_stats_t *cell_stats); + +#ifdef TOR_UNIT_TESTS +MOCK_DECL(STATIC void, + send_control_event_string,(uint16_t event, const char *msg)); + +MOCK_DECL(STATIC void, + queue_control_event_string,(uint16_t event, char *msg)); + +void control_testing_set_global_event_mask(uint64_t mask); + +#endif /* defined(TOR_UNIT_TESTS) */ + +#endif /* defined(CONTROL_EVENTS_PRIVATE) */ + +#endif /* !defined(TOR_CONTROL_EVENTS_H) */ diff --git a/src/feature/control/control_fmt.c b/src/feature/control/control_fmt.c new file mode 100644 index 0000000000..a6d7919915 --- /dev/null +++ b/src/feature/control/control_fmt.c @@ -0,0 +1,289 @@ +/* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file control.c + * \brief Formatting functions for controller data. + */ + +#include "core/or/or.h" + +#include "core/mainloop/connection.h" +#include "core/or/circuitbuild.h" +#include "core/or/circuitlist.h" +#include "core/or/connection_edge.h" +#include "feature/control/control_fmt.h" +#include "feature/nodelist/nodelist.h" + +#include "core/or/cpath_build_state_st.h" +#include "core/or/entry_connection_st.h" +#include "core/or/or_connection_st.h" +#include "core/or/origin_circuit_st.h" +#include "core/or/socks_request_st.h" +#include "feature/control/control_connection_st.h" + +/** Acts like sprintf, but writes its formatted string to the end of + * conn-\>outbuf. */ +void +connection_printf_to_buf(control_connection_t *conn, const char *format, ...) +{ + va_list ap; + char *buf = NULL; + int len; + + va_start(ap,format); + len = tor_vasprintf(&buf, format, ap); + va_end(ap); + + if (len < 0) { + log_err(LD_BUG, "Unable to format string for controller."); + tor_assert(0); + } + + connection_buf_add(buf, (size_t)len, TO_CONN(conn)); + + tor_free(buf); +} + +/** Given an AP connection conn and a len-character buffer + * buf, determine the address:port combination requested on + * conn, and write it to buf. Return 0 on success, -1 on + * failure. */ +int +write_stream_target_to_buf(entry_connection_t *conn, char *buf, size_t len) +{ + char buf2[256]; + if (conn->chosen_exit_name) + if (tor_snprintf(buf2, sizeof(buf2), ".%s.exit", conn->chosen_exit_name)<0) + return -1; + if (!conn->socks_request) + return -1; + if (tor_snprintf(buf, len, "%s%s%s:%d", + conn->socks_request->address, + conn->chosen_exit_name ? buf2 : "", + !conn->chosen_exit_name && connection_edge_is_rendezvous_stream( + ENTRY_TO_EDGE_CONN(conn)) ? ".onion" : "", + conn->socks_request->port)<0) + return -1; + return 0; +} + +/** Figure out the best name for the target router of an OR connection + * conn, and write it into the len-character buffer + * name. */ +void +orconn_target_get_name(char *name, size_t len, or_connection_t *conn) +{ + const node_t *node = node_get_by_id(conn->identity_digest); + if (node) { + tor_assert(len > MAX_VERBOSE_NICKNAME_LEN); + node_get_verbose_nickname(node, name); + } else if (! tor_digest_is_zero(conn->identity_digest)) { + name[0] = '$'; + base16_encode(name+1, len-1, conn->identity_digest, + DIGEST_LEN); + } else { + tor_snprintf(name, len, "%s:%d", + conn->base_.address, conn->base_.port); + } +} + +/** Allocate and return a description of circ's current status, + * including its path (if any). */ +char * +circuit_describe_status_for_controller(origin_circuit_t *circ) +{ + char *rv; + smartlist_t *descparts = smartlist_new(); + + { + char *vpath = circuit_list_path_for_controller(circ); + if (*vpath) { + smartlist_add(descparts, vpath); + } else { + tor_free(vpath); /* empty path; don't put an extra space in the result */ + } + } + + { + cpath_build_state_t *build_state = circ->build_state; + smartlist_t *flaglist = smartlist_new(); + char *flaglist_joined; + + if (build_state->onehop_tunnel) + smartlist_add(flaglist, (void *)"ONEHOP_TUNNEL"); + if (build_state->is_internal) + smartlist_add(flaglist, (void *)"IS_INTERNAL"); + if (build_state->need_capacity) + smartlist_add(flaglist, (void *)"NEED_CAPACITY"); + if (build_state->need_uptime) + smartlist_add(flaglist, (void *)"NEED_UPTIME"); + + /* Only emit a BUILD_FLAGS argument if it will have a non-empty value. */ + if (smartlist_len(flaglist)) { + flaglist_joined = smartlist_join_strings(flaglist, ",", 0, NULL); + + smartlist_add_asprintf(descparts, "BUILD_FLAGS=%s", flaglist_joined); + + tor_free(flaglist_joined); + } + + smartlist_free(flaglist); + } + + smartlist_add_asprintf(descparts, "PURPOSE=%s", + circuit_purpose_to_controller_string(circ->base_.purpose)); + + { + const char *hs_state = + circuit_purpose_to_controller_hs_state_string(circ->base_.purpose); + + if (hs_state != NULL) { + smartlist_add_asprintf(descparts, "HS_STATE=%s", hs_state); + } + } + + if (circ->rend_data != NULL || circ->hs_ident != NULL) { + char addr[HS_SERVICE_ADDR_LEN_BASE32 + 1]; + const char *onion_address; + if (circ->rend_data) { + onion_address = rend_data_get_address(circ->rend_data); + } else { + hs_build_address(&circ->hs_ident->identity_pk, HS_VERSION_THREE, addr); + onion_address = addr; + } + smartlist_add_asprintf(descparts, "REND_QUERY=%s", onion_address); + } + + { + char tbuf[ISO_TIME_USEC_LEN+1]; + format_iso_time_nospace_usec(tbuf, &circ->base_.timestamp_created); + + smartlist_add_asprintf(descparts, "TIME_CREATED=%s", tbuf); + } + + // Show username and/or password if available. + if (circ->socks_username_len > 0) { + char* socks_username_escaped = esc_for_log_len(circ->socks_username, + (size_t) circ->socks_username_len); + smartlist_add_asprintf(descparts, "SOCKS_USERNAME=%s", + socks_username_escaped); + tor_free(socks_username_escaped); + } + if (circ->socks_password_len > 0) { + char* socks_password_escaped = esc_for_log_len(circ->socks_password, + (size_t) circ->socks_password_len); + smartlist_add_asprintf(descparts, "SOCKS_PASSWORD=%s", + socks_password_escaped); + tor_free(socks_password_escaped); + } + + rv = smartlist_join_strings(descparts, " ", 0, NULL); + + SMARTLIST_FOREACH(descparts, char *, cp, tor_free(cp)); + smartlist_free(descparts); + + return rv; +} + +/** Given a len-character string in data, made of lines + * terminated by CRLF, allocate a new string in *out, and copy the + * contents of data into *out, adding a period before any period + * that appears at the start of a line, and adding a period-CRLF line at + * the end. Replace all LF characters sequences with CRLF. Return the number + * of bytes in *out. + */ +size_t +write_escaped_data(const char *data, size_t len, char **out) +{ + tor_assert(len < SIZE_MAX - 9); + size_t sz_out = len+8+1; + char *outp; + const char *start = data, *end; + size_t i; + int start_of_line; + for (i=0; i < len; ++i) { + if (data[i] == '\n') { + sz_out += 2; /* Maybe add a CR; maybe add a dot. */ + if (sz_out >= SIZE_T_CEILING) { + log_warn(LD_BUG, "Input to write_escaped_data was too long"); + *out = tor_strdup(".\r\n"); + return 3; + } + } + } + *out = outp = tor_malloc(sz_out); + end = data+len; + start_of_line = 1; + while (data < end) { + if (*data == '\n') { + if (data > start && data[-1] != '\r') + *outp++ = '\r'; + start_of_line = 1; + } else if (*data == '.') { + if (start_of_line) { + start_of_line = 0; + *outp++ = '.'; + } + } else { + start_of_line = 0; + } + *outp++ = *data++; + } + if (outp < *out+2 || fast_memcmp(outp-2, "\r\n", 2)) { + *outp++ = '\r'; + *outp++ = '\n'; + } + *outp++ = '.'; + *outp++ = '\r'; + *outp++ = '\n'; + *outp = '\0'; /* NUL-terminate just in case. */ + tor_assert(outp >= *out); + tor_assert((size_t)(outp - *out) <= sz_out); + return outp - *out; +} + +/** Given a len-character string in data, made of lines + * terminated by CRLF, allocate a new string in *out, and copy + * the contents of data into *out, removing any period + * that appears at the start of a line, and replacing all CRLF sequences + * with LF. Return the number of + * bytes in *out. */ +size_t +read_escaped_data(const char *data, size_t len, char **out) +{ + char *outp; + const char *next; + const char *end; + + *out = outp = tor_malloc(len+1); + + end = data+len; + + while (data < end) { + /* we're at the start of a line. */ + if (*data == '.') + ++data; + next = memchr(data, '\n', end-data); + if (next) { + size_t n_to_copy = next-data; + /* Don't copy a CR that precedes this LF. */ + if (n_to_copy && *(next-1) == '\r') + --n_to_copy; + memcpy(outp, data, n_to_copy); + outp += n_to_copy; + data = next+1; /* This will point at the start of the next line, + * or the end of the string, or a period. */ + } else { + memcpy(outp, data, end-data); + outp += (end-data); + *outp = '\0'; + return outp - *out; + } + *outp++ = '\n'; + } + + *outp = '\0'; + return outp - *out; +} diff --git a/src/feature/control/control_fmt.h b/src/feature/control/control_fmt.h new file mode 100644 index 0000000000..e36edd2a46 --- /dev/null +++ b/src/feature/control/control_fmt.h @@ -0,0 +1,28 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file control_fmt.h + * \brief Header file for control_fmt.c. + **/ + +#ifndef TOR_CONTROL_FMT_H +#define TOR_CONTROL_FMT_H + +void connection_printf_to_buf(control_connection_t *conn, + const char *format, ...) + CHECK_PRINTF(2,3); + +int write_stream_target_to_buf(entry_connection_t *conn, char *buf, + size_t len); +void orconn_target_get_name(char *buf, size_t len, + or_connection_t *conn); +char *circuit_describe_status_for_controller(origin_circuit_t *circ); + +size_t write_escaped_data(const char *data, size_t len, char **out); +size_t read_escaped_data(const char *data, size_t len, char **out); + +#endif /* !defined(TOR_CONTROL_FMT_H) */ diff --git a/src/feature/dirclient/dirclient.c b/src/feature/dirclient/dirclient.c index 70b6a20028..0b79b07799 100644 --- a/src/feature/dirclient/dirclient.c +++ b/src/feature/dirclient/dirclient.c @@ -14,7 +14,7 @@ #include "core/or/policies.h" #include "feature/client/bridges.h" #include "feature/client/entrynodes.h" -#include "feature/control/control.h" +#include "feature/control/control_events.h" #include "feature/dirauth/authmode.h" #include "feature/dirauth/dirvote.h" #include "feature/dirauth/shared_random.h" diff --git a/src/feature/hibernate/hibernate.c b/src/feature/hibernate/hibernate.c index 70c2b4f69f..7351e5e002 100644 --- a/src/feature/hibernate/hibernate.c +++ b/src/feature/hibernate/hibernate.c @@ -35,7 +35,7 @@ hibernating, phase 2: #include "core/mainloop/connection.h" #include "core/or/connection_edge.h" #include "core/or/connection_or.h" -#include "feature/control/control.h" +#include "feature/control/control_events.h" #include "lib/crypt_ops/crypto_rand.h" #include "lib/defs/time.h" #include "feature/hibernate/hibernate.h" diff --git a/src/feature/hs/hs_control.c b/src/feature/hs/hs_control.c index d837c5beed..20a1061609 100644 --- a/src/feature/hs/hs_control.c +++ b/src/feature/hs/hs_control.c @@ -7,7 +7,7 @@ **/ #include "core/or/or.h" -#include "feature/control/control.h" +#include "feature/control/control_events.h" #include "lib/crypt_ops/crypto_format.h" #include "lib/crypt_ops/crypto_util.h" #include "feature/hs/hs_client.h" diff --git a/src/feature/nodelist/dirlist.c b/src/feature/nodelist/dirlist.c index 93baa6e4e0..e2a1d6a9fa 100644 --- a/src/feature/nodelist/dirlist.c +++ b/src/feature/nodelist/dirlist.c @@ -28,7 +28,7 @@ #include "app/config/config.h" #include "core/or/policies.h" -#include "feature/control/control.h" +#include "feature/control/control_events.h" #include "feature/dirauth/authmode.h" #include "feature/dircommon/directory.h" #include "feature/nodelist/dirlist.h" diff --git a/src/feature/nodelist/networkstatus.c b/src/feature/nodelist/networkstatus.c index 023115978c..ea9f12367f 100644 --- a/src/feature/nodelist/networkstatus.c +++ b/src/feature/nodelist/networkstatus.c @@ -58,7 +58,7 @@ #include "feature/client/bridges.h" #include "feature/client/entrynodes.h" #include "feature/client/transports.h" -#include "feature/control/control.h" +#include "feature/control/control_events.h" #include "feature/dirauth/reachability.h" #include "feature/dircache/consdiffmgr.h" #include "feature/dircache/dirserv.h" diff --git a/src/feature/nodelist/nodelist.c b/src/feature/nodelist/nodelist.c index 8371d787fd..f878d47fd7 100644 --- a/src/feature/nodelist/nodelist.c +++ b/src/feature/nodelist/nodelist.c @@ -49,7 +49,7 @@ #include "core/or/protover.h" #include "feature/client/bridges.h" #include "feature/client/entrynodes.h" -#include "feature/control/control.h" +#include "feature/control/control_events.h" #include "feature/dirauth/process_descs.h" #include "feature/dircache/dirserv.h" #include "feature/hs/hs_client.h" diff --git a/src/feature/nodelist/routerlist.c b/src/feature/nodelist/routerlist.c index d1220f553a..48f448ad1e 100644 --- a/src/feature/nodelist/routerlist.c +++ b/src/feature/nodelist/routerlist.c @@ -67,7 +67,7 @@ #include "core/mainloop/mainloop.h" #include "core/or/policies.h" #include "feature/client/bridges.h" -#include "feature/control/control.h" +#include "feature/control/control_events.h" #include "feature/dirauth/authmode.h" #include "feature/dirauth/process_descs.h" #include "feature/dirauth/reachability.h" diff --git a/src/feature/relay/dns.c b/src/feature/relay/dns.c index fa0a1b5910..664edf96a9 100644 --- a/src/feature/relay/dns.c +++ b/src/feature/relay/dns.c @@ -59,7 +59,7 @@ #include "core/or/connection_edge.h" #include "core/or/policies.h" #include "core/or/relay.h" -#include "feature/control/control.h" +#include "feature/control/control_events.h" #include "feature/relay/dns.h" #include "feature/relay/router.h" #include "feature/relay/routermode.h" diff --git a/src/feature/relay/ext_orport.c b/src/feature/relay/ext_orport.c index 8589efb48d..c343d19b8d 100644 --- a/src/feature/relay/ext_orport.c +++ b/src/feature/relay/ext_orport.c @@ -20,7 +20,7 @@ #include "core/or/or.h" #include "core/mainloop/connection.h" #include "core/or/connection_or.h" -#include "feature/control/control.h" +#include "feature/control/control_events.h" #include "app/config/config.h" #include "lib/crypt_ops/crypto_rand.h" #include "lib/crypt_ops/crypto_util.h" @@ -659,4 +659,3 @@ ext_orport_free_all(void) if (ext_or_auth_cookie) /* Free the auth cookie */ tor_free(ext_or_auth_cookie); } - diff --git a/src/feature/relay/router.c b/src/feature/relay/router.c index cdd032f78d..fcc84730bd 100644 --- a/src/feature/relay/router.c +++ b/src/feature/relay/router.c @@ -16,7 +16,7 @@ #include "core/or/policies.h" #include "core/or/protover.h" #include "feature/client/transports.h" -#include "feature/control/control.h" +#include "feature/control/control_events.h" #include "feature/dirauth/process_descs.h" #include "feature/dircache/dirserv.h" #include "feature/dirclient/dirclient.h" diff --git a/src/feature/relay/selftest.c b/src/feature/relay/selftest.c index 064eea6c46..eeddd09b63 100644 --- a/src/feature/relay/selftest.c +++ b/src/feature/relay/selftest.c @@ -26,7 +26,7 @@ #include "core/or/crypt_path_st.h" #include "core/or/origin_circuit_st.h" #include "core/or/relay.h" -#include "feature/control/control.h" +#include "feature/control/control_events.h" #include "feature/dirclient/dirclient.h" #include "feature/dircommon/directory.h" #include "feature/nodelist/authority_cert_st.h" diff --git a/src/feature/rend/rendclient.c b/src/feature/rend/rendclient.c index 4ca783c7c3..5a8b234544 100644 --- a/src/feature/rend/rendclient.c +++ b/src/feature/rend/rendclient.c @@ -17,7 +17,7 @@ #include "core/or/connection_edge.h" #include "core/or/relay.h" #include "feature/client/circpathbias.h" -#include "feature/control/control.h" +#include "feature/control/control_events.h" #include "feature/dirclient/dirclient.h" #include "feature/dircommon/directory.h" #include "feature/hs/hs_circuit.h" diff --git a/src/feature/rend/rendcommon.c b/src/feature/rend/rendcommon.c index 5cc054f454..777de2984c 100644 --- a/src/feature/rend/rendcommon.c +++ b/src/feature/rend/rendcommon.c @@ -15,7 +15,7 @@ #include "core/or/circuitlist.h" #include "core/or/circuituse.h" #include "app/config/config.h" -#include "feature/control/control.h" +#include "feature/control/control_events.h" #include "lib/crypt_ops/crypto_rand.h" #include "lib/crypt_ops/crypto_util.h" #include "feature/hs/hs_client.h" diff --git a/src/feature/rend/rendservice.c b/src/feature/rend/rendservice.c index 73edcaccf5..57475a64b0 100644 --- a/src/feature/rend/rendservice.c +++ b/src/feature/rend/rendservice.c @@ -19,7 +19,7 @@ #include "core/or/policies.h" #include "core/or/relay.h" #include "feature/client/circpathbias.h" -#include "feature/control/control.h" +#include "feature/control/control_events.h" #include "feature/dirclient/dirclient.h" #include "feature/dircommon/directory.h" #include "feature/hs/hs_common.h" diff --git a/src/feature/stats/geoip_stats.c b/src/feature/stats/geoip_stats.c index 5119da19a0..6fb21f4f79 100644 --- a/src/feature/stats/geoip_stats.c +++ b/src/feature/stats/geoip_stats.c @@ -32,7 +32,7 @@ #include "ht.h" #include "lib/buf/buffers.h" #include "app/config/config.h" -#include "feature/control/control.h" +#include "feature/control/control_events.h" #include "feature/client/dnsserv.h" #include "core/or/dos.h" #include "lib/geoip/geoip.h" diff --git a/src/test/test_controller_events.c b/src/test/test_controller_events.c index 647eac43c7..910aacace3 100644 --- a/src/test/test_controller_events.c +++ b/src/test/test_controller_events.c @@ -4,6 +4,7 @@ #define CONNECTION_PRIVATE #define TOR_CHANNEL_INTERNAL_ #define CONTROL_PRIVATE +#define CONTROL_EVENTS_PRIVATE #define OCIRC_EVENT_PRIVATE #define ORCONN_EVENT_PRIVATE #include "core/or/or.h" @@ -13,7 +14,7 @@ #include "core/or/ocirc_event.h" #include "core/or/orconn_event.h" #include "core/mainloop/connection.h" -#include "feature/control/control.h" +#include "feature/control/control_events.h" #include "test/test.h" #include "core/or/or_circuit_st.h" diff --git a/src/test/test_extorport.c b/src/test/test_extorport.c index aeb71ec583..f5d16af921 100644 --- a/src/test/test_extorport.c +++ b/src/test/test_extorport.c @@ -9,7 +9,7 @@ #include "core/mainloop/connection.h" #include "core/or/connection_or.h" #include "app/config/config.h" -#include "feature/control/control.h" +#include "feature/control/control_events.h" #include "lib/crypt_ops/crypto_rand.h" #include "feature/relay/ext_orport.h" #include "core/mainloop/mainloop.h" diff --git a/src/test/test_hs.c b/src/test/test_hs.c index a611b46ca6..de10a10d81 100644 --- a/src/test/test_hs.c +++ b/src/test/test_hs.c @@ -6,7 +6,7 @@ * \brief Unit tests for hidden service. **/ -#define CONTROL_PRIVATE +#define CONTROL_EVENTS_PRIVATE #define CIRCUITBUILD_PRIVATE #define RENDCOMMON_PRIVATE #define RENDSERVICE_PRIVATE @@ -15,6 +15,7 @@ #include "core/or/or.h" #include "test/test.h" #include "feature/control/control.h" +#include "feature/control/control_events.h" #include "app/config/config.h" #include "feature/hs/hs_common.h" #include "feature/rend/rendcommon.h" diff --git a/src/test/test_hs_control.c b/src/test/test_hs_control.c index ba67712f1b..d23d31954b 100644 --- a/src/test/test_hs_control.c +++ b/src/test/test_hs_control.c @@ -6,11 +6,12 @@ * \brief Unit tests for hidden service control port event and command. **/ -#define CONTROL_PRIVATE +#define CONTROL_EVENTS_PRIVATE #include "core/or/or.h" #include "test/test.h" #include "feature/control/control.h" +#include "feature/control/control_events.h" #include "app/config/config.h" #include "feature/hs/hs_common.h" #include "feature/hs/hs_control.h" diff --git a/src/test/test_pt.c b/src/test/test_pt.c index d2996f4cc3..612006906d 100644 --- a/src/test/test_pt.c +++ b/src/test/test_pt.c @@ -7,12 +7,14 @@ #define PT_PRIVATE #define UTIL_PRIVATE #define STATEFILE_PRIVATE -#define CONTROL_PRIVATE +//#define CONTROL_PRIVATE +#define CONTROL_EVENTS_PRIVATE #define PROCESS_PRIVATE #include "core/or/or.h" #include "app/config/config.h" #include "app/config/confparse.h" #include "feature/control/control.h" +#include "feature/control/control_events.h" #include "feature/client/transports.h" #include "core/or/circuitbuild.h" #include "app/config/statefile.h" diff --git a/src/test/test_util.c b/src/test/test_util.c index 4990aa709a..4875a7c943 100644 --- a/src/test/test_util.c +++ b/src/test/test_util.c @@ -16,6 +16,7 @@ #include "lib/buf/buffers.h" #include "app/config/config.h" #include "feature/control/control.h" +#include "feature/control/control_fmt.h" #include "feature/client/transports.h" #include "lib/crypt_ops/crypto_format.h" #include "lib/crypt_ops/crypto_rand.h" diff --git a/src/test/testing_common.c b/src/test/testing_common.c index 8fc8ef7830..1c2a2e8960 100644 --- a/src/test/testing_common.c +++ b/src/test/testing_common.c @@ -12,6 +12,7 @@ #include "orconfig.h" #include "core/or/or.h" #include "feature/control/control.h" +#include "feature/control/control_events.h" #include "app/config/config.h" #include "lib/crypt_ops/crypto_dh.h" #include "lib/crypt_ops/crypto_ed25519.h" From 4754e9058b7521967dd92092e54483cec456bb2f Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 25 Mar 2019 12:48:52 -0400 Subject: [PATCH 0634/2557] Split getinfo handling into a new control_getinfo.c --- scripts/maint/practracker/exceptions.txt | 7 +- src/core/include.am | 4 +- src/feature/control/control.c | 1661 +--------------------- src/feature/control/control.h | 44 +- src/feature/control/control_events.c | 37 + src/feature/control/control_events.h | 8 + src/feature/control/control_fmt.c | 10 + src/feature/control/control_fmt.h | 1 + src/feature/control/control_getinfo.c | 1661 ++++++++++++++++++++++ src/feature/control/control_getinfo.h | 57 + src/test/test_controller.c | 2 + src/test/test_dir.c | 4 +- 12 files changed, 1803 insertions(+), 1693 deletions(-) create mode 100644 src/feature/control/control_getinfo.c create mode 100644 src/feature/control/control_getinfo.h diff --git a/scripts/maint/practracker/exceptions.txt b/scripts/maint/practracker/exceptions.txt index 15f16fdb4e..142fda5806 100644 --- a/scripts/maint/practracker/exceptions.txt +++ b/scripts/maint/practracker/exceptions.txt @@ -112,9 +112,9 @@ problem function-size /src/feature/dircommon/consdiff.c:apply_ed_diff() 159 problem file-size /src/feature/control/control.c 7592 problem include-count /src/feature/control/control.c 90 problem function-size /src/feature/control/control.c:handle_control_authenticate() 188 -problem function-size /src/feature/control/control.c:getinfo_helper_misc() 109 -problem function-size /src/feature/control/control.c:getinfo_helper_dir() 304 -problem function-size /src/feature/control/control.c:getinfo_helper_events() 236 +problem function-size /src/feature/control/control_getinfo.c:getinfo_helper_misc() 109 +problem function-size /src/feature/control/control_getinfo.c:getinfo_helper_dir() 304 +problem function-size /src/feature/control/control_getinfo.c:getinfo_helper_events() 236 problem function-size /src/feature/control/control.c:handle_control_extendcircuit() 151 problem function-size /src/feature/control/control.c:handle_control_authchallenge() 115 problem function-size /src/feature/control/control.c:handle_control_hsfetch() 114 @@ -123,6 +123,7 @@ problem function-size /src/feature/control/control.c:handle_control_add_onion() problem function-size /src/feature/control/control.c:add_onion_helper_keyarg() 125 problem function-size /src/feature/control/control.c:connection_control_process_inbuf() 239 problem function-size /src/feature/control/control_events.c:control_event_stream_status() 119 +problem include-count /src/feature/control/control_getinfo.c 51 problem function-size /src/feature/stats/rephist.c:rep_hist_load_mtbf_data() 185 problem function-size /src/feature/stats/rephist.c:rep_hist_format_exit_stats() 148 problem function-size /src/feature/dircache/consdiffmgr.c:consdiffmgr_cleanup() 115 diff --git a/src/core/include.am b/src/core/include.am index 58fcb784fd..a8ec360592 100644 --- a/src/core/include.am +++ b/src/core/include.am @@ -73,6 +73,7 @@ LIBTOR_APP_A_SOURCES = \ src/feature/control/control_bootstrap.c \ src/feature/control/control_events.c \ src/feature/control/control_fmt.c \ + src/feature/control/control_getinfo.c \ src/feature/control/fmt_serverstatus.c \ src/feature/control/getinfo_geoip.c \ src/feature/dirauth/keypin.c \ @@ -291,7 +292,8 @@ noinst_HEADERS += \ src/feature/control/control.h \ src/feature/control/control_connection_st.h \ src/feature/control/control_events.h \ - src/feature/control/control_fmt.h \ + src/feature/control/control_fmt.h \ + src/feature/control/control_getinfo.h \ src/feature/control/fmt_serverstatus.h \ src/feature/control/getinfo_geoip.h \ src/feature/dirauth/authmode.h \ diff --git a/src/feature/control/control.c b/src/feature/control/control.c index d56ff41b0e..94643efde8 100644 --- a/src/feature/control/control.c +++ b/src/feature/control/control.c @@ -65,6 +65,7 @@ #include "feature/control/control.h" #include "feature/control/control_events.h" #include "feature/control/control_fmt.h" +#include "feature/control/control_getinfo.h" #include "feature/control/fmt_serverstatus.h" #include "feature/control/getinfo_geoip.h" #include "feature/dircache/dirserv.h" @@ -178,9 +179,6 @@ static int handle_control_signal(control_connection_t *conn, uint32_t len, const char *body); static int handle_control_mapaddress(control_connection_t *conn, uint32_t len, const char *body); -static char *list_getinfo_options(void); -static int handle_control_getinfo(control_connection_t *conn, uint32_t len, - const char *body); static int handle_control_extendcircuit(control_connection_t *conn, uint32_t len, const char *body); @@ -214,8 +212,6 @@ static int handle_control_add_onion(control_connection_t *conn, uint32_t len, static int handle_control_del_onion(control_connection_t *conn, uint32_t len, const char *body); -static char * download_status_to_string(const download_status_t *dl); - /** Convert a connection_t* to an control_connection_t*; assert if the cast is * invalid. */ control_connection_t * @@ -225,16 +221,6 @@ TO_CONTROL_CONN(connection_t *c) return DOWNCAST(control_connection_t, c); } -/** Append a NUL-terminated string s to the end of - * conn-\>outbuf. - */ -static inline void -connection_write_str_to_buf(const char *s, control_connection_t *conn) -{ - size_t len = strlen(s); - connection_buf_add(s, len, TO_CONN(conn)); -} - /** If the first in_len_max characters in start contain a * double-quoted string with escaped characters, return the length of that * string (as encoded, including quotes). Otherwise return -1. */ @@ -684,48 +670,6 @@ handle_control_loadconf(control_connection_t *conn, uint32_t len, return 0; } -/** Helper structure: maps event values to their names. */ -struct control_event_t { - uint16_t event_code; - const char *event_name; -}; -/** Table mapping event values to their names. Used to implement SETEVENTS - * and GETINFO events/names, and to keep they in sync. */ -static const struct control_event_t control_event_table[] = { - { EVENT_CIRCUIT_STATUS, "CIRC" }, - { EVENT_CIRCUIT_STATUS_MINOR, "CIRC_MINOR" }, - { EVENT_STREAM_STATUS, "STREAM" }, - { EVENT_OR_CONN_STATUS, "ORCONN" }, - { EVENT_BANDWIDTH_USED, "BW" }, - { EVENT_DEBUG_MSG, "DEBUG" }, - { EVENT_INFO_MSG, "INFO" }, - { EVENT_NOTICE_MSG, "NOTICE" }, - { EVENT_WARN_MSG, "WARN" }, - { EVENT_ERR_MSG, "ERR" }, - { EVENT_NEW_DESC, "NEWDESC" }, - { EVENT_ADDRMAP, "ADDRMAP" }, - { EVENT_DESCCHANGED, "DESCCHANGED" }, - { EVENT_NS, "NS" }, - { EVENT_STATUS_GENERAL, "STATUS_GENERAL" }, - { EVENT_STATUS_CLIENT, "STATUS_CLIENT" }, - { EVENT_STATUS_SERVER, "STATUS_SERVER" }, - { EVENT_GUARD, "GUARD" }, - { EVENT_STREAM_BANDWIDTH_USED, "STREAM_BW" }, - { EVENT_CLIENTS_SEEN, "CLIENTS_SEEN" }, - { EVENT_NEWCONSENSUS, "NEWCONSENSUS" }, - { EVENT_BUILDTIMEOUT_SET, "BUILDTIMEOUT_SET" }, - { EVENT_GOT_SIGNAL, "SIGNAL" }, - { EVENT_CONF_CHANGED, "CONF_CHANGED"}, - { EVENT_CONN_BW, "CONN_BW" }, - { EVENT_CELL_STATS, "CELL_STATS" }, - { EVENT_CIRC_BANDWIDTH_USED, "CIRC_BW" }, - { EVENT_TRANSPORT_LAUNCHED, "TRANSPORT_LAUNCHED" }, - { EVENT_HS_DESC, "HS_DESC" }, - { EVENT_HS_DESC_CONTENT, "HS_DESC_CONTENT" }, - { EVENT_NETWORK_LIVENESS, "NETWORK_LIVENESS" }, - { 0, NULL }, -}; - /** Called when we get a SETEVENTS message: update conn->event_mask, * and reply with DONE or ERROR. */ static int @@ -1034,12 +978,7 @@ handle_control_saveconf(control_connection_t *conn, uint32_t len, return 0; } -struct signal_t { - int sig; - const char *signal_name; -}; - -static const struct signal_t signal_table[] = { +const struct signal_name_t signal_table[] = { { SIGHUP, "RELOAD" }, { SIGHUP, "HUP" }, { SIGINT, "SHUTDOWN" }, @@ -1240,1593 +1179,6 @@ handle_control_mapaddress(control_connection_t *conn, uint32_t len, return 0; } -/** Implementation helper for GETINFO: knows the answers for various - * trivial-to-implement questions. */ -static int -getinfo_helper_misc(control_connection_t *conn, const char *question, - char **answer, const char **errmsg) -{ - (void) conn; - if (!strcmp(question, "version")) { - *answer = tor_strdup(get_version()); - } else if (!strcmp(question, "bw-event-cache")) { - *answer = get_bw_samples(); - } else if (!strcmp(question, "config-file")) { - const char *a = get_torrc_fname(0); - if (a) - *answer = tor_strdup(a); - } else if (!strcmp(question, "config-defaults-file")) { - const char *a = get_torrc_fname(1); - if (a) - *answer = tor_strdup(a); - } else if (!strcmp(question, "config-text")) { - *answer = options_dump(get_options(), OPTIONS_DUMP_MINIMAL); - } else if (!strcmp(question, "config-can-saveconf")) { - *answer = tor_strdup(get_options()->IncludeUsed ? "0" : "1"); - } else if (!strcmp(question, "info/names")) { - *answer = list_getinfo_options(); - } else if (!strcmp(question, "dormant")) { - int dormant = rep_hist_circbuilding_dormant(time(NULL)); - *answer = tor_strdup(dormant ? "1" : "0"); - } else if (!strcmp(question, "events/names")) { - int i; - smartlist_t *event_names = smartlist_new(); - - for (i = 0; control_event_table[i].event_name != NULL; ++i) { - smartlist_add(event_names, (char *)control_event_table[i].event_name); - } - - *answer = smartlist_join_strings(event_names, " ", 0, NULL); - - smartlist_free(event_names); - } else if (!strcmp(question, "signal/names")) { - smartlist_t *signal_names = smartlist_new(); - int j; - for (j = 0; signal_table[j].signal_name != NULL; ++j) { - smartlist_add(signal_names, (char*)signal_table[j].signal_name); - } - - *answer = smartlist_join_strings(signal_names, " ", 0, NULL); - - smartlist_free(signal_names); - } else if (!strcmp(question, "features/names")) { - *answer = tor_strdup("VERBOSE_NAMES EXTENDED_EVENTS"); - } else if (!strcmp(question, "address")) { - uint32_t addr; - if (router_pick_published_address(get_options(), &addr, 0) < 0) { - *errmsg = "Address unknown"; - return -1; - } - *answer = tor_dup_ip(addr); - } else if (!strcmp(question, "traffic/read")) { - tor_asprintf(answer, "%"PRIu64, (get_bytes_read())); - } else if (!strcmp(question, "traffic/written")) { - tor_asprintf(answer, "%"PRIu64, (get_bytes_written())); - } else if (!strcmp(question, "uptime")) { - long uptime_secs = get_uptime(); - tor_asprintf(answer, "%ld", uptime_secs); - } else if (!strcmp(question, "process/pid")) { - int myPid = -1; - -#ifdef _WIN32 - myPid = _getpid(); -#else - myPid = getpid(); -#endif - - tor_asprintf(answer, "%d", myPid); - } else if (!strcmp(question, "process/uid")) { -#ifdef _WIN32 - *answer = tor_strdup("-1"); -#else - int myUid = geteuid(); - tor_asprintf(answer, "%d", myUid); -#endif /* defined(_WIN32) */ - } else if (!strcmp(question, "process/user")) { -#ifdef _WIN32 - *answer = tor_strdup(""); -#else - int myUid = geteuid(); - const struct passwd *myPwEntry = tor_getpwuid(myUid); - - if (myPwEntry) { - *answer = tor_strdup(myPwEntry->pw_name); - } else { - *answer = tor_strdup(""); - } -#endif /* defined(_WIN32) */ - } else if (!strcmp(question, "process/descriptor-limit")) { - int max_fds = get_max_sockets(); - tor_asprintf(answer, "%d", max_fds); - } else if (!strcmp(question, "limits/max-mem-in-queues")) { - tor_asprintf(answer, "%"PRIu64, - (get_options()->MaxMemInQueues)); - } else if (!strcmp(question, "fingerprint")) { - crypto_pk_t *server_key; - if (!server_mode(get_options())) { - *errmsg = "Not running in server mode"; - return -1; - } - server_key = get_server_identity_key(); - *answer = tor_malloc(HEX_DIGEST_LEN+1); - crypto_pk_get_fingerprint(server_key, *answer, 0); - } - return 0; -} - -/** Awful hack: return a newly allocated string based on a routerinfo and - * (possibly) an extrainfo, sticking the read-history and write-history from - * ei into the resulting string. The thing you get back won't - * necessarily have a valid signature. - * - * New code should never use this; it's for backward compatibility. - * - * NOTE: ri_body is as returned by signed_descriptor_get_body: it might - * not be NUL-terminated. */ -static char * -munge_extrainfo_into_routerinfo(const char *ri_body, - const signed_descriptor_t *ri, - const signed_descriptor_t *ei) -{ - char *out = NULL, *outp; - int i; - const char *router_sig; - const char *ei_body = signed_descriptor_get_body(ei); - size_t ri_len = ri->signed_descriptor_len; - size_t ei_len = ei->signed_descriptor_len; - if (!ei_body) - goto bail; - - outp = out = tor_malloc(ri_len+ei_len+1); - if (!(router_sig = tor_memstr(ri_body, ri_len, "\nrouter-signature"))) - goto bail; - ++router_sig; - memcpy(out, ri_body, router_sig-ri_body); - outp += router_sig-ri_body; - - for (i=0; i < 2; ++i) { - const char *kwd = i ? "\nwrite-history " : "\nread-history "; - const char *cp, *eol; - if (!(cp = tor_memstr(ei_body, ei_len, kwd))) - continue; - ++cp; - if (!(eol = memchr(cp, '\n', ei_len - (cp-ei_body)))) - continue; - memcpy(outp, cp, eol-cp+1); - outp += eol-cp+1; - } - memcpy(outp, router_sig, ri_len - (router_sig-ri_body)); - *outp++ = '\0'; - tor_assert(outp-out < (int)(ri_len+ei_len+1)); - - return out; - bail: - tor_free(out); - return tor_strndup(ri_body, ri->signed_descriptor_len); -} - -/** Implementation helper for GETINFO: answers requests for information about - * which ports are bound. */ -static int -getinfo_helper_listeners(control_connection_t *control_conn, - const char *question, - char **answer, const char **errmsg) -{ - int type; - smartlist_t *res; - - (void)control_conn; - (void)errmsg; - - if (!strcmp(question, "net/listeners/or")) - type = CONN_TYPE_OR_LISTENER; - else if (!strcmp(question, "net/listeners/extor")) - type = CONN_TYPE_EXT_OR_LISTENER; - else if (!strcmp(question, "net/listeners/dir")) - type = CONN_TYPE_DIR_LISTENER; - else if (!strcmp(question, "net/listeners/socks")) - type = CONN_TYPE_AP_LISTENER; - else if (!strcmp(question, "net/listeners/trans")) - type = CONN_TYPE_AP_TRANS_LISTENER; - else if (!strcmp(question, "net/listeners/natd")) - type = CONN_TYPE_AP_NATD_LISTENER; - else if (!strcmp(question, "net/listeners/httptunnel")) - type = CONN_TYPE_AP_HTTP_CONNECT_LISTENER; - else if (!strcmp(question, "net/listeners/dns")) - type = CONN_TYPE_AP_DNS_LISTENER; - else if (!strcmp(question, "net/listeners/control")) - type = CONN_TYPE_CONTROL_LISTENER; - else - return 0; /* unknown key */ - - res = smartlist_new(); - SMARTLIST_FOREACH_BEGIN(get_connection_array(), connection_t *, conn) { - struct sockaddr_storage ss; - socklen_t ss_len = sizeof(ss); - - if (conn->type != type || conn->marked_for_close || !SOCKET_OK(conn->s)) - continue; - - if (getsockname(conn->s, (struct sockaddr *)&ss, &ss_len) < 0) { - smartlist_add_asprintf(res, "%s:%d", conn->address, (int)conn->port); - } else { - char *tmp = tor_sockaddr_to_str((struct sockaddr *)&ss); - smartlist_add(res, esc_for_log(tmp)); - tor_free(tmp); - } - - } SMARTLIST_FOREACH_END(conn); - - *answer = smartlist_join_strings(res, " ", 0, NULL); - - SMARTLIST_FOREACH(res, char *, cp, tor_free(cp)); - smartlist_free(res); - return 0; -} - -/** Implementation helper for GETINFO: answers requests for information about - * the current time in both local and UTC forms. */ -STATIC int -getinfo_helper_current_time(control_connection_t *control_conn, - const char *question, - char **answer, const char **errmsg) -{ - (void)control_conn; - (void)errmsg; - - struct timeval now; - tor_gettimeofday(&now); - char timebuf[ISO_TIME_LEN+1]; - - if (!strcmp(question, "current-time/local")) - format_local_iso_time_nospace(timebuf, (time_t)now.tv_sec); - else if (!strcmp(question, "current-time/utc")) - format_iso_time_nospace(timebuf, (time_t)now.tv_sec); - else - return 0; - - *answer = tor_strdup(timebuf); - return 0; -} - -/** Implementation helper for GETINFO: knows the answers for questions about - * directory information. */ -STATIC int -getinfo_helper_dir(control_connection_t *control_conn, - const char *question, char **answer, - const char **errmsg) -{ - (void) control_conn; - if (!strcmpstart(question, "desc/id/")) { - const routerinfo_t *ri = NULL; - const node_t *node = node_get_by_hex_id(question+strlen("desc/id/"), 0); - if (node) - ri = node->ri; - if (ri) { - const char *body = signed_descriptor_get_body(&ri->cache_info); - if (body) - *answer = tor_strndup(body, ri->cache_info.signed_descriptor_len); - } else if (! we_fetch_router_descriptors(get_options())) { - /* Descriptors won't be available, provide proper error */ - *errmsg = "We fetch microdescriptors, not router " - "descriptors. You'll need to use md/id/* " - "instead of desc/id/*."; - return 0; - } - } else if (!strcmpstart(question, "desc/name/")) { - const routerinfo_t *ri = NULL; - /* XXX Setting 'warn_if_unnamed' here is a bit silly -- the - * warning goes to the user, not to the controller. */ - const node_t *node = - node_get_by_nickname(question+strlen("desc/name/"), 0); - if (node) - ri = node->ri; - if (ri) { - const char *body = signed_descriptor_get_body(&ri->cache_info); - if (body) - *answer = tor_strndup(body, ri->cache_info.signed_descriptor_len); - } else if (! we_fetch_router_descriptors(get_options())) { - /* Descriptors won't be available, provide proper error */ - *errmsg = "We fetch microdescriptors, not router " - "descriptors. You'll need to use md/name/* " - "instead of desc/name/*."; - return 0; - } - } else if (!strcmp(question, "desc/download-enabled")) { - int r = we_fetch_router_descriptors(get_options()); - tor_asprintf(answer, "%d", !!r); - } else if (!strcmp(question, "desc/all-recent")) { - routerlist_t *routerlist = router_get_routerlist(); - smartlist_t *sl = smartlist_new(); - if (routerlist && routerlist->routers) { - SMARTLIST_FOREACH(routerlist->routers, const routerinfo_t *, ri, - { - const char *body = signed_descriptor_get_body(&ri->cache_info); - if (body) - smartlist_add(sl, - tor_strndup(body, ri->cache_info.signed_descriptor_len)); - }); - } - *answer = smartlist_join_strings(sl, "", 0, NULL); - SMARTLIST_FOREACH(sl, char *, c, tor_free(c)); - smartlist_free(sl); - } else if (!strcmp(question, "desc/all-recent-extrainfo-hack")) { - /* XXXX Remove this once Torstat asks for extrainfos. */ - routerlist_t *routerlist = router_get_routerlist(); - smartlist_t *sl = smartlist_new(); - if (routerlist && routerlist->routers) { - SMARTLIST_FOREACH_BEGIN(routerlist->routers, const routerinfo_t *, ri) { - const char *body = signed_descriptor_get_body(&ri->cache_info); - signed_descriptor_t *ei = extrainfo_get_by_descriptor_digest( - ri->cache_info.extra_info_digest); - if (ei && body) { - smartlist_add(sl, munge_extrainfo_into_routerinfo(body, - &ri->cache_info, ei)); - } else if (body) { - smartlist_add(sl, - tor_strndup(body, ri->cache_info.signed_descriptor_len)); - } - } SMARTLIST_FOREACH_END(ri); - } - *answer = smartlist_join_strings(sl, "", 0, NULL); - SMARTLIST_FOREACH(sl, char *, c, tor_free(c)); - smartlist_free(sl); - } else if (!strcmpstart(question, "hs/client/desc/id/")) { - hostname_type_t addr_type; - - question += strlen("hs/client/desc/id/"); - if (rend_valid_v2_service_id(question)) { - addr_type = ONION_V2_HOSTNAME; - } else if (hs_address_is_valid(question)) { - addr_type = ONION_V3_HOSTNAME; - } else { - *errmsg = "Invalid address"; - return -1; - } - - if (addr_type == ONION_V2_HOSTNAME) { - rend_cache_entry_t *e = NULL; - if (!rend_cache_lookup_entry(question, -1, &e)) { - /* Descriptor found in cache */ - *answer = tor_strdup(e->desc); - } else { - *errmsg = "Not found in cache"; - return -1; - } - } else { - ed25519_public_key_t service_pk; - const char *desc; - - /* The check before this if/else makes sure of this. */ - tor_assert(addr_type == ONION_V3_HOSTNAME); - - if (hs_parse_address(question, &service_pk, NULL, NULL) < 0) { - *errmsg = "Invalid v3 address"; - return -1; - } - - desc = hs_cache_lookup_encoded_as_client(&service_pk); - if (desc) { - *answer = tor_strdup(desc); - } else { - *errmsg = "Not found in cache"; - return -1; - } - } - } else if (!strcmpstart(question, "hs/service/desc/id/")) { - hostname_type_t addr_type; - - question += strlen("hs/service/desc/id/"); - if (rend_valid_v2_service_id(question)) { - addr_type = ONION_V2_HOSTNAME; - } else if (hs_address_is_valid(question)) { - addr_type = ONION_V3_HOSTNAME; - } else { - *errmsg = "Invalid address"; - return -1; - } - rend_cache_entry_t *e = NULL; - - if (addr_type == ONION_V2_HOSTNAME) { - if (!rend_cache_lookup_v2_desc_as_service(question, &e)) { - /* Descriptor found in cache */ - *answer = tor_strdup(e->desc); - } else { - *errmsg = "Not found in cache"; - return -1; - } - } else { - ed25519_public_key_t service_pk; - char *desc; - - /* The check before this if/else makes sure of this. */ - tor_assert(addr_type == ONION_V3_HOSTNAME); - - if (hs_parse_address(question, &service_pk, NULL, NULL) < 0) { - *errmsg = "Invalid v3 address"; - return -1; - } - - desc = hs_service_lookup_current_desc(&service_pk); - if (desc) { - /* Newly allocated string, we have ownership. */ - *answer = desc; - } else { - *errmsg = "Not found in cache"; - return -1; - } - } - } else if (!strcmp(question, "md/all")) { - const smartlist_t *nodes = nodelist_get_list(); - tor_assert(nodes); - - if (smartlist_len(nodes) == 0) { - *answer = tor_strdup(""); - return 0; - } - - smartlist_t *microdescs = smartlist_new(); - - SMARTLIST_FOREACH_BEGIN(nodes, node_t *, n) { - if (n->md && n->md->body) { - char *copy = tor_strndup(n->md->body, n->md->bodylen); - smartlist_add(microdescs, copy); - } - } SMARTLIST_FOREACH_END(n); - - *answer = smartlist_join_strings(microdescs, "", 0, NULL); - SMARTLIST_FOREACH(microdescs, char *, md, tor_free(md)); - smartlist_free(microdescs); - } else if (!strcmpstart(question, "md/id/")) { - const node_t *node = node_get_by_hex_id(question+strlen("md/id/"), 0); - const microdesc_t *md = NULL; - if (node) md = node->md; - if (md && md->body) { - *answer = tor_strndup(md->body, md->bodylen); - } - } else if (!strcmpstart(question, "md/name/")) { - /* XXX Setting 'warn_if_unnamed' here is a bit silly -- the - * warning goes to the user, not to the controller. */ - const node_t *node = node_get_by_nickname(question+strlen("md/name/"), 0); - /* XXXX duplicated code */ - const microdesc_t *md = NULL; - if (node) md = node->md; - if (md && md->body) { - *answer = tor_strndup(md->body, md->bodylen); - } - } else if (!strcmp(question, "md/download-enabled")) { - int r = we_fetch_microdescriptors(get_options()); - tor_asprintf(answer, "%d", !!r); - } else if (!strcmpstart(question, "desc-annotations/id/")) { - const routerinfo_t *ri = NULL; - const node_t *node = - node_get_by_hex_id(question+strlen("desc-annotations/id/"), 0); - if (node) - ri = node->ri; - if (ri) { - const char *annotations = - signed_descriptor_get_annotations(&ri->cache_info); - if (annotations) - *answer = tor_strndup(annotations, - ri->cache_info.annotations_len); - } - } else if (!strcmpstart(question, "dir/server/")) { - size_t answer_len = 0; - char *url = NULL; - smartlist_t *descs = smartlist_new(); - const char *msg; - int res; - char *cp; - tor_asprintf(&url, "/tor/%s", question+4); - res = dirserv_get_routerdescs(descs, url, &msg); - if (res) { - log_warn(LD_CONTROL, "getinfo '%s': %s", question, msg); - smartlist_free(descs); - tor_free(url); - *errmsg = msg; - return -1; - } - SMARTLIST_FOREACH(descs, signed_descriptor_t *, sd, - answer_len += sd->signed_descriptor_len); - cp = *answer = tor_malloc(answer_len+1); - SMARTLIST_FOREACH(descs, signed_descriptor_t *, sd, - { - memcpy(cp, signed_descriptor_get_body(sd), - sd->signed_descriptor_len); - cp += sd->signed_descriptor_len; - }); - *cp = '\0'; - tor_free(url); - smartlist_free(descs); - } else if (!strcmpstart(question, "dir/status/")) { - *answer = tor_strdup(""); - } else if (!strcmp(question, "dir/status-vote/current/consensus")) { /* v3 */ - if (we_want_to_fetch_flavor(get_options(), FLAV_NS)) { - const cached_dir_t *consensus = dirserv_get_consensus("ns"); - if (consensus) - *answer = tor_strdup(consensus->dir); - } - if (!*answer) { /* try loading it from disk */ - tor_mmap_t *mapped = networkstatus_map_cached_consensus("ns"); - if (mapped) { - *answer = tor_memdup_nulterm(mapped->data, mapped->size); - tor_munmap_file(mapped); - } - if (!*answer) { /* generate an error */ - *errmsg = "Could not open cached consensus. " - "Make sure FetchUselessDescriptors is set to 1."; - return -1; - } - } - } else if (!strcmp(question, "network-status")) { /* v1 */ - static int network_status_warned = 0; - if (!network_status_warned) { - log_warn(LD_CONTROL, "GETINFO network-status is deprecated; it will " - "go away in a future version of Tor."); - network_status_warned = 1; - } - routerlist_t *routerlist = router_get_routerlist(); - if (!routerlist || !routerlist->routers || - list_server_status_v1(routerlist->routers, answer, 1) < 0) { - return -1; - } - } else if (!strcmpstart(question, "extra-info/digest/")) { - question += strlen("extra-info/digest/"); - if (strlen(question) == HEX_DIGEST_LEN) { - char d[DIGEST_LEN]; - signed_descriptor_t *sd = NULL; - if (base16_decode(d, sizeof(d), question, strlen(question)) - == sizeof(d)) { - /* XXXX this test should move into extrainfo_get_by_descriptor_digest, - * but I don't want to risk affecting other parts of the code, - * especially since the rules for using our own extrainfo (including - * when it might be freed) are different from those for using one - * we have downloaded. */ - if (router_extrainfo_digest_is_me(d)) - sd = &(router_get_my_extrainfo()->cache_info); - else - sd = extrainfo_get_by_descriptor_digest(d); - } - if (sd) { - const char *body = signed_descriptor_get_body(sd); - if (body) - *answer = tor_strndup(body, sd->signed_descriptor_len); - } - } - } - - return 0; -} - -/** Given a smartlist of 20-byte digests, return a newly allocated string - * containing each of those digests in order, formatted in HEX, and terminated - * with a newline. */ -static char * -digest_list_to_string(const smartlist_t *sl) -{ - int len; - char *result, *s; - - /* Allow for newlines, and a \0 at the end */ - len = smartlist_len(sl) * (HEX_DIGEST_LEN + 1) + 1; - result = tor_malloc_zero(len); - - s = result; - SMARTLIST_FOREACH_BEGIN(sl, const char *, digest) { - base16_encode(s, HEX_DIGEST_LEN + 1, digest, DIGEST_LEN); - s[HEX_DIGEST_LEN] = '\n'; - s += HEX_DIGEST_LEN + 1; - } SMARTLIST_FOREACH_END(digest); - *s = '\0'; - - return result; -} - -/** Turn a download_status_t into a human-readable description in a newly - * allocated string. The format is specified in control-spec.txt, under - * the documentation for "GETINFO download/..." . */ -static char * -download_status_to_string(const download_status_t *dl) -{ - char *rv = NULL; - char tbuf[ISO_TIME_LEN+1]; - const char *schedule_str, *want_authority_str; - const char *increment_on_str, *backoff_str; - - if (dl) { - /* Get some substrings of the eventual output ready */ - format_iso_time(tbuf, download_status_get_next_attempt_at(dl)); - - switch (dl->schedule) { - case DL_SCHED_GENERIC: - schedule_str = "DL_SCHED_GENERIC"; - break; - case DL_SCHED_CONSENSUS: - schedule_str = "DL_SCHED_CONSENSUS"; - break; - case DL_SCHED_BRIDGE: - schedule_str = "DL_SCHED_BRIDGE"; - break; - default: - schedule_str = "unknown"; - break; - } - - switch (dl->want_authority) { - case DL_WANT_ANY_DIRSERVER: - want_authority_str = "DL_WANT_ANY_DIRSERVER"; - break; - case DL_WANT_AUTHORITY: - want_authority_str = "DL_WANT_AUTHORITY"; - break; - default: - want_authority_str = "unknown"; - break; - } - - switch (dl->increment_on) { - case DL_SCHED_INCREMENT_FAILURE: - increment_on_str = "DL_SCHED_INCREMENT_FAILURE"; - break; - case DL_SCHED_INCREMENT_ATTEMPT: - increment_on_str = "DL_SCHED_INCREMENT_ATTEMPT"; - break; - default: - increment_on_str = "unknown"; - break; - } - - backoff_str = "DL_SCHED_RANDOM_EXPONENTIAL"; - - /* Now assemble them */ - tor_asprintf(&rv, - "next-attempt-at %s\n" - "n-download-failures %u\n" - "n-download-attempts %u\n" - "schedule %s\n" - "want-authority %s\n" - "increment-on %s\n" - "backoff %s\n" - "last-backoff-position %u\n" - "last-delay-used %d\n", - tbuf, - dl->n_download_failures, - dl->n_download_attempts, - schedule_str, - want_authority_str, - increment_on_str, - backoff_str, - dl->last_backoff_position, - dl->last_delay_used); - } - - return rv; -} - -/** Handle the consensus download cases for getinfo_helper_downloads() */ -STATIC void -getinfo_helper_downloads_networkstatus(const char *flavor, - download_status_t **dl_to_emit, - const char **errmsg) -{ - /* - * We get the one for the current bootstrapped status by default, or - * take an extra /bootstrap or /running suffix - */ - if (strcmp(flavor, "ns") == 0) { - *dl_to_emit = networkstatus_get_dl_status_by_flavor(FLAV_NS); - } else if (strcmp(flavor, "ns/bootstrap") == 0) { - *dl_to_emit = networkstatus_get_dl_status_by_flavor_bootstrap(FLAV_NS); - } else if (strcmp(flavor, "ns/running") == 0 ) { - *dl_to_emit = networkstatus_get_dl_status_by_flavor_running(FLAV_NS); - } else if (strcmp(flavor, "microdesc") == 0) { - *dl_to_emit = networkstatus_get_dl_status_by_flavor(FLAV_MICRODESC); - } else if (strcmp(flavor, "microdesc/bootstrap") == 0) { - *dl_to_emit = - networkstatus_get_dl_status_by_flavor_bootstrap(FLAV_MICRODESC); - } else if (strcmp(flavor, "microdesc/running") == 0) { - *dl_to_emit = - networkstatus_get_dl_status_by_flavor_running(FLAV_MICRODESC); - } else { - *errmsg = "Unknown flavor"; - } -} - -/** Handle the cert download cases for getinfo_helper_downloads() */ -STATIC void -getinfo_helper_downloads_cert(const char *fp_sk_req, - download_status_t **dl_to_emit, - smartlist_t **digest_list, - const char **errmsg) -{ - const char *sk_req; - char id_digest[DIGEST_LEN]; - char sk_digest[DIGEST_LEN]; - - /* - * We have to handle four cases; fp_sk_req is the request with - * a prefix of "downloads/cert/" snipped off. - * - * Case 1: fp_sk_req = "fps" - * - We should emit a digest_list with a list of all the identity - * fingerprints that can be queried for certificate download status; - * get it by calling list_authority_ids_with_downloads(). - * - * Case 2: fp_sk_req = "fp/" for some fingerprint fp - * - We want the default certificate for this identity fingerprint's - * download status; this is the download we get from URLs starting - * in /fp/ on the directory server. We can get it with - * id_only_download_status_for_authority_id(). - * - * Case 3: fp_sk_req = "fp//sks" for some fingerprint fp - * - We want a list of all signing key digests for this identity - * fingerprint which can be queried for certificate download status. - * Get it with list_sk_digests_for_authority_id(). - * - * Case 4: fp_sk_req = "fp//" for some fingerprint fp and - * signing key digest sk - * - We want the download status for the certificate for this specific - * signing key and fingerprint. These correspond to the ones we get - * from URLs starting in /fp-sk/ on the directory server. Get it with - * list_sk_digests_for_authority_id(). - */ - - if (strcmp(fp_sk_req, "fps") == 0) { - *digest_list = list_authority_ids_with_downloads(); - if (!(*digest_list)) { - *errmsg = "Failed to get list of authority identity digests (!)"; - } - } else if (!strcmpstart(fp_sk_req, "fp/")) { - fp_sk_req += strlen("fp/"); - /* Okay, look for another / to tell the fp from fp-sk cases */ - sk_req = strchr(fp_sk_req, '/'); - if (sk_req) { - /* okay, split it here and try to parse */ - if (base16_decode(id_digest, DIGEST_LEN, - fp_sk_req, sk_req - fp_sk_req) == DIGEST_LEN) { - /* Skip past the '/' */ - ++sk_req; - if (strcmp(sk_req, "sks") == 0) { - /* We're asking for the list of signing key fingerprints */ - *digest_list = list_sk_digests_for_authority_id(id_digest); - if (!(*digest_list)) { - *errmsg = "Failed to get list of signing key digests for this " - "authority identity digest"; - } - } else { - /* We've got a signing key digest */ - if (base16_decode(sk_digest, DIGEST_LEN, - sk_req, strlen(sk_req)) == DIGEST_LEN) { - *dl_to_emit = - download_status_for_authority_id_and_sk(id_digest, sk_digest); - if (!(*dl_to_emit)) { - *errmsg = "Failed to get download status for this identity/" - "signing key digest pair"; - } - } else { - *errmsg = "That didn't look like a signing key digest"; - } - } - } else { - *errmsg = "That didn't look like an identity digest"; - } - } else { - /* We're either in downloads/certs/fp/, or we can't parse */ - if (strlen(fp_sk_req) == HEX_DIGEST_LEN) { - if (base16_decode(id_digest, DIGEST_LEN, - fp_sk_req, strlen(fp_sk_req)) == DIGEST_LEN) { - *dl_to_emit = id_only_download_status_for_authority_id(id_digest); - if (!(*dl_to_emit)) { - *errmsg = "Failed to get download status for this authority " - "identity digest"; - } - } else { - *errmsg = "That didn't look like a digest"; - } - } else { - *errmsg = "That didn't look like a digest"; - } - } - } else { - *errmsg = "Unknown certificate download status query"; - } -} - -/** Handle the routerdesc download cases for getinfo_helper_downloads() */ -STATIC void -getinfo_helper_downloads_desc(const char *desc_req, - download_status_t **dl_to_emit, - smartlist_t **digest_list, - const char **errmsg) -{ - char desc_digest[DIGEST_LEN]; - /* - * Two cases to handle here: - * - * Case 1: desc_req = "descs" - * - Emit a list of all router descriptor digests, which we get by - * calling router_get_descriptor_digests(); this can return NULL - * if we have no current ns-flavor consensus. - * - * Case 2: desc_req = - * - Check on the specified fingerprint and emit its download_status_t - * using router_get_dl_status_by_descriptor_digest(). - */ - - if (strcmp(desc_req, "descs") == 0) { - *digest_list = router_get_descriptor_digests(); - if (!(*digest_list)) { - *errmsg = "We don't seem to have a networkstatus-flavored consensus"; - } - /* - * Microdescs don't use the download_status_t mechanism, so we don't - * answer queries about their downloads here; see microdesc.c. - */ - } else if (strlen(desc_req) == HEX_DIGEST_LEN) { - if (base16_decode(desc_digest, DIGEST_LEN, - desc_req, strlen(desc_req)) == DIGEST_LEN) { - /* Okay we got a digest-shaped thing; try asking for it */ - *dl_to_emit = router_get_dl_status_by_descriptor_digest(desc_digest); - if (!(*dl_to_emit)) { - *errmsg = "No such descriptor digest found"; - } - } else { - *errmsg = "That didn't look like a digest"; - } - } else { - *errmsg = "Unknown router descriptor download status query"; - } -} - -/** Handle the bridge download cases for getinfo_helper_downloads() */ -STATIC void -getinfo_helper_downloads_bridge(const char *bridge_req, - download_status_t **dl_to_emit, - smartlist_t **digest_list, - const char **errmsg) -{ - char bridge_digest[DIGEST_LEN]; - /* - * Two cases to handle here: - * - * Case 1: bridge_req = "bridges" - * - Emit a list of all bridge identity digests, which we get by - * calling list_bridge_identities(); this can return NULL if we are - * not using bridges. - * - * Case 2: bridge_req = - * - Check on the specified fingerprint and emit its download_status_t - * using get_bridge_dl_status_by_id(). - */ - - if (strcmp(bridge_req, "bridges") == 0) { - *digest_list = list_bridge_identities(); - if (!(*digest_list)) { - *errmsg = "We don't seem to be using bridges"; - } - } else if (strlen(bridge_req) == HEX_DIGEST_LEN) { - if (base16_decode(bridge_digest, DIGEST_LEN, - bridge_req, strlen(bridge_req)) == DIGEST_LEN) { - /* Okay we got a digest-shaped thing; try asking for it */ - *dl_to_emit = get_bridge_dl_status_by_id(bridge_digest); - if (!(*dl_to_emit)) { - *errmsg = "No such bridge identity digest found"; - } - } else { - *errmsg = "That didn't look like a digest"; - } - } else { - *errmsg = "Unknown bridge descriptor download status query"; - } -} - -/** Implementation helper for GETINFO: knows the answers for questions about - * download status information. */ -STATIC int -getinfo_helper_downloads(control_connection_t *control_conn, - const char *question, char **answer, - const char **errmsg) -{ - download_status_t *dl_to_emit = NULL; - smartlist_t *digest_list = NULL; - - /* Assert args are sane */ - tor_assert(control_conn != NULL); - tor_assert(question != NULL); - tor_assert(answer != NULL); - tor_assert(errmsg != NULL); - - /* We check for this later to see if we should supply a default */ - *errmsg = NULL; - - /* Are we after networkstatus downloads? */ - if (!strcmpstart(question, "downloads/networkstatus/")) { - getinfo_helper_downloads_networkstatus( - question + strlen("downloads/networkstatus/"), - &dl_to_emit, errmsg); - /* Certificates? */ - } else if (!strcmpstart(question, "downloads/cert/")) { - getinfo_helper_downloads_cert( - question + strlen("downloads/cert/"), - &dl_to_emit, &digest_list, errmsg); - /* Router descriptors? */ - } else if (!strcmpstart(question, "downloads/desc/")) { - getinfo_helper_downloads_desc( - question + strlen("downloads/desc/"), - &dl_to_emit, &digest_list, errmsg); - /* Bridge descriptors? */ - } else if (!strcmpstart(question, "downloads/bridge/")) { - getinfo_helper_downloads_bridge( - question + strlen("downloads/bridge/"), - &dl_to_emit, &digest_list, errmsg); - } else { - *errmsg = "Unknown download status query"; - } - - if (dl_to_emit) { - *answer = download_status_to_string(dl_to_emit); - - return 0; - } else if (digest_list) { - *answer = digest_list_to_string(digest_list); - SMARTLIST_FOREACH(digest_list, void *, s, tor_free(s)); - smartlist_free(digest_list); - - return 0; - } else { - if (!(*errmsg)) { - *errmsg = "Unknown error"; - } - - return -1; - } -} - -/** Implementation helper for GETINFO: knows how to generate summaries of the - * current states of things we send events about. */ -static int -getinfo_helper_events(control_connection_t *control_conn, - const char *question, char **answer, - const char **errmsg) -{ - const or_options_t *options = get_options(); - (void) control_conn; - if (!strcmp(question, "circuit-status")) { - smartlist_t *status = smartlist_new(); - SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ_) { - origin_circuit_t *circ; - char *circdesc; - const char *state; - if (! CIRCUIT_IS_ORIGIN(circ_) || circ_->marked_for_close) - continue; - circ = TO_ORIGIN_CIRCUIT(circ_); - - if (circ->base_.state == CIRCUIT_STATE_OPEN) - state = "BUILT"; - else if (circ->base_.state == CIRCUIT_STATE_GUARD_WAIT) - state = "GUARD_WAIT"; - else if (circ->cpath) - state = "EXTENDED"; - else - state = "LAUNCHED"; - - circdesc = circuit_describe_status_for_controller(circ); - - smartlist_add_asprintf(status, "%lu %s%s%s", - (unsigned long)circ->global_identifier, - state, *circdesc ? " " : "", circdesc); - tor_free(circdesc); - } - SMARTLIST_FOREACH_END(circ_); - *answer = smartlist_join_strings(status, "\r\n", 0, NULL); - SMARTLIST_FOREACH(status, char *, cp, tor_free(cp)); - smartlist_free(status); - } else if (!strcmp(question, "stream-status")) { - smartlist_t *conns = get_connection_array(); - smartlist_t *status = smartlist_new(); - char buf[256]; - SMARTLIST_FOREACH_BEGIN(conns, connection_t *, base_conn) { - const char *state; - entry_connection_t *conn; - circuit_t *circ; - origin_circuit_t *origin_circ = NULL; - if (base_conn->type != CONN_TYPE_AP || - base_conn->marked_for_close || - base_conn->state == AP_CONN_STATE_SOCKS_WAIT || - base_conn->state == AP_CONN_STATE_NATD_WAIT) - continue; - conn = TO_ENTRY_CONN(base_conn); - switch (base_conn->state) - { - case AP_CONN_STATE_CONTROLLER_WAIT: - case AP_CONN_STATE_CIRCUIT_WAIT: - if (conn->socks_request && - SOCKS_COMMAND_IS_RESOLVE(conn->socks_request->command)) - state = "NEWRESOLVE"; - else - state = "NEW"; - break; - case AP_CONN_STATE_RENDDESC_WAIT: - case AP_CONN_STATE_CONNECT_WAIT: - state = "SENTCONNECT"; break; - case AP_CONN_STATE_RESOLVE_WAIT: - state = "SENTRESOLVE"; break; - case AP_CONN_STATE_OPEN: - state = "SUCCEEDED"; break; - default: - log_warn(LD_BUG, "Asked for stream in unknown state %d", - base_conn->state); - continue; - } - circ = circuit_get_by_edge_conn(ENTRY_TO_EDGE_CONN(conn)); - if (circ && CIRCUIT_IS_ORIGIN(circ)) - origin_circ = TO_ORIGIN_CIRCUIT(circ); - write_stream_target_to_buf(conn, buf, sizeof(buf)); - smartlist_add_asprintf(status, "%lu %s %lu %s", - (unsigned long) base_conn->global_identifier,state, - origin_circ? - (unsigned long)origin_circ->global_identifier : 0ul, - buf); - } SMARTLIST_FOREACH_END(base_conn); - *answer = smartlist_join_strings(status, "\r\n", 0, NULL); - SMARTLIST_FOREACH(status, char *, cp, tor_free(cp)); - smartlist_free(status); - } else if (!strcmp(question, "orconn-status")) { - smartlist_t *conns = get_connection_array(); - smartlist_t *status = smartlist_new(); - SMARTLIST_FOREACH_BEGIN(conns, connection_t *, base_conn) { - const char *state; - char name[128]; - or_connection_t *conn; - if (base_conn->type != CONN_TYPE_OR || base_conn->marked_for_close) - continue; - conn = TO_OR_CONN(base_conn); - if (conn->base_.state == OR_CONN_STATE_OPEN) - state = "CONNECTED"; - else if (conn->nickname) - state = "LAUNCHED"; - else - state = "NEW"; - orconn_target_get_name(name, sizeof(name), conn); - smartlist_add_asprintf(status, "%s %s", name, state); - } SMARTLIST_FOREACH_END(base_conn); - *answer = smartlist_join_strings(status, "\r\n", 0, NULL); - SMARTLIST_FOREACH(status, char *, cp, tor_free(cp)); - smartlist_free(status); - } else if (!strcmpstart(question, "address-mappings/")) { - time_t min_e, max_e; - smartlist_t *mappings; - question += strlen("address-mappings/"); - if (!strcmp(question, "all")) { - min_e = 0; max_e = TIME_MAX; - } else if (!strcmp(question, "cache")) { - min_e = 2; max_e = TIME_MAX; - } else if (!strcmp(question, "config")) { - min_e = 0; max_e = 0; - } else if (!strcmp(question, "control")) { - min_e = 1; max_e = 1; - } else { - return 0; - } - mappings = smartlist_new(); - addressmap_get_mappings(mappings, min_e, max_e, 1); - *answer = smartlist_join_strings(mappings, "\r\n", 0, NULL); - SMARTLIST_FOREACH(mappings, char *, cp, tor_free(cp)); - smartlist_free(mappings); - } else if (!strcmpstart(question, "status/")) { - /* Note that status/ is not a catch-all for events; there's only supposed - * to be a status GETINFO if there's a corresponding STATUS event. */ - if (!strcmp(question, "status/circuit-established")) { - *answer = tor_strdup(have_completed_a_circuit() ? "1" : "0"); - } else if (!strcmp(question, "status/enough-dir-info")) { - *answer = tor_strdup(router_have_minimum_dir_info() ? "1" : "0"); - } else if (!strcmp(question, "status/good-server-descriptor") || - !strcmp(question, "status/accepted-server-descriptor")) { - /* They're equivalent for now, until we can figure out how to make - * good-server-descriptor be what we want. See comment in - * control-spec.txt. */ - *answer = tor_strdup(directories_have_accepted_server_descriptor() - ? "1" : "0"); - } else if (!strcmp(question, "status/reachability-succeeded/or")) { - *answer = tor_strdup(check_whether_orport_reachable(options) ? - "1" : "0"); - } else if (!strcmp(question, "status/reachability-succeeded/dir")) { - *answer = tor_strdup(check_whether_dirport_reachable(options) ? - "1" : "0"); - } else if (!strcmp(question, "status/reachability-succeeded")) { - tor_asprintf(answer, "OR=%d DIR=%d", - check_whether_orport_reachable(options) ? 1 : 0, - check_whether_dirport_reachable(options) ? 1 : 0); - } else if (!strcmp(question, "status/bootstrap-phase")) { - *answer = control_event_boot_last_msg(); - } else if (!strcmpstart(question, "status/version/")) { - int is_server = server_mode(options); - networkstatus_t *c = networkstatus_get_latest_consensus(); - version_status_t status; - const char *recommended; - if (c) { - recommended = is_server ? c->server_versions : c->client_versions; - status = tor_version_is_obsolete(VERSION, recommended); - } else { - recommended = "?"; - status = VS_UNKNOWN; - } - - if (!strcmp(question, "status/version/recommended")) { - *answer = tor_strdup(recommended); - return 0; - } - if (!strcmp(question, "status/version/current")) { - switch (status) - { - case VS_RECOMMENDED: *answer = tor_strdup("recommended"); break; - case VS_OLD: *answer = tor_strdup("obsolete"); break; - case VS_NEW: *answer = tor_strdup("new"); break; - case VS_NEW_IN_SERIES: *answer = tor_strdup("new in series"); break; - case VS_UNRECOMMENDED: *answer = tor_strdup("unrecommended"); break; - case VS_EMPTY: *answer = tor_strdup("none recommended"); break; - case VS_UNKNOWN: *answer = tor_strdup("unknown"); break; - default: tor_fragile_assert(); - } - } - } else if (!strcmp(question, "status/clients-seen")) { - char *bridge_stats = geoip_get_bridge_stats_controller(time(NULL)); - if (!bridge_stats) { - *errmsg = "No bridge-client stats available"; - return -1; - } - *answer = bridge_stats; - } else if (!strcmp(question, "status/fresh-relay-descs")) { - if (!server_mode(options)) { - *errmsg = "Only relays have descriptors"; - return -1; - } - routerinfo_t *r; - extrainfo_t *e; - if (router_build_fresh_descriptor(&r, &e) < 0) { - *errmsg = "Error generating descriptor"; - return -1; - } - size_t size = r->cache_info.signed_descriptor_len + 1; - if (e) { - size += e->cache_info.signed_descriptor_len + 1; - } - tor_assert(r->cache_info.signed_descriptor_len); - char *descs = tor_malloc(size); - char *cp = descs; - memcpy(cp, signed_descriptor_get_body(&r->cache_info), - r->cache_info.signed_descriptor_len); - cp += r->cache_info.signed_descriptor_len - 1; - if (e) { - if (cp[0] == '\0') { - cp[0] = '\n'; - } else if (cp[0] != '\n') { - cp[1] = '\n'; - cp++; - } - memcpy(cp, signed_descriptor_get_body(&e->cache_info), - e->cache_info.signed_descriptor_len); - cp += e->cache_info.signed_descriptor_len - 1; - } - if (cp[0] == '\n') { - cp[0] = '\0'; - } else if (cp[0] != '\0') { - cp[1] = '\0'; - } - *answer = descs; - routerinfo_free(r); - extrainfo_free(e); - } else { - return 0; - } - } - return 0; -} - -/** Implementation helper for GETINFO: knows how to enumerate hidden services - * created via the control port. */ -STATIC int -getinfo_helper_onions(control_connection_t *control_conn, - const char *question, char **answer, - const char **errmsg) -{ - smartlist_t *onion_list = NULL; - (void) errmsg; /* no errors from this method */ - - if (control_conn && !strcmp(question, "onions/current")) { - onion_list = control_conn->ephemeral_onion_services; - } else if (!strcmp(question, "onions/detached")) { - onion_list = detached_onion_services; - } else { - return 0; - } - if (!onion_list || smartlist_len(onion_list) == 0) { - if (answer) { - *answer = tor_strdup(""); - } - } else { - if (answer) { - *answer = smartlist_join_strings(onion_list, "\r\n", 0, NULL); - } - } - - return 0; -} - -/** Implementation helper for GETINFO: answers queries about network - * liveness. */ -static int -getinfo_helper_liveness(control_connection_t *control_conn, - const char *question, char **answer, - const char **errmsg) -{ - (void)control_conn; - (void)errmsg; - if (strcmp(question, "network-liveness") == 0) { - if (get_cached_network_liveness()) { - *answer = tor_strdup("up"); - } else { - *answer = tor_strdup("down"); - } - } - - return 0; -} - -/** Implementation helper for GETINFO: answers queries about shared random - * value. */ -static int -getinfo_helper_sr(control_connection_t *control_conn, - const char *question, char **answer, - const char **errmsg) -{ - (void) control_conn; - (void) errmsg; - - if (!strcmp(question, "sr/current")) { - *answer = sr_get_current_for_control(); - } else if (!strcmp(question, "sr/previous")) { - *answer = sr_get_previous_for_control(); - } - /* Else statement here is unrecognized key so do nothing. */ - - return 0; -} - -/** Callback function for GETINFO: on a given control connection, try to - * answer the question q and store the newly-allocated answer in - * *a. If an internal error occurs, return -1 and optionally set - * *error_out to point to an error message to be delivered to the - * controller. On success, _or if the key is not recognized_, return 0. Do not - * set a if the key is not recognized but you may set error_out - * to improve the error message. - */ -typedef int (*getinfo_helper_t)(control_connection_t *, - const char *q, char **a, - const char **error_out); - -/** A single item for the GETINFO question-to-answer-function table. */ -typedef struct getinfo_item_t { - const char *varname; /**< The value (or prefix) of the question. */ - getinfo_helper_t fn; /**< The function that knows the answer: NULL if - * this entry is documentation-only. */ - const char *desc; /**< Description of the variable. */ - int is_prefix; /** Must varname match exactly, or must it be a prefix? */ -} getinfo_item_t; - -#define ITEM(name, fn, desc) { name, getinfo_helper_##fn, desc, 0 } -#define PREFIX(name, fn, desc) { name, getinfo_helper_##fn, desc, 1 } -#define DOC(name, desc) { name, NULL, desc, 0 } - -/** Table mapping questions accepted by GETINFO to the functions that know how - * to answer them. */ -static const getinfo_item_t getinfo_items[] = { - ITEM("version", misc, "The current version of Tor."), - ITEM("bw-event-cache", misc, "Cached BW events for a short interval."), - ITEM("config-file", misc, "Current location of the \"torrc\" file."), - ITEM("config-defaults-file", misc, "Current location of the defaults file."), - ITEM("config-text", misc, - "Return the string that would be written by a saveconf command."), - ITEM("config-can-saveconf", misc, - "Is it possible to save the configuration to the \"torrc\" file?"), - ITEM("accounting/bytes", accounting, - "Number of bytes read/written so far in the accounting interval."), - ITEM("accounting/bytes-left", accounting, - "Number of bytes left to write/read so far in the accounting interval."), - ITEM("accounting/enabled", accounting, "Is accounting currently enabled?"), - ITEM("accounting/hibernating", accounting, "Are we hibernating or awake?"), - ITEM("accounting/interval-start", accounting, - "Time when the accounting period starts."), - ITEM("accounting/interval-end", accounting, - "Time when the accounting period ends."), - ITEM("accounting/interval-wake", accounting, - "Time to wake up in this accounting period."), - ITEM("helper-nodes", entry_guards, NULL), /* deprecated */ - ITEM("entry-guards", entry_guards, - "Which nodes are we using as entry guards?"), - ITEM("fingerprint", misc, NULL), - PREFIX("config/", config, "Current configuration values."), - DOC("config/names", - "List of configuration options, types, and documentation."), - DOC("config/defaults", - "List of default values for configuration options. " - "See also config/names"), - PREFIX("current-time/", current_time, "Current time."), - DOC("current-time/local", "Current time on the local system."), - DOC("current-time/utc", "Current UTC time."), - PREFIX("downloads/networkstatus/", downloads, - "Download statuses for networkstatus objects"), - DOC("downloads/networkstatus/ns", - "Download status for current-mode networkstatus download"), - DOC("downloads/networkstatus/ns/bootstrap", - "Download status for bootstrap-time networkstatus download"), - DOC("downloads/networkstatus/ns/running", - "Download status for run-time networkstatus download"), - DOC("downloads/networkstatus/microdesc", - "Download status for current-mode microdesc download"), - DOC("downloads/networkstatus/microdesc/bootstrap", - "Download status for bootstrap-time microdesc download"), - DOC("downloads/networkstatus/microdesc/running", - "Download status for run-time microdesc download"), - PREFIX("downloads/cert/", downloads, - "Download statuses for certificates, by id fingerprint and " - "signing key"), - DOC("downloads/cert/fps", - "List of authority fingerprints for which any download statuses " - "exist"), - DOC("downloads/cert/fp/", - "Download status for with the default signing key; corresponds " - "to /fp/ URLs on directory server."), - DOC("downloads/cert/fp//sks", - "List of signing keys for which specific download statuses are " - "available for this id fingerprint"), - DOC("downloads/cert/fp//", - "Download status for with signing key ; corresponds " - "to /fp-sk/ URLs on directory server."), - PREFIX("downloads/desc/", downloads, - "Download statuses for router descriptors, by descriptor digest"), - DOC("downloads/desc/descs", - "Return a list of known router descriptor digests"), - DOC("downloads/desc/", - "Return a download status for a given descriptor digest"), - PREFIX("downloads/bridge/", downloads, - "Download statuses for bridge descriptors, by bridge identity " - "digest"), - DOC("downloads/bridge/bridges", - "Return a list of configured bridge identity digests with download " - "statuses"), - DOC("downloads/bridge/", - "Return a download status for a given bridge identity digest"), - ITEM("info/names", misc, - "List of GETINFO options, types, and documentation."), - ITEM("events/names", misc, - "Events that the controller can ask for with SETEVENTS."), - ITEM("signal/names", misc, "Signal names recognized by the SIGNAL command"), - ITEM("features/names", misc, "What arguments can USEFEATURE take?"), - PREFIX("desc/id/", dir, "Router descriptors by ID."), - PREFIX("desc/name/", dir, "Router descriptors by nickname."), - ITEM("desc/all-recent", dir, - "All non-expired, non-superseded router descriptors."), - ITEM("desc/download-enabled", dir, - "Do we try to download router descriptors?"), - ITEM("desc/all-recent-extrainfo-hack", dir, NULL), /* Hack. */ - ITEM("md/all", dir, "All known microdescriptors."), - PREFIX("md/id/", dir, "Microdescriptors by ID"), - PREFIX("md/name/", dir, "Microdescriptors by name"), - ITEM("md/download-enabled", dir, - "Do we try to download microdescriptors?"), - PREFIX("extra-info/digest/", dir, "Extra-info documents by digest."), - PREFIX("hs/client/desc/id", dir, - "Hidden Service descriptor in client's cache by onion."), - PREFIX("hs/service/desc/id/", dir, - "Hidden Service descriptor in services's cache by onion."), - PREFIX("net/listeners/", listeners, "Bound addresses by type"), - ITEM("ns/all", networkstatus, - "Brief summary of router status (v2 directory format)"), - PREFIX("ns/id/", networkstatus, - "Brief summary of router status by ID (v2 directory format)."), - PREFIX("ns/name/", networkstatus, - "Brief summary of router status by nickname (v2 directory format)."), - PREFIX("ns/purpose/", networkstatus, - "Brief summary of router status by purpose (v2 directory format)."), - PREFIX("consensus/", networkstatus, - "Information about and from the ns consensus."), - ITEM("network-status", dir, - "Brief summary of router status (v1 directory format)"), - ITEM("network-liveness", liveness, - "Current opinion on whether the network is live"), - ITEM("circuit-status", events, "List of current circuits originating here."), - ITEM("stream-status", events,"List of current streams."), - ITEM("orconn-status", events, "A list of current OR connections."), - ITEM("dormant", misc, - "Is Tor dormant (not building circuits because it's idle)?"), - PREFIX("address-mappings/", events, NULL), - DOC("address-mappings/all", "Current address mappings."), - DOC("address-mappings/cache", "Current cached DNS replies."), - DOC("address-mappings/config", - "Current address mappings from configuration."), - DOC("address-mappings/control", "Current address mappings from controller."), - PREFIX("status/", events, NULL), - DOC("status/circuit-established", - "Whether we think client functionality is working."), - DOC("status/enough-dir-info", - "Whether we have enough up-to-date directory information to build " - "circuits."), - DOC("status/bootstrap-phase", - "The last bootstrap phase status event that Tor sent."), - DOC("status/clients-seen", - "Breakdown of client countries seen by a bridge."), - DOC("status/fresh-relay-descs", - "A fresh relay/ei descriptor pair for Tor's current state. Not stored."), - DOC("status/version/recommended", "List of currently recommended versions."), - DOC("status/version/current", "Status of the current version."), - ITEM("address", misc, "IP address of this Tor host, if we can guess it."), - ITEM("traffic/read", misc,"Bytes read since the process was started."), - ITEM("traffic/written", misc, - "Bytes written since the process was started."), - ITEM("uptime", misc, "Uptime of the Tor daemon in seconds."), - ITEM("process/pid", misc, "Process id belonging to the main tor process."), - ITEM("process/uid", misc, "User id running the tor process."), - ITEM("process/user", misc, - "Username under which the tor process is running."), - ITEM("process/descriptor-limit", misc, "File descriptor limit."), - ITEM("limits/max-mem-in-queues", misc, "Actual limit on memory in queues"), - PREFIX("desc-annotations/id/", dir, "Router annotations by hexdigest."), - PREFIX("dir/server/", dir,"Router descriptors as retrieved from a DirPort."), - PREFIX("dir/status/", dir, - "v2 networkstatus docs as retrieved from a DirPort."), - ITEM("dir/status-vote/current/consensus", dir, - "v3 Networkstatus consensus as retrieved from a DirPort."), - ITEM("exit-policy/default", policies, - "The default value appended to the configured exit policy."), - ITEM("exit-policy/reject-private/default", policies, - "The default rules appended to the configured exit policy by" - " ExitPolicyRejectPrivate."), - ITEM("exit-policy/reject-private/relay", policies, - "The relay-specific rules appended to the configured exit policy by" - " ExitPolicyRejectPrivate and/or ExitPolicyRejectLocalInterfaces."), - ITEM("exit-policy/full", policies, "The entire exit policy of onion router"), - ITEM("exit-policy/ipv4", policies, "IPv4 parts of exit policy"), - ITEM("exit-policy/ipv6", policies, "IPv6 parts of exit policy"), - PREFIX("ip-to-country/", geoip, "Perform a GEOIP lookup"), - ITEM("onions/current", onions, - "Onion services owned by the current control connection."), - ITEM("onions/detached", onions, - "Onion services detached from the control connection."), - ITEM("sr/current", sr, "Get current shared random value."), - ITEM("sr/previous", sr, "Get previous shared random value."), - { NULL, NULL, NULL, 0 } -}; - -/** Allocate and return a list of recognized GETINFO options. */ -static char * -list_getinfo_options(void) -{ - int i; - smartlist_t *lines = smartlist_new(); - char *ans; - for (i = 0; getinfo_items[i].varname; ++i) { - if (!getinfo_items[i].desc) - continue; - - smartlist_add_asprintf(lines, "%s%s -- %s\n", - getinfo_items[i].varname, - getinfo_items[i].is_prefix ? "*" : "", - getinfo_items[i].desc); - } - smartlist_sort_strings(lines); - - ans = smartlist_join_strings(lines, "", 0, NULL); - SMARTLIST_FOREACH(lines, char *, cp, tor_free(cp)); - smartlist_free(lines); - - return ans; -} - -/** Lookup the 'getinfo' entry question, and return - * the answer in *answer (or NULL if key not recognized). - * Return 0 if success or unrecognized, or -1 if recognized but - * internal error. */ -static int -handle_getinfo_helper(control_connection_t *control_conn, - const char *question, char **answer, - const char **err_out) -{ - int i; - *answer = NULL; /* unrecognized key by default */ - - for (i = 0; getinfo_items[i].varname; ++i) { - int match; - if (getinfo_items[i].is_prefix) - match = !strcmpstart(question, getinfo_items[i].varname); - else - match = !strcmp(question, getinfo_items[i].varname); - if (match) { - tor_assert(getinfo_items[i].fn); - return getinfo_items[i].fn(control_conn, question, answer, err_out); - } - } - - return 0; /* unrecognized */ -} - -/** Called when we receive a GETINFO command. Try to fetch all requested - * information, and reply with information or error message. */ -static int -handle_control_getinfo(control_connection_t *conn, uint32_t len, - const char *body) -{ - smartlist_t *questions = smartlist_new(); - smartlist_t *answers = smartlist_new(); - smartlist_t *unrecognized = smartlist_new(); - char *ans = NULL; - int i; - (void) len; /* body is NUL-terminated, so it's safe to ignore the length. */ - - smartlist_split_string(questions, body, " ", - SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); - SMARTLIST_FOREACH_BEGIN(questions, const char *, q) { - const char *errmsg = NULL; - - if (handle_getinfo_helper(conn, q, &ans, &errmsg) < 0) { - if (!errmsg) - errmsg = "Internal error"; - connection_printf_to_buf(conn, "551 %s\r\n", errmsg); - goto done; - } - if (!ans) { - if (errmsg) /* use provided error message */ - smartlist_add_strdup(unrecognized, errmsg); - else /* use default error message */ - smartlist_add_asprintf(unrecognized, "Unrecognized key \"%s\"", q); - } else { - smartlist_add_strdup(answers, q); - smartlist_add(answers, ans); - } - } SMARTLIST_FOREACH_END(q); - - if (smartlist_len(unrecognized)) { - /* control-spec section 2.3, mid-reply '-' or end of reply ' ' */ - for (i=0; i < smartlist_len(unrecognized)-1; ++i) - connection_printf_to_buf(conn, - "552-%s\r\n", - (char *)smartlist_get(unrecognized, i)); - - connection_printf_to_buf(conn, - "552 %s\r\n", - (char *)smartlist_get(unrecognized, i)); - goto done; - } - - for (i = 0; i < smartlist_len(answers); i += 2) { - char *k = smartlist_get(answers, i); - char *v = smartlist_get(answers, i+1); - if (!strchr(v, '\n') && !strchr(v, '\r')) { - connection_printf_to_buf(conn, "250-%s=", k); - connection_write_str_to_buf(v, conn); - connection_write_str_to_buf("\r\n", conn); - } else { - char *esc = NULL; - size_t esc_len; - esc_len = write_escaped_data(v, strlen(v), &esc); - connection_printf_to_buf(conn, "250+%s=\r\n", k); - connection_buf_add(esc, esc_len, TO_CONN(conn)); - tor_free(esc); - } - } - connection_write_str_to_buf("250 OK\r\n", conn); - - done: - SMARTLIST_FOREACH(answers, char *, cp, tor_free(cp)); - smartlist_free(answers); - SMARTLIST_FOREACH(questions, char *, cp, tor_free(cp)); - smartlist_free(questions); - SMARTLIST_FOREACH(unrecognized, char *, cp, tor_free(cp)); - smartlist_free(unrecognized); - - return 0; -} - /** Given a string, convert it to a circuit purpose. */ static uint8_t circuit_purpose_from_string(const char *string) @@ -3977,6 +2329,15 @@ add_onion_helper_add_service(int hs_version, return ret; } +/** + * + **/ +smartlist_t * +get_detached_onion_services(void) +{ + return detached_onion_services; +} + /** Called when we get a ADD_ONION command; parse the body, and set up * the new ephemeral Onion Service. */ static int diff --git a/src/feature/control/control.h b/src/feature/control/control.h index f9742d555e..810fe67843 100644 --- a/src/feature/control/control.h +++ b/src/feature/control/control.h @@ -77,48 +77,18 @@ STATIC int add_onion_helper_keyarg(const char *arg, int discard_pk, STATIC rend_authorized_client_t * add_onion_helper_clientauth(const char *arg, int *created, char **err_msg_out); -STATIC int getinfo_helper_onions( - control_connection_t *control_conn, - const char *question, - char **answer, - const char **errmsg); -STATIC void getinfo_helper_downloads_networkstatus( - const char *flavor, - download_status_t **dl_to_emit, - const char **errmsg); -STATIC void getinfo_helper_downloads_cert( - const char *fp_sk_req, - download_status_t **dl_to_emit, - smartlist_t **digest_list, - const char **errmsg); -STATIC void getinfo_helper_downloads_desc( - const char *desc_req, - download_status_t **dl_to_emit, - smartlist_t **digest_list, - const char **errmsg); -STATIC void getinfo_helper_downloads_bridge( - const char *bridge_req, - download_status_t **dl_to_emit, - smartlist_t **digest_list, - const char **errmsg); -STATIC int getinfo_helper_downloads( - control_connection_t *control_conn, - const char *question, char **answer, - const char **errmsg); -STATIC int getinfo_helper_dir( - control_connection_t *control_conn, - const char *question, char **answer, - const char **errmsg); -STATIC int getinfo_helper_current_time( - control_connection_t *control_conn, - const char *question, char **answer, - const char **errmsg); - #endif /* defined(CONTROL_PRIVATE) */ #ifdef CONTROL_MODULE_PRIVATE +struct signal_name_t { + int sig; + const char *signal_name; +}; +extern const struct signal_name_t signal_table[]; + int get_cached_network_liveness(void); void set_cached_network_liveness(int liveness); +smartlist_t * get_detached_onion_services(void); #endif /* defined(CONTROL_MODULE_PRIVATE) */ #endif /* !defined(TOR_CONTROL_H) */ diff --git a/src/feature/control/control_events.c b/src/feature/control/control_events.c index 07e1a9d94a..129776f49f 100644 --- a/src/feature/control/control_events.c +++ b/src/feature/control/control_events.c @@ -72,6 +72,43 @@ static void send_control_event(uint16_t event, const char *format, ...) CHECK_PRINTF(2,3); +/** Table mapping event values to their names. Used to implement SETEVENTS + * and GETINFO events/names, and to keep they in sync. */ +const struct control_event_t control_event_table[] = { + { EVENT_CIRCUIT_STATUS, "CIRC" }, + { EVENT_CIRCUIT_STATUS_MINOR, "CIRC_MINOR" }, + { EVENT_STREAM_STATUS, "STREAM" }, + { EVENT_OR_CONN_STATUS, "ORCONN" }, + { EVENT_BANDWIDTH_USED, "BW" }, + { EVENT_DEBUG_MSG, "DEBUG" }, + { EVENT_INFO_MSG, "INFO" }, + { EVENT_NOTICE_MSG, "NOTICE" }, + { EVENT_WARN_MSG, "WARN" }, + { EVENT_ERR_MSG, "ERR" }, + { EVENT_NEW_DESC, "NEWDESC" }, + { EVENT_ADDRMAP, "ADDRMAP" }, + { EVENT_DESCCHANGED, "DESCCHANGED" }, + { EVENT_NS, "NS" }, + { EVENT_STATUS_GENERAL, "STATUS_GENERAL" }, + { EVENT_STATUS_CLIENT, "STATUS_CLIENT" }, + { EVENT_STATUS_SERVER, "STATUS_SERVER" }, + { EVENT_GUARD, "GUARD" }, + { EVENT_STREAM_BANDWIDTH_USED, "STREAM_BW" }, + { EVENT_CLIENTS_SEEN, "CLIENTS_SEEN" }, + { EVENT_NEWCONSENSUS, "NEWCONSENSUS" }, + { EVENT_BUILDTIMEOUT_SET, "BUILDTIMEOUT_SET" }, + { EVENT_GOT_SIGNAL, "SIGNAL" }, + { EVENT_CONF_CHANGED, "CONF_CHANGED"}, + { EVENT_CONN_BW, "CONN_BW" }, + { EVENT_CELL_STATS, "CELL_STATS" }, + { EVENT_CIRC_BANDWIDTH_USED, "CIRC_BW" }, + { EVENT_TRANSPORT_LAUNCHED, "TRANSPORT_LAUNCHED" }, + { EVENT_HS_DESC, "HS_DESC" }, + { EVENT_HS_DESC_CONTENT, "HS_DESC_CONTENT" }, + { EVENT_NETWORK_LIVENESS, "NETWORK_LIVENESS" }, + { 0, NULL }, +}; + /** Given a log severity, return the corresponding control event code. */ static inline int log_severity_to_event(int severity) diff --git a/src/feature/control/control_events.h b/src/feature/control/control_events.h index 9e79346714..0bdbb9cfd2 100644 --- a/src/feature/control/control_events.h +++ b/src/feature/control/control_events.h @@ -327,6 +327,14 @@ void append_cell_stats_by_command(smartlist_t *event_parts, void format_cell_stats(char **event_string, circuit_t *circ, cell_stats_t *cell_stats); +/** Helper structure: maps event values to their names. */ +struct control_event_t { + uint16_t event_code; + const char *event_name; +}; + +extern const struct control_event_t control_event_table[]; + #ifdef TOR_UNIT_TESTS MOCK_DECL(STATIC void, send_control_event_string,(uint16_t event, const char *msg)); diff --git a/src/feature/control/control_fmt.c b/src/feature/control/control_fmt.c index a6d7919915..c87a224ac5 100644 --- a/src/feature/control/control_fmt.c +++ b/src/feature/control/control_fmt.c @@ -23,6 +23,16 @@ #include "core/or/socks_request_st.h" #include "feature/control/control_connection_st.h" +/** Append a NUL-terminated string s to the end of + * conn-\>outbuf. + */ +void +connection_write_str_to_buf(const char *s, control_connection_t *conn) +{ + size_t len = strlen(s); + connection_buf_add(s, len, TO_CONN(conn)); +} + /** Acts like sprintf, but writes its formatted string to the end of * conn-\>outbuf. */ void diff --git a/src/feature/control/control_fmt.h b/src/feature/control/control_fmt.h index e36edd2a46..1b4534cc3b 100644 --- a/src/feature/control/control_fmt.h +++ b/src/feature/control/control_fmt.h @@ -12,6 +12,7 @@ #ifndef TOR_CONTROL_FMT_H #define TOR_CONTROL_FMT_H +void connection_write_str_to_buf(const char *s, control_connection_t *conn); void connection_printf_to_buf(control_connection_t *conn, const char *format, ...) CHECK_PRINTF(2,3); diff --git a/src/feature/control/control_getinfo.c b/src/feature/control/control_getinfo.c new file mode 100644 index 0000000000..67a6e27316 --- /dev/null +++ b/src/feature/control/control_getinfo.c @@ -0,0 +1,1661 @@ +/* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file control_getinfo.c + * \brief Implementation for miscellaneous controller getinfo commands. + */ + +#define CONTROL_EVENTS_PRIVATE +#define CONTROL_MODULE_PRIVATE +#define CONTROL_GETINFO_PRIVATE + +#include "core/or/or.h" +#include "app/config/config.h" +#include "core/mainloop/connection.h" +#include "core/mainloop/mainloop.h" +#include "core/or/circuitlist.h" +#include "core/or/connection_edge.h" +#include "core/or/connection_or.h" +#include "core/or/policies.h" +#include "core/or/versions.h" +#include "feature/client/addressmap.h" +#include "feature/client/bridges.h" +#include "feature/client/entrynodes.h" +#include "feature/control/control.h" +#include "feature/control/control_events.h" +#include "feature/control/control_fmt.h" +#include "feature/control/control_getinfo.h" +#include "feature/control/fmt_serverstatus.h" +#include "feature/control/getinfo_geoip.h" +#include "feature/dircache/dirserv.h" +#include "feature/dirclient/dirclient.h" +#include "feature/dirclient/dlstatus.h" +#include "feature/hibernate/hibernate.h" +#include "feature/hs/hs_cache.h" +#include "feature/hs_common/shared_random_client.h" +#include "feature/nodelist/authcert.h" +#include "feature/nodelist/microdesc.h" +#include "feature/nodelist/networkstatus.h" +#include "feature/nodelist/nodelist.h" +#include "feature/nodelist/routerinfo.h" +#include "feature/nodelist/routerlist.h" +#include "feature/relay/router.h" +#include "feature/relay/routermode.h" +#include "feature/relay/selftest.h" +#include "feature/rend/rendcache.h" +#include "feature/stats/geoip_stats.h" +#include "feature/stats/predict_ports.h" +#include "lib/version/torversion.h" + +#include "core/or/entry_connection_st.h" +#include "core/or/or_connection_st.h" +#include "core/or/origin_circuit_st.h" +#include "core/or/socks_request_st.h" +#include "feature/control/control_connection_st.h" +#include "feature/dircache/cached_dir_st.h" +#include "feature/nodelist/extrainfo_st.h" +#include "feature/nodelist/microdesc_st.h" +#include "feature/nodelist/networkstatus_st.h" +#include "feature/nodelist/node_st.h" +#include "feature/nodelist/routerinfo_st.h" +#include "feature/nodelist/routerlist_st.h" + +#ifdef HAVE_UNISTD_H +#include +#endif + +#ifndef _WIN32 +#include +#endif + +static char *list_getinfo_options(void); +static char *download_status_to_string(const download_status_t *dl); + +/** Implementation helper for GETINFO: knows the answers for various + * trivial-to-implement questions. */ +static int +getinfo_helper_misc(control_connection_t *conn, const char *question, + char **answer, const char **errmsg) +{ + (void) conn; + if (!strcmp(question, "version")) { + *answer = tor_strdup(get_version()); + } else if (!strcmp(question, "bw-event-cache")) { + *answer = get_bw_samples(); + } else if (!strcmp(question, "config-file")) { + const char *a = get_torrc_fname(0); + if (a) + *answer = tor_strdup(a); + } else if (!strcmp(question, "config-defaults-file")) { + const char *a = get_torrc_fname(1); + if (a) + *answer = tor_strdup(a); + } else if (!strcmp(question, "config-text")) { + *answer = options_dump(get_options(), OPTIONS_DUMP_MINIMAL); + } else if (!strcmp(question, "config-can-saveconf")) { + *answer = tor_strdup(get_options()->IncludeUsed ? "0" : "1"); + } else if (!strcmp(question, "info/names")) { + *answer = list_getinfo_options(); + } else if (!strcmp(question, "dormant")) { + int dormant = rep_hist_circbuilding_dormant(time(NULL)); + *answer = tor_strdup(dormant ? "1" : "0"); + } else if (!strcmp(question, "events/names")) { + int i; + smartlist_t *event_names = smartlist_new(); + + for (i = 0; control_event_table[i].event_name != NULL; ++i) { + smartlist_add(event_names, (char *)control_event_table[i].event_name); + } + + *answer = smartlist_join_strings(event_names, " ", 0, NULL); + + smartlist_free(event_names); + } else if (!strcmp(question, "signal/names")) { + smartlist_t *signal_names = smartlist_new(); + int j; + for (j = 0; signal_table[j].signal_name != NULL; ++j) { + smartlist_add(signal_names, (char*)signal_table[j].signal_name); + } + + *answer = smartlist_join_strings(signal_names, " ", 0, NULL); + + smartlist_free(signal_names); + } else if (!strcmp(question, "features/names")) { + *answer = tor_strdup("VERBOSE_NAMES EXTENDED_EVENTS"); + } else if (!strcmp(question, "address")) { + uint32_t addr; + if (router_pick_published_address(get_options(), &addr, 0) < 0) { + *errmsg = "Address unknown"; + return -1; + } + *answer = tor_dup_ip(addr); + } else if (!strcmp(question, "traffic/read")) { + tor_asprintf(answer, "%"PRIu64, (get_bytes_read())); + } else if (!strcmp(question, "traffic/written")) { + tor_asprintf(answer, "%"PRIu64, (get_bytes_written())); + } else if (!strcmp(question, "uptime")) { + long uptime_secs = get_uptime(); + tor_asprintf(answer, "%ld", uptime_secs); + } else if (!strcmp(question, "process/pid")) { + int myPid = -1; + +#ifdef _WIN32 + myPid = _getpid(); +#else + myPid = getpid(); +#endif + + tor_asprintf(answer, "%d", myPid); + } else if (!strcmp(question, "process/uid")) { +#ifdef _WIN32 + *answer = tor_strdup("-1"); +#else + int myUid = geteuid(); + tor_asprintf(answer, "%d", myUid); +#endif /* defined(_WIN32) */ + } else if (!strcmp(question, "process/user")) { +#ifdef _WIN32 + *answer = tor_strdup(""); +#else + int myUid = geteuid(); + const struct passwd *myPwEntry = tor_getpwuid(myUid); + + if (myPwEntry) { + *answer = tor_strdup(myPwEntry->pw_name); + } else { + *answer = tor_strdup(""); + } +#endif /* defined(_WIN32) */ + } else if (!strcmp(question, "process/descriptor-limit")) { + int max_fds = get_max_sockets(); + tor_asprintf(answer, "%d", max_fds); + } else if (!strcmp(question, "limits/max-mem-in-queues")) { + tor_asprintf(answer, "%"PRIu64, + (get_options()->MaxMemInQueues)); + } else if (!strcmp(question, "fingerprint")) { + crypto_pk_t *server_key; + if (!server_mode(get_options())) { + *errmsg = "Not running in server mode"; + return -1; + } + server_key = get_server_identity_key(); + *answer = tor_malloc(HEX_DIGEST_LEN+1); + crypto_pk_get_fingerprint(server_key, *answer, 0); + } + return 0; +} + +/** Awful hack: return a newly allocated string based on a routerinfo and + * (possibly) an extrainfo, sticking the read-history and write-history from + * ei into the resulting string. The thing you get back won't + * necessarily have a valid signature. + * + * New code should never use this; it's for backward compatibility. + * + * NOTE: ri_body is as returned by signed_descriptor_get_body: it might + * not be NUL-terminated. */ +static char * +munge_extrainfo_into_routerinfo(const char *ri_body, + const signed_descriptor_t *ri, + const signed_descriptor_t *ei) +{ + char *out = NULL, *outp; + int i; + const char *router_sig; + const char *ei_body = signed_descriptor_get_body(ei); + size_t ri_len = ri->signed_descriptor_len; + size_t ei_len = ei->signed_descriptor_len; + if (!ei_body) + goto bail; + + outp = out = tor_malloc(ri_len+ei_len+1); + if (!(router_sig = tor_memstr(ri_body, ri_len, "\nrouter-signature"))) + goto bail; + ++router_sig; + memcpy(out, ri_body, router_sig-ri_body); + outp += router_sig-ri_body; + + for (i=0; i < 2; ++i) { + const char *kwd = i ? "\nwrite-history " : "\nread-history "; + const char *cp, *eol; + if (!(cp = tor_memstr(ei_body, ei_len, kwd))) + continue; + ++cp; + if (!(eol = memchr(cp, '\n', ei_len - (cp-ei_body)))) + continue; + memcpy(outp, cp, eol-cp+1); + outp += eol-cp+1; + } + memcpy(outp, router_sig, ri_len - (router_sig-ri_body)); + *outp++ = '\0'; + tor_assert(outp-out < (int)(ri_len+ei_len+1)); + + return out; + bail: + tor_free(out); + return tor_strndup(ri_body, ri->signed_descriptor_len); +} + +/** Implementation helper for GETINFO: answers requests for information about + * which ports are bound. */ +static int +getinfo_helper_listeners(control_connection_t *control_conn, + const char *question, + char **answer, const char **errmsg) +{ + int type; + smartlist_t *res; + + (void)control_conn; + (void)errmsg; + + if (!strcmp(question, "net/listeners/or")) + type = CONN_TYPE_OR_LISTENER; + else if (!strcmp(question, "net/listeners/extor")) + type = CONN_TYPE_EXT_OR_LISTENER; + else if (!strcmp(question, "net/listeners/dir")) + type = CONN_TYPE_DIR_LISTENER; + else if (!strcmp(question, "net/listeners/socks")) + type = CONN_TYPE_AP_LISTENER; + else if (!strcmp(question, "net/listeners/trans")) + type = CONN_TYPE_AP_TRANS_LISTENER; + else if (!strcmp(question, "net/listeners/natd")) + type = CONN_TYPE_AP_NATD_LISTENER; + else if (!strcmp(question, "net/listeners/httptunnel")) + type = CONN_TYPE_AP_HTTP_CONNECT_LISTENER; + else if (!strcmp(question, "net/listeners/dns")) + type = CONN_TYPE_AP_DNS_LISTENER; + else if (!strcmp(question, "net/listeners/control")) + type = CONN_TYPE_CONTROL_LISTENER; + else + return 0; /* unknown key */ + + res = smartlist_new(); + SMARTLIST_FOREACH_BEGIN(get_connection_array(), connection_t *, conn) { + struct sockaddr_storage ss; + socklen_t ss_len = sizeof(ss); + + if (conn->type != type || conn->marked_for_close || !SOCKET_OK(conn->s)) + continue; + + if (getsockname(conn->s, (struct sockaddr *)&ss, &ss_len) < 0) { + smartlist_add_asprintf(res, "%s:%d", conn->address, (int)conn->port); + } else { + char *tmp = tor_sockaddr_to_str((struct sockaddr *)&ss); + smartlist_add(res, esc_for_log(tmp)); + tor_free(tmp); + } + + } SMARTLIST_FOREACH_END(conn); + + *answer = smartlist_join_strings(res, " ", 0, NULL); + + SMARTLIST_FOREACH(res, char *, cp, tor_free(cp)); + smartlist_free(res); + return 0; +} + +/** Implementation helper for GETINFO: answers requests for information about + * the current time in both local and UTC forms. */ +STATIC int +getinfo_helper_current_time(control_connection_t *control_conn, + const char *question, + char **answer, const char **errmsg) +{ + (void)control_conn; + (void)errmsg; + + struct timeval now; + tor_gettimeofday(&now); + char timebuf[ISO_TIME_LEN+1]; + + if (!strcmp(question, "current-time/local")) + format_local_iso_time_nospace(timebuf, (time_t)now.tv_sec); + else if (!strcmp(question, "current-time/utc")) + format_iso_time_nospace(timebuf, (time_t)now.tv_sec); + else + return 0; + + *answer = tor_strdup(timebuf); + return 0; +} + +/** Implementation helper for GETINFO: knows the answers for questions about + * directory information. */ +STATIC int +getinfo_helper_dir(control_connection_t *control_conn, + const char *question, char **answer, + const char **errmsg) +{ + (void) control_conn; + if (!strcmpstart(question, "desc/id/")) { + const routerinfo_t *ri = NULL; + const node_t *node = node_get_by_hex_id(question+strlen("desc/id/"), 0); + if (node) + ri = node->ri; + if (ri) { + const char *body = signed_descriptor_get_body(&ri->cache_info); + if (body) + *answer = tor_strndup(body, ri->cache_info.signed_descriptor_len); + } else if (! we_fetch_router_descriptors(get_options())) { + /* Descriptors won't be available, provide proper error */ + *errmsg = "We fetch microdescriptors, not router " + "descriptors. You'll need to use md/id/* " + "instead of desc/id/*."; + return 0; + } + } else if (!strcmpstart(question, "desc/name/")) { + const routerinfo_t *ri = NULL; + /* XXX Setting 'warn_if_unnamed' here is a bit silly -- the + * warning goes to the user, not to the controller. */ + const node_t *node = + node_get_by_nickname(question+strlen("desc/name/"), 0); + if (node) + ri = node->ri; + if (ri) { + const char *body = signed_descriptor_get_body(&ri->cache_info); + if (body) + *answer = tor_strndup(body, ri->cache_info.signed_descriptor_len); + } else if (! we_fetch_router_descriptors(get_options())) { + /* Descriptors won't be available, provide proper error */ + *errmsg = "We fetch microdescriptors, not router " + "descriptors. You'll need to use md/name/* " + "instead of desc/name/*."; + return 0; + } + } else if (!strcmp(question, "desc/download-enabled")) { + int r = we_fetch_router_descriptors(get_options()); + tor_asprintf(answer, "%d", !!r); + } else if (!strcmp(question, "desc/all-recent")) { + routerlist_t *routerlist = router_get_routerlist(); + smartlist_t *sl = smartlist_new(); + if (routerlist && routerlist->routers) { + SMARTLIST_FOREACH(routerlist->routers, const routerinfo_t *, ri, + { + const char *body = signed_descriptor_get_body(&ri->cache_info); + if (body) + smartlist_add(sl, + tor_strndup(body, ri->cache_info.signed_descriptor_len)); + }); + } + *answer = smartlist_join_strings(sl, "", 0, NULL); + SMARTLIST_FOREACH(sl, char *, c, tor_free(c)); + smartlist_free(sl); + } else if (!strcmp(question, "desc/all-recent-extrainfo-hack")) { + /* XXXX Remove this once Torstat asks for extrainfos. */ + routerlist_t *routerlist = router_get_routerlist(); + smartlist_t *sl = smartlist_new(); + if (routerlist && routerlist->routers) { + SMARTLIST_FOREACH_BEGIN(routerlist->routers, const routerinfo_t *, ri) { + const char *body = signed_descriptor_get_body(&ri->cache_info); + signed_descriptor_t *ei = extrainfo_get_by_descriptor_digest( + ri->cache_info.extra_info_digest); + if (ei && body) { + smartlist_add(sl, munge_extrainfo_into_routerinfo(body, + &ri->cache_info, ei)); + } else if (body) { + smartlist_add(sl, + tor_strndup(body, ri->cache_info.signed_descriptor_len)); + } + } SMARTLIST_FOREACH_END(ri); + } + *answer = smartlist_join_strings(sl, "", 0, NULL); + SMARTLIST_FOREACH(sl, char *, c, tor_free(c)); + smartlist_free(sl); + } else if (!strcmpstart(question, "hs/client/desc/id/")) { + hostname_type_t addr_type; + + question += strlen("hs/client/desc/id/"); + if (rend_valid_v2_service_id(question)) { + addr_type = ONION_V2_HOSTNAME; + } else if (hs_address_is_valid(question)) { + addr_type = ONION_V3_HOSTNAME; + } else { + *errmsg = "Invalid address"; + return -1; + } + + if (addr_type == ONION_V2_HOSTNAME) { + rend_cache_entry_t *e = NULL; + if (!rend_cache_lookup_entry(question, -1, &e)) { + /* Descriptor found in cache */ + *answer = tor_strdup(e->desc); + } else { + *errmsg = "Not found in cache"; + return -1; + } + } else { + ed25519_public_key_t service_pk; + const char *desc; + + /* The check before this if/else makes sure of this. */ + tor_assert(addr_type == ONION_V3_HOSTNAME); + + if (hs_parse_address(question, &service_pk, NULL, NULL) < 0) { + *errmsg = "Invalid v3 address"; + return -1; + } + + desc = hs_cache_lookup_encoded_as_client(&service_pk); + if (desc) { + *answer = tor_strdup(desc); + } else { + *errmsg = "Not found in cache"; + return -1; + } + } + } else if (!strcmpstart(question, "hs/service/desc/id/")) { + hostname_type_t addr_type; + + question += strlen("hs/service/desc/id/"); + if (rend_valid_v2_service_id(question)) { + addr_type = ONION_V2_HOSTNAME; + } else if (hs_address_is_valid(question)) { + addr_type = ONION_V3_HOSTNAME; + } else { + *errmsg = "Invalid address"; + return -1; + } + rend_cache_entry_t *e = NULL; + + if (addr_type == ONION_V2_HOSTNAME) { + if (!rend_cache_lookup_v2_desc_as_service(question, &e)) { + /* Descriptor found in cache */ + *answer = tor_strdup(e->desc); + } else { + *errmsg = "Not found in cache"; + return -1; + } + } else { + ed25519_public_key_t service_pk; + char *desc; + + /* The check before this if/else makes sure of this. */ + tor_assert(addr_type == ONION_V3_HOSTNAME); + + if (hs_parse_address(question, &service_pk, NULL, NULL) < 0) { + *errmsg = "Invalid v3 address"; + return -1; + } + + desc = hs_service_lookup_current_desc(&service_pk); + if (desc) { + /* Newly allocated string, we have ownership. */ + *answer = desc; + } else { + *errmsg = "Not found in cache"; + return -1; + } + } + } else if (!strcmp(question, "md/all")) { + const smartlist_t *nodes = nodelist_get_list(); + tor_assert(nodes); + + if (smartlist_len(nodes) == 0) { + *answer = tor_strdup(""); + return 0; + } + + smartlist_t *microdescs = smartlist_new(); + + SMARTLIST_FOREACH_BEGIN(nodes, node_t *, n) { + if (n->md && n->md->body) { + char *copy = tor_strndup(n->md->body, n->md->bodylen); + smartlist_add(microdescs, copy); + } + } SMARTLIST_FOREACH_END(n); + + *answer = smartlist_join_strings(microdescs, "", 0, NULL); + SMARTLIST_FOREACH(microdescs, char *, md, tor_free(md)); + smartlist_free(microdescs); + } else if (!strcmpstart(question, "md/id/")) { + const node_t *node = node_get_by_hex_id(question+strlen("md/id/"), 0); + const microdesc_t *md = NULL; + if (node) md = node->md; + if (md && md->body) { + *answer = tor_strndup(md->body, md->bodylen); + } + } else if (!strcmpstart(question, "md/name/")) { + /* XXX Setting 'warn_if_unnamed' here is a bit silly -- the + * warning goes to the user, not to the controller. */ + const node_t *node = node_get_by_nickname(question+strlen("md/name/"), 0); + /* XXXX duplicated code */ + const microdesc_t *md = NULL; + if (node) md = node->md; + if (md && md->body) { + *answer = tor_strndup(md->body, md->bodylen); + } + } else if (!strcmp(question, "md/download-enabled")) { + int r = we_fetch_microdescriptors(get_options()); + tor_asprintf(answer, "%d", !!r); + } else if (!strcmpstart(question, "desc-annotations/id/")) { + const routerinfo_t *ri = NULL; + const node_t *node = + node_get_by_hex_id(question+strlen("desc-annotations/id/"), 0); + if (node) + ri = node->ri; + if (ri) { + const char *annotations = + signed_descriptor_get_annotations(&ri->cache_info); + if (annotations) + *answer = tor_strndup(annotations, + ri->cache_info.annotations_len); + } + } else if (!strcmpstart(question, "dir/server/")) { + size_t answer_len = 0; + char *url = NULL; + smartlist_t *descs = smartlist_new(); + const char *msg; + int res; + char *cp; + tor_asprintf(&url, "/tor/%s", question+4); + res = dirserv_get_routerdescs(descs, url, &msg); + if (res) { + log_warn(LD_CONTROL, "getinfo '%s': %s", question, msg); + smartlist_free(descs); + tor_free(url); + *errmsg = msg; + return -1; + } + SMARTLIST_FOREACH(descs, signed_descriptor_t *, sd, + answer_len += sd->signed_descriptor_len); + cp = *answer = tor_malloc(answer_len+1); + SMARTLIST_FOREACH(descs, signed_descriptor_t *, sd, + { + memcpy(cp, signed_descriptor_get_body(sd), + sd->signed_descriptor_len); + cp += sd->signed_descriptor_len; + }); + *cp = '\0'; + tor_free(url); + smartlist_free(descs); + } else if (!strcmpstart(question, "dir/status/")) { + *answer = tor_strdup(""); + } else if (!strcmp(question, "dir/status-vote/current/consensus")) { /* v3 */ + if (we_want_to_fetch_flavor(get_options(), FLAV_NS)) { + const cached_dir_t *consensus = dirserv_get_consensus("ns"); + if (consensus) + *answer = tor_strdup(consensus->dir); + } + if (!*answer) { /* try loading it from disk */ + tor_mmap_t *mapped = networkstatus_map_cached_consensus("ns"); + if (mapped) { + *answer = tor_memdup_nulterm(mapped->data, mapped->size); + tor_munmap_file(mapped); + } + if (!*answer) { /* generate an error */ + *errmsg = "Could not open cached consensus. " + "Make sure FetchUselessDescriptors is set to 1."; + return -1; + } + } + } else if (!strcmp(question, "network-status")) { /* v1 */ + static int network_status_warned = 0; + if (!network_status_warned) { + log_warn(LD_CONTROL, "GETINFO network-status is deprecated; it will " + "go away in a future version of Tor."); + network_status_warned = 1; + } + routerlist_t *routerlist = router_get_routerlist(); + if (!routerlist || !routerlist->routers || + list_server_status_v1(routerlist->routers, answer, 1) < 0) { + return -1; + } + } else if (!strcmpstart(question, "extra-info/digest/")) { + question += strlen("extra-info/digest/"); + if (strlen(question) == HEX_DIGEST_LEN) { + char d[DIGEST_LEN]; + signed_descriptor_t *sd = NULL; + if (base16_decode(d, sizeof(d), question, strlen(question)) + == sizeof(d)) { + /* XXXX this test should move into extrainfo_get_by_descriptor_digest, + * but I don't want to risk affecting other parts of the code, + * especially since the rules for using our own extrainfo (including + * when it might be freed) are different from those for using one + * we have downloaded. */ + if (router_extrainfo_digest_is_me(d)) + sd = &(router_get_my_extrainfo()->cache_info); + else + sd = extrainfo_get_by_descriptor_digest(d); + } + if (sd) { + const char *body = signed_descriptor_get_body(sd); + if (body) + *answer = tor_strndup(body, sd->signed_descriptor_len); + } + } + } + + return 0; +} + +/** Given a smartlist of 20-byte digests, return a newly allocated string + * containing each of those digests in order, formatted in HEX, and terminated + * with a newline. */ +static char * +digest_list_to_string(const smartlist_t *sl) +{ + int len; + char *result, *s; + + /* Allow for newlines, and a \0 at the end */ + len = smartlist_len(sl) * (HEX_DIGEST_LEN + 1) + 1; + result = tor_malloc_zero(len); + + s = result; + SMARTLIST_FOREACH_BEGIN(sl, const char *, digest) { + base16_encode(s, HEX_DIGEST_LEN + 1, digest, DIGEST_LEN); + s[HEX_DIGEST_LEN] = '\n'; + s += HEX_DIGEST_LEN + 1; + } SMARTLIST_FOREACH_END(digest); + *s = '\0'; + + return result; +} + +/** Turn a download_status_t into a human-readable description in a newly + * allocated string. The format is specified in control-spec.txt, under + * the documentation for "GETINFO download/..." . */ +static char * +download_status_to_string(const download_status_t *dl) +{ + char *rv = NULL; + char tbuf[ISO_TIME_LEN+1]; + const char *schedule_str, *want_authority_str; + const char *increment_on_str, *backoff_str; + + if (dl) { + /* Get some substrings of the eventual output ready */ + format_iso_time(tbuf, download_status_get_next_attempt_at(dl)); + + switch (dl->schedule) { + case DL_SCHED_GENERIC: + schedule_str = "DL_SCHED_GENERIC"; + break; + case DL_SCHED_CONSENSUS: + schedule_str = "DL_SCHED_CONSENSUS"; + break; + case DL_SCHED_BRIDGE: + schedule_str = "DL_SCHED_BRIDGE"; + break; + default: + schedule_str = "unknown"; + break; + } + + switch (dl->want_authority) { + case DL_WANT_ANY_DIRSERVER: + want_authority_str = "DL_WANT_ANY_DIRSERVER"; + break; + case DL_WANT_AUTHORITY: + want_authority_str = "DL_WANT_AUTHORITY"; + break; + default: + want_authority_str = "unknown"; + break; + } + + switch (dl->increment_on) { + case DL_SCHED_INCREMENT_FAILURE: + increment_on_str = "DL_SCHED_INCREMENT_FAILURE"; + break; + case DL_SCHED_INCREMENT_ATTEMPT: + increment_on_str = "DL_SCHED_INCREMENT_ATTEMPT"; + break; + default: + increment_on_str = "unknown"; + break; + } + + backoff_str = "DL_SCHED_RANDOM_EXPONENTIAL"; + + /* Now assemble them */ + tor_asprintf(&rv, + "next-attempt-at %s\n" + "n-download-failures %u\n" + "n-download-attempts %u\n" + "schedule %s\n" + "want-authority %s\n" + "increment-on %s\n" + "backoff %s\n" + "last-backoff-position %u\n" + "last-delay-used %d\n", + tbuf, + dl->n_download_failures, + dl->n_download_attempts, + schedule_str, + want_authority_str, + increment_on_str, + backoff_str, + dl->last_backoff_position, + dl->last_delay_used); + } + + return rv; +} + +/** Handle the consensus download cases for getinfo_helper_downloads() */ +STATIC void +getinfo_helper_downloads_networkstatus(const char *flavor, + download_status_t **dl_to_emit, + const char **errmsg) +{ + /* + * We get the one for the current bootstrapped status by default, or + * take an extra /bootstrap or /running suffix + */ + if (strcmp(flavor, "ns") == 0) { + *dl_to_emit = networkstatus_get_dl_status_by_flavor(FLAV_NS); + } else if (strcmp(flavor, "ns/bootstrap") == 0) { + *dl_to_emit = networkstatus_get_dl_status_by_flavor_bootstrap(FLAV_NS); + } else if (strcmp(flavor, "ns/running") == 0 ) { + *dl_to_emit = networkstatus_get_dl_status_by_flavor_running(FLAV_NS); + } else if (strcmp(flavor, "microdesc") == 0) { + *dl_to_emit = networkstatus_get_dl_status_by_flavor(FLAV_MICRODESC); + } else if (strcmp(flavor, "microdesc/bootstrap") == 0) { + *dl_to_emit = + networkstatus_get_dl_status_by_flavor_bootstrap(FLAV_MICRODESC); + } else if (strcmp(flavor, "microdesc/running") == 0) { + *dl_to_emit = + networkstatus_get_dl_status_by_flavor_running(FLAV_MICRODESC); + } else { + *errmsg = "Unknown flavor"; + } +} + +/** Handle the cert download cases for getinfo_helper_downloads() */ +STATIC void +getinfo_helper_downloads_cert(const char *fp_sk_req, + download_status_t **dl_to_emit, + smartlist_t **digest_list, + const char **errmsg) +{ + const char *sk_req; + char id_digest[DIGEST_LEN]; + char sk_digest[DIGEST_LEN]; + + /* + * We have to handle four cases; fp_sk_req is the request with + * a prefix of "downloads/cert/" snipped off. + * + * Case 1: fp_sk_req = "fps" + * - We should emit a digest_list with a list of all the identity + * fingerprints that can be queried for certificate download status; + * get it by calling list_authority_ids_with_downloads(). + * + * Case 2: fp_sk_req = "fp/" for some fingerprint fp + * - We want the default certificate for this identity fingerprint's + * download status; this is the download we get from URLs starting + * in /fp/ on the directory server. We can get it with + * id_only_download_status_for_authority_id(). + * + * Case 3: fp_sk_req = "fp//sks" for some fingerprint fp + * - We want a list of all signing key digests for this identity + * fingerprint which can be queried for certificate download status. + * Get it with list_sk_digests_for_authority_id(). + * + * Case 4: fp_sk_req = "fp//" for some fingerprint fp and + * signing key digest sk + * - We want the download status for the certificate for this specific + * signing key and fingerprint. These correspond to the ones we get + * from URLs starting in /fp-sk/ on the directory server. Get it with + * list_sk_digests_for_authority_id(). + */ + + if (strcmp(fp_sk_req, "fps") == 0) { + *digest_list = list_authority_ids_with_downloads(); + if (!(*digest_list)) { + *errmsg = "Failed to get list of authority identity digests (!)"; + } + } else if (!strcmpstart(fp_sk_req, "fp/")) { + fp_sk_req += strlen("fp/"); + /* Okay, look for another / to tell the fp from fp-sk cases */ + sk_req = strchr(fp_sk_req, '/'); + if (sk_req) { + /* okay, split it here and try to parse */ + if (base16_decode(id_digest, DIGEST_LEN, + fp_sk_req, sk_req - fp_sk_req) == DIGEST_LEN) { + /* Skip past the '/' */ + ++sk_req; + if (strcmp(sk_req, "sks") == 0) { + /* We're asking for the list of signing key fingerprints */ + *digest_list = list_sk_digests_for_authority_id(id_digest); + if (!(*digest_list)) { + *errmsg = "Failed to get list of signing key digests for this " + "authority identity digest"; + } + } else { + /* We've got a signing key digest */ + if (base16_decode(sk_digest, DIGEST_LEN, + sk_req, strlen(sk_req)) == DIGEST_LEN) { + *dl_to_emit = + download_status_for_authority_id_and_sk(id_digest, sk_digest); + if (!(*dl_to_emit)) { + *errmsg = "Failed to get download status for this identity/" + "signing key digest pair"; + } + } else { + *errmsg = "That didn't look like a signing key digest"; + } + } + } else { + *errmsg = "That didn't look like an identity digest"; + } + } else { + /* We're either in downloads/certs/fp/, or we can't parse */ + if (strlen(fp_sk_req) == HEX_DIGEST_LEN) { + if (base16_decode(id_digest, DIGEST_LEN, + fp_sk_req, strlen(fp_sk_req)) == DIGEST_LEN) { + *dl_to_emit = id_only_download_status_for_authority_id(id_digest); + if (!(*dl_to_emit)) { + *errmsg = "Failed to get download status for this authority " + "identity digest"; + } + } else { + *errmsg = "That didn't look like a digest"; + } + } else { + *errmsg = "That didn't look like a digest"; + } + } + } else { + *errmsg = "Unknown certificate download status query"; + } +} + +/** Handle the routerdesc download cases for getinfo_helper_downloads() */ +STATIC void +getinfo_helper_downloads_desc(const char *desc_req, + download_status_t **dl_to_emit, + smartlist_t **digest_list, + const char **errmsg) +{ + char desc_digest[DIGEST_LEN]; + /* + * Two cases to handle here: + * + * Case 1: desc_req = "descs" + * - Emit a list of all router descriptor digests, which we get by + * calling router_get_descriptor_digests(); this can return NULL + * if we have no current ns-flavor consensus. + * + * Case 2: desc_req = + * - Check on the specified fingerprint and emit its download_status_t + * using router_get_dl_status_by_descriptor_digest(). + */ + + if (strcmp(desc_req, "descs") == 0) { + *digest_list = router_get_descriptor_digests(); + if (!(*digest_list)) { + *errmsg = "We don't seem to have a networkstatus-flavored consensus"; + } + /* + * Microdescs don't use the download_status_t mechanism, so we don't + * answer queries about their downloads here; see microdesc.c. + */ + } else if (strlen(desc_req) == HEX_DIGEST_LEN) { + if (base16_decode(desc_digest, DIGEST_LEN, + desc_req, strlen(desc_req)) == DIGEST_LEN) { + /* Okay we got a digest-shaped thing; try asking for it */ + *dl_to_emit = router_get_dl_status_by_descriptor_digest(desc_digest); + if (!(*dl_to_emit)) { + *errmsg = "No such descriptor digest found"; + } + } else { + *errmsg = "That didn't look like a digest"; + } + } else { + *errmsg = "Unknown router descriptor download status query"; + } +} + +/** Handle the bridge download cases for getinfo_helper_downloads() */ +STATIC void +getinfo_helper_downloads_bridge(const char *bridge_req, + download_status_t **dl_to_emit, + smartlist_t **digest_list, + const char **errmsg) +{ + char bridge_digest[DIGEST_LEN]; + /* + * Two cases to handle here: + * + * Case 1: bridge_req = "bridges" + * - Emit a list of all bridge identity digests, which we get by + * calling list_bridge_identities(); this can return NULL if we are + * not using bridges. + * + * Case 2: bridge_req = + * - Check on the specified fingerprint and emit its download_status_t + * using get_bridge_dl_status_by_id(). + */ + + if (strcmp(bridge_req, "bridges") == 0) { + *digest_list = list_bridge_identities(); + if (!(*digest_list)) { + *errmsg = "We don't seem to be using bridges"; + } + } else if (strlen(bridge_req) == HEX_DIGEST_LEN) { + if (base16_decode(bridge_digest, DIGEST_LEN, + bridge_req, strlen(bridge_req)) == DIGEST_LEN) { + /* Okay we got a digest-shaped thing; try asking for it */ + *dl_to_emit = get_bridge_dl_status_by_id(bridge_digest); + if (!(*dl_to_emit)) { + *errmsg = "No such bridge identity digest found"; + } + } else { + *errmsg = "That didn't look like a digest"; + } + } else { + *errmsg = "Unknown bridge descriptor download status query"; + } +} + +/** Implementation helper for GETINFO: knows the answers for questions about + * download status information. */ +STATIC int +getinfo_helper_downloads(control_connection_t *control_conn, + const char *question, char **answer, + const char **errmsg) +{ + download_status_t *dl_to_emit = NULL; + smartlist_t *digest_list = NULL; + + /* Assert args are sane */ + tor_assert(control_conn != NULL); + tor_assert(question != NULL); + tor_assert(answer != NULL); + tor_assert(errmsg != NULL); + + /* We check for this later to see if we should supply a default */ + *errmsg = NULL; + + /* Are we after networkstatus downloads? */ + if (!strcmpstart(question, "downloads/networkstatus/")) { + getinfo_helper_downloads_networkstatus( + question + strlen("downloads/networkstatus/"), + &dl_to_emit, errmsg); + /* Certificates? */ + } else if (!strcmpstart(question, "downloads/cert/")) { + getinfo_helper_downloads_cert( + question + strlen("downloads/cert/"), + &dl_to_emit, &digest_list, errmsg); + /* Router descriptors? */ + } else if (!strcmpstart(question, "downloads/desc/")) { + getinfo_helper_downloads_desc( + question + strlen("downloads/desc/"), + &dl_to_emit, &digest_list, errmsg); + /* Bridge descriptors? */ + } else if (!strcmpstart(question, "downloads/bridge/")) { + getinfo_helper_downloads_bridge( + question + strlen("downloads/bridge/"), + &dl_to_emit, &digest_list, errmsg); + } else { + *errmsg = "Unknown download status query"; + } + + if (dl_to_emit) { + *answer = download_status_to_string(dl_to_emit); + + return 0; + } else if (digest_list) { + *answer = digest_list_to_string(digest_list); + SMARTLIST_FOREACH(digest_list, void *, s, tor_free(s)); + smartlist_free(digest_list); + + return 0; + } else { + if (!(*errmsg)) { + *errmsg = "Unknown error"; + } + + return -1; + } +} + +/** Implementation helper for GETINFO: knows how to generate summaries of the + * current states of things we send events about. */ +static int +getinfo_helper_events(control_connection_t *control_conn, + const char *question, char **answer, + const char **errmsg) +{ + const or_options_t *options = get_options(); + (void) control_conn; + if (!strcmp(question, "circuit-status")) { + smartlist_t *status = smartlist_new(); + SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ_) { + origin_circuit_t *circ; + char *circdesc; + const char *state; + if (! CIRCUIT_IS_ORIGIN(circ_) || circ_->marked_for_close) + continue; + circ = TO_ORIGIN_CIRCUIT(circ_); + + if (circ->base_.state == CIRCUIT_STATE_OPEN) + state = "BUILT"; + else if (circ->base_.state == CIRCUIT_STATE_GUARD_WAIT) + state = "GUARD_WAIT"; + else if (circ->cpath) + state = "EXTENDED"; + else + state = "LAUNCHED"; + + circdesc = circuit_describe_status_for_controller(circ); + + smartlist_add_asprintf(status, "%lu %s%s%s", + (unsigned long)circ->global_identifier, + state, *circdesc ? " " : "", circdesc); + tor_free(circdesc); + } + SMARTLIST_FOREACH_END(circ_); + *answer = smartlist_join_strings(status, "\r\n", 0, NULL); + SMARTLIST_FOREACH(status, char *, cp, tor_free(cp)); + smartlist_free(status); + } else if (!strcmp(question, "stream-status")) { + smartlist_t *conns = get_connection_array(); + smartlist_t *status = smartlist_new(); + char buf[256]; + SMARTLIST_FOREACH_BEGIN(conns, connection_t *, base_conn) { + const char *state; + entry_connection_t *conn; + circuit_t *circ; + origin_circuit_t *origin_circ = NULL; + if (base_conn->type != CONN_TYPE_AP || + base_conn->marked_for_close || + base_conn->state == AP_CONN_STATE_SOCKS_WAIT || + base_conn->state == AP_CONN_STATE_NATD_WAIT) + continue; + conn = TO_ENTRY_CONN(base_conn); + switch (base_conn->state) + { + case AP_CONN_STATE_CONTROLLER_WAIT: + case AP_CONN_STATE_CIRCUIT_WAIT: + if (conn->socks_request && + SOCKS_COMMAND_IS_RESOLVE(conn->socks_request->command)) + state = "NEWRESOLVE"; + else + state = "NEW"; + break; + case AP_CONN_STATE_RENDDESC_WAIT: + case AP_CONN_STATE_CONNECT_WAIT: + state = "SENTCONNECT"; break; + case AP_CONN_STATE_RESOLVE_WAIT: + state = "SENTRESOLVE"; break; + case AP_CONN_STATE_OPEN: + state = "SUCCEEDED"; break; + default: + log_warn(LD_BUG, "Asked for stream in unknown state %d", + base_conn->state); + continue; + } + circ = circuit_get_by_edge_conn(ENTRY_TO_EDGE_CONN(conn)); + if (circ && CIRCUIT_IS_ORIGIN(circ)) + origin_circ = TO_ORIGIN_CIRCUIT(circ); + write_stream_target_to_buf(conn, buf, sizeof(buf)); + smartlist_add_asprintf(status, "%lu %s %lu %s", + (unsigned long) base_conn->global_identifier,state, + origin_circ? + (unsigned long)origin_circ->global_identifier : 0ul, + buf); + } SMARTLIST_FOREACH_END(base_conn); + *answer = smartlist_join_strings(status, "\r\n", 0, NULL); + SMARTLIST_FOREACH(status, char *, cp, tor_free(cp)); + smartlist_free(status); + } else if (!strcmp(question, "orconn-status")) { + smartlist_t *conns = get_connection_array(); + smartlist_t *status = smartlist_new(); + SMARTLIST_FOREACH_BEGIN(conns, connection_t *, base_conn) { + const char *state; + char name[128]; + or_connection_t *conn; + if (base_conn->type != CONN_TYPE_OR || base_conn->marked_for_close) + continue; + conn = TO_OR_CONN(base_conn); + if (conn->base_.state == OR_CONN_STATE_OPEN) + state = "CONNECTED"; + else if (conn->nickname) + state = "LAUNCHED"; + else + state = "NEW"; + orconn_target_get_name(name, sizeof(name), conn); + smartlist_add_asprintf(status, "%s %s", name, state); + } SMARTLIST_FOREACH_END(base_conn); + *answer = smartlist_join_strings(status, "\r\n", 0, NULL); + SMARTLIST_FOREACH(status, char *, cp, tor_free(cp)); + smartlist_free(status); + } else if (!strcmpstart(question, "address-mappings/")) { + time_t min_e, max_e; + smartlist_t *mappings; + question += strlen("address-mappings/"); + if (!strcmp(question, "all")) { + min_e = 0; max_e = TIME_MAX; + } else if (!strcmp(question, "cache")) { + min_e = 2; max_e = TIME_MAX; + } else if (!strcmp(question, "config")) { + min_e = 0; max_e = 0; + } else if (!strcmp(question, "control")) { + min_e = 1; max_e = 1; + } else { + return 0; + } + mappings = smartlist_new(); + addressmap_get_mappings(mappings, min_e, max_e, 1); + *answer = smartlist_join_strings(mappings, "\r\n", 0, NULL); + SMARTLIST_FOREACH(mappings, char *, cp, tor_free(cp)); + smartlist_free(mappings); + } else if (!strcmpstart(question, "status/")) { + /* Note that status/ is not a catch-all for events; there's only supposed + * to be a status GETINFO if there's a corresponding STATUS event. */ + if (!strcmp(question, "status/circuit-established")) { + *answer = tor_strdup(have_completed_a_circuit() ? "1" : "0"); + } else if (!strcmp(question, "status/enough-dir-info")) { + *answer = tor_strdup(router_have_minimum_dir_info() ? "1" : "0"); + } else if (!strcmp(question, "status/good-server-descriptor") || + !strcmp(question, "status/accepted-server-descriptor")) { + /* They're equivalent for now, until we can figure out how to make + * good-server-descriptor be what we want. See comment in + * control-spec.txt. */ + *answer = tor_strdup(directories_have_accepted_server_descriptor() + ? "1" : "0"); + } else if (!strcmp(question, "status/reachability-succeeded/or")) { + *answer = tor_strdup(check_whether_orport_reachable(options) ? + "1" : "0"); + } else if (!strcmp(question, "status/reachability-succeeded/dir")) { + *answer = tor_strdup(check_whether_dirport_reachable(options) ? + "1" : "0"); + } else if (!strcmp(question, "status/reachability-succeeded")) { + tor_asprintf(answer, "OR=%d DIR=%d", + check_whether_orport_reachable(options) ? 1 : 0, + check_whether_dirport_reachable(options) ? 1 : 0); + } else if (!strcmp(question, "status/bootstrap-phase")) { + *answer = control_event_boot_last_msg(); + } else if (!strcmpstart(question, "status/version/")) { + int is_server = server_mode(options); + networkstatus_t *c = networkstatus_get_latest_consensus(); + version_status_t status; + const char *recommended; + if (c) { + recommended = is_server ? c->server_versions : c->client_versions; + status = tor_version_is_obsolete(VERSION, recommended); + } else { + recommended = "?"; + status = VS_UNKNOWN; + } + + if (!strcmp(question, "status/version/recommended")) { + *answer = tor_strdup(recommended); + return 0; + } + if (!strcmp(question, "status/version/current")) { + switch (status) + { + case VS_RECOMMENDED: *answer = tor_strdup("recommended"); break; + case VS_OLD: *answer = tor_strdup("obsolete"); break; + case VS_NEW: *answer = tor_strdup("new"); break; + case VS_NEW_IN_SERIES: *answer = tor_strdup("new in series"); break; + case VS_UNRECOMMENDED: *answer = tor_strdup("unrecommended"); break; + case VS_EMPTY: *answer = tor_strdup("none recommended"); break; + case VS_UNKNOWN: *answer = tor_strdup("unknown"); break; + default: tor_fragile_assert(); + } + } + } else if (!strcmp(question, "status/clients-seen")) { + char *bridge_stats = geoip_get_bridge_stats_controller(time(NULL)); + if (!bridge_stats) { + *errmsg = "No bridge-client stats available"; + return -1; + } + *answer = bridge_stats; + } else if (!strcmp(question, "status/fresh-relay-descs")) { + if (!server_mode(options)) { + *errmsg = "Only relays have descriptors"; + return -1; + } + routerinfo_t *r; + extrainfo_t *e; + if (router_build_fresh_descriptor(&r, &e) < 0) { + *errmsg = "Error generating descriptor"; + return -1; + } + size_t size = r->cache_info.signed_descriptor_len + 1; + if (e) { + size += e->cache_info.signed_descriptor_len + 1; + } + tor_assert(r->cache_info.signed_descriptor_len); + char *descs = tor_malloc(size); + char *cp = descs; + memcpy(cp, signed_descriptor_get_body(&r->cache_info), + r->cache_info.signed_descriptor_len); + cp += r->cache_info.signed_descriptor_len - 1; + if (e) { + if (cp[0] == '\0') { + cp[0] = '\n'; + } else if (cp[0] != '\n') { + cp[1] = '\n'; + cp++; + } + memcpy(cp, signed_descriptor_get_body(&e->cache_info), + e->cache_info.signed_descriptor_len); + cp += e->cache_info.signed_descriptor_len - 1; + } + if (cp[0] == '\n') { + cp[0] = '\0'; + } else if (cp[0] != '\0') { + cp[1] = '\0'; + } + *answer = descs; + routerinfo_free(r); + extrainfo_free(e); + } else { + return 0; + } + } + return 0; +} + +/** Implementation helper for GETINFO: knows how to enumerate hidden services + * created via the control port. */ +STATIC int +getinfo_helper_onions(control_connection_t *control_conn, + const char *question, char **answer, + const char **errmsg) +{ + smartlist_t *onion_list = NULL; + (void) errmsg; /* no errors from this method */ + + if (control_conn && !strcmp(question, "onions/current")) { + onion_list = control_conn->ephemeral_onion_services; + } else if (!strcmp(question, "onions/detached")) { + onion_list = get_detached_onion_services(); + } else { + return 0; + } + if (!onion_list || smartlist_len(onion_list) == 0) { + if (answer) { + *answer = tor_strdup(""); + } + } else { + if (answer) { + *answer = smartlist_join_strings(onion_list, "\r\n", 0, NULL); + } + } + + return 0; +} + +/** Implementation helper for GETINFO: answers queries about network + * liveness. */ +static int +getinfo_helper_liveness(control_connection_t *control_conn, + const char *question, char **answer, + const char **errmsg) +{ + (void)control_conn; + (void)errmsg; + if (strcmp(question, "network-liveness") == 0) { + if (get_cached_network_liveness()) { + *answer = tor_strdup("up"); + } else { + *answer = tor_strdup("down"); + } + } + + return 0; +} + +/** Implementation helper for GETINFO: answers queries about shared random + * value. */ +static int +getinfo_helper_sr(control_connection_t *control_conn, + const char *question, char **answer, + const char **errmsg) +{ + (void) control_conn; + (void) errmsg; + + if (!strcmp(question, "sr/current")) { + *answer = sr_get_current_for_control(); + } else if (!strcmp(question, "sr/previous")) { + *answer = sr_get_previous_for_control(); + } + /* Else statement here is unrecognized key so do nothing. */ + + return 0; +} + +/** Callback function for GETINFO: on a given control connection, try to + * answer the question q and store the newly-allocated answer in + * *a. If an internal error occurs, return -1 and optionally set + * *error_out to point to an error message to be delivered to the + * controller. On success, _or if the key is not recognized_, return 0. Do not + * set a if the key is not recognized but you may set error_out + * to improve the error message. + */ +typedef int (*getinfo_helper_t)(control_connection_t *, + const char *q, char **a, + const char **error_out); + +/** A single item for the GETINFO question-to-answer-function table. */ +typedef struct getinfo_item_t { + const char *varname; /**< The value (or prefix) of the question. */ + getinfo_helper_t fn; /**< The function that knows the answer: NULL if + * this entry is documentation-only. */ + const char *desc; /**< Description of the variable. */ + int is_prefix; /** Must varname match exactly, or must it be a prefix? */ +} getinfo_item_t; + +#define ITEM(name, fn, desc) { name, getinfo_helper_##fn, desc, 0 } +#define PREFIX(name, fn, desc) { name, getinfo_helper_##fn, desc, 1 } +#define DOC(name, desc) { name, NULL, desc, 0 } + +/** Table mapping questions accepted by GETINFO to the functions that know how + * to answer them. */ +static const getinfo_item_t getinfo_items[] = { + ITEM("version", misc, "The current version of Tor."), + ITEM("bw-event-cache", misc, "Cached BW events for a short interval."), + ITEM("config-file", misc, "Current location of the \"torrc\" file."), + ITEM("config-defaults-file", misc, "Current location of the defaults file."), + ITEM("config-text", misc, + "Return the string that would be written by a saveconf command."), + ITEM("config-can-saveconf", misc, + "Is it possible to save the configuration to the \"torrc\" file?"), + ITEM("accounting/bytes", accounting, + "Number of bytes read/written so far in the accounting interval."), + ITEM("accounting/bytes-left", accounting, + "Number of bytes left to write/read so far in the accounting interval."), + ITEM("accounting/enabled", accounting, "Is accounting currently enabled?"), + ITEM("accounting/hibernating", accounting, "Are we hibernating or awake?"), + ITEM("accounting/interval-start", accounting, + "Time when the accounting period starts."), + ITEM("accounting/interval-end", accounting, + "Time when the accounting period ends."), + ITEM("accounting/interval-wake", accounting, + "Time to wake up in this accounting period."), + ITEM("helper-nodes", entry_guards, NULL), /* deprecated */ + ITEM("entry-guards", entry_guards, + "Which nodes are we using as entry guards?"), + ITEM("fingerprint", misc, NULL), + PREFIX("config/", config, "Current configuration values."), + DOC("config/names", + "List of configuration options, types, and documentation."), + DOC("config/defaults", + "List of default values for configuration options. " + "See also config/names"), + PREFIX("current-time/", current_time, "Current time."), + DOC("current-time/local", "Current time on the local system."), + DOC("current-time/utc", "Current UTC time."), + PREFIX("downloads/networkstatus/", downloads, + "Download statuses for networkstatus objects"), + DOC("downloads/networkstatus/ns", + "Download status for current-mode networkstatus download"), + DOC("downloads/networkstatus/ns/bootstrap", + "Download status for bootstrap-time networkstatus download"), + DOC("downloads/networkstatus/ns/running", + "Download status for run-time networkstatus download"), + DOC("downloads/networkstatus/microdesc", + "Download status for current-mode microdesc download"), + DOC("downloads/networkstatus/microdesc/bootstrap", + "Download status for bootstrap-time microdesc download"), + DOC("downloads/networkstatus/microdesc/running", + "Download status for run-time microdesc download"), + PREFIX("downloads/cert/", downloads, + "Download statuses for certificates, by id fingerprint and " + "signing key"), + DOC("downloads/cert/fps", + "List of authority fingerprints for which any download statuses " + "exist"), + DOC("downloads/cert/fp/", + "Download status for with the default signing key; corresponds " + "to /fp/ URLs on directory server."), + DOC("downloads/cert/fp//sks", + "List of signing keys for which specific download statuses are " + "available for this id fingerprint"), + DOC("downloads/cert/fp//", + "Download status for with signing key ; corresponds " + "to /fp-sk/ URLs on directory server."), + PREFIX("downloads/desc/", downloads, + "Download statuses for router descriptors, by descriptor digest"), + DOC("downloads/desc/descs", + "Return a list of known router descriptor digests"), + DOC("downloads/desc/", + "Return a download status for a given descriptor digest"), + PREFIX("downloads/bridge/", downloads, + "Download statuses for bridge descriptors, by bridge identity " + "digest"), + DOC("downloads/bridge/bridges", + "Return a list of configured bridge identity digests with download " + "statuses"), + DOC("downloads/bridge/", + "Return a download status for a given bridge identity digest"), + ITEM("info/names", misc, + "List of GETINFO options, types, and documentation."), + ITEM("events/names", misc, + "Events that the controller can ask for with SETEVENTS."), + ITEM("signal/names", misc, "Signal names recognized by the SIGNAL command"), + ITEM("features/names", misc, "What arguments can USEFEATURE take?"), + PREFIX("desc/id/", dir, "Router descriptors by ID."), + PREFIX("desc/name/", dir, "Router descriptors by nickname."), + ITEM("desc/all-recent", dir, + "All non-expired, non-superseded router descriptors."), + ITEM("desc/download-enabled", dir, + "Do we try to download router descriptors?"), + ITEM("desc/all-recent-extrainfo-hack", dir, NULL), /* Hack. */ + ITEM("md/all", dir, "All known microdescriptors."), + PREFIX("md/id/", dir, "Microdescriptors by ID"), + PREFIX("md/name/", dir, "Microdescriptors by name"), + ITEM("md/download-enabled", dir, + "Do we try to download microdescriptors?"), + PREFIX("extra-info/digest/", dir, "Extra-info documents by digest."), + PREFIX("hs/client/desc/id", dir, + "Hidden Service descriptor in client's cache by onion."), + PREFIX("hs/service/desc/id/", dir, + "Hidden Service descriptor in services's cache by onion."), + PREFIX("net/listeners/", listeners, "Bound addresses by type"), + ITEM("ns/all", networkstatus, + "Brief summary of router status (v2 directory format)"), + PREFIX("ns/id/", networkstatus, + "Brief summary of router status by ID (v2 directory format)."), + PREFIX("ns/name/", networkstatus, + "Brief summary of router status by nickname (v2 directory format)."), + PREFIX("ns/purpose/", networkstatus, + "Brief summary of router status by purpose (v2 directory format)."), + PREFIX("consensus/", networkstatus, + "Information about and from the ns consensus."), + ITEM("network-status", dir, + "Brief summary of router status (v1 directory format)"), + ITEM("network-liveness", liveness, + "Current opinion on whether the network is live"), + ITEM("circuit-status", events, "List of current circuits originating here."), + ITEM("stream-status", events,"List of current streams."), + ITEM("orconn-status", events, "A list of current OR connections."), + ITEM("dormant", misc, + "Is Tor dormant (not building circuits because it's idle)?"), + PREFIX("address-mappings/", events, NULL), + DOC("address-mappings/all", "Current address mappings."), + DOC("address-mappings/cache", "Current cached DNS replies."), + DOC("address-mappings/config", + "Current address mappings from configuration."), + DOC("address-mappings/control", "Current address mappings from controller."), + PREFIX("status/", events, NULL), + DOC("status/circuit-established", + "Whether we think client functionality is working."), + DOC("status/enough-dir-info", + "Whether we have enough up-to-date directory information to build " + "circuits."), + DOC("status/bootstrap-phase", + "The last bootstrap phase status event that Tor sent."), + DOC("status/clients-seen", + "Breakdown of client countries seen by a bridge."), + DOC("status/fresh-relay-descs", + "A fresh relay/ei descriptor pair for Tor's current state. Not stored."), + DOC("status/version/recommended", "List of currently recommended versions."), + DOC("status/version/current", "Status of the current version."), + ITEM("address", misc, "IP address of this Tor host, if we can guess it."), + ITEM("traffic/read", misc,"Bytes read since the process was started."), + ITEM("traffic/written", misc, + "Bytes written since the process was started."), + ITEM("uptime", misc, "Uptime of the Tor daemon in seconds."), + ITEM("process/pid", misc, "Process id belonging to the main tor process."), + ITEM("process/uid", misc, "User id running the tor process."), + ITEM("process/user", misc, + "Username under which the tor process is running."), + ITEM("process/descriptor-limit", misc, "File descriptor limit."), + ITEM("limits/max-mem-in-queues", misc, "Actual limit on memory in queues"), + PREFIX("desc-annotations/id/", dir, "Router annotations by hexdigest."), + PREFIX("dir/server/", dir,"Router descriptors as retrieved from a DirPort."), + PREFIX("dir/status/", dir, + "v2 networkstatus docs as retrieved from a DirPort."), + ITEM("dir/status-vote/current/consensus", dir, + "v3 Networkstatus consensus as retrieved from a DirPort."), + ITEM("exit-policy/default", policies, + "The default value appended to the configured exit policy."), + ITEM("exit-policy/reject-private/default", policies, + "The default rules appended to the configured exit policy by" + " ExitPolicyRejectPrivate."), + ITEM("exit-policy/reject-private/relay", policies, + "The relay-specific rules appended to the configured exit policy by" + " ExitPolicyRejectPrivate and/or ExitPolicyRejectLocalInterfaces."), + ITEM("exit-policy/full", policies, "The entire exit policy of onion router"), + ITEM("exit-policy/ipv4", policies, "IPv4 parts of exit policy"), + ITEM("exit-policy/ipv6", policies, "IPv6 parts of exit policy"), + PREFIX("ip-to-country/", geoip, "Perform a GEOIP lookup"), + ITEM("onions/current", onions, + "Onion services owned by the current control connection."), + ITEM("onions/detached", onions, + "Onion services detached from the control connection."), + ITEM("sr/current", sr, "Get current shared random value."), + ITEM("sr/previous", sr, "Get previous shared random value."), + { NULL, NULL, NULL, 0 } +}; + +/** Allocate and return a list of recognized GETINFO options. */ +static char * +list_getinfo_options(void) +{ + int i; + smartlist_t *lines = smartlist_new(); + char *ans; + for (i = 0; getinfo_items[i].varname; ++i) { + if (!getinfo_items[i].desc) + continue; + + smartlist_add_asprintf(lines, "%s%s -- %s\n", + getinfo_items[i].varname, + getinfo_items[i].is_prefix ? "*" : "", + getinfo_items[i].desc); + } + smartlist_sort_strings(lines); + + ans = smartlist_join_strings(lines, "", 0, NULL); + SMARTLIST_FOREACH(lines, char *, cp, tor_free(cp)); + smartlist_free(lines); + + return ans; +} + +/** Lookup the 'getinfo' entry question, and return + * the answer in *answer (or NULL if key not recognized). + * Return 0 if success or unrecognized, or -1 if recognized but + * internal error. */ +static int +handle_getinfo_helper(control_connection_t *control_conn, + const char *question, char **answer, + const char **err_out) +{ + int i; + *answer = NULL; /* unrecognized key by default */ + + for (i = 0; getinfo_items[i].varname; ++i) { + int match; + if (getinfo_items[i].is_prefix) + match = !strcmpstart(question, getinfo_items[i].varname); + else + match = !strcmp(question, getinfo_items[i].varname); + if (match) { + tor_assert(getinfo_items[i].fn); + return getinfo_items[i].fn(control_conn, question, answer, err_out); + } + } + + return 0; /* unrecognized */ +} + +/** Called when we receive a GETINFO command. Try to fetch all requested + * information, and reply with information or error message. */ +int +handle_control_getinfo(control_connection_t *conn, uint32_t len, + const char *body) +{ + smartlist_t *questions = smartlist_new(); + smartlist_t *answers = smartlist_new(); + smartlist_t *unrecognized = smartlist_new(); + char *ans = NULL; + int i; + (void) len; /* body is NUL-terminated, so it's safe to ignore the length. */ + + smartlist_split_string(questions, body, " ", + SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); + SMARTLIST_FOREACH_BEGIN(questions, const char *, q) { + const char *errmsg = NULL; + + if (handle_getinfo_helper(conn, q, &ans, &errmsg) < 0) { + if (!errmsg) + errmsg = "Internal error"; + connection_printf_to_buf(conn, "551 %s\r\n", errmsg); + goto done; + } + if (!ans) { + if (errmsg) /* use provided error message */ + smartlist_add_strdup(unrecognized, errmsg); + else /* use default error message */ + smartlist_add_asprintf(unrecognized, "Unrecognized key \"%s\"", q); + } else { + smartlist_add_strdup(answers, q); + smartlist_add(answers, ans); + } + } SMARTLIST_FOREACH_END(q); + + if (smartlist_len(unrecognized)) { + /* control-spec section 2.3, mid-reply '-' or end of reply ' ' */ + for (i=0; i < smartlist_len(unrecognized)-1; ++i) + connection_printf_to_buf(conn, + "552-%s\r\n", + (char *)smartlist_get(unrecognized, i)); + + connection_printf_to_buf(conn, + "552 %s\r\n", + (char *)smartlist_get(unrecognized, i)); + goto done; + } + + for (i = 0; i < smartlist_len(answers); i += 2) { + char *k = smartlist_get(answers, i); + char *v = smartlist_get(answers, i+1); + if (!strchr(v, '\n') && !strchr(v, '\r')) { + connection_printf_to_buf(conn, "250-%s=", k); + connection_write_str_to_buf(v, conn); + connection_write_str_to_buf("\r\n", conn); + } else { + char *esc = NULL; + size_t esc_len; + esc_len = write_escaped_data(v, strlen(v), &esc); + connection_printf_to_buf(conn, "250+%s=\r\n", k); + connection_buf_add(esc, esc_len, TO_CONN(conn)); + tor_free(esc); + } + } + connection_write_str_to_buf("250 OK\r\n", conn); + + done: + SMARTLIST_FOREACH(answers, char *, cp, tor_free(cp)); + smartlist_free(answers); + SMARTLIST_FOREACH(questions, char *, cp, tor_free(cp)); + smartlist_free(questions); + SMARTLIST_FOREACH(unrecognized, char *, cp, tor_free(cp)); + smartlist_free(unrecognized); + + return 0; +} diff --git a/src/feature/control/control_getinfo.h b/src/feature/control/control_getinfo.h new file mode 100644 index 0000000000..d5a2feb3e0 --- /dev/null +++ b/src/feature/control/control_getinfo.h @@ -0,0 +1,57 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file control.h + * \brief Header file for control.c. + **/ + +#ifndef TOR_CONTROL_GETINFO_H +#define TOR_CONTROL_GETINFO_H + +int handle_control_getinfo(control_connection_t *conn, uint32_t len, + const char *body); + +#ifdef CONTROL_GETINFO_PRIVATE +STATIC int getinfo_helper_onions( + control_connection_t *control_conn, + const char *question, + char **answer, + const char **errmsg); +STATIC void getinfo_helper_downloads_networkstatus( + const char *flavor, + download_status_t **dl_to_emit, + const char **errmsg); +STATIC void getinfo_helper_downloads_cert( + const char *fp_sk_req, + download_status_t **dl_to_emit, + smartlist_t **digest_list, + const char **errmsg); +STATIC void getinfo_helper_downloads_desc( + const char *desc_req, + download_status_t **dl_to_emit, + smartlist_t **digest_list, + const char **errmsg); +STATIC void getinfo_helper_downloads_bridge( + const char *bridge_req, + download_status_t **dl_to_emit, + smartlist_t **digest_list, + const char **errmsg); +STATIC int getinfo_helper_downloads( + control_connection_t *control_conn, + const char *question, char **answer, + const char **errmsg); +STATIC int getinfo_helper_dir( + control_connection_t *control_conn, + const char *question, char **answer, + const char **errmsg); +STATIC int getinfo_helper_current_time( + control_connection_t *control_conn, + const char *question, char **answer, + const char **errmsg); +#endif /* defined(CONTROL_GETINFO_PRIVATE) */ + +#endif /* !defined(TOR_CONTROL_GETINFO) */ diff --git a/src/test/test_controller.c b/src/test/test_controller.c index 5b406e159b..82922d2028 100644 --- a/src/test/test_controller.c +++ b/src/test/test_controller.c @@ -2,10 +2,12 @@ /* See LICENSE for licensing information */ #define CONTROL_PRIVATE +#define CONTROL_GETINFO_PRIVATE #include "core/or/or.h" #include "lib/crypt_ops/crypto_ed25519.h" #include "feature/client/bridges.h" #include "feature/control/control.h" +#include "feature/control/control_getinfo.h" #include "feature/client/entrynodes.h" #include "feature/hs/hs_common.h" #include "feature/nodelist/networkstatus.h" diff --git a/src/test/test_dir.c b/src/test/test_dir.c index 4132d42d12..bb8c8970ea 100644 --- a/src/test/test_dir.c +++ b/src/test/test_dir.c @@ -8,7 +8,7 @@ #define BWAUTH_PRIVATE #define CONFIG_PRIVATE -#define CONTROL_PRIVATE +#define CONTROL_GETINFO_PRIVATE #define DIRCACHE_PRIVATE #define DIRCLIENT_PRIVATE #define DIRSERV_PRIVATE @@ -32,7 +32,7 @@ #include "core/or/versions.h" #include "feature/client/bridges.h" #include "feature/client/entrynodes.h" -#include "feature/control/control.h" +#include "feature/control/control_getinfo.h" #include "feature/dirauth/bwauth.h" #include "feature/dirauth/dirvote.h" #include "feature/dirauth/dsigs_parse.h" From 2917ecaa970c8fabcc0e9875b1e87c0d279ab1b3 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 25 Mar 2019 14:03:49 -0400 Subject: [PATCH 0635/2557] Split command-handling and authentication from control.c --- scripts/maint/practracker/exceptions.txt | 23 +- src/app/config/config.c | 1 + src/app/main/main.c | 1 + src/core/include.am | 4 + src/feature/control/control.c | 2819 +--------------------- src/feature/control/control.h | 30 - src/feature/control/control_auth.c | 439 ++++ src/feature/control/control_auth.h | 27 + src/feature/control/control_cmd.c | 2323 ++++++++++++++++++ src/feature/control/control_cmd.h | 48 + src/feature/control/control_fmt.c | 95 + src/feature/control/control_fmt.h | 5 + src/feature/control/control_getinfo.c | 1 + src/test/test_controller.c | 3 +- src/test/test_pt.c | 1 - src/test/test_util.c | 1 - 16 files changed, 2965 insertions(+), 2856 deletions(-) create mode 100644 src/feature/control/control_auth.c create mode 100644 src/feature/control/control_auth.h create mode 100644 src/feature/control/control_cmd.c create mode 100644 src/feature/control/control_cmd.h diff --git a/scripts/maint/practracker/exceptions.txt b/scripts/maint/practracker/exceptions.txt index 142fda5806..fe91e0c345 100644 --- a/scripts/maint/practracker/exceptions.txt +++ b/scripts/maint/practracker/exceptions.txt @@ -79,8 +79,8 @@ problem function-size /src/core/mainloop/connection.c:connection_handle_write_im problem function-size /src/core/mainloop/connection.c:assert_connection_ok() 143 problem function-size /src/app/config/confparse.c:config_assign_value() 205 problem function-size /src/app/config/confparse.c:config_get_assigned_option() 129 -problem file-size /src/app/config/config.c 8489 -problem include-count /src/app/config/config.c 85 +problem file-size /src/app/config/config.c 8490 +problem include-count /src/app/config/config.c 86 problem function-size /src/app/config/config.c:options_act_reversible() 296 problem function-size /src/app/config/config.c:options_act() 588 problem function-size /src/app/config/config.c:resolve_my_address() 192 @@ -96,7 +96,7 @@ problem function-size /src/app/config/config.c:parse_port_config() 452 problem function-size /src/app/config/config.c:parse_ports() 170 problem function-size /src/app/config/config.c:getinfo_helper_config() 116 problem function-size /src/app/main/ntmain.c:nt_service_install() 125 -problem include-count /src/app/main/main.c 84 +problem include-count /src/app/main/main.c 85 problem function-size /src/app/main/main.c:dumpstats() 102 problem function-size /src/app/main/main.c:tor_init() 136 problem function-size /src/app/main/main.c:sandbox_init_filter() 291 @@ -111,19 +111,20 @@ problem function-size /src/feature/dircommon/consdiff.c:gen_ed_diff() 204 problem function-size /src/feature/dircommon/consdiff.c:apply_ed_diff() 159 problem file-size /src/feature/control/control.c 7592 problem include-count /src/feature/control/control.c 90 -problem function-size /src/feature/control/control.c:handle_control_authenticate() 188 +problem function-size /src/feature/control/control_auth.c:handle_control_authenticate() 188 +problem function-size /src/feature/control/control_auth.c:handle_control_authchallenge() 115 problem function-size /src/feature/control/control_getinfo.c:getinfo_helper_misc() 109 problem function-size /src/feature/control/control_getinfo.c:getinfo_helper_dir() 304 problem function-size /src/feature/control/control_getinfo.c:getinfo_helper_events() 236 -problem function-size /src/feature/control/control.c:handle_control_extendcircuit() 151 -problem function-size /src/feature/control/control.c:handle_control_authchallenge() 115 -problem function-size /src/feature/control/control.c:handle_control_hsfetch() 114 -problem function-size /src/feature/control/control.c:handle_control_hspost() 117 -problem function-size /src/feature/control/control.c:handle_control_add_onion() 293 -problem function-size /src/feature/control/control.c:add_onion_helper_keyarg() 125 +problem function-size /src/feature/control/control_cmd.c:handle_control_command() 104 +problem function-size /src/feature/control/control_cmd.c:handle_control_extendcircuit() 151 +problem function-size /src/feature/control/control_cmd.c:handle_control_hsfetch() 114 +problem function-size /src/feature/control/control_cmd.c:handle_control_hspost() 117 +problem function-size /src/feature/control/control_cmd.c:handle_control_add_onion() 293 +problem function-size /src/feature/control/control_cmd.c:add_onion_helper_keyarg() 125 problem function-size /src/feature/control/control.c:connection_control_process_inbuf() 239 problem function-size /src/feature/control/control_events.c:control_event_stream_status() 119 -problem include-count /src/feature/control/control_getinfo.c 51 +problem include-count /src/feature/control/control_getinfo.c 52 problem function-size /src/feature/stats/rephist.c:rep_hist_load_mtbf_data() 185 problem function-size /src/feature/stats/rephist.c:rep_hist_format_exit_stats() 148 problem function-size /src/feature/dircache/consdiffmgr.c:consdiffmgr_cleanup() 115 diff --git a/src/app/config/config.c b/src/app/config/config.c index 1966910a81..07521ea0ae 100644 --- a/src/app/config/config.c +++ b/src/app/config/config.c @@ -86,6 +86,7 @@ #include "feature/client/entrynodes.h" #include "feature/client/transports.h" #include "feature/control/control.h" +#include "feature/control/control_auth.h" #include "feature/control/control_events.h" #include "feature/dirauth/bwauth.h" #include "feature/dirauth/guardfraction.h" diff --git a/src/app/main/main.c b/src/app/main/main.c index 56978f9bab..83eef0dc70 100644 --- a/src/app/main/main.c +++ b/src/app/main/main.c @@ -43,6 +43,7 @@ #include "feature/client/entrynodes.h" #include "feature/client/transports.h" #include "feature/control/control.h" +#include "feature/control/control_auth.h" #include "feature/control/control_events.h" #include "feature/dirauth/bwauth.h" #include "feature/dirauth/keypin.h" diff --git a/src/core/include.am b/src/core/include.am index a8ec360592..27954af5f5 100644 --- a/src/core/include.am +++ b/src/core/include.am @@ -70,7 +70,9 @@ LIBTOR_APP_A_SOURCES = \ src/feature/control/btrack_orconn_cevent.c \ src/feature/control/btrack_orconn_maps.c \ src/feature/control/control.c \ + src/feature/control/control_auth.c \ src/feature/control/control_bootstrap.c \ + src/feature/control/control_cmd.c \ src/feature/control/control_events.c \ src/feature/control/control_fmt.c \ src/feature/control/control_getinfo.c \ @@ -290,6 +292,8 @@ noinst_HEADERS += \ src/feature/control/btrack_orconn_maps.h \ src/feature/control/btrack_sys.h \ src/feature/control/control.h \ + src/feature/control/control_auth.h \ + src/feature/control/control_cmd.h \ src/feature/control/control_connection_st.h \ src/feature/control/control_events.h \ src/feature/control/control_fmt.h \ diff --git a/src/feature/control/control.c b/src/feature/control/control.c index 94643efde8..70d70bedac 100644 --- a/src/feature/control/control.c +++ b/src/feature/control/control.c @@ -33,7 +33,6 @@ **/ #define CONTROL_MODULE_PRIVATE -#define CONTROL_PRIVATE #define CONTROL_EVENTS_PRIVATE #define OCIRC_EVENT_PRIVATE @@ -63,6 +62,8 @@ #include "feature/client/dnsserv.h" #include "feature/client/entrynodes.h" #include "feature/control/control.h" +#include "feature/control/control_auth.h" +#include "feature/control/control_cmd.h" #include "feature/control/control_events.h" #include "feature/control/control_fmt.h" #include "feature/control/control_getinfo.h" @@ -134,84 +135,6 @@ #include "lib/evloop/procmon.h" #include "lib/evloop/compat_libevent.h" -/** Yield true iff s is the state of a control_connection_t that has - * finished authentication and is accepting commands. */ -#define STATE_IS_OPEN(s) ((s) == CONTROL_CONN_STATE_OPEN) - -/** If we're using cookie-type authentication, how long should our cookies be? - */ -#define AUTHENTICATION_COOKIE_LEN 32 - -/** If true, we've set authentication_cookie to a secret code and - * stored it to disk. */ -static int authentication_cookie_is_set = 0; -/** If authentication_cookie_is_set, a secret cookie that we've stored to disk - * and which we're using to authenticate controllers. (If the controller can - * read it off disk, it has permission to connect.) */ -static uint8_t *authentication_cookie = NULL; - -#define SAFECOOKIE_SERVER_TO_CONTROLLER_CONSTANT \ - "Tor safe cookie authentication server-to-controller hash" -#define SAFECOOKIE_CONTROLLER_TO_SERVER_CONSTANT \ - "Tor safe cookie authentication controller-to-server hash" -#define SAFECOOKIE_SERVER_NONCE_LEN DIGEST256_LEN - -/** The list of onion services that have been added via ADD_ONION that do not - * belong to any particular control connection. - */ -static smartlist_t *detached_onion_services = NULL; - -static void send_control_done(control_connection_t *conn); -static int handle_control_setconf(control_connection_t *conn, uint32_t len, - char *body); -static int handle_control_resetconf(control_connection_t *conn, uint32_t len, - char *body); -static int handle_control_getconf(control_connection_t *conn, uint32_t len, - const char *body); -static int handle_control_loadconf(control_connection_t *conn, uint32_t len, - const char *body); -static int handle_control_setevents(control_connection_t *conn, uint32_t len, - const char *body); -static int handle_control_authenticate(control_connection_t *conn, - uint32_t len, - const char *body); -static int handle_control_signal(control_connection_t *conn, uint32_t len, - const char *body); -static int handle_control_mapaddress(control_connection_t *conn, uint32_t len, - const char *body); -static int handle_control_extendcircuit(control_connection_t *conn, - uint32_t len, - const char *body); -static int handle_control_setcircuitpurpose(control_connection_t *conn, - uint32_t len, const char *body); -static int handle_control_attachstream(control_connection_t *conn, - uint32_t len, - const char *body); -static int handle_control_postdescriptor(control_connection_t *conn, - uint32_t len, - const char *body); -static int handle_control_redirectstream(control_connection_t *conn, - uint32_t len, - const char *body); -static int handle_control_closestream(control_connection_t *conn, uint32_t len, - const char *body); -static int handle_control_closecircuit(control_connection_t *conn, - uint32_t len, - const char *body); -static int handle_control_resolve(control_connection_t *conn, uint32_t len, - const char *body); -static int handle_control_usefeature(control_connection_t *conn, - uint32_t len, - const char *body); -static int handle_control_hsfetch(control_connection_t *conn, uint32_t len, - const char *body); -static int handle_control_hspost(control_connection_t *conn, uint32_t len, - const char *body); -static int handle_control_add_onion(control_connection_t *conn, uint32_t len, - const char *body); -static int handle_control_del_onion(control_connection_t *conn, uint32_t len, - const char *body); - /** Convert a connection_t* to an control_connection_t*; assert if the cast is * invalid. */ control_connection_t * @@ -221,94 +144,6 @@ TO_CONTROL_CONN(connection_t *c) return DOWNCAST(control_connection_t, c); } -/** If the first in_len_max characters in start contain a - * double-quoted string with escaped characters, return the length of that - * string (as encoded, including quotes). Otherwise return -1. */ -static inline int -get_escaped_string_length(const char *start, size_t in_len_max, - int *chars_out) -{ - const char *cp, *end; - int chars = 0; - - if (*start != '\"') - return -1; - - cp = start+1; - end = start+in_len_max; - - /* Calculate length. */ - while (1) { - if (cp >= end) { - return -1; /* Too long. */ - } else if (*cp == '\\') { - if (++cp == end) - return -1; /* Can't escape EOS. */ - ++cp; - ++chars; - } else if (*cp == '\"') { - break; - } else { - ++cp; - ++chars; - } - } - if (chars_out) - *chars_out = chars; - return (int)(cp - start+1); -} - -/** As decode_escaped_string, but does not decode the string: copies the - * entire thing, including quotation marks. */ -static const char * -extract_escaped_string(const char *start, size_t in_len_max, - char **out, size_t *out_len) -{ - int length = get_escaped_string_length(start, in_len_max, NULL); - if (length<0) - return NULL; - *out_len = length; - *out = tor_strndup(start, *out_len); - return start+length; -} - -/** Given a pointer to a string starting at start containing - * in_len_max characters, decode a string beginning with one double - * quote, containing any number of non-quote characters or characters escaped - * with a backslash, and ending with a final double quote. Place the resulting - * string (unquoted, unescaped) into a newly allocated string in *out; - * store its length in out_len. On success, return a pointer to the - * character immediately following the escaped string. On failure, return - * NULL. */ -static const char * -decode_escaped_string(const char *start, size_t in_len_max, - char **out, size_t *out_len) -{ - const char *cp, *end; - char *outp; - int len, n_chars = 0; - - len = get_escaped_string_length(start, in_len_max, &n_chars); - if (len<0) - return NULL; - - end = start+len-1; /* Index of last quote. */ - tor_assert(*end == '\"'); - outp = *out = tor_malloc(len+1); - *out_len = n_chars; - - cp = start+1; - while (cp < end) { - if (*cp == '\\') - ++cp; - *outp++ = *cp++; - } - *outp = '\0'; - tor_assert((outp - *out) == (int)*out_len); - - return end+1; -} - /** Create and add a new controller connection on sock. If * CC_LOCAL_FD_IS_OWNER is set in flags, this Tor process should * exit when the connection closes. If CC_LOCAL_FD_IS_AUTHENTICATED @@ -396,588 +231,6 @@ control_ports_write_to_file(void) smartlist_free(lines); } -/** Send a "DONE" message down the control connection conn. */ -static void -send_control_done(control_connection_t *conn) -{ - connection_write_str_to_buf("250 OK\r\n", conn); -} - -/** Given a text circuit id, return the corresponding circuit. */ -static origin_circuit_t * -get_circ(const char *id) -{ - uint32_t n_id; - int ok; - n_id = (uint32_t) tor_parse_ulong(id, 10, 0, UINT32_MAX, &ok, NULL); - if (!ok) - return NULL; - return circuit_get_by_global_id(n_id); -} - -/** Given a text stream id, return the corresponding AP connection. */ -static entry_connection_t * -get_stream(const char *id) -{ - uint64_t n_id; - int ok; - connection_t *conn; - n_id = tor_parse_uint64(id, 10, 0, UINT64_MAX, &ok, NULL); - if (!ok) - return NULL; - conn = connection_get_by_global_id(n_id); - if (!conn || conn->type != CONN_TYPE_AP || conn->marked_for_close) - return NULL; - return TO_ENTRY_CONN(conn); -} - -/** Helper for setconf and resetconf. Acts like setconf, except - * it passes use_defaults on to options_trial_assign(). Modifies the - * contents of body. - */ -static int -control_setconf_helper(control_connection_t *conn, uint32_t len, char *body, - int use_defaults) -{ - setopt_err_t opt_err; - config_line_t *lines=NULL; - char *start = body; - char *errstring = NULL; - const unsigned flags = - CAL_CLEAR_FIRST | (use_defaults ? CAL_USE_DEFAULTS : 0); - - char *config; - smartlist_t *entries = smartlist_new(); - - /* We have a string, "body", of the format '(key(=val|="val")?)' entries - * separated by space. break it into a list of configuration entries. */ - while (*body) { - char *eq = body; - char *key; - char *entry; - while (!TOR_ISSPACE(*eq) && *eq != '=') - ++eq; - key = tor_strndup(body, eq-body); - body = eq+1; - if (*eq == '=') { - char *val=NULL; - size_t val_len=0; - if (*body != '\"') { - char *val_start = body; - while (!TOR_ISSPACE(*body)) - body++; - val = tor_strndup(val_start, body-val_start); - val_len = strlen(val); - } else { - body = (char*)extract_escaped_string(body, (len - (body-start)), - &val, &val_len); - if (!body) { - connection_write_str_to_buf("551 Couldn't parse string\r\n", conn); - SMARTLIST_FOREACH(entries, char *, cp, tor_free(cp)); - smartlist_free(entries); - tor_free(key); - return 0; - } - } - tor_asprintf(&entry, "%s %s", key, val); - tor_free(key); - tor_free(val); - } else { - entry = key; - } - smartlist_add(entries, entry); - while (TOR_ISSPACE(*body)) - ++body; - } - - smartlist_add_strdup(entries, ""); - config = smartlist_join_strings(entries, "\n", 0, NULL); - SMARTLIST_FOREACH(entries, char *, cp, tor_free(cp)); - smartlist_free(entries); - - if (config_get_lines(config, &lines, 0) < 0) { - log_warn(LD_CONTROL,"Controller gave us config lines we can't parse."); - connection_write_str_to_buf("551 Couldn't parse configuration\r\n", - conn); - tor_free(config); - return 0; - } - tor_free(config); - - opt_err = options_trial_assign(lines, flags, &errstring); - { - const char *msg; - switch (opt_err) { - case SETOPT_ERR_MISC: - msg = "552 Unrecognized option"; - break; - case SETOPT_ERR_PARSE: - msg = "513 Unacceptable option value"; - break; - case SETOPT_ERR_TRANSITION: - msg = "553 Transition not allowed"; - break; - case SETOPT_ERR_SETTING: - default: - msg = "553 Unable to set option"; - break; - case SETOPT_OK: - config_free_lines(lines); - send_control_done(conn); - return 0; - } - log_warn(LD_CONTROL, - "Controller gave us config lines that didn't validate: %s", - errstring); - connection_printf_to_buf(conn, "%s: %s\r\n", msg, errstring); - config_free_lines(lines); - tor_free(errstring); - return 0; - } -} - -/** Called when we receive a SETCONF message: parse the body and try - * to update our configuration. Reply with a DONE or ERROR message. - * Modifies the contents of body.*/ -static int -handle_control_setconf(control_connection_t *conn, uint32_t len, char *body) -{ - return control_setconf_helper(conn, len, body, 0); -} - -/** Called when we receive a RESETCONF message: parse the body and try - * to update our configuration. Reply with a DONE or ERROR message. - * Modifies the contents of body. */ -static int -handle_control_resetconf(control_connection_t *conn, uint32_t len, char *body) -{ - return control_setconf_helper(conn, len, body, 1); -} - -/** Called when we receive a GETCONF message. Parse the request, and - * reply with a CONFVALUE or an ERROR message */ -static int -handle_control_getconf(control_connection_t *conn, uint32_t body_len, - const char *body) -{ - smartlist_t *questions = smartlist_new(); - smartlist_t *answers = smartlist_new(); - smartlist_t *unrecognized = smartlist_new(); - char *msg = NULL; - size_t msg_len; - const or_options_t *options = get_options(); - int i, len; - - (void) body_len; /* body is NUL-terminated; so we can ignore len. */ - smartlist_split_string(questions, body, " ", - SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); - SMARTLIST_FOREACH_BEGIN(questions, const char *, q) { - if (!option_is_recognized(q)) { - smartlist_add(unrecognized, (char*) q); - } else { - config_line_t *answer = option_get_assignment(options,q); - if (!answer) { - const char *name = option_get_canonical_name(q); - smartlist_add_asprintf(answers, "250-%s\r\n", name); - } - - while (answer) { - config_line_t *next; - smartlist_add_asprintf(answers, "250-%s=%s\r\n", - answer->key, answer->value); - - next = answer->next; - tor_free(answer->key); - tor_free(answer->value); - tor_free(answer); - answer = next; - } - } - } SMARTLIST_FOREACH_END(q); - - if ((len = smartlist_len(unrecognized))) { - for (i=0; i < len-1; ++i) - connection_printf_to_buf(conn, - "552-Unrecognized configuration key \"%s\"\r\n", - (char*)smartlist_get(unrecognized, i)); - connection_printf_to_buf(conn, - "552 Unrecognized configuration key \"%s\"\r\n", - (char*)smartlist_get(unrecognized, len-1)); - } else if ((len = smartlist_len(answers))) { - char *tmp = smartlist_get(answers, len-1); - tor_assert(strlen(tmp)>4); - tmp[3] = ' '; - msg = smartlist_join_strings(answers, "", 0, &msg_len); - connection_buf_add(msg, msg_len, TO_CONN(conn)); - } else { - connection_write_str_to_buf("250 OK\r\n", conn); - } - - SMARTLIST_FOREACH(answers, char *, cp, tor_free(cp)); - smartlist_free(answers); - SMARTLIST_FOREACH(questions, char *, cp, tor_free(cp)); - smartlist_free(questions); - smartlist_free(unrecognized); - - tor_free(msg); - - return 0; -} - -/** Called when we get a +LOADCONF message. */ -static int -handle_control_loadconf(control_connection_t *conn, uint32_t len, - const char *body) -{ - setopt_err_t retval; - char *errstring = NULL; - const char *msg = NULL; - (void) len; - - retval = options_init_from_string(NULL, body, CMD_RUN_TOR, NULL, &errstring); - - if (retval != SETOPT_OK) - log_warn(LD_CONTROL, - "Controller gave us config file that didn't validate: %s", - errstring); - - switch (retval) { - case SETOPT_ERR_PARSE: - msg = "552 Invalid config file"; - break; - case SETOPT_ERR_TRANSITION: - msg = "553 Transition not allowed"; - break; - case SETOPT_ERR_SETTING: - msg = "553 Unable to set option"; - break; - case SETOPT_ERR_MISC: - default: - msg = "550 Unable to load config"; - break; - case SETOPT_OK: - break; - } - if (msg) { - if (errstring) - connection_printf_to_buf(conn, "%s: %s\r\n", msg, errstring); - else - connection_printf_to_buf(conn, "%s\r\n", msg); - } else { - send_control_done(conn); - } - tor_free(errstring); - return 0; -} - -/** Called when we get a SETEVENTS message: update conn->event_mask, - * and reply with DONE or ERROR. */ -static int -handle_control_setevents(control_connection_t *conn, uint32_t len, - const char *body) -{ - int event_code; - event_mask_t event_mask = 0; - smartlist_t *events = smartlist_new(); - - (void) len; - - smartlist_split_string(events, body, " ", - SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); - SMARTLIST_FOREACH_BEGIN(events, const char *, ev) - { - if (!strcasecmp(ev, "EXTENDED") || - !strcasecmp(ev, "AUTHDIR_NEWDESCS")) { - log_warn(LD_CONTROL, "The \"%s\" SETEVENTS argument is no longer " - "supported.", ev); - continue; - } else { - int i; - event_code = -1; - - for (i = 0; control_event_table[i].event_name != NULL; ++i) { - if (!strcasecmp(ev, control_event_table[i].event_name)) { - event_code = control_event_table[i].event_code; - break; - } - } - - if (event_code == -1) { - connection_printf_to_buf(conn, "552 Unrecognized event \"%s\"\r\n", - ev); - SMARTLIST_FOREACH(events, char *, e, tor_free(e)); - smartlist_free(events); - return 0; - } - } - event_mask |= (((event_mask_t)1) << event_code); - } - SMARTLIST_FOREACH_END(ev); - SMARTLIST_FOREACH(events, char *, e, tor_free(e)); - smartlist_free(events); - - conn->event_mask = event_mask; - - control_update_global_event_mask(); - send_control_done(conn); - return 0; -} - -/** Decode the hashed, base64'd passwords stored in passwords. - * Return a smartlist of acceptable passwords (unterminated strings of - * length S2K_RFC2440_SPECIFIER_LEN+DIGEST_LEN) on success, or NULL on - * failure. - */ -smartlist_t * -decode_hashed_passwords(config_line_t *passwords) -{ - char decoded[64]; - config_line_t *cl; - smartlist_t *sl = smartlist_new(); - - tor_assert(passwords); - - for (cl = passwords; cl; cl = cl->next) { - const char *hashed = cl->value; - - if (!strcmpstart(hashed, "16:")) { - if (base16_decode(decoded, sizeof(decoded), hashed+3, strlen(hashed+3)) - != S2K_RFC2440_SPECIFIER_LEN + DIGEST_LEN - || strlen(hashed+3) != (S2K_RFC2440_SPECIFIER_LEN+DIGEST_LEN)*2) { - goto err; - } - } else { - if (base64_decode(decoded, sizeof(decoded), hashed, strlen(hashed)) - != S2K_RFC2440_SPECIFIER_LEN+DIGEST_LEN) { - goto err; - } - } - smartlist_add(sl, - tor_memdup(decoded, S2K_RFC2440_SPECIFIER_LEN+DIGEST_LEN)); - } - - return sl; - - err: - SMARTLIST_FOREACH(sl, char*, cp, tor_free(cp)); - smartlist_free(sl); - return NULL; -} - -/** Called when we get an AUTHENTICATE message. Check whether the - * authentication is valid, and if so, update the connection's state to - * OPEN. Reply with DONE or ERROR. - */ -static int -handle_control_authenticate(control_connection_t *conn, uint32_t len, - const char *body) -{ - int used_quoted_string = 0; - const or_options_t *options = get_options(); - const char *errstr = "Unknown error"; - char *password; - size_t password_len; - const char *cp; - int i; - int bad_cookie=0, bad_password=0; - smartlist_t *sl = NULL; - - if (!len) { - password = tor_strdup(""); - password_len = 0; - } else if (TOR_ISXDIGIT(body[0])) { - cp = body; - while (TOR_ISXDIGIT(*cp)) - ++cp; - i = (int)(cp - body); - tor_assert(i>0); - password_len = i/2; - password = tor_malloc(password_len + 1); - if (base16_decode(password, password_len+1, body, i) - != (int) password_len) { - connection_write_str_to_buf( - "551 Invalid hexadecimal encoding. Maybe you tried a plain text " - "password? If so, the standard requires that you put it in " - "double quotes.\r\n", conn); - connection_mark_for_close(TO_CONN(conn)); - tor_free(password); - return 0; - } - } else { - if (!decode_escaped_string(body, len, &password, &password_len)) { - connection_write_str_to_buf("551 Invalid quoted string. You need " - "to put the password in double quotes.\r\n", conn); - connection_mark_for_close(TO_CONN(conn)); - return 0; - } - used_quoted_string = 1; - } - - if (conn->safecookie_client_hash != NULL) { - /* The controller has chosen safe cookie authentication; the only - * acceptable authentication value is the controller-to-server - * response. */ - - tor_assert(authentication_cookie_is_set); - - if (password_len != DIGEST256_LEN) { - log_warn(LD_CONTROL, - "Got safe cookie authentication response with wrong length " - "(%d)", (int)password_len); - errstr = "Wrong length for safe cookie response."; - goto err; - } - - if (tor_memneq(conn->safecookie_client_hash, password, DIGEST256_LEN)) { - log_warn(LD_CONTROL, - "Got incorrect safe cookie authentication response"); - errstr = "Safe cookie response did not match expected value."; - goto err; - } - - tor_free(conn->safecookie_client_hash); - goto ok; - } - - if (!options->CookieAuthentication && !options->HashedControlPassword && - !options->HashedControlSessionPassword) { - /* if Tor doesn't demand any stronger authentication, then - * the controller can get in with anything. */ - goto ok; - } - - if (options->CookieAuthentication) { - int also_password = options->HashedControlPassword != NULL || - options->HashedControlSessionPassword != NULL; - if (password_len != AUTHENTICATION_COOKIE_LEN) { - if (!also_password) { - log_warn(LD_CONTROL, "Got authentication cookie with wrong length " - "(%d)", (int)password_len); - errstr = "Wrong length on authentication cookie."; - goto err; - } - bad_cookie = 1; - } else if (tor_memneq(authentication_cookie, password, password_len)) { - if (!also_password) { - log_warn(LD_CONTROL, "Got mismatched authentication cookie"); - errstr = "Authentication cookie did not match expected value."; - goto err; - } - bad_cookie = 1; - } else { - goto ok; - } - } - - if (options->HashedControlPassword || - options->HashedControlSessionPassword) { - int bad = 0; - smartlist_t *sl_tmp; - char received[DIGEST_LEN]; - int also_cookie = options->CookieAuthentication; - sl = smartlist_new(); - if (options->HashedControlPassword) { - sl_tmp = decode_hashed_passwords(options->HashedControlPassword); - if (!sl_tmp) - bad = 1; - else { - smartlist_add_all(sl, sl_tmp); - smartlist_free(sl_tmp); - } - } - if (options->HashedControlSessionPassword) { - sl_tmp = decode_hashed_passwords(options->HashedControlSessionPassword); - if (!sl_tmp) - bad = 1; - else { - smartlist_add_all(sl, sl_tmp); - smartlist_free(sl_tmp); - } - } - if (bad) { - if (!also_cookie) { - log_warn(LD_BUG, - "Couldn't decode HashedControlPassword: invalid base16"); - errstr="Couldn't decode HashedControlPassword value in configuration."; - goto err; - } - bad_password = 1; - SMARTLIST_FOREACH(sl, char *, str, tor_free(str)); - smartlist_free(sl); - sl = NULL; - } else { - SMARTLIST_FOREACH(sl, char *, expected, - { - secret_to_key_rfc2440(received,DIGEST_LEN, - password,password_len,expected); - if (tor_memeq(expected + S2K_RFC2440_SPECIFIER_LEN, - received, DIGEST_LEN)) - goto ok; - }); - SMARTLIST_FOREACH(sl, char *, str, tor_free(str)); - smartlist_free(sl); - sl = NULL; - - if (used_quoted_string) - errstr = "Password did not match HashedControlPassword value from " - "configuration"; - else - errstr = "Password did not match HashedControlPassword value from " - "configuration. Maybe you tried a plain text password? " - "If so, the standard requires that you put it in double quotes."; - bad_password = 1; - if (!also_cookie) - goto err; - } - } - - /** We only get here if both kinds of authentication failed. */ - tor_assert(bad_password && bad_cookie); - log_warn(LD_CONTROL, "Bad password or authentication cookie on controller."); - errstr = "Password did not match HashedControlPassword *or* authentication " - "cookie."; - - err: - tor_free(password); - connection_printf_to_buf(conn, "515 Authentication failed: %s\r\n", errstr); - connection_mark_for_close(TO_CONN(conn)); - if (sl) { /* clean up */ - SMARTLIST_FOREACH(sl, char *, str, tor_free(str)); - smartlist_free(sl); - } - return 0; - ok: - log_info(LD_CONTROL, "Authenticated control connection ("TOR_SOCKET_T_FORMAT - ")", conn->base_.s); - send_control_done(conn); - conn->base_.state = CONTROL_CONN_STATE_OPEN; - tor_free(password); - if (sl) { /* clean up */ - SMARTLIST_FOREACH(sl, char *, str, tor_free(str)); - smartlist_free(sl); - } - return 0; -} - -/** Called when we get a SAVECONF command. Try to flush the current options to - * disk, and report success or failure. */ -static int -handle_control_saveconf(control_connection_t *conn, uint32_t len, - const char *body) -{ - (void) len; - - int force = !strcmpstart(body, "FORCE"); - const or_options_t *options = get_options(); - if ((!force && options->IncludeUsed) || options_save_current() < 0) { - connection_write_str_to_buf( - "551 Unable to write configuration to disk.\r\n", conn); - } else { - send_control_done(conn); - } - return 0; -} - const struct signal_name_t signal_table[] = { { SIGHUP, "RELOAD" }, { SIGHUP, "HUP" }, @@ -997,1927 +250,6 @@ const struct signal_name_t signal_table[] = { { 0, NULL }, }; -/** Called when we get a SIGNAL command. React to the provided signal, and - * report success or failure. (If the signal results in a shutdown, success - * may not be reported.) */ -static int -handle_control_signal(control_connection_t *conn, uint32_t len, - const char *body) -{ - int sig = -1; - int i; - int n = 0; - char *s; - - (void) len; - - while (body[n] && ! TOR_ISSPACE(body[n])) - ++n; - s = tor_strndup(body, n); - - for (i = 0; signal_table[i].signal_name != NULL; ++i) { - if (!strcasecmp(s, signal_table[i].signal_name)) { - sig = signal_table[i].sig; - break; - } - } - - if (sig < 0) - connection_printf_to_buf(conn, "552 Unrecognized signal code \"%s\"\r\n", - s); - tor_free(s); - if (sig < 0) - return 0; - - send_control_done(conn); - /* Flush the "done" first if the signal might make us shut down. */ - if (sig == SIGTERM || sig == SIGINT) - connection_flush(TO_CONN(conn)); - - activate_signal(sig); - - return 0; -} - -/** Called when we get a TAKEOWNERSHIP command. Mark this connection - * as an owning connection, so that we will exit if the connection - * closes. */ -static int -handle_control_takeownership(control_connection_t *conn, uint32_t len, - const char *body) -{ - (void)len; - (void)body; - - conn->is_owning_control_connection = 1; - - log_info(LD_CONTROL, "Control connection %d has taken ownership of this " - "Tor instance.", - (int)(conn->base_.s)); - - send_control_done(conn); - return 0; -} - -/** Called when we get a DROPOWNERSHIP command. Mark this connection - * as a non-owning connection, so that we will not exit if the connection - * closes. */ -static int -handle_control_dropownership(control_connection_t *conn, uint32_t len, - const char *body) -{ - (void)len; - (void)body; - - conn->is_owning_control_connection = 0; - - log_info(LD_CONTROL, "Control connection %d has dropped ownership of this " - "Tor instance.", - (int)(conn->base_.s)); - - send_control_done(conn); - return 0; -} - -/** Return true iff addr is unusable as a mapaddress target because of - * containing funny characters. */ -static int -address_is_invalid_mapaddress_target(const char *addr) -{ - if (!strcmpstart(addr, "*.")) - return address_is_invalid_destination(addr+2, 1); - else - return address_is_invalid_destination(addr, 1); -} - -/** Called when we get a MAPADDRESS command; try to bind all listed addresses, - * and report success or failure. */ -static int -handle_control_mapaddress(control_connection_t *conn, uint32_t len, - const char *body) -{ - smartlist_t *elts; - smartlist_t *lines; - smartlist_t *reply; - char *r; - size_t sz; - (void) len; /* body is NUL-terminated, so it's safe to ignore the length. */ - - lines = smartlist_new(); - elts = smartlist_new(); - reply = smartlist_new(); - smartlist_split_string(lines, body, " ", - SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); - SMARTLIST_FOREACH_BEGIN(lines, char *, line) { - tor_strlower(line); - smartlist_split_string(elts, line, "=", 0, 2); - if (smartlist_len(elts) == 2) { - const char *from = smartlist_get(elts,0); - const char *to = smartlist_get(elts,1); - if (address_is_invalid_mapaddress_target(to)) { - smartlist_add_asprintf(reply, - "512-syntax error: invalid address '%s'", to); - log_warn(LD_CONTROL, - "Skipping invalid argument '%s' in MapAddress msg", to); - } else if (!strcmp(from, ".") || !strcmp(from, "0.0.0.0") || - !strcmp(from, "::")) { - const char type = - !strcmp(from,".") ? RESOLVED_TYPE_HOSTNAME : - (!strcmp(from, "0.0.0.0") ? RESOLVED_TYPE_IPV4 : RESOLVED_TYPE_IPV6); - const char *address = addressmap_register_virtual_address( - type, tor_strdup(to)); - if (!address) { - smartlist_add_asprintf(reply, - "451-resource exhausted: skipping '%s'", line); - log_warn(LD_CONTROL, - "Unable to allocate address for '%s' in MapAddress msg", - safe_str_client(line)); - } else { - smartlist_add_asprintf(reply, "250-%s=%s", address, to); - } - } else { - const char *msg; - if (addressmap_register_auto(from, to, 1, - ADDRMAPSRC_CONTROLLER, &msg) < 0) { - smartlist_add_asprintf(reply, - "512-syntax error: invalid address mapping " - " '%s': %s", line, msg); - log_warn(LD_CONTROL, - "Skipping invalid argument '%s' in MapAddress msg: %s", - line, msg); - } else { - smartlist_add_asprintf(reply, "250-%s", line); - } - } - } else { - smartlist_add_asprintf(reply, "512-syntax error: mapping '%s' is " - "not of expected form 'foo=bar'.", line); - log_info(LD_CONTROL, "Skipping MapAddress '%s': wrong " - "number of items.", - safe_str_client(line)); - } - SMARTLIST_FOREACH(elts, char *, cp, tor_free(cp)); - smartlist_clear(elts); - } SMARTLIST_FOREACH_END(line); - SMARTLIST_FOREACH(lines, char *, cp, tor_free(cp)); - smartlist_free(lines); - smartlist_free(elts); - - if (smartlist_len(reply)) { - ((char*)smartlist_get(reply,smartlist_len(reply)-1))[3] = ' '; - r = smartlist_join_strings(reply, "\r\n", 1, &sz); - connection_buf_add(r, sz, TO_CONN(conn)); - tor_free(r); - } else { - const char *response = - "512 syntax error: not enough arguments to mapaddress.\r\n"; - connection_buf_add(response, strlen(response), TO_CONN(conn)); - } - - SMARTLIST_FOREACH(reply, char *, cp, tor_free(cp)); - smartlist_free(reply); - return 0; -} - -/** Given a string, convert it to a circuit purpose. */ -static uint8_t -circuit_purpose_from_string(const char *string) -{ - if (!strcasecmpstart(string, "purpose=")) - string += strlen("purpose="); - - if (!strcasecmp(string, "general")) - return CIRCUIT_PURPOSE_C_GENERAL; - else if (!strcasecmp(string, "controller")) - return CIRCUIT_PURPOSE_CONTROLLER; - else - return CIRCUIT_PURPOSE_UNKNOWN; -} - -/** Return a newly allocated smartlist containing the arguments to the command - * waiting in body. If there are fewer than min_args arguments, - * or if max_args is nonnegative and there are more than - * max_args arguments, send a 512 error to the controller, using - * command as the command name in the error message. */ -static smartlist_t * -getargs_helper(const char *command, control_connection_t *conn, - const char *body, int min_args, int max_args) -{ - smartlist_t *args = smartlist_new(); - smartlist_split_string(args, body, " ", - SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); - if (smartlist_len(args) < min_args) { - connection_printf_to_buf(conn, "512 Missing argument to %s\r\n",command); - goto err; - } else if (max_args >= 0 && smartlist_len(args) > max_args) { - connection_printf_to_buf(conn, "512 Too many arguments to %s\r\n",command); - goto err; - } - return args; - err: - SMARTLIST_FOREACH(args, char *, s, tor_free(s)); - smartlist_free(args); - return NULL; -} - -/** Helper. Return the first element of sl at index start_at or - * higher that starts with prefix, case-insensitive. Return NULL if no - * such element exists. */ -static const char * -find_element_starting_with(smartlist_t *sl, int start_at, const char *prefix) -{ - int i; - for (i = start_at; i < smartlist_len(sl); ++i) { - const char *elt = smartlist_get(sl, i); - if (!strcasecmpstart(elt, prefix)) - return elt; - } - return NULL; -} - -/** Helper. Return true iff s is an argument that we should treat as a - * key-value pair. */ -static int -is_keyval_pair(const char *s) -{ - /* An argument is a key-value pair if it has an =, and it isn't of the form - * $fingeprint=name */ - return strchr(s, '=') && s[0] != '$'; -} - -/** Called when we get an EXTENDCIRCUIT message. Try to extend the listed - * circuit, and report success or failure. */ -static int -handle_control_extendcircuit(control_connection_t *conn, uint32_t len, - const char *body) -{ - smartlist_t *router_nicknames=NULL, *nodes=NULL; - origin_circuit_t *circ = NULL; - int zero_circ; - uint8_t intended_purpose = CIRCUIT_PURPOSE_C_GENERAL; - smartlist_t *args; - (void) len; - - router_nicknames = smartlist_new(); - - args = getargs_helper("EXTENDCIRCUIT", conn, body, 1, -1); - if (!args) - goto done; - - zero_circ = !strcmp("0", (char*)smartlist_get(args,0)); - - if (zero_circ) { - const char *purp = find_element_starting_with(args, 1, "PURPOSE="); - - if (purp) { - intended_purpose = circuit_purpose_from_string(purp); - if (intended_purpose == CIRCUIT_PURPOSE_UNKNOWN) { - connection_printf_to_buf(conn, "552 Unknown purpose \"%s\"\r\n", purp); - SMARTLIST_FOREACH(args, char *, cp, tor_free(cp)); - smartlist_free(args); - goto done; - } - } - - if ((smartlist_len(args) == 1) || - (smartlist_len(args) >= 2 && is_keyval_pair(smartlist_get(args, 1)))) { - // "EXTENDCIRCUIT 0" || EXTENDCIRCUIT 0 foo=bar" - circ = circuit_launch(intended_purpose, CIRCLAUNCH_NEED_CAPACITY); - if (!circ) { - connection_write_str_to_buf("551 Couldn't start circuit\r\n", conn); - } else { - connection_printf_to_buf(conn, "250 EXTENDED %lu\r\n", - (unsigned long)circ->global_identifier); - } - SMARTLIST_FOREACH(args, char *, cp, tor_free(cp)); - smartlist_free(args); - goto done; - } - // "EXTENDCIRCUIT 0 router1,router2" || - // "EXTENDCIRCUIT 0 router1,router2 PURPOSE=foo" - } - - if (!zero_circ && !(circ = get_circ(smartlist_get(args,0)))) { - connection_printf_to_buf(conn, "552 Unknown circuit \"%s\"\r\n", - (char*)smartlist_get(args, 0)); - SMARTLIST_FOREACH(args, char *, cp, tor_free(cp)); - smartlist_free(args); - goto done; - } - - if (smartlist_len(args) < 2) { - connection_printf_to_buf(conn, - "512 syntax error: not enough arguments.\r\n"); - SMARTLIST_FOREACH(args, char *, cp, tor_free(cp)); - smartlist_free(args); - goto done; - } - - smartlist_split_string(router_nicknames, smartlist_get(args,1), ",", 0, 0); - - SMARTLIST_FOREACH(args, char *, cp, tor_free(cp)); - smartlist_free(args); - - nodes = smartlist_new(); - int first_node = zero_circ; - SMARTLIST_FOREACH_BEGIN(router_nicknames, const char *, n) { - const node_t *node = node_get_by_nickname(n, 0); - if (!node) { - connection_printf_to_buf(conn, "552 No such router \"%s\"\r\n", n); - goto done; - } - if (!node_has_preferred_descriptor(node, first_node)) { - connection_printf_to_buf(conn, "552 No descriptor for \"%s\"\r\n", n); - goto done; - } - smartlist_add(nodes, (void*)node); - first_node = 0; - } SMARTLIST_FOREACH_END(n); - if (!smartlist_len(nodes)) { - connection_write_str_to_buf("512 No router names provided\r\n", conn); - goto done; - } - - if (zero_circ) { - /* start a new circuit */ - circ = origin_circuit_init(intended_purpose, 0); - } - - /* now circ refers to something that is ready to be extended */ - first_node = zero_circ; - SMARTLIST_FOREACH(nodes, const node_t *, node, - { - extend_info_t *info = extend_info_from_node(node, first_node); - if (!info) { - tor_assert_nonfatal(first_node); - log_warn(LD_CONTROL, - "controller tried to connect to a node that lacks a suitable " - "descriptor, or which doesn't have any " - "addresses that are allowed by the firewall configuration; " - "circuit marked for closing."); - circuit_mark_for_close(TO_CIRCUIT(circ), -END_CIRC_REASON_CONNECTFAILED); - connection_write_str_to_buf("551 Couldn't start circuit\r\n", conn); - goto done; - } - circuit_append_new_exit(circ, info); - if (circ->build_state->desired_path_len > 1) { - circ->build_state->onehop_tunnel = 0; - } - extend_info_free(info); - first_node = 0; - }); - - /* now that we've populated the cpath, start extending */ - if (zero_circ) { - int err_reason = 0; - if ((err_reason = circuit_handle_first_hop(circ)) < 0) { - circuit_mark_for_close(TO_CIRCUIT(circ), -err_reason); - connection_write_str_to_buf("551 Couldn't start circuit\r\n", conn); - goto done; - } - } else { - if (circ->base_.state == CIRCUIT_STATE_OPEN || - circ->base_.state == CIRCUIT_STATE_GUARD_WAIT) { - int err_reason = 0; - circuit_set_state(TO_CIRCUIT(circ), CIRCUIT_STATE_BUILDING); - if ((err_reason = circuit_send_next_onion_skin(circ)) < 0) { - log_info(LD_CONTROL, - "send_next_onion_skin failed; circuit marked for closing."); - circuit_mark_for_close(TO_CIRCUIT(circ), -err_reason); - connection_write_str_to_buf("551 Couldn't send onion skin\r\n", conn); - goto done; - } - } - } - - connection_printf_to_buf(conn, "250 EXTENDED %lu\r\n", - (unsigned long)circ->global_identifier); - if (zero_circ) /* send a 'launched' event, for completeness */ - circuit_event_status(circ, CIRC_EVENT_LAUNCHED, 0); - done: - SMARTLIST_FOREACH(router_nicknames, char *, n, tor_free(n)); - smartlist_free(router_nicknames); - smartlist_free(nodes); - return 0; -} - -/** Called when we get a SETCIRCUITPURPOSE message. If we can find the - * circuit and it's a valid purpose, change it. */ -static int -handle_control_setcircuitpurpose(control_connection_t *conn, - uint32_t len, const char *body) -{ - origin_circuit_t *circ = NULL; - uint8_t new_purpose; - smartlist_t *args; - (void) len; /* body is NUL-terminated, so it's safe to ignore the length. */ - - args = getargs_helper("SETCIRCUITPURPOSE", conn, body, 2, -1); - if (!args) - goto done; - - if (!(circ = get_circ(smartlist_get(args,0)))) { - connection_printf_to_buf(conn, "552 Unknown circuit \"%s\"\r\n", - (char*)smartlist_get(args, 0)); - goto done; - } - - { - const char *purp = find_element_starting_with(args,1,"PURPOSE="); - if (!purp) { - connection_write_str_to_buf("552 No purpose given\r\n", conn); - goto done; - } - new_purpose = circuit_purpose_from_string(purp); - if (new_purpose == CIRCUIT_PURPOSE_UNKNOWN) { - connection_printf_to_buf(conn, "552 Unknown purpose \"%s\"\r\n", purp); - goto done; - } - } - - circuit_change_purpose(TO_CIRCUIT(circ), new_purpose); - connection_write_str_to_buf("250 OK\r\n", conn); - - done: - if (args) { - SMARTLIST_FOREACH(args, char *, cp, tor_free(cp)); - smartlist_free(args); - } - return 0; -} - -/** Called when we get an ATTACHSTREAM message. Try to attach the requested - * stream, and report success or failure. */ -static int -handle_control_attachstream(control_connection_t *conn, uint32_t len, - const char *body) -{ - entry_connection_t *ap_conn = NULL; - origin_circuit_t *circ = NULL; - int zero_circ; - smartlist_t *args; - crypt_path_t *cpath=NULL; - int hop=0, hop_line_ok=1; - (void) len; - - args = getargs_helper("ATTACHSTREAM", conn, body, 2, -1); - if (!args) - return 0; - - zero_circ = !strcmp("0", (char*)smartlist_get(args,1)); - - if (!(ap_conn = get_stream(smartlist_get(args, 0)))) { - connection_printf_to_buf(conn, "552 Unknown stream \"%s\"\r\n", - (char*)smartlist_get(args, 0)); - } else if (!zero_circ && !(circ = get_circ(smartlist_get(args, 1)))) { - connection_printf_to_buf(conn, "552 Unknown circuit \"%s\"\r\n", - (char*)smartlist_get(args, 1)); - } else if (circ) { - const char *hopstring = find_element_starting_with(args,2,"HOP="); - if (hopstring) { - hopstring += strlen("HOP="); - hop = (int) tor_parse_ulong(hopstring, 10, 0, INT_MAX, - &hop_line_ok, NULL); - if (!hop_line_ok) { /* broken hop line */ - connection_printf_to_buf(conn, "552 Bad value hop=%s\r\n", hopstring); - } - } - } - SMARTLIST_FOREACH(args, char *, cp, tor_free(cp)); - smartlist_free(args); - if (!ap_conn || (!zero_circ && !circ) || !hop_line_ok) - return 0; - - if (ENTRY_TO_CONN(ap_conn)->state != AP_CONN_STATE_CONTROLLER_WAIT && - ENTRY_TO_CONN(ap_conn)->state != AP_CONN_STATE_CONNECT_WAIT && - ENTRY_TO_CONN(ap_conn)->state != AP_CONN_STATE_RESOLVE_WAIT) { - connection_write_str_to_buf( - "555 Connection is not managed by controller.\r\n", - conn); - return 0; - } - - /* Do we need to detach it first? */ - if (ENTRY_TO_CONN(ap_conn)->state != AP_CONN_STATE_CONTROLLER_WAIT) { - edge_connection_t *edge_conn = ENTRY_TO_EDGE_CONN(ap_conn); - circuit_t *tmpcirc = circuit_get_by_edge_conn(edge_conn); - connection_edge_end(edge_conn, END_STREAM_REASON_TIMEOUT); - /* Un-mark it as ending, since we're going to reuse it. */ - edge_conn->edge_has_sent_end = 0; - edge_conn->end_reason = 0; - if (tmpcirc) - circuit_detach_stream(tmpcirc, edge_conn); - CONNECTION_AP_EXPECT_NONPENDING(ap_conn); - TO_CONN(edge_conn)->state = AP_CONN_STATE_CONTROLLER_WAIT; - } - - if (circ && (circ->base_.state != CIRCUIT_STATE_OPEN)) { - connection_write_str_to_buf( - "551 Can't attach stream to non-open origin circuit\r\n", - conn); - return 0; - } - /* Is this a single hop circuit? */ - if (circ && (circuit_get_cpath_len(circ)<2 || hop==1)) { - connection_write_str_to_buf( - "551 Can't attach stream to this one-hop circuit.\r\n", conn); - return 0; - } - - if (circ && hop>0) { - /* find this hop in the circuit, and set cpath */ - cpath = circuit_get_cpath_hop(circ, hop); - if (!cpath) { - connection_printf_to_buf(conn, - "551 Circuit doesn't have %d hops.\r\n", hop); - return 0; - } - } - if (connection_ap_handshake_rewrite_and_attach(ap_conn, circ, cpath) < 0) { - connection_write_str_to_buf("551 Unable to attach stream\r\n", conn); - return 0; - } - send_control_done(conn); - return 0; -} - -/** Called when we get a POSTDESCRIPTOR message. Try to learn the provided - * descriptor, and report success or failure. */ -static int -handle_control_postdescriptor(control_connection_t *conn, uint32_t len, - const char *body) -{ - char *desc; - const char *msg=NULL; - uint8_t purpose = ROUTER_PURPOSE_GENERAL; - int cache = 0; /* eventually, we may switch this to 1 */ - - const char *cp = memchr(body, '\n', len); - - if (cp == NULL) { - connection_printf_to_buf(conn, "251 Empty body\r\n"); - return 0; - } - ++cp; - - char *cmdline = tor_memdup_nulterm(body, cp-body); - smartlist_t *args = smartlist_new(); - smartlist_split_string(args, cmdline, " ", - SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); - SMARTLIST_FOREACH_BEGIN(args, char *, option) { - if (!strcasecmpstart(option, "purpose=")) { - option += strlen("purpose="); - purpose = router_purpose_from_string(option); - if (purpose == ROUTER_PURPOSE_UNKNOWN) { - connection_printf_to_buf(conn, "552 Unknown purpose \"%s\"\r\n", - option); - goto done; - } - } else if (!strcasecmpstart(option, "cache=")) { - option += strlen("cache="); - if (!strcasecmp(option, "no")) - cache = 0; - else if (!strcasecmp(option, "yes")) - cache = 1; - else { - connection_printf_to_buf(conn, "552 Unknown cache request \"%s\"\r\n", - option); - goto done; - } - } else { /* unrecognized argument? */ - connection_printf_to_buf(conn, - "512 Unexpected argument \"%s\" to postdescriptor\r\n", option); - goto done; - } - } SMARTLIST_FOREACH_END(option); - - read_escaped_data(cp, len-(cp-body), &desc); - - switch (router_load_single_router(desc, purpose, cache, &msg)) { - case -1: - if (!msg) msg = "Could not parse descriptor"; - connection_printf_to_buf(conn, "554 %s\r\n", msg); - break; - case 0: - if (!msg) msg = "Descriptor not added"; - connection_printf_to_buf(conn, "251 %s\r\n",msg); - break; - case 1: - send_control_done(conn); - break; - } - - tor_free(desc); - done: - SMARTLIST_FOREACH(args, char *, arg, tor_free(arg)); - smartlist_free(args); - tor_free(cmdline); - return 0; -} - -/** Called when we receive a REDIRECTSTERAM command. Try to change the target - * address of the named AP stream, and report success or failure. */ -static int -handle_control_redirectstream(control_connection_t *conn, uint32_t len, - const char *body) -{ - entry_connection_t *ap_conn = NULL; - char *new_addr = NULL; - uint16_t new_port = 0; - smartlist_t *args; - (void) len; - - args = getargs_helper("REDIRECTSTREAM", conn, body, 2, -1); - if (!args) - return 0; - - if (!(ap_conn = get_stream(smartlist_get(args, 0))) - || !ap_conn->socks_request) { - connection_printf_to_buf(conn, "552 Unknown stream \"%s\"\r\n", - (char*)smartlist_get(args, 0)); - } else { - int ok = 1; - if (smartlist_len(args) > 2) { /* they included a port too */ - new_port = (uint16_t) tor_parse_ulong(smartlist_get(args, 2), - 10, 1, 65535, &ok, NULL); - } - if (!ok) { - connection_printf_to_buf(conn, "512 Cannot parse port \"%s\"\r\n", - (char*)smartlist_get(args, 2)); - } else { - new_addr = tor_strdup(smartlist_get(args, 1)); - } - } - - SMARTLIST_FOREACH(args, char *, cp, tor_free(cp)); - smartlist_free(args); - if (!new_addr) - return 0; - - strlcpy(ap_conn->socks_request->address, new_addr, - sizeof(ap_conn->socks_request->address)); - if (new_port) - ap_conn->socks_request->port = new_port; - tor_free(new_addr); - send_control_done(conn); - return 0; -} - -/** Called when we get a CLOSESTREAM command; try to close the named stream - * and report success or failure. */ -static int -handle_control_closestream(control_connection_t *conn, uint32_t len, - const char *body) -{ - entry_connection_t *ap_conn=NULL; - uint8_t reason=0; - smartlist_t *args; - int ok; - (void) len; - - args = getargs_helper("CLOSESTREAM", conn, body, 2, -1); - if (!args) - return 0; - - else if (!(ap_conn = get_stream(smartlist_get(args, 0)))) - connection_printf_to_buf(conn, "552 Unknown stream \"%s\"\r\n", - (char*)smartlist_get(args, 0)); - else { - reason = (uint8_t) tor_parse_ulong(smartlist_get(args,1), 10, 0, 255, - &ok, NULL); - if (!ok) { - connection_printf_to_buf(conn, "552 Unrecognized reason \"%s\"\r\n", - (char*)smartlist_get(args, 1)); - ap_conn = NULL; - } - } - SMARTLIST_FOREACH(args, char *, cp, tor_free(cp)); - smartlist_free(args); - if (!ap_conn) - return 0; - - connection_mark_unattached_ap(ap_conn, reason); - send_control_done(conn); - return 0; -} - -/** Called when we get a CLOSECIRCUIT command; try to close the named circuit - * and report success or failure. */ -static int -handle_control_closecircuit(control_connection_t *conn, uint32_t len, - const char *body) -{ - origin_circuit_t *circ = NULL; - int safe = 0; - smartlist_t *args; - (void) len; - - args = getargs_helper("CLOSECIRCUIT", conn, body, 1, -1); - if (!args) - return 0; - - if (!(circ=get_circ(smartlist_get(args, 0)))) - connection_printf_to_buf(conn, "552 Unknown circuit \"%s\"\r\n", - (char*)smartlist_get(args, 0)); - else { - int i; - for (i=1; i < smartlist_len(args); ++i) { - if (!strcasecmp(smartlist_get(args, i), "IfUnused")) - safe = 1; - else - log_info(LD_CONTROL, "Skipping unknown option %s", - (char*)smartlist_get(args,i)); - } - } - SMARTLIST_FOREACH(args, char *, cp, tor_free(cp)); - smartlist_free(args); - if (!circ) - return 0; - - if (!safe || !circ->p_streams) { - circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_REQUESTED); - } - - send_control_done(conn); - return 0; -} - -/** Called when we get a RESOLVE command: start trying to resolve - * the listed addresses. */ -static int -handle_control_resolve(control_connection_t *conn, uint32_t len, - const char *body) -{ - smartlist_t *args, *failed; - int is_reverse = 0; - (void) len; /* body is nul-terminated; it's safe to ignore the length */ - - if (!(conn->event_mask & (((event_mask_t)1)<have_sent_protocolinfo = 1; - args = smartlist_new(); - smartlist_split_string(args, body, " ", - SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); - SMARTLIST_FOREACH(args, const char *, arg, { - int ok; - tor_parse_long(arg, 10, 0, LONG_MAX, &ok, NULL); - if (!ok) { - bad_arg = arg; - break; - } - }); - if (bad_arg) { - connection_printf_to_buf(conn, "513 No such version %s\r\n", - escaped(bad_arg)); - /* Don't tolerate bad arguments when not authenticated. */ - if (!STATE_IS_OPEN(TO_CONN(conn)->state)) - connection_mark_for_close(TO_CONN(conn)); - goto done; - } else { - const or_options_t *options = get_options(); - int cookies = options->CookieAuthentication; - char *cfile = get_controller_cookie_file_name(); - char *abs_cfile; - char *esc_cfile; - char *methods; - abs_cfile = make_path_absolute(cfile); - esc_cfile = esc_for_log(abs_cfile); - { - int passwd = (options->HashedControlPassword != NULL || - options->HashedControlSessionPassword != NULL); - smartlist_t *mlist = smartlist_new(); - if (cookies) { - smartlist_add(mlist, (char*)"COOKIE"); - smartlist_add(mlist, (char*)"SAFECOOKIE"); - } - if (passwd) - smartlist_add(mlist, (char*)"HASHEDPASSWORD"); - if (!cookies && !passwd) - smartlist_add(mlist, (char*)"NULL"); - methods = smartlist_join_strings(mlist, ",", 0, NULL); - smartlist_free(mlist); - } - - connection_printf_to_buf(conn, - "250-PROTOCOLINFO 1\r\n" - "250-AUTH METHODS=%s%s%s\r\n" - "250-VERSION Tor=%s\r\n" - "250 OK\r\n", - methods, - cookies?" COOKIEFILE=":"", - cookies?esc_cfile:"", - escaped(VERSION)); - tor_free(methods); - tor_free(cfile); - tor_free(abs_cfile); - tor_free(esc_cfile); - } - done: - SMARTLIST_FOREACH(args, char *, cp, tor_free(cp)); - smartlist_free(args); - return 0; -} - -/** Called when we get an AUTHCHALLENGE command. */ -static int -handle_control_authchallenge(control_connection_t *conn, uint32_t len, - const char *body) -{ - const char *cp = body; - char *client_nonce; - size_t client_nonce_len; - char server_hash[DIGEST256_LEN]; - char server_hash_encoded[HEX_DIGEST256_LEN+1]; - char server_nonce[SAFECOOKIE_SERVER_NONCE_LEN]; - char server_nonce_encoded[(2*SAFECOOKIE_SERVER_NONCE_LEN) + 1]; - - cp += strspn(cp, " \t\n\r"); - if (!strcasecmpstart(cp, "SAFECOOKIE")) { - cp += strlen("SAFECOOKIE"); - } else { - connection_write_str_to_buf("513 AUTHCHALLENGE only supports SAFECOOKIE " - "authentication\r\n", conn); - connection_mark_for_close(TO_CONN(conn)); - return -1; - } - - if (!authentication_cookie_is_set) { - connection_write_str_to_buf("515 Cookie authentication is disabled\r\n", - conn); - connection_mark_for_close(TO_CONN(conn)); - return -1; - } - - cp += strspn(cp, " \t\n\r"); - if (*cp == '"') { - const char *newcp = - decode_escaped_string(cp, len - (cp - body), - &client_nonce, &client_nonce_len); - if (newcp == NULL) { - connection_write_str_to_buf("513 Invalid quoted client nonce\r\n", - conn); - connection_mark_for_close(TO_CONN(conn)); - return -1; - } - cp = newcp; - } else { - size_t client_nonce_encoded_len = strspn(cp, "0123456789ABCDEFabcdef"); - - client_nonce_len = client_nonce_encoded_len / 2; - client_nonce = tor_malloc_zero(client_nonce_len); - - if (base16_decode(client_nonce, client_nonce_len, - cp, client_nonce_encoded_len) - != (int) client_nonce_len) { - connection_write_str_to_buf("513 Invalid base16 client nonce\r\n", - conn); - connection_mark_for_close(TO_CONN(conn)); - tor_free(client_nonce); - return -1; - } - - cp += client_nonce_encoded_len; - } - - cp += strspn(cp, " \t\n\r"); - if (*cp != '\0' || - cp != body + len) { - connection_write_str_to_buf("513 Junk at end of AUTHCHALLENGE command\r\n", - conn); - connection_mark_for_close(TO_CONN(conn)); - tor_free(client_nonce); - return -1; - } - crypto_rand(server_nonce, SAFECOOKIE_SERVER_NONCE_LEN); - - /* Now compute and send the server-to-controller response, and the - * server's nonce. */ - tor_assert(authentication_cookie != NULL); - - { - size_t tmp_len = (AUTHENTICATION_COOKIE_LEN + - client_nonce_len + - SAFECOOKIE_SERVER_NONCE_LEN); - char *tmp = tor_malloc_zero(tmp_len); - char *client_hash = tor_malloc_zero(DIGEST256_LEN); - memcpy(tmp, authentication_cookie, AUTHENTICATION_COOKIE_LEN); - memcpy(tmp + AUTHENTICATION_COOKIE_LEN, client_nonce, client_nonce_len); - memcpy(tmp + AUTHENTICATION_COOKIE_LEN + client_nonce_len, - server_nonce, SAFECOOKIE_SERVER_NONCE_LEN); - - crypto_hmac_sha256(server_hash, - SAFECOOKIE_SERVER_TO_CONTROLLER_CONSTANT, - strlen(SAFECOOKIE_SERVER_TO_CONTROLLER_CONSTANT), - tmp, - tmp_len); - - crypto_hmac_sha256(client_hash, - SAFECOOKIE_CONTROLLER_TO_SERVER_CONSTANT, - strlen(SAFECOOKIE_CONTROLLER_TO_SERVER_CONSTANT), - tmp, - tmp_len); - - conn->safecookie_client_hash = client_hash; - - tor_free(tmp); - } - - base16_encode(server_hash_encoded, sizeof(server_hash_encoded), - server_hash, sizeof(server_hash)); - base16_encode(server_nonce_encoded, sizeof(server_nonce_encoded), - server_nonce, sizeof(server_nonce)); - - connection_printf_to_buf(conn, - "250 AUTHCHALLENGE SERVERHASH=%s " - "SERVERNONCE=%s\r\n", - server_hash_encoded, - server_nonce_encoded); - - tor_free(client_nonce); - return 0; -} - -/** Called when we get a USEFEATURE command: parse the feature list, and - * set up the control_connection's options properly. */ -static int -handle_control_usefeature(control_connection_t *conn, - uint32_t len, - const char *body) -{ - smartlist_t *args; - int bad = 0; - (void) len; /* body is nul-terminated; it's safe to ignore the length */ - args = smartlist_new(); - smartlist_split_string(args, body, " ", - SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); - SMARTLIST_FOREACH_BEGIN(args, const char *, arg) { - if (!strcasecmp(arg, "VERBOSE_NAMES")) - ; - else if (!strcasecmp(arg, "EXTENDED_EVENTS")) - ; - else { - connection_printf_to_buf(conn, "552 Unrecognized feature \"%s\"\r\n", - arg); - bad = 1; - break; - } - } SMARTLIST_FOREACH_END(arg); - - if (!bad) { - send_control_done(conn); - } - - SMARTLIST_FOREACH(args, char *, cp, tor_free(cp)); - smartlist_free(args); - return 0; -} - -/** Implementation for the DROPGUARDS command. */ -static int -handle_control_dropguards(control_connection_t *conn, - uint32_t len, - const char *body) -{ - smartlist_t *args; - (void) len; /* body is nul-terminated; it's safe to ignore the length */ - args = smartlist_new(); - smartlist_split_string(args, body, " ", - SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); - - static int have_warned = 0; - if (! have_warned) { - log_warn(LD_CONTROL, "DROPGUARDS is dangerous; make sure you understand " - "the risks before using it. It may be removed in a future " - "version of Tor."); - have_warned = 1; - } - - if (smartlist_len(args)) { - connection_printf_to_buf(conn, "512 Too many arguments to DROPGUARDS\r\n"); - } else { - remove_all_entry_guards(); - send_control_done(conn); - } - - SMARTLIST_FOREACH(args, char *, cp, tor_free(cp)); - smartlist_free(args); - return 0; -} - -/** Implementation for the HSFETCH command. */ -static int -handle_control_hsfetch(control_connection_t *conn, uint32_t len, - const char *body) -{ - int i; - char digest[DIGEST_LEN], *hsaddress = NULL, *arg1 = NULL, *desc_id = NULL; - smartlist_t *args = NULL, *hsdirs = NULL; - (void) len; /* body is nul-terminated; it's safe to ignore the length */ - static const char *hsfetch_command = "HSFETCH"; - static const char *v2_str = "v2-"; - const size_t v2_str_len = strlen(v2_str); - rend_data_t *rend_query = NULL; - ed25519_public_key_t v3_pk; - uint32_t version; - - /* Make sure we have at least one argument, the HSAddress. */ - args = getargs_helper(hsfetch_command, conn, body, 1, -1); - if (!args) { - goto exit; - } - - /* Extract the first argument (either HSAddress or DescID). */ - arg1 = smartlist_get(args, 0); - /* Test if it's an HS address without the .onion part. */ - if (rend_valid_v2_service_id(arg1)) { - hsaddress = arg1; - version = HS_VERSION_TWO; - } else if (strcmpstart(arg1, v2_str) == 0 && - rend_valid_descriptor_id(arg1 + v2_str_len) && - base32_decode(digest, sizeof(digest), arg1 + v2_str_len, - REND_DESC_ID_V2_LEN_BASE32) == - REND_DESC_ID_V2_LEN_BASE32) { - /* We have a well formed version 2 descriptor ID. Keep the decoded value - * of the id. */ - desc_id = digest; - version = HS_VERSION_TWO; - } else if (hs_address_is_valid(arg1)) { - hsaddress = arg1; - version = HS_VERSION_THREE; - hs_parse_address(hsaddress, &v3_pk, NULL, NULL); - } else { - connection_printf_to_buf(conn, "513 Invalid argument \"%s\"\r\n", - arg1); - goto done; - } - - static const char *opt_server = "SERVER="; - - /* Skip first argument because it's the HSAddress or DescID. */ - for (i = 1; i < smartlist_len(args); ++i) { - const char *arg = smartlist_get(args, i); - const node_t *node; - - if (!strcasecmpstart(arg, opt_server)) { - const char *server; - - server = arg + strlen(opt_server); - node = node_get_by_hex_id(server, 0); - if (!node) { - connection_printf_to_buf(conn, "552 Server \"%s\" not found\r\n", - server); - goto done; - } - if (!hsdirs) { - /* Stores routerstatus_t object for each specified server. */ - hsdirs = smartlist_new(); - } - /* Valid server, add it to our local list. */ - smartlist_add(hsdirs, node->rs); - } else { - connection_printf_to_buf(conn, "513 Unexpected argument \"%s\"\r\n", - arg); - goto done; - } - } - - if (version == HS_VERSION_TWO) { - rend_query = rend_data_client_create(hsaddress, desc_id, NULL, - REND_NO_AUTH); - if (rend_query == NULL) { - connection_printf_to_buf(conn, "551 Error creating the HS query\r\n"); - goto done; - } - } - - /* Using a descriptor ID, we force the user to provide at least one - * hsdir server using the SERVER= option. */ - if (desc_id && (!hsdirs || !smartlist_len(hsdirs))) { - connection_printf_to_buf(conn, "512 %s option is required\r\n", - opt_server); - goto done; - } - - /* We are about to trigger HSDir fetch so send the OK now because after - * that 650 event(s) are possible so better to have the 250 OK before them - * to avoid out of order replies. */ - send_control_done(conn); - - /* Trigger the fetch using the built rend query and possibly a list of HS - * directory to use. This function ignores the client cache thus this will - * always send a fetch command. */ - if (version == HS_VERSION_TWO) { - rend_client_fetch_v2_desc(rend_query, hsdirs); - } else if (version == HS_VERSION_THREE) { - hs_control_hsfetch_command(&v3_pk, hsdirs); - } - - done: - SMARTLIST_FOREACH(args, char *, cp, tor_free(cp)); - smartlist_free(args); - /* Contains data pointer that we don't own thus no cleanup. */ - smartlist_free(hsdirs); - rend_data_free(rend_query); - exit: - return 0; -} - -/** Implementation for the HSPOST command. */ -static int -handle_control_hspost(control_connection_t *conn, - uint32_t len, - const char *body) -{ - static const char *opt_server = "SERVER="; - static const char *opt_hsaddress = "HSADDRESS="; - smartlist_t *hs_dirs = NULL; - const char *encoded_desc = body; - size_t encoded_desc_len = len; - const char *onion_address = NULL; - - char *cp = memchr(body, '\n', len); - if (cp == NULL) { - connection_printf_to_buf(conn, "251 Empty body\r\n"); - return 0; - } - char *argline = tor_strndup(body, cp-body); - - smartlist_t *args = smartlist_new(); - - /* If any SERVER= or HSADDRESS= options were specified, try to parse - * the options line. */ - if (!strcasecmpstart(argline, opt_server) || - !strcasecmpstart(argline, opt_hsaddress)) { - /* encoded_desc begins after a newline character */ - cp = cp + 1; - encoded_desc = cp; - encoded_desc_len = len-(cp-body); - - smartlist_split_string(args, argline, " ", - SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); - SMARTLIST_FOREACH_BEGIN(args, const char *, arg) { - if (!strcasecmpstart(arg, opt_server)) { - const char *server = arg + strlen(opt_server); - const node_t *node = node_get_by_hex_id(server, 0); - - if (!node || !node->rs) { - connection_printf_to_buf(conn, "552 Server \"%s\" not found\r\n", - server); - goto done; - } - /* Valid server, add it to our local list. */ - if (!hs_dirs) - hs_dirs = smartlist_new(); - smartlist_add(hs_dirs, node->rs); - } else if (!strcasecmpstart(arg, opt_hsaddress)) { - const char *address = arg + strlen(opt_hsaddress); - if (!hs_address_is_valid(address)) { - connection_printf_to_buf(conn, "512 Malformed onion address\r\n"); - goto done; - } - onion_address = address; - } else { - connection_printf_to_buf(conn, "512 Unexpected argument \"%s\"\r\n", - arg); - goto done; - } - } SMARTLIST_FOREACH_END(arg); - } - - /* Handle the v3 case. */ - if (onion_address) { - char *desc_str = NULL; - read_escaped_data(encoded_desc, encoded_desc_len, &desc_str); - if (hs_control_hspost_command(desc_str, onion_address, hs_dirs) < 0) { - connection_printf_to_buf(conn, "554 Invalid descriptor\r\n"); - } else { - send_control_done(conn); - } - tor_free(desc_str); - goto done; - } - - /* From this point on, it is only v2. */ - - /* Read the dot encoded descriptor, and parse it. */ - rend_encoded_v2_service_descriptor_t *desc = - tor_malloc_zero(sizeof(rend_encoded_v2_service_descriptor_t)); - read_escaped_data(encoded_desc, encoded_desc_len, &desc->desc_str); - - rend_service_descriptor_t *parsed = NULL; - char *intro_content = NULL; - size_t intro_size; - size_t encoded_size; - const char *next_desc; - if (!rend_parse_v2_service_descriptor(&parsed, desc->desc_id, &intro_content, - &intro_size, &encoded_size, - &next_desc, desc->desc_str, 1)) { - /* Post the descriptor. */ - char serviceid[REND_SERVICE_ID_LEN_BASE32+1]; - if (!rend_get_service_id(parsed->pk, serviceid)) { - smartlist_t *descs = smartlist_new(); - smartlist_add(descs, desc); - - /* We are about to trigger HS descriptor upload so send the OK now - * because after that 650 event(s) are possible so better to have the - * 250 OK before them to avoid out of order replies. */ - send_control_done(conn); - - /* Trigger the descriptor upload */ - directory_post_to_hs_dir(parsed, descs, hs_dirs, serviceid, 0); - smartlist_free(descs); - } - - rend_service_descriptor_free(parsed); - } else { - connection_printf_to_buf(conn, "554 Invalid descriptor\r\n"); - } - - tor_free(intro_content); - rend_encoded_v2_service_descriptor_free(desc); - done: - tor_free(argline); - smartlist_free(hs_dirs); /* Contents belong to the rend service code. */ - SMARTLIST_FOREACH(args, char *, arg, tor_free(arg)); - smartlist_free(args); - return 0; -} - -/* Helper function for ADD_ONION that adds an ephemeral service depending on - * the given hs_version. - * - * The secret key in pk depends on the hs_version. The ownership of the key - * used in pk is given to the HS subsystem so the caller must stop accessing - * it after. - * - * The port_cfgs is a list of service port. Ownership transferred to service. - * The max_streams refers to the MaxStreams= key. - * The max_streams_close_circuit refers to the MaxStreamsCloseCircuit key. - * The auth_type is the authentication type of the clients in auth_clients. - * The ownership of that list is transferred to the service. - * - * On success (RSAE_OKAY), the address_out points to a newly allocated string - * containing the onion address without the .onion part. On error, address_out - * is untouched. */ -static hs_service_add_ephemeral_status_t -add_onion_helper_add_service(int hs_version, - add_onion_secret_key_t *pk, - smartlist_t *port_cfgs, int max_streams, - int max_streams_close_circuit, int auth_type, - smartlist_t *auth_clients, char **address_out) -{ - hs_service_add_ephemeral_status_t ret; - - tor_assert(pk); - tor_assert(port_cfgs); - tor_assert(address_out); - - switch (hs_version) { - case HS_VERSION_TWO: - ret = rend_service_add_ephemeral(pk->v2, port_cfgs, max_streams, - max_streams_close_circuit, auth_type, - auth_clients, address_out); - break; - case HS_VERSION_THREE: - ret = hs_service_add_ephemeral(pk->v3, port_cfgs, max_streams, - max_streams_close_circuit, address_out); - break; - default: - tor_assert_unreached(); - } - - return ret; -} - -/** - * - **/ -smartlist_t * -get_detached_onion_services(void) -{ - return detached_onion_services; -} - -/** Called when we get a ADD_ONION command; parse the body, and set up - * the new ephemeral Onion Service. */ -static int -handle_control_add_onion(control_connection_t *conn, - uint32_t len, - const char *body) -{ - smartlist_t *args; - int arg_len; - (void) len; /* body is nul-terminated; it's safe to ignore the length */ - args = getargs_helper("ADD_ONION", conn, body, 2, -1); - if (!args) - return 0; - arg_len = smartlist_len(args); - - /* Parse all of the arguments that do not involve handling cryptographic - * material first, since there's no reason to touch that at all if any of - * the other arguments are malformed. - */ - smartlist_t *port_cfgs = smartlist_new(); - smartlist_t *auth_clients = NULL; - smartlist_t *auth_created_clients = NULL; - int discard_pk = 0; - int detach = 0; - int max_streams = 0; - int max_streams_close_circuit = 0; - rend_auth_type_t auth_type = REND_NO_AUTH; - /* Default to adding an anonymous hidden service if no flag is given */ - int non_anonymous = 0; - for (int i = 1; i < arg_len; i++) { - static const char *port_prefix = "Port="; - static const char *flags_prefix = "Flags="; - static const char *max_s_prefix = "MaxStreams="; - static const char *auth_prefix = "ClientAuth="; - - const char *arg = smartlist_get(args, (int)i); - if (!strcasecmpstart(arg, port_prefix)) { - /* "Port=VIRTPORT[,TARGET]". */ - const char *port_str = arg + strlen(port_prefix); - - rend_service_port_config_t *cfg = - rend_service_parse_port_config(port_str, ",", NULL); - if (!cfg) { - connection_printf_to_buf(conn, "512 Invalid VIRTPORT/TARGET\r\n"); - goto out; - } - smartlist_add(port_cfgs, cfg); - } else if (!strcasecmpstart(arg, max_s_prefix)) { - /* "MaxStreams=[0..65535]". */ - const char *max_s_str = arg + strlen(max_s_prefix); - int ok = 0; - max_streams = (int)tor_parse_long(max_s_str, 10, 0, 65535, &ok, NULL); - if (!ok) { - connection_printf_to_buf(conn, "512 Invalid MaxStreams\r\n"); - goto out; - } - } else if (!strcasecmpstart(arg, flags_prefix)) { - /* "Flags=Flag[,Flag]", where Flag can be: - * * 'DiscardPK' - If tor generates the keypair, do not include it in - * the response. - * * 'Detach' - Do not tie this onion service to any particular control - * connection. - * * 'MaxStreamsCloseCircuit' - Close the circuit if MaxStreams is - * exceeded. - * * 'BasicAuth' - Client authorization using the 'basic' method. - * * 'NonAnonymous' - Add a non-anonymous Single Onion Service. If this - * flag is present, tor must be in non-anonymous - * hidden service mode. If this flag is absent, - * tor must be in anonymous hidden service mode. - */ - static const char *discard_flag = "DiscardPK"; - static const char *detach_flag = "Detach"; - static const char *max_s_close_flag = "MaxStreamsCloseCircuit"; - static const char *basicauth_flag = "BasicAuth"; - static const char *non_anonymous_flag = "NonAnonymous"; - - smartlist_t *flags = smartlist_new(); - int bad = 0; - - smartlist_split_string(flags, arg + strlen(flags_prefix), ",", - SPLIT_IGNORE_BLANK, 0); - if (smartlist_len(flags) < 1) { - connection_printf_to_buf(conn, "512 Invalid 'Flags' argument\r\n"); - bad = 1; - } - SMARTLIST_FOREACH_BEGIN(flags, const char *, flag) - { - if (!strcasecmp(flag, discard_flag)) { - discard_pk = 1; - } else if (!strcasecmp(flag, detach_flag)) { - detach = 1; - } else if (!strcasecmp(flag, max_s_close_flag)) { - max_streams_close_circuit = 1; - } else if (!strcasecmp(flag, basicauth_flag)) { - auth_type = REND_BASIC_AUTH; - } else if (!strcasecmp(flag, non_anonymous_flag)) { - non_anonymous = 1; - } else { - connection_printf_to_buf(conn, - "512 Invalid 'Flags' argument: %s\r\n", - escaped(flag)); - bad = 1; - break; - } - } SMARTLIST_FOREACH_END(flag); - SMARTLIST_FOREACH(flags, char *, cp, tor_free(cp)); - smartlist_free(flags); - if (bad) - goto out; - } else if (!strcasecmpstart(arg, auth_prefix)) { - char *err_msg = NULL; - int created = 0; - rend_authorized_client_t *client = - add_onion_helper_clientauth(arg + strlen(auth_prefix), - &created, &err_msg); - if (!client) { - if (err_msg) { - connection_write_str_to_buf(err_msg, conn); - tor_free(err_msg); - } - goto out; - } - - if (auth_clients != NULL) { - int bad = 0; - SMARTLIST_FOREACH_BEGIN(auth_clients, rend_authorized_client_t *, ac) { - if (strcmp(ac->client_name, client->client_name) == 0) { - bad = 1; - break; - } - } SMARTLIST_FOREACH_END(ac); - if (bad) { - connection_printf_to_buf(conn, - "512 Duplicate name in ClientAuth\r\n"); - rend_authorized_client_free(client); - goto out; - } - } else { - auth_clients = smartlist_new(); - auth_created_clients = smartlist_new(); - } - smartlist_add(auth_clients, client); - if (created) { - smartlist_add(auth_created_clients, client); - } - } else { - connection_printf_to_buf(conn, "513 Invalid argument\r\n"); - goto out; - } - } - if (smartlist_len(port_cfgs) == 0) { - connection_printf_to_buf(conn, "512 Missing 'Port' argument\r\n"); - goto out; - } else if (auth_type == REND_NO_AUTH && auth_clients != NULL) { - connection_printf_to_buf(conn, "512 No auth type specified\r\n"); - goto out; - } else if (auth_type != REND_NO_AUTH && auth_clients == NULL) { - connection_printf_to_buf(conn, "512 No auth clients specified\r\n"); - goto out; - } else if ((auth_type == REND_BASIC_AUTH && - smartlist_len(auth_clients) > 512) || - (auth_type == REND_STEALTH_AUTH && - smartlist_len(auth_clients) > 16)) { - connection_printf_to_buf(conn, "512 Too many auth clients\r\n"); - goto out; - } else if (non_anonymous != rend_service_non_anonymous_mode_enabled( - get_options())) { - /* If we failed, and the non-anonymous flag is set, Tor must be in - * anonymous hidden service mode. - * The error message changes based on the current Tor config: - * 512 Tor is in anonymous hidden service mode - * 512 Tor is in non-anonymous hidden service mode - * (I've deliberately written them out in full here to aid searchability.) - */ - connection_printf_to_buf(conn, "512 Tor is in %sanonymous hidden service " - "mode\r\n", - non_anonymous ? "" : "non-"); - goto out; - } - - /* Parse the "keytype:keyblob" argument. */ - int hs_version = 0; - add_onion_secret_key_t pk = { NULL }; - const char *key_new_alg = NULL; - char *key_new_blob = NULL; - char *err_msg = NULL; - - if (add_onion_helper_keyarg(smartlist_get(args, 0), discard_pk, - &key_new_alg, &key_new_blob, &pk, &hs_version, - &err_msg) < 0) { - if (err_msg) { - connection_write_str_to_buf(err_msg, conn); - tor_free(err_msg); - } - goto out; - } - tor_assert(!err_msg); - - /* Hidden service version 3 don't have client authentication support so if - * ClientAuth was given, send back an error. */ - if (hs_version == HS_VERSION_THREE && auth_clients) { - connection_printf_to_buf(conn, "513 ClientAuth not supported\r\n"); - goto out; - } - - /* Create the HS, using private key pk, client authentication auth_type, - * the list of auth_clients, and port config port_cfg. - * rend_service_add_ephemeral() will take ownership of pk and port_cfg, - * regardless of success/failure. - */ - char *service_id = NULL; - int ret = add_onion_helper_add_service(hs_version, &pk, port_cfgs, - max_streams, - max_streams_close_circuit, auth_type, - auth_clients, &service_id); - port_cfgs = NULL; /* port_cfgs is now owned by the rendservice code. */ - auth_clients = NULL; /* so is auth_clients */ - switch (ret) { - case RSAE_OKAY: - { - if (detach) { - if (!detached_onion_services) - detached_onion_services = smartlist_new(); - smartlist_add(detached_onion_services, service_id); - } else { - if (!conn->ephemeral_onion_services) - conn->ephemeral_onion_services = smartlist_new(); - smartlist_add(conn->ephemeral_onion_services, service_id); - } - - tor_assert(service_id); - connection_printf_to_buf(conn, "250-ServiceID=%s\r\n", service_id); - if (key_new_alg) { - tor_assert(key_new_blob); - connection_printf_to_buf(conn, "250-PrivateKey=%s:%s\r\n", - key_new_alg, key_new_blob); - } - if (auth_created_clients) { - SMARTLIST_FOREACH(auth_created_clients, rend_authorized_client_t *, ac, { - char *encoded = rend_auth_encode_cookie(ac->descriptor_cookie, - auth_type); - tor_assert(encoded); - connection_printf_to_buf(conn, "250-ClientAuth=%s:%s\r\n", - ac->client_name, encoded); - memwipe(encoded, 0, strlen(encoded)); - tor_free(encoded); - }); - } - - connection_printf_to_buf(conn, "250 OK\r\n"); - break; - } - case RSAE_BADPRIVKEY: - connection_printf_to_buf(conn, "551 Failed to generate onion address\r\n"); - break; - case RSAE_ADDREXISTS: - connection_printf_to_buf(conn, "550 Onion address collision\r\n"); - break; - case RSAE_BADVIRTPORT: - connection_printf_to_buf(conn, "512 Invalid VIRTPORT/TARGET\r\n"); - break; - case RSAE_BADAUTH: - connection_printf_to_buf(conn, "512 Invalid client authorization\r\n"); - break; - case RSAE_INTERNAL: /* FALLSTHROUGH */ - default: - connection_printf_to_buf(conn, "551 Failed to add Onion Service\r\n"); - } - if (key_new_blob) { - memwipe(key_new_blob, 0, strlen(key_new_blob)); - tor_free(key_new_blob); - } - - out: - if (port_cfgs) { - SMARTLIST_FOREACH(port_cfgs, rend_service_port_config_t*, p, - rend_service_port_config_free(p)); - smartlist_free(port_cfgs); - } - - if (auth_clients) { - SMARTLIST_FOREACH(auth_clients, rend_authorized_client_t *, ac, - rend_authorized_client_free(ac)); - smartlist_free(auth_clients); - } - if (auth_created_clients) { - // Do not free entries; they are the same as auth_clients - smartlist_free(auth_created_clients); - } - - SMARTLIST_FOREACH(args, char *, cp, { - memwipe(cp, 0, strlen(cp)); - tor_free(cp); - }); - smartlist_free(args); - return 0; -} - -/** Helper function to handle parsing the KeyType:KeyBlob argument to the - * ADD_ONION command. Return a new crypto_pk_t and if a new key was generated - * and the private key not discarded, the algorithm and serialized private key, - * or NULL and an optional control protocol error message on failure. The - * caller is responsible for freeing the returned key_new_blob and err_msg. - * - * Note: The error messages returned are deliberately vague to avoid echoing - * key material. - */ -STATIC int -add_onion_helper_keyarg(const char *arg, int discard_pk, - const char **key_new_alg_out, char **key_new_blob_out, - add_onion_secret_key_t *decoded_key, int *hs_version, - char **err_msg_out) -{ - smartlist_t *key_args = smartlist_new(); - crypto_pk_t *pk = NULL; - const char *key_new_alg = NULL; - char *key_new_blob = NULL; - char *err_msg = NULL; - int ret = -1; - - smartlist_split_string(key_args, arg, ":", SPLIT_IGNORE_BLANK, 0); - if (smartlist_len(key_args) != 2) { - err_msg = tor_strdup("512 Invalid key type/blob\r\n"); - goto err; - } - - /* The format is "KeyType:KeyBlob". */ - static const char *key_type_new = "NEW"; - static const char *key_type_best = "BEST"; - static const char *key_type_rsa1024 = "RSA1024"; - static const char *key_type_ed25519_v3 = "ED25519-V3"; - - const char *key_type = smartlist_get(key_args, 0); - const char *key_blob = smartlist_get(key_args, 1); - - if (!strcasecmp(key_type_rsa1024, key_type)) { - /* "RSA:" - Loading a pre-existing RSA1024 key. */ - pk = crypto_pk_base64_decode_private(key_blob, strlen(key_blob)); - if (!pk) { - err_msg = tor_strdup("512 Failed to decode RSA key\r\n"); - goto err; - } - if (crypto_pk_num_bits(pk) != PK_BYTES*8) { - crypto_pk_free(pk); - err_msg = tor_strdup("512 Invalid RSA key size\r\n"); - goto err; - } - decoded_key->v2 = pk; - *hs_version = HS_VERSION_TWO; - } else if (!strcasecmp(key_type_ed25519_v3, key_type)) { - /* "ED25519-V3:" - Loading a pre-existing ed25519 key. */ - ed25519_secret_key_t *sk = tor_malloc_zero(sizeof(*sk)); - if (base64_decode((char *) sk->seckey, sizeof(sk->seckey), key_blob, - strlen(key_blob)) != sizeof(sk->seckey)) { - tor_free(sk); - err_msg = tor_strdup("512 Failed to decode ED25519-V3 key\r\n"); - goto err; - } - decoded_key->v3 = sk; - *hs_version = HS_VERSION_THREE; - } else if (!strcasecmp(key_type_new, key_type)) { - /* "NEW:" - Generating a new key, blob as algorithm. */ - if (!strcasecmp(key_type_rsa1024, key_blob) || - !strcasecmp(key_type_best, key_blob)) { - /* "RSA1024", RSA 1024 bit, also currently "BEST" by default. */ - pk = crypto_pk_new(); - if (crypto_pk_generate_key(pk)) { - tor_asprintf(&err_msg, "551 Failed to generate %s key\r\n", - key_type_rsa1024); - goto err; - } - if (!discard_pk) { - if (crypto_pk_base64_encode_private(pk, &key_new_blob)) { - crypto_pk_free(pk); - tor_asprintf(&err_msg, "551 Failed to encode %s key\r\n", - key_type_rsa1024); - goto err; - } - key_new_alg = key_type_rsa1024; - } - decoded_key->v2 = pk; - *hs_version = HS_VERSION_TWO; - } else if (!strcasecmp(key_type_ed25519_v3, key_blob)) { - ed25519_secret_key_t *sk = tor_malloc_zero(sizeof(*sk)); - if (ed25519_secret_key_generate(sk, 1) < 0) { - tor_free(sk); - tor_asprintf(&err_msg, "551 Failed to generate %s key\r\n", - key_type_ed25519_v3); - goto err; - } - if (!discard_pk) { - ssize_t len = base64_encode_size(sizeof(sk->seckey), 0) + 1; - key_new_blob = tor_malloc_zero(len); - if (base64_encode(key_new_blob, len, (const char *) sk->seckey, - sizeof(sk->seckey), 0) != (len - 1)) { - tor_free(sk); - tor_free(key_new_blob); - tor_asprintf(&err_msg, "551 Failed to encode %s key\r\n", - key_type_ed25519_v3); - goto err; - } - key_new_alg = key_type_ed25519_v3; - } - decoded_key->v3 = sk; - *hs_version = HS_VERSION_THREE; - } else { - err_msg = tor_strdup("513 Invalid key type\r\n"); - goto err; - } - } else { - err_msg = tor_strdup("513 Invalid key type\r\n"); - goto err; - } - - /* Succeeded in loading or generating a private key. */ - ret = 0; - - err: - SMARTLIST_FOREACH(key_args, char *, cp, { - memwipe(cp, 0, strlen(cp)); - tor_free(cp); - }); - smartlist_free(key_args); - - if (err_msg_out) { - *err_msg_out = err_msg; - } else { - tor_free(err_msg); - } - *key_new_alg_out = key_new_alg; - *key_new_blob_out = key_new_blob; - - return ret; -} - -/** Helper function to handle parsing a ClientAuth argument to the - * ADD_ONION command. Return a new rend_authorized_client_t, or NULL - * and an optional control protocol error message on failure. The - * caller is responsible for freeing the returned auth_client and err_msg. - * - * If 'created' is specified, it will be set to 1 when a new cookie has - * been generated. - */ -STATIC rend_authorized_client_t * -add_onion_helper_clientauth(const char *arg, int *created, char **err_msg) -{ - int ok = 0; - - tor_assert(arg); - tor_assert(created); - tor_assert(err_msg); - *err_msg = NULL; - - smartlist_t *auth_args = smartlist_new(); - rend_authorized_client_t *client = - tor_malloc_zero(sizeof(rend_authorized_client_t)); - smartlist_split_string(auth_args, arg, ":", 0, 0); - if (smartlist_len(auth_args) < 1 || smartlist_len(auth_args) > 2) { - *err_msg = tor_strdup("512 Invalid ClientAuth syntax\r\n"); - goto err; - } - client->client_name = tor_strdup(smartlist_get(auth_args, 0)); - if (smartlist_len(auth_args) == 2) { - char *decode_err_msg = NULL; - if (rend_auth_decode_cookie(smartlist_get(auth_args, 1), - client->descriptor_cookie, - NULL, &decode_err_msg) < 0) { - tor_assert(decode_err_msg); - tor_asprintf(err_msg, "512 %s\r\n", decode_err_msg); - tor_free(decode_err_msg); - goto err; - } - *created = 0; - } else { - crypto_rand((char *) client->descriptor_cookie, REND_DESC_COOKIE_LEN); - *created = 1; - } - - if (!rend_valid_client_name(client->client_name)) { - *err_msg = tor_strdup("512 Invalid name in ClientAuth\r\n"); - goto err; - } - - ok = 1; - err: - SMARTLIST_FOREACH(auth_args, char *, item, tor_free(item)); - smartlist_free(auth_args); - if (!ok) { - rend_authorized_client_free(client); - client = NULL; - } - return client; -} - -/** Called when we get a DEL_ONION command; parse the body, and remove - * the existing ephemeral Onion Service. */ -static int -handle_control_del_onion(control_connection_t *conn, - uint32_t len, - const char *body) -{ - int hs_version = 0; - smartlist_t *args; - (void) len; /* body is nul-terminated; it's safe to ignore the length */ - args = getargs_helper("DEL_ONION", conn, body, 1, 1); - if (!args) - return 0; - - const char *service_id = smartlist_get(args, 0); - if (rend_valid_v2_service_id(service_id)) { - hs_version = HS_VERSION_TWO; - } else if (hs_address_is_valid(service_id)) { - hs_version = HS_VERSION_THREE; - } else { - connection_printf_to_buf(conn, "512 Malformed Onion Service id\r\n"); - goto out; - } - - /* Determine if the onion service belongs to this particular control - * connection, or if it is in the global list of detached services. If it - * is in neither, either the service ID is invalid in some way, or it - * explicitly belongs to a different control connection, and an error - * should be returned. - */ - smartlist_t *services[2] = { - conn->ephemeral_onion_services, - detached_onion_services - }; - smartlist_t *onion_services = NULL; - int idx = -1; - for (size_t i = 0; i < ARRAY_LENGTH(services); i++) { - idx = smartlist_string_pos(services[i], service_id); - if (idx != -1) { - onion_services = services[i]; - break; - } - } - if (onion_services == NULL) { - connection_printf_to_buf(conn, "552 Unknown Onion Service id\r\n"); - } else { - int ret = -1; - switch (hs_version) { - case HS_VERSION_TWO: - ret = rend_service_del_ephemeral(service_id); - break; - case HS_VERSION_THREE: - ret = hs_service_del_ephemeral(service_id); - break; - default: - /* The ret value will be -1 thus hitting the warning below. This should - * never happen because of the check at the start of the function. */ - break; - } - if (ret < 0) { - /* This should *NEVER* fail, since the service is on either the - * per-control connection list, or the global one. - */ - log_warn(LD_BUG, "Failed to remove Onion Service %s.", - escaped(service_id)); - tor_fragile_assert(); - } - - /* Remove/scrub the service_id from the appropriate list. */ - char *cp = smartlist_get(onion_services, idx); - smartlist_del(onion_services, idx); - memwipe(cp, 0, strlen(cp)); - tor_free(cp); - - send_control_done(conn); - } - - out: - SMARTLIST_FOREACH(args, char *, cp, { - memwipe(cp, 0, strlen(cp)); - tor_free(cp); - }); - smartlist_free(args); - return 0; -} - /** Called when conn has no more bytes left on its outbuf. */ int connection_control_finished_flushing(control_connection_t *conn) @@ -3185,108 +517,14 @@ connection_control_process_inbuf(control_connection_t *conn) return 0; } - /* XXXX Why is this not implemented as a table like the GETINFO - * items are? Even handling the plus signs at the beginnings of - * commands wouldn't be very hard with proper macros. */ cmd_data_len = (uint32_t)data_len; - if (!strcasecmp(conn->incoming_cmd, "SETCONF")) { - if (handle_control_setconf(conn, cmd_data_len, args)) - return -1; - } else if (!strcasecmp(conn->incoming_cmd, "RESETCONF")) { - if (handle_control_resetconf(conn, cmd_data_len, args)) - return -1; - } else if (!strcasecmp(conn->incoming_cmd, "GETCONF")) { - if (handle_control_getconf(conn, cmd_data_len, args)) - return -1; - } else if (!strcasecmp(conn->incoming_cmd, "+LOADCONF")) { - if (handle_control_loadconf(conn, cmd_data_len, args)) - return -1; - } else if (!strcasecmp(conn->incoming_cmd, "SETEVENTS")) { - if (handle_control_setevents(conn, cmd_data_len, args)) - return -1; - } else if (!strcasecmp(conn->incoming_cmd, "AUTHENTICATE")) { - if (handle_control_authenticate(conn, cmd_data_len, args)) - return -1; - } else if (!strcasecmp(conn->incoming_cmd, "SAVECONF")) { - if (handle_control_saveconf(conn, cmd_data_len, args)) - return -1; - } else if (!strcasecmp(conn->incoming_cmd, "SIGNAL")) { - if (handle_control_signal(conn, cmd_data_len, args)) - return -1; - } else if (!strcasecmp(conn->incoming_cmd, "TAKEOWNERSHIP")) { - if (handle_control_takeownership(conn, cmd_data_len, args)) - return -1; - } else if (!strcasecmp(conn->incoming_cmd, "DROPOWNERSHIP")) { - if (handle_control_dropownership(conn, cmd_data_len, args)) - return -1; - } else if (!strcasecmp(conn->incoming_cmd, "MAPADDRESS")) { - if (handle_control_mapaddress(conn, cmd_data_len, args)) - return -1; - } else if (!strcasecmp(conn->incoming_cmd, "GETINFO")) { - if (handle_control_getinfo(conn, cmd_data_len, args)) - return -1; - } else if (!strcasecmp(conn->incoming_cmd, "EXTENDCIRCUIT")) { - if (handle_control_extendcircuit(conn, cmd_data_len, args)) - return -1; - } else if (!strcasecmp(conn->incoming_cmd, "SETCIRCUITPURPOSE")) { - if (handle_control_setcircuitpurpose(conn, cmd_data_len, args)) - return -1; - } else if (!strcasecmp(conn->incoming_cmd, "SETROUTERPURPOSE")) { - connection_write_str_to_buf("511 SETROUTERPURPOSE is obsolete.\r\n", conn); - } else if (!strcasecmp(conn->incoming_cmd, "ATTACHSTREAM")) { - if (handle_control_attachstream(conn, cmd_data_len, args)) - return -1; - } else if (!strcasecmp(conn->incoming_cmd, "+POSTDESCRIPTOR")) { - if (handle_control_postdescriptor(conn, cmd_data_len, args)) - return -1; - } else if (!strcasecmp(conn->incoming_cmd, "REDIRECTSTREAM")) { - if (handle_control_redirectstream(conn, cmd_data_len, args)) - return -1; - } else if (!strcasecmp(conn->incoming_cmd, "CLOSESTREAM")) { - if (handle_control_closestream(conn, cmd_data_len, args)) - return -1; - } else if (!strcasecmp(conn->incoming_cmd, "CLOSECIRCUIT")) { - if (handle_control_closecircuit(conn, cmd_data_len, args)) - return -1; - } else if (!strcasecmp(conn->incoming_cmd, "USEFEATURE")) { - if (handle_control_usefeature(conn, cmd_data_len, args)) - return -1; - } else if (!strcasecmp(conn->incoming_cmd, "RESOLVE")) { - if (handle_control_resolve(conn, cmd_data_len, args)) - return -1; - } else if (!strcasecmp(conn->incoming_cmd, "PROTOCOLINFO")) { - if (handle_control_protocolinfo(conn, cmd_data_len, args)) - return -1; - } else if (!strcasecmp(conn->incoming_cmd, "AUTHCHALLENGE")) { - if (handle_control_authchallenge(conn, cmd_data_len, args)) - return -1; - } else if (!strcasecmp(conn->incoming_cmd, "DROPGUARDS")) { - if (handle_control_dropguards(conn, cmd_data_len, args)) - return -1; - } else if (!strcasecmp(conn->incoming_cmd, "HSFETCH")) { - if (handle_control_hsfetch(conn, cmd_data_len, args)) - return -1; - } else if (!strcasecmp(conn->incoming_cmd, "+HSPOST")) { - if (handle_control_hspost(conn, cmd_data_len, args)) - return -1; - } else if (!strcasecmp(conn->incoming_cmd, "ADD_ONION")) { - int ret = handle_control_add_onion(conn, cmd_data_len, args); - memwipe(args, 0, cmd_data_len); /* Scrub the private key. */ - if (ret) - return -1; - } else if (!strcasecmp(conn->incoming_cmd, "DEL_ONION")) { - int ret = handle_control_del_onion(conn, cmd_data_len, args); - memwipe(args, 0, cmd_data_len); /* Scrub the service id/pk. */ - if (ret) - return -1; - } else { - connection_printf_to_buf(conn, "510 Unrecognized command \"%s\"\r\n", - conn->incoming_cmd); - } + if (handle_control_command(conn, cmd_data_len, args) < 0) + return -1; conn->incoming_cmd_cur_len = 0; goto again; } + /** Cached liveness for network liveness events and GETINFO */ @@ -3304,43 +542,6 @@ set_cached_network_liveness(int liveness) network_is_live = liveness; } -/** Helper: Return a newly allocated string containing a path to the - * file where we store our authentication cookie. */ -char * -get_controller_cookie_file_name(void) -{ - const or_options_t *options = get_options(); - if (options->CookieAuthFile && strlen(options->CookieAuthFile)) { - return tor_strdup(options->CookieAuthFile); - } else { - return get_datadir_fname("control_auth_cookie"); - } -} - -/* Initialize the cookie-based authentication system of the - * ControlPort. If enabled is 0, then disable the cookie - * authentication system. */ -int -init_control_cookie_authentication(int enabled) -{ - char *fname = NULL; - int retval; - - if (!enabled) { - authentication_cookie_is_set = 0; - return 0; - } - - fname = get_controller_cookie_file_name(); - retval = init_cookie_authentication(fname, "", /* no header */ - AUTHENTICATION_COOKIE_LEN, - get_options()->CookieAuthFileGroupReadable, - &authentication_cookie, - &authentication_cookie_is_set); - tor_free(fname); - return retval; -} - /** A copy of the process specifier of Tor's owning controller, or * NULL if this Tor instance is not currently owned by a process. */ static char *owning_controller_process_spec = NULL; @@ -3428,14 +629,8 @@ node_describe_longname_by_id,(const char *id_digest)) void control_free_all(void) { + control_auth_free_all(); control_events_free_all(); - - if (authentication_cookie) /* Free the auth cookie */ - tor_free(authentication_cookie); - if (detached_onion_services) { /* Free the detached onion services */ - SMARTLIST_FOREACH(detached_onion_services, char *, cp, tor_free(cp)); - smartlist_free(detached_onion_services); - } + control_cmd_free_all(); control_event_bootstrap_reset(); - authentication_cookie_is_set = 0; } diff --git a/src/feature/control/control.h b/src/feature/control/control.h index 810fe67843..d20bc6fa2b 100644 --- a/src/feature/control/control.h +++ b/src/feature/control/control.h @@ -42,10 +42,6 @@ void connection_control_closed(control_connection_t *conn); int connection_control_process_inbuf(control_connection_t *conn); -int init_control_cookie_authentication(int enabled); -char *get_controller_cookie_file_name(void); -struct config_line_t; -smartlist_t *decode_hashed_passwords(struct config_line_t *passwords); void disable_control_logging(void); void enable_control_logging(void); @@ -55,40 +51,14 @@ const char *rend_auth_type_to_string(rend_auth_type_t auth_type); MOCK_DECL(const char *, node_describe_longname_by_id,(const char *id_digest)); void control_free_all(void); -#ifdef CONTROL_PRIVATE -#include "lib/crypt_ops/crypto_ed25519.h" - -/* ADD_ONION secret key to create an ephemeral service. The command supports - * multiple versions so this union stores the key and passes it to the HS - * subsystem depending on the requested version. */ -typedef union add_onion_secret_key_t { - /* Hidden service v2 secret key. */ - crypto_pk_t *v2; - /* Hidden service v3 secret key. */ - ed25519_secret_key_t *v3; -} add_onion_secret_key_t; - -STATIC int add_onion_helper_keyarg(const char *arg, int discard_pk, - const char **key_new_alg_out, - char **key_new_blob_out, - add_onion_secret_key_t *decoded_key, - int *hs_version, char **err_msg_out); - -STATIC rend_authorized_client_t * -add_onion_helper_clientauth(const char *arg, int *created, char **err_msg_out); - -#endif /* defined(CONTROL_PRIVATE) */ - #ifdef CONTROL_MODULE_PRIVATE struct signal_name_t { int sig; const char *signal_name; }; extern const struct signal_name_t signal_table[]; - int get_cached_network_liveness(void); void set_cached_network_liveness(int liveness); -smartlist_t * get_detached_onion_services(void); #endif /* defined(CONTROL_MODULE_PRIVATE) */ #endif /* !defined(TOR_CONTROL_H) */ diff --git a/src/feature/control/control_auth.c b/src/feature/control/control_auth.c new file mode 100644 index 0000000000..927115a308 --- /dev/null +++ b/src/feature/control/control_auth.c @@ -0,0 +1,439 @@ +/* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file control_auth.c + * \brief Authentication for Tor's control-socket interface. + **/ + +#include "core/or/or.h" +#include "app/config/config.h" +#include "core/mainloop/connection.h" +#include "feature/control/control.h" +#include "feature/control/control_auth.h" +#include "feature/control/control_connection_st.h" +#include "feature/control/control_fmt.h" +#include "lib/crypt_ops/crypto_rand.h" +#include "lib/crypt_ops/crypto_util.h" +#include "lib/encoding/confline.h" + +#include "lib/crypt_ops/crypto_s2k.h" + +/** If we're using cookie-type authentication, how long should our cookies be? + */ +#define AUTHENTICATION_COOKIE_LEN 32 + +/** If true, we've set authentication_cookie to a secret code and + * stored it to disk. */ +static int authentication_cookie_is_set = 0; +/** If authentication_cookie_is_set, a secret cookie that we've stored to disk + * and which we're using to authenticate controllers. (If the controller can + * read it off disk, it has permission to connect.) */ +static uint8_t *authentication_cookie = NULL; + +#define SAFECOOKIE_SERVER_TO_CONTROLLER_CONSTANT \ + "Tor safe cookie authentication server-to-controller hash" +#define SAFECOOKIE_CONTROLLER_TO_SERVER_CONSTANT \ + "Tor safe cookie authentication controller-to-server hash" +#define SAFECOOKIE_SERVER_NONCE_LEN DIGEST256_LEN + +/** Helper: Return a newly allocated string containing a path to the + * file where we store our authentication cookie. */ +char * +get_controller_cookie_file_name(void) +{ + const or_options_t *options = get_options(); + if (options->CookieAuthFile && strlen(options->CookieAuthFile)) { + return tor_strdup(options->CookieAuthFile); + } else { + return get_datadir_fname("control_auth_cookie"); + } +} + +/* Initialize the cookie-based authentication system of the + * ControlPort. If enabled is 0, then disable the cookie + * authentication system. */ +int +init_control_cookie_authentication(int enabled) +{ + char *fname = NULL; + int retval; + + if (!enabled) { + authentication_cookie_is_set = 0; + return 0; + } + + fname = get_controller_cookie_file_name(); + retval = init_cookie_authentication(fname, "", /* no header */ + AUTHENTICATION_COOKIE_LEN, + get_options()->CookieAuthFileGroupReadable, + &authentication_cookie, + &authentication_cookie_is_set); + tor_free(fname); + return retval; +} + +/** Decode the hashed, base64'd passwords stored in passwords. + * Return a smartlist of acceptable passwords (unterminated strings of + * length S2K_RFC2440_SPECIFIER_LEN+DIGEST_LEN) on success, or NULL on + * failure. + */ +smartlist_t * +decode_hashed_passwords(config_line_t *passwords) +{ + char decoded[64]; + config_line_t *cl; + smartlist_t *sl = smartlist_new(); + + tor_assert(passwords); + + for (cl = passwords; cl; cl = cl->next) { + const char *hashed = cl->value; + + if (!strcmpstart(hashed, "16:")) { + if (base16_decode(decoded, sizeof(decoded), hashed+3, strlen(hashed+3)) + != S2K_RFC2440_SPECIFIER_LEN + DIGEST_LEN + || strlen(hashed+3) != (S2K_RFC2440_SPECIFIER_LEN+DIGEST_LEN)*2) { + goto err; + } + } else { + if (base64_decode(decoded, sizeof(decoded), hashed, strlen(hashed)) + != S2K_RFC2440_SPECIFIER_LEN+DIGEST_LEN) { + goto err; + } + } + smartlist_add(sl, + tor_memdup(decoded, S2K_RFC2440_SPECIFIER_LEN+DIGEST_LEN)); + } + + return sl; + + err: + SMARTLIST_FOREACH(sl, char*, cp, tor_free(cp)); + smartlist_free(sl); + return NULL; +} + +/** Called when we get an AUTHCHALLENGE command. */ +int +handle_control_authchallenge(control_connection_t *conn, uint32_t len, + const char *body) +{ + const char *cp = body; + char *client_nonce; + size_t client_nonce_len; + char server_hash[DIGEST256_LEN]; + char server_hash_encoded[HEX_DIGEST256_LEN+1]; + char server_nonce[SAFECOOKIE_SERVER_NONCE_LEN]; + char server_nonce_encoded[(2*SAFECOOKIE_SERVER_NONCE_LEN) + 1]; + + cp += strspn(cp, " \t\n\r"); + if (!strcasecmpstart(cp, "SAFECOOKIE")) { + cp += strlen("SAFECOOKIE"); + } else { + connection_write_str_to_buf("513 AUTHCHALLENGE only supports SAFECOOKIE " + "authentication\r\n", conn); + connection_mark_for_close(TO_CONN(conn)); + return -1; + } + + if (!authentication_cookie_is_set) { + connection_write_str_to_buf("515 Cookie authentication is disabled\r\n", + conn); + connection_mark_for_close(TO_CONN(conn)); + return -1; + } + + cp += strspn(cp, " \t\n\r"); + if (*cp == '"') { + const char *newcp = + decode_escaped_string(cp, len - (cp - body), + &client_nonce, &client_nonce_len); + if (newcp == NULL) { + connection_write_str_to_buf("513 Invalid quoted client nonce\r\n", + conn); + connection_mark_for_close(TO_CONN(conn)); + return -1; + } + cp = newcp; + } else { + size_t client_nonce_encoded_len = strspn(cp, "0123456789ABCDEFabcdef"); + + client_nonce_len = client_nonce_encoded_len / 2; + client_nonce = tor_malloc_zero(client_nonce_len); + + if (base16_decode(client_nonce, client_nonce_len, + cp, client_nonce_encoded_len) + != (int) client_nonce_len) { + connection_write_str_to_buf("513 Invalid base16 client nonce\r\n", + conn); + connection_mark_for_close(TO_CONN(conn)); + tor_free(client_nonce); + return -1; + } + + cp += client_nonce_encoded_len; + } + + cp += strspn(cp, " \t\n\r"); + if (*cp != '\0' || + cp != body + len) { + connection_write_str_to_buf("513 Junk at end of AUTHCHALLENGE command\r\n", + conn); + connection_mark_for_close(TO_CONN(conn)); + tor_free(client_nonce); + return -1; + } + crypto_rand(server_nonce, SAFECOOKIE_SERVER_NONCE_LEN); + + /* Now compute and send the server-to-controller response, and the + * server's nonce. */ + tor_assert(authentication_cookie != NULL); + + { + size_t tmp_len = (AUTHENTICATION_COOKIE_LEN + + client_nonce_len + + SAFECOOKIE_SERVER_NONCE_LEN); + char *tmp = tor_malloc_zero(tmp_len); + char *client_hash = tor_malloc_zero(DIGEST256_LEN); + memcpy(tmp, authentication_cookie, AUTHENTICATION_COOKIE_LEN); + memcpy(tmp + AUTHENTICATION_COOKIE_LEN, client_nonce, client_nonce_len); + memcpy(tmp + AUTHENTICATION_COOKIE_LEN + client_nonce_len, + server_nonce, SAFECOOKIE_SERVER_NONCE_LEN); + + crypto_hmac_sha256(server_hash, + SAFECOOKIE_SERVER_TO_CONTROLLER_CONSTANT, + strlen(SAFECOOKIE_SERVER_TO_CONTROLLER_CONSTANT), + tmp, + tmp_len); + + crypto_hmac_sha256(client_hash, + SAFECOOKIE_CONTROLLER_TO_SERVER_CONSTANT, + strlen(SAFECOOKIE_CONTROLLER_TO_SERVER_CONSTANT), + tmp, + tmp_len); + + conn->safecookie_client_hash = client_hash; + + tor_free(tmp); + } + + base16_encode(server_hash_encoded, sizeof(server_hash_encoded), + server_hash, sizeof(server_hash)); + base16_encode(server_nonce_encoded, sizeof(server_nonce_encoded), + server_nonce, sizeof(server_nonce)); + + connection_printf_to_buf(conn, + "250 AUTHCHALLENGE SERVERHASH=%s " + "SERVERNONCE=%s\r\n", + server_hash_encoded, + server_nonce_encoded); + + tor_free(client_nonce); + return 0; +} + +/** Called when we get an AUTHENTICATE message. Check whether the + * authentication is valid, and if so, update the connection's state to + * OPEN. Reply with DONE or ERROR. + */ +int +handle_control_authenticate(control_connection_t *conn, uint32_t len, + const char *body) +{ + int used_quoted_string = 0; + const or_options_t *options = get_options(); + const char *errstr = "Unknown error"; + char *password; + size_t password_len; + const char *cp; + int i; + int bad_cookie=0, bad_password=0; + smartlist_t *sl = NULL; + + if (!len) { + password = tor_strdup(""); + password_len = 0; + } else if (TOR_ISXDIGIT(body[0])) { + cp = body; + while (TOR_ISXDIGIT(*cp)) + ++cp; + i = (int)(cp - body); + tor_assert(i>0); + password_len = i/2; + password = tor_malloc(password_len + 1); + if (base16_decode(password, password_len+1, body, i) + != (int) password_len) { + connection_write_str_to_buf( + "551 Invalid hexadecimal encoding. Maybe you tried a plain text " + "password? If so, the standard requires that you put it in " + "double quotes.\r\n", conn); + connection_mark_for_close(TO_CONN(conn)); + tor_free(password); + return 0; + } + } else { + if (!decode_escaped_string(body, len, &password, &password_len)) { + connection_write_str_to_buf("551 Invalid quoted string. You need " + "to put the password in double quotes.\r\n", conn); + connection_mark_for_close(TO_CONN(conn)); + return 0; + } + used_quoted_string = 1; + } + + if (conn->safecookie_client_hash != NULL) { + /* The controller has chosen safe cookie authentication; the only + * acceptable authentication value is the controller-to-server + * response. */ + + tor_assert(authentication_cookie_is_set); + + if (password_len != DIGEST256_LEN) { + log_warn(LD_CONTROL, + "Got safe cookie authentication response with wrong length " + "(%d)", (int)password_len); + errstr = "Wrong length for safe cookie response."; + goto err; + } + + if (tor_memneq(conn->safecookie_client_hash, password, DIGEST256_LEN)) { + log_warn(LD_CONTROL, + "Got incorrect safe cookie authentication response"); + errstr = "Safe cookie response did not match expected value."; + goto err; + } + + tor_free(conn->safecookie_client_hash); + goto ok; + } + + if (!options->CookieAuthentication && !options->HashedControlPassword && + !options->HashedControlSessionPassword) { + /* if Tor doesn't demand any stronger authentication, then + * the controller can get in with anything. */ + goto ok; + } + + if (options->CookieAuthentication) { + int also_password = options->HashedControlPassword != NULL || + options->HashedControlSessionPassword != NULL; + if (password_len != AUTHENTICATION_COOKIE_LEN) { + if (!also_password) { + log_warn(LD_CONTROL, "Got authentication cookie with wrong length " + "(%d)", (int)password_len); + errstr = "Wrong length on authentication cookie."; + goto err; + } + bad_cookie = 1; + } else if (tor_memneq(authentication_cookie, password, password_len)) { + if (!also_password) { + log_warn(LD_CONTROL, "Got mismatched authentication cookie"); + errstr = "Authentication cookie did not match expected value."; + goto err; + } + bad_cookie = 1; + } else { + goto ok; + } + } + + if (options->HashedControlPassword || + options->HashedControlSessionPassword) { + int bad = 0; + smartlist_t *sl_tmp; + char received[DIGEST_LEN]; + int also_cookie = options->CookieAuthentication; + sl = smartlist_new(); + if (options->HashedControlPassword) { + sl_tmp = decode_hashed_passwords(options->HashedControlPassword); + if (!sl_tmp) + bad = 1; + else { + smartlist_add_all(sl, sl_tmp); + smartlist_free(sl_tmp); + } + } + if (options->HashedControlSessionPassword) { + sl_tmp = decode_hashed_passwords(options->HashedControlSessionPassword); + if (!sl_tmp) + bad = 1; + else { + smartlist_add_all(sl, sl_tmp); + smartlist_free(sl_tmp); + } + } + if (bad) { + if (!also_cookie) { + log_warn(LD_BUG, + "Couldn't decode HashedControlPassword: invalid base16"); + errstr="Couldn't decode HashedControlPassword value in configuration."; + goto err; + } + bad_password = 1; + SMARTLIST_FOREACH(sl, char *, str, tor_free(str)); + smartlist_free(sl); + sl = NULL; + } else { + SMARTLIST_FOREACH(sl, char *, expected, + { + secret_to_key_rfc2440(received,DIGEST_LEN, + password,password_len,expected); + if (tor_memeq(expected + S2K_RFC2440_SPECIFIER_LEN, + received, DIGEST_LEN)) + goto ok; + }); + SMARTLIST_FOREACH(sl, char *, str, tor_free(str)); + smartlist_free(sl); + sl = NULL; + + if (used_quoted_string) + errstr = "Password did not match HashedControlPassword value from " + "configuration"; + else + errstr = "Password did not match HashedControlPassword value from " + "configuration. Maybe you tried a plain text password? " + "If so, the standard requires that you put it in double quotes."; + bad_password = 1; + if (!also_cookie) + goto err; + } + } + + /** We only get here if both kinds of authentication failed. */ + tor_assert(bad_password && bad_cookie); + log_warn(LD_CONTROL, "Bad password or authentication cookie on controller."); + errstr = "Password did not match HashedControlPassword *or* authentication " + "cookie."; + + err: + tor_free(password); + connection_printf_to_buf(conn, "515 Authentication failed: %s\r\n", errstr); + connection_mark_for_close(TO_CONN(conn)); + if (sl) { /* clean up */ + SMARTLIST_FOREACH(sl, char *, str, tor_free(str)); + smartlist_free(sl); + } + return 0; + ok: + log_info(LD_CONTROL, "Authenticated control connection ("TOR_SOCKET_T_FORMAT + ")", conn->base_.s); + send_control_done(conn); + conn->base_.state = CONTROL_CONN_STATE_OPEN; + tor_free(password); + if (sl) { /* clean up */ + SMARTLIST_FOREACH(sl, char *, str, tor_free(str)); + smartlist_free(sl); + } + return 0; +} + +void +control_auth_free_all(void) +{ + if (authentication_cookie) /* Free the auth cookie */ + tor_free(authentication_cookie); + authentication_cookie_is_set = 0; +} diff --git a/src/feature/control/control_auth.h b/src/feature/control/control_auth.h new file mode 100644 index 0000000000..f436482e4a --- /dev/null +++ b/src/feature/control/control_auth.h @@ -0,0 +1,27 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file control_auth.h + * \brief Header file for control_auth.c. + **/ + +#ifndef TOR_CONTROL_AUTH_H +#define TOR_CONTROL_AUTH_H + +int init_control_cookie_authentication(int enabled); +char *get_controller_cookie_file_name(void); +struct config_line_t; +smartlist_t *decode_hashed_passwords(struct config_line_t *passwords); + +int handle_control_authchallenge(control_connection_t *conn, uint32_t len, + const char *body); +int handle_control_authenticate(control_connection_t *conn, + uint32_t cmd_data_len, + const char *args); +void control_auth_free_all(void); + +#endif /* !defined(TOR_CONTROL_AUTH_H) */ diff --git a/src/feature/control/control_cmd.c b/src/feature/control/control_cmd.c new file mode 100644 index 0000000000..95cf0d561e --- /dev/null +++ b/src/feature/control/control_cmd.c @@ -0,0 +1,2323 @@ +/* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file control_cmd.c + * \brief Implement various commands for Tor's control-socket interface. + **/ + +#define CONTROL_MODULE_PRIVATE +#define CONTROL_CMD_PRIVATE +#define CONTROL_EVENTS_PRIVATE + +#include "core/or/or.h" +#include "app/config/config.h" +#include "app/config/confparse.h" +#include "app/main/main.h" +#include "core/mainloop/connection.h" +#include "core/or/circuitbuild.h" +#include "core/or/circuitlist.h" +#include "core/or/circuituse.h" +#include "core/or/connection_edge.h" +#include "feature/client/addressmap.h" +#include "feature/client/dnsserv.h" +#include "feature/client/entrynodes.h" +#include "feature/control/control.h" +#include "feature/control/control_auth.h" +#include "feature/control/control_cmd.h" +#include "feature/control/control_events.h" +#include "feature/control/control_fmt.h" +#include "feature/control/control_getinfo.h" +#include "feature/hs/hs_control.h" +#include "feature/nodelist/nodelist.h" +#include "feature/nodelist/routerinfo.h" +#include "feature/nodelist/routerlist.h" +#include "feature/rend/rendclient.h" +#include "feature/rend/rendcommon.h" +#include "feature/rend/rendparse.h" +#include "feature/rend/rendservice.h" +#include "lib/crypt_ops/crypto_rand.h" +#include "lib/crypt_ops/crypto_util.h" +#include "lib/encoding/confline.h" + +#include "core/or/cpath_build_state_st.h" +#include "core/or/entry_connection_st.h" +#include "core/or/origin_circuit_st.h" +#include "core/or/socks_request_st.h" +#include "feature/control/control_connection_st.h" +#include "feature/nodelist/node_st.h" +#include "feature/nodelist/routerinfo_st.h" +#include "feature/rend/rend_authorized_client_st.h" +#include "feature/rend/rend_encoded_v2_service_descriptor_st.h" +#include "feature/rend/rend_service_descriptor_st.h" + +static int control_setconf_helper(control_connection_t *conn, uint32_t len, + char *body, + int use_defaults); + +/** Yield true iff s is the state of a control_connection_t that has + * finished authentication and is accepting commands. */ +#define STATE_IS_OPEN(s) ((s) == CONTROL_CONN_STATE_OPEN) + +/** Called when we receive a SETCONF message: parse the body and try + * to update our configuration. Reply with a DONE or ERROR message. + * Modifies the contents of body.*/ +static int +handle_control_setconf(control_connection_t *conn, uint32_t len, char *body) +{ + return control_setconf_helper(conn, len, body, 0); +} + +/** Called when we receive a RESETCONF message: parse the body and try + * to update our configuration. Reply with a DONE or ERROR message. + * Modifies the contents of body. */ +static int +handle_control_resetconf(control_connection_t *conn, uint32_t len, char *body) +{ + return control_setconf_helper(conn, len, body, 1); +} + +/** Called when we receive a GETCONF message. Parse the request, and + * reply with a CONFVALUE or an ERROR message */ +static int +handle_control_getconf(control_connection_t *conn, uint32_t body_len, + const char *body) +{ + smartlist_t *questions = smartlist_new(); + smartlist_t *answers = smartlist_new(); + smartlist_t *unrecognized = smartlist_new(); + char *msg = NULL; + size_t msg_len; + const or_options_t *options = get_options(); + int i, len; + + (void) body_len; /* body is NUL-terminated; so we can ignore len. */ + smartlist_split_string(questions, body, " ", + SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); + SMARTLIST_FOREACH_BEGIN(questions, const char *, q) { + if (!option_is_recognized(q)) { + smartlist_add(unrecognized, (char*) q); + } else { + config_line_t *answer = option_get_assignment(options,q); + if (!answer) { + const char *name = option_get_canonical_name(q); + smartlist_add_asprintf(answers, "250-%s\r\n", name); + } + + while (answer) { + config_line_t *next; + smartlist_add_asprintf(answers, "250-%s=%s\r\n", + answer->key, answer->value); + + next = answer->next; + tor_free(answer->key); + tor_free(answer->value); + tor_free(answer); + answer = next; + } + } + } SMARTLIST_FOREACH_END(q); + + if ((len = smartlist_len(unrecognized))) { + for (i=0; i < len-1; ++i) + connection_printf_to_buf(conn, + "552-Unrecognized configuration key \"%s\"\r\n", + (char*)smartlist_get(unrecognized, i)); + connection_printf_to_buf(conn, + "552 Unrecognized configuration key \"%s\"\r\n", + (char*)smartlist_get(unrecognized, len-1)); + } else if ((len = smartlist_len(answers))) { + char *tmp = smartlist_get(answers, len-1); + tor_assert(strlen(tmp)>4); + tmp[3] = ' '; + msg = smartlist_join_strings(answers, "", 0, &msg_len); + connection_buf_add(msg, msg_len, TO_CONN(conn)); + } else { + connection_write_str_to_buf("250 OK\r\n", conn); + } + + SMARTLIST_FOREACH(answers, char *, cp, tor_free(cp)); + smartlist_free(answers); + SMARTLIST_FOREACH(questions, char *, cp, tor_free(cp)); + smartlist_free(questions); + smartlist_free(unrecognized); + + tor_free(msg); + + return 0; +} + +/** Called when we get a +LOADCONF message. */ +static int +handle_control_loadconf(control_connection_t *conn, uint32_t len, + const char *body) +{ + setopt_err_t retval; + char *errstring = NULL; + const char *msg = NULL; + (void) len; + + retval = options_init_from_string(NULL, body, CMD_RUN_TOR, NULL, &errstring); + + if (retval != SETOPT_OK) + log_warn(LD_CONTROL, + "Controller gave us config file that didn't validate: %s", + errstring); + + switch (retval) { + case SETOPT_ERR_PARSE: + msg = "552 Invalid config file"; + break; + case SETOPT_ERR_TRANSITION: + msg = "553 Transition not allowed"; + break; + case SETOPT_ERR_SETTING: + msg = "553 Unable to set option"; + break; + case SETOPT_ERR_MISC: + default: + msg = "550 Unable to load config"; + break; + case SETOPT_OK: + break; + } + if (msg) { + if (errstring) + connection_printf_to_buf(conn, "%s: %s\r\n", msg, errstring); + else + connection_printf_to_buf(conn, "%s\r\n", msg); + } else { + send_control_done(conn); + } + tor_free(errstring); + return 0; +} + +/** Called when we get a SETEVENTS message: update conn->event_mask, + * and reply with DONE or ERROR. */ +static int +handle_control_setevents(control_connection_t *conn, uint32_t len, + const char *body) +{ + int event_code; + event_mask_t event_mask = 0; + smartlist_t *events = smartlist_new(); + + (void) len; + + smartlist_split_string(events, body, " ", + SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); + SMARTLIST_FOREACH_BEGIN(events, const char *, ev) + { + if (!strcasecmp(ev, "EXTENDED") || + !strcasecmp(ev, "AUTHDIR_NEWDESCS")) { + log_warn(LD_CONTROL, "The \"%s\" SETEVENTS argument is no longer " + "supported.", ev); + continue; + } else { + int i; + event_code = -1; + + for (i = 0; control_event_table[i].event_name != NULL; ++i) { + if (!strcasecmp(ev, control_event_table[i].event_name)) { + event_code = control_event_table[i].event_code; + break; + } + } + + if (event_code == -1) { + connection_printf_to_buf(conn, "552 Unrecognized event \"%s\"\r\n", + ev); + SMARTLIST_FOREACH(events, char *, e, tor_free(e)); + smartlist_free(events); + return 0; + } + } + event_mask |= (((event_mask_t)1) << event_code); + } + SMARTLIST_FOREACH_END(ev); + SMARTLIST_FOREACH(events, char *, e, tor_free(e)); + smartlist_free(events); + + conn->event_mask = event_mask; + + control_update_global_event_mask(); + send_control_done(conn); + return 0; +} + +/** Called when we get a SAVECONF command. Try to flush the current options to + * disk, and report success or failure. */ +static int +handle_control_saveconf(control_connection_t *conn, uint32_t len, + const char *body) +{ + (void) len; + + int force = !strcmpstart(body, "FORCE"); + const or_options_t *options = get_options(); + if ((!force && options->IncludeUsed) || options_save_current() < 0) { + connection_write_str_to_buf( + "551 Unable to write configuration to disk.\r\n", conn); + } else { + send_control_done(conn); + } + return 0; +} + +/** Called when we get a SIGNAL command. React to the provided signal, and + * report success or failure. (If the signal results in a shutdown, success + * may not be reported.) */ +static int +handle_control_signal(control_connection_t *conn, uint32_t len, + const char *body) +{ + int sig = -1; + int i; + int n = 0; + char *s; + + (void) len; + + while (body[n] && ! TOR_ISSPACE(body[n])) + ++n; + s = tor_strndup(body, n); + + for (i = 0; signal_table[i].signal_name != NULL; ++i) { + if (!strcasecmp(s, signal_table[i].signal_name)) { + sig = signal_table[i].sig; + break; + } + } + + if (sig < 0) + connection_printf_to_buf(conn, "552 Unrecognized signal code \"%s\"\r\n", + s); + tor_free(s); + if (sig < 0) + return 0; + + send_control_done(conn); + /* Flush the "done" first if the signal might make us shut down. */ + if (sig == SIGTERM || sig == SIGINT) + connection_flush(TO_CONN(conn)); + + activate_signal(sig); + + return 0; +} + +/** Called when we get a TAKEOWNERSHIP command. Mark this connection + * as an owning connection, so that we will exit if the connection + * closes. */ +static int +handle_control_takeownership(control_connection_t *conn, uint32_t len, + const char *body) +{ + (void)len; + (void)body; + + conn->is_owning_control_connection = 1; + + log_info(LD_CONTROL, "Control connection %d has taken ownership of this " + "Tor instance.", + (int)(conn->base_.s)); + + send_control_done(conn); + return 0; +} + +/** Called when we get a DROPOWNERSHIP command. Mark this connection + * as a non-owning connection, so that we will not exit if the connection + * closes. */ +static int +handle_control_dropownership(control_connection_t *conn, uint32_t len, + const char *body) +{ + (void)len; + (void)body; + + conn->is_owning_control_connection = 0; + + log_info(LD_CONTROL, "Control connection %d has dropped ownership of this " + "Tor instance.", + (int)(conn->base_.s)); + + send_control_done(conn); + return 0; +} + +/** Given a text circuit id, return the corresponding circuit. */ +static origin_circuit_t * +get_circ(const char *id) +{ + uint32_t n_id; + int ok; + n_id = (uint32_t) tor_parse_ulong(id, 10, 0, UINT32_MAX, &ok, NULL); + if (!ok) + return NULL; + return circuit_get_by_global_id(n_id); +} + +/** Given a text stream id, return the corresponding AP connection. */ +static entry_connection_t * +get_stream(const char *id) +{ + uint64_t n_id; + int ok; + connection_t *conn; + n_id = tor_parse_uint64(id, 10, 0, UINT64_MAX, &ok, NULL); + if (!ok) + return NULL; + conn = connection_get_by_global_id(n_id); + if (!conn || conn->type != CONN_TYPE_AP || conn->marked_for_close) + return NULL; + return TO_ENTRY_CONN(conn); +} + +/** Helper for setconf and resetconf. Acts like setconf, except + * it passes use_defaults on to options_trial_assign(). Modifies the + * contents of body. + */ +static int +control_setconf_helper(control_connection_t *conn, uint32_t len, char *body, + int use_defaults) +{ + setopt_err_t opt_err; + config_line_t *lines=NULL; + char *start = body; + char *errstring = NULL; + const unsigned flags = + CAL_CLEAR_FIRST | (use_defaults ? CAL_USE_DEFAULTS : 0); + + char *config; + smartlist_t *entries = smartlist_new(); + + /* We have a string, "body", of the format '(key(=val|="val")?)' entries + * separated by space. break it into a list of configuration entries. */ + while (*body) { + char *eq = body; + char *key; + char *entry; + while (!TOR_ISSPACE(*eq) && *eq != '=') + ++eq; + key = tor_strndup(body, eq-body); + body = eq+1; + if (*eq == '=') { + char *val=NULL; + size_t val_len=0; + if (*body != '\"') { + char *val_start = body; + while (!TOR_ISSPACE(*body)) + body++; + val = tor_strndup(val_start, body-val_start); + val_len = strlen(val); + } else { + body = (char*)extract_escaped_string(body, (len - (body-start)), + &val, &val_len); + if (!body) { + connection_write_str_to_buf("551 Couldn't parse string\r\n", conn); + SMARTLIST_FOREACH(entries, char *, cp, tor_free(cp)); + smartlist_free(entries); + tor_free(key); + return 0; + } + } + tor_asprintf(&entry, "%s %s", key, val); + tor_free(key); + tor_free(val); + } else { + entry = key; + } + smartlist_add(entries, entry); + while (TOR_ISSPACE(*body)) + ++body; + } + + smartlist_add_strdup(entries, ""); + config = smartlist_join_strings(entries, "\n", 0, NULL); + SMARTLIST_FOREACH(entries, char *, cp, tor_free(cp)); + smartlist_free(entries); + + if (config_get_lines(config, &lines, 0) < 0) { + log_warn(LD_CONTROL,"Controller gave us config lines we can't parse."); + connection_write_str_to_buf("551 Couldn't parse configuration\r\n", + conn); + tor_free(config); + return 0; + } + tor_free(config); + + opt_err = options_trial_assign(lines, flags, &errstring); + { + const char *msg; + switch (opt_err) { + case SETOPT_ERR_MISC: + msg = "552 Unrecognized option"; + break; + case SETOPT_ERR_PARSE: + msg = "513 Unacceptable option value"; + break; + case SETOPT_ERR_TRANSITION: + msg = "553 Transition not allowed"; + break; + case SETOPT_ERR_SETTING: + default: + msg = "553 Unable to set option"; + break; + case SETOPT_OK: + config_free_lines(lines); + send_control_done(conn); + return 0; + } + log_warn(LD_CONTROL, + "Controller gave us config lines that didn't validate: %s", + errstring); + connection_printf_to_buf(conn, "%s: %s\r\n", msg, errstring); + config_free_lines(lines); + tor_free(errstring); + return 0; + } +} + +/** Return true iff addr is unusable as a mapaddress target because of + * containing funny characters. */ +static int +address_is_invalid_mapaddress_target(const char *addr) +{ + if (!strcmpstart(addr, "*.")) + return address_is_invalid_destination(addr+2, 1); + else + return address_is_invalid_destination(addr, 1); +} + +/** Called when we get a MAPADDRESS command; try to bind all listed addresses, + * and report success or failure. */ +static int +handle_control_mapaddress(control_connection_t *conn, uint32_t len, + const char *body) +{ + smartlist_t *elts; + smartlist_t *lines; + smartlist_t *reply; + char *r; + size_t sz; + (void) len; /* body is NUL-terminated, so it's safe to ignore the length. */ + + lines = smartlist_new(); + elts = smartlist_new(); + reply = smartlist_new(); + smartlist_split_string(lines, body, " ", + SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); + SMARTLIST_FOREACH_BEGIN(lines, char *, line) { + tor_strlower(line); + smartlist_split_string(elts, line, "=", 0, 2); + if (smartlist_len(elts) == 2) { + const char *from = smartlist_get(elts,0); + const char *to = smartlist_get(elts,1); + if (address_is_invalid_mapaddress_target(to)) { + smartlist_add_asprintf(reply, + "512-syntax error: invalid address '%s'", to); + log_warn(LD_CONTROL, + "Skipping invalid argument '%s' in MapAddress msg", to); + } else if (!strcmp(from, ".") || !strcmp(from, "0.0.0.0") || + !strcmp(from, "::")) { + const char type = + !strcmp(from,".") ? RESOLVED_TYPE_HOSTNAME : + (!strcmp(from, "0.0.0.0") ? RESOLVED_TYPE_IPV4 : RESOLVED_TYPE_IPV6); + const char *address = addressmap_register_virtual_address( + type, tor_strdup(to)); + if (!address) { + smartlist_add_asprintf(reply, + "451-resource exhausted: skipping '%s'", line); + log_warn(LD_CONTROL, + "Unable to allocate address for '%s' in MapAddress msg", + safe_str_client(line)); + } else { + smartlist_add_asprintf(reply, "250-%s=%s", address, to); + } + } else { + const char *msg; + if (addressmap_register_auto(from, to, 1, + ADDRMAPSRC_CONTROLLER, &msg) < 0) { + smartlist_add_asprintf(reply, + "512-syntax error: invalid address mapping " + " '%s': %s", line, msg); + log_warn(LD_CONTROL, + "Skipping invalid argument '%s' in MapAddress msg: %s", + line, msg); + } else { + smartlist_add_asprintf(reply, "250-%s", line); + } + } + } else { + smartlist_add_asprintf(reply, "512-syntax error: mapping '%s' is " + "not of expected form 'foo=bar'.", line); + log_info(LD_CONTROL, "Skipping MapAddress '%s': wrong " + "number of items.", + safe_str_client(line)); + } + SMARTLIST_FOREACH(elts, char *, cp, tor_free(cp)); + smartlist_clear(elts); + } SMARTLIST_FOREACH_END(line); + SMARTLIST_FOREACH(lines, char *, cp, tor_free(cp)); + smartlist_free(lines); + smartlist_free(elts); + + if (smartlist_len(reply)) { + ((char*)smartlist_get(reply,smartlist_len(reply)-1))[3] = ' '; + r = smartlist_join_strings(reply, "\r\n", 1, &sz); + connection_buf_add(r, sz, TO_CONN(conn)); + tor_free(r); + } else { + const char *response = + "512 syntax error: not enough arguments to mapaddress.\r\n"; + connection_buf_add(response, strlen(response), TO_CONN(conn)); + } + + SMARTLIST_FOREACH(reply, char *, cp, tor_free(cp)); + smartlist_free(reply); + return 0; +} + +/** Given a string, convert it to a circuit purpose. */ +static uint8_t +circuit_purpose_from_string(const char *string) +{ + if (!strcasecmpstart(string, "purpose=")) + string += strlen("purpose="); + + if (!strcasecmp(string, "general")) + return CIRCUIT_PURPOSE_C_GENERAL; + else if (!strcasecmp(string, "controller")) + return CIRCUIT_PURPOSE_CONTROLLER; + else + return CIRCUIT_PURPOSE_UNKNOWN; +} + +/** Return a newly allocated smartlist containing the arguments to the command + * waiting in body. If there are fewer than min_args arguments, + * or if max_args is nonnegative and there are more than + * max_args arguments, send a 512 error to the controller, using + * command as the command name in the error message. */ +static smartlist_t * +getargs_helper(const char *command, control_connection_t *conn, + const char *body, int min_args, int max_args) +{ + smartlist_t *args = smartlist_new(); + smartlist_split_string(args, body, " ", + SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); + if (smartlist_len(args) < min_args) { + connection_printf_to_buf(conn, "512 Missing argument to %s\r\n",command); + goto err; + } else if (max_args >= 0 && smartlist_len(args) > max_args) { + connection_printf_to_buf(conn, "512 Too many arguments to %s\r\n",command); + goto err; + } + return args; + err: + SMARTLIST_FOREACH(args, char *, s, tor_free(s)); + smartlist_free(args); + return NULL; +} + +/** Helper. Return the first element of sl at index start_at or + * higher that starts with prefix, case-insensitive. Return NULL if no + * such element exists. */ +static const char * +find_element_starting_with(smartlist_t *sl, int start_at, const char *prefix) +{ + int i; + for (i = start_at; i < smartlist_len(sl); ++i) { + const char *elt = smartlist_get(sl, i); + if (!strcasecmpstart(elt, prefix)) + return elt; + } + return NULL; +} + +/** Helper. Return true iff s is an argument that we should treat as a + * key-value pair. */ +static int +is_keyval_pair(const char *s) +{ + /* An argument is a key-value pair if it has an =, and it isn't of the form + * $fingeprint=name */ + return strchr(s, '=') && s[0] != '$'; +} + +/** Called when we get an EXTENDCIRCUIT message. Try to extend the listed + * circuit, and report success or failure. */ +static int +handle_control_extendcircuit(control_connection_t *conn, uint32_t len, + const char *body) +{ + smartlist_t *router_nicknames=NULL, *nodes=NULL; + origin_circuit_t *circ = NULL; + int zero_circ; + uint8_t intended_purpose = CIRCUIT_PURPOSE_C_GENERAL; + smartlist_t *args; + (void) len; + + router_nicknames = smartlist_new(); + + args = getargs_helper("EXTENDCIRCUIT", conn, body, 1, -1); + if (!args) + goto done; + + zero_circ = !strcmp("0", (char*)smartlist_get(args,0)); + + if (zero_circ) { + const char *purp = find_element_starting_with(args, 1, "PURPOSE="); + + if (purp) { + intended_purpose = circuit_purpose_from_string(purp); + if (intended_purpose == CIRCUIT_PURPOSE_UNKNOWN) { + connection_printf_to_buf(conn, "552 Unknown purpose \"%s\"\r\n", purp); + SMARTLIST_FOREACH(args, char *, cp, tor_free(cp)); + smartlist_free(args); + goto done; + } + } + + if ((smartlist_len(args) == 1) || + (smartlist_len(args) >= 2 && is_keyval_pair(smartlist_get(args, 1)))) { + // "EXTENDCIRCUIT 0" || EXTENDCIRCUIT 0 foo=bar" + circ = circuit_launch(intended_purpose, CIRCLAUNCH_NEED_CAPACITY); + if (!circ) { + connection_write_str_to_buf("551 Couldn't start circuit\r\n", conn); + } else { + connection_printf_to_buf(conn, "250 EXTENDED %lu\r\n", + (unsigned long)circ->global_identifier); + } + SMARTLIST_FOREACH(args, char *, cp, tor_free(cp)); + smartlist_free(args); + goto done; + } + // "EXTENDCIRCUIT 0 router1,router2" || + // "EXTENDCIRCUIT 0 router1,router2 PURPOSE=foo" + } + + if (!zero_circ && !(circ = get_circ(smartlist_get(args,0)))) { + connection_printf_to_buf(conn, "552 Unknown circuit \"%s\"\r\n", + (char*)smartlist_get(args, 0)); + SMARTLIST_FOREACH(args, char *, cp, tor_free(cp)); + smartlist_free(args); + goto done; + } + + if (smartlist_len(args) < 2) { + connection_printf_to_buf(conn, + "512 syntax error: not enough arguments.\r\n"); + SMARTLIST_FOREACH(args, char *, cp, tor_free(cp)); + smartlist_free(args); + goto done; + } + + smartlist_split_string(router_nicknames, smartlist_get(args,1), ",", 0, 0); + + SMARTLIST_FOREACH(args, char *, cp, tor_free(cp)); + smartlist_free(args); + + nodes = smartlist_new(); + int first_node = zero_circ; + SMARTLIST_FOREACH_BEGIN(router_nicknames, const char *, n) { + const node_t *node = node_get_by_nickname(n, 0); + if (!node) { + connection_printf_to_buf(conn, "552 No such router \"%s\"\r\n", n); + goto done; + } + if (!node_has_preferred_descriptor(node, first_node)) { + connection_printf_to_buf(conn, "552 No descriptor for \"%s\"\r\n", n); + goto done; + } + smartlist_add(nodes, (void*)node); + first_node = 0; + } SMARTLIST_FOREACH_END(n); + if (!smartlist_len(nodes)) { + connection_write_str_to_buf("512 No router names provided\r\n", conn); + goto done; + } + + if (zero_circ) { + /* start a new circuit */ + circ = origin_circuit_init(intended_purpose, 0); + } + + /* now circ refers to something that is ready to be extended */ + first_node = zero_circ; + SMARTLIST_FOREACH(nodes, const node_t *, node, + { + extend_info_t *info = extend_info_from_node(node, first_node); + if (!info) { + tor_assert_nonfatal(first_node); + log_warn(LD_CONTROL, + "controller tried to connect to a node that lacks a suitable " + "descriptor, or which doesn't have any " + "addresses that are allowed by the firewall configuration; " + "circuit marked for closing."); + circuit_mark_for_close(TO_CIRCUIT(circ), -END_CIRC_REASON_CONNECTFAILED); + connection_write_str_to_buf("551 Couldn't start circuit\r\n", conn); + goto done; + } + circuit_append_new_exit(circ, info); + if (circ->build_state->desired_path_len > 1) { + circ->build_state->onehop_tunnel = 0; + } + extend_info_free(info); + first_node = 0; + }); + + /* now that we've populated the cpath, start extending */ + if (zero_circ) { + int err_reason = 0; + if ((err_reason = circuit_handle_first_hop(circ)) < 0) { + circuit_mark_for_close(TO_CIRCUIT(circ), -err_reason); + connection_write_str_to_buf("551 Couldn't start circuit\r\n", conn); + goto done; + } + } else { + if (circ->base_.state == CIRCUIT_STATE_OPEN || + circ->base_.state == CIRCUIT_STATE_GUARD_WAIT) { + int err_reason = 0; + circuit_set_state(TO_CIRCUIT(circ), CIRCUIT_STATE_BUILDING); + if ((err_reason = circuit_send_next_onion_skin(circ)) < 0) { + log_info(LD_CONTROL, + "send_next_onion_skin failed; circuit marked for closing."); + circuit_mark_for_close(TO_CIRCUIT(circ), -err_reason); + connection_write_str_to_buf("551 Couldn't send onion skin\r\n", conn); + goto done; + } + } + } + + connection_printf_to_buf(conn, "250 EXTENDED %lu\r\n", + (unsigned long)circ->global_identifier); + if (zero_circ) /* send a 'launched' event, for completeness */ + circuit_event_status(circ, CIRC_EVENT_LAUNCHED, 0); + done: + SMARTLIST_FOREACH(router_nicknames, char *, n, tor_free(n)); + smartlist_free(router_nicknames); + smartlist_free(nodes); + return 0; +} + +/** Called when we get a SETCIRCUITPURPOSE message. If we can find the + * circuit and it's a valid purpose, change it. */ +static int +handle_control_setcircuitpurpose(control_connection_t *conn, + uint32_t len, const char *body) +{ + origin_circuit_t *circ = NULL; + uint8_t new_purpose; + smartlist_t *args; + (void) len; /* body is NUL-terminated, so it's safe to ignore the length. */ + + args = getargs_helper("SETCIRCUITPURPOSE", conn, body, 2, -1); + if (!args) + goto done; + + if (!(circ = get_circ(smartlist_get(args,0)))) { + connection_printf_to_buf(conn, "552 Unknown circuit \"%s\"\r\n", + (char*)smartlist_get(args, 0)); + goto done; + } + + { + const char *purp = find_element_starting_with(args,1,"PURPOSE="); + if (!purp) { + connection_write_str_to_buf("552 No purpose given\r\n", conn); + goto done; + } + new_purpose = circuit_purpose_from_string(purp); + if (new_purpose == CIRCUIT_PURPOSE_UNKNOWN) { + connection_printf_to_buf(conn, "552 Unknown purpose \"%s\"\r\n", purp); + goto done; + } + } + + circuit_change_purpose(TO_CIRCUIT(circ), new_purpose); + connection_write_str_to_buf("250 OK\r\n", conn); + + done: + if (args) { + SMARTLIST_FOREACH(args, char *, cp, tor_free(cp)); + smartlist_free(args); + } + return 0; +} + +/** Called when we get an ATTACHSTREAM message. Try to attach the requested + * stream, and report success or failure. */ +static int +handle_control_attachstream(control_connection_t *conn, uint32_t len, + const char *body) +{ + entry_connection_t *ap_conn = NULL; + origin_circuit_t *circ = NULL; + int zero_circ; + smartlist_t *args; + crypt_path_t *cpath=NULL; + int hop=0, hop_line_ok=1; + (void) len; + + args = getargs_helper("ATTACHSTREAM", conn, body, 2, -1); + if (!args) + return 0; + + zero_circ = !strcmp("0", (char*)smartlist_get(args,1)); + + if (!(ap_conn = get_stream(smartlist_get(args, 0)))) { + connection_printf_to_buf(conn, "552 Unknown stream \"%s\"\r\n", + (char*)smartlist_get(args, 0)); + } else if (!zero_circ && !(circ = get_circ(smartlist_get(args, 1)))) { + connection_printf_to_buf(conn, "552 Unknown circuit \"%s\"\r\n", + (char*)smartlist_get(args, 1)); + } else if (circ) { + const char *hopstring = find_element_starting_with(args,2,"HOP="); + if (hopstring) { + hopstring += strlen("HOP="); + hop = (int) tor_parse_ulong(hopstring, 10, 0, INT_MAX, + &hop_line_ok, NULL); + if (!hop_line_ok) { /* broken hop line */ + connection_printf_to_buf(conn, "552 Bad value hop=%s\r\n", hopstring); + } + } + } + SMARTLIST_FOREACH(args, char *, cp, tor_free(cp)); + smartlist_free(args); + if (!ap_conn || (!zero_circ && !circ) || !hop_line_ok) + return 0; + + if (ENTRY_TO_CONN(ap_conn)->state != AP_CONN_STATE_CONTROLLER_WAIT && + ENTRY_TO_CONN(ap_conn)->state != AP_CONN_STATE_CONNECT_WAIT && + ENTRY_TO_CONN(ap_conn)->state != AP_CONN_STATE_RESOLVE_WAIT) { + connection_write_str_to_buf( + "555 Connection is not managed by controller.\r\n", + conn); + return 0; + } + + /* Do we need to detach it first? */ + if (ENTRY_TO_CONN(ap_conn)->state != AP_CONN_STATE_CONTROLLER_WAIT) { + edge_connection_t *edge_conn = ENTRY_TO_EDGE_CONN(ap_conn); + circuit_t *tmpcirc = circuit_get_by_edge_conn(edge_conn); + connection_edge_end(edge_conn, END_STREAM_REASON_TIMEOUT); + /* Un-mark it as ending, since we're going to reuse it. */ + edge_conn->edge_has_sent_end = 0; + edge_conn->end_reason = 0; + if (tmpcirc) + circuit_detach_stream(tmpcirc, edge_conn); + CONNECTION_AP_EXPECT_NONPENDING(ap_conn); + TO_CONN(edge_conn)->state = AP_CONN_STATE_CONTROLLER_WAIT; + } + + if (circ && (circ->base_.state != CIRCUIT_STATE_OPEN)) { + connection_write_str_to_buf( + "551 Can't attach stream to non-open origin circuit\r\n", + conn); + return 0; + } + /* Is this a single hop circuit? */ + if (circ && (circuit_get_cpath_len(circ)<2 || hop==1)) { + connection_write_str_to_buf( + "551 Can't attach stream to this one-hop circuit.\r\n", conn); + return 0; + } + + if (circ && hop>0) { + /* find this hop in the circuit, and set cpath */ + cpath = circuit_get_cpath_hop(circ, hop); + if (!cpath) { + connection_printf_to_buf(conn, + "551 Circuit doesn't have %d hops.\r\n", hop); + return 0; + } + } + if (connection_ap_handshake_rewrite_and_attach(ap_conn, circ, cpath) < 0) { + connection_write_str_to_buf("551 Unable to attach stream\r\n", conn); + return 0; + } + send_control_done(conn); + return 0; +} + +/** Called when we get a POSTDESCRIPTOR message. Try to learn the provided + * descriptor, and report success or failure. */ +static int +handle_control_postdescriptor(control_connection_t *conn, uint32_t len, + const char *body) +{ + char *desc; + const char *msg=NULL; + uint8_t purpose = ROUTER_PURPOSE_GENERAL; + int cache = 0; /* eventually, we may switch this to 1 */ + + const char *cp = memchr(body, '\n', len); + + if (cp == NULL) { + connection_printf_to_buf(conn, "251 Empty body\r\n"); + return 0; + } + ++cp; + + char *cmdline = tor_memdup_nulterm(body, cp-body); + smartlist_t *args = smartlist_new(); + smartlist_split_string(args, cmdline, " ", + SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); + SMARTLIST_FOREACH_BEGIN(args, char *, option) { + if (!strcasecmpstart(option, "purpose=")) { + option += strlen("purpose="); + purpose = router_purpose_from_string(option); + if (purpose == ROUTER_PURPOSE_UNKNOWN) { + connection_printf_to_buf(conn, "552 Unknown purpose \"%s\"\r\n", + option); + goto done; + } + } else if (!strcasecmpstart(option, "cache=")) { + option += strlen("cache="); + if (!strcasecmp(option, "no")) + cache = 0; + else if (!strcasecmp(option, "yes")) + cache = 1; + else { + connection_printf_to_buf(conn, "552 Unknown cache request \"%s\"\r\n", + option); + goto done; + } + } else { /* unrecognized argument? */ + connection_printf_to_buf(conn, + "512 Unexpected argument \"%s\" to postdescriptor\r\n", option); + goto done; + } + } SMARTLIST_FOREACH_END(option); + + read_escaped_data(cp, len-(cp-body), &desc); + + switch (router_load_single_router(desc, purpose, cache, &msg)) { + case -1: + if (!msg) msg = "Could not parse descriptor"; + connection_printf_to_buf(conn, "554 %s\r\n", msg); + break; + case 0: + if (!msg) msg = "Descriptor not added"; + connection_printf_to_buf(conn, "251 %s\r\n",msg); + break; + case 1: + send_control_done(conn); + break; + } + + tor_free(desc); + done: + SMARTLIST_FOREACH(args, char *, arg, tor_free(arg)); + smartlist_free(args); + tor_free(cmdline); + return 0; +} + +/** Called when we receive a REDIRECTSTERAM command. Try to change the target + * address of the named AP stream, and report success or failure. */ +static int +handle_control_redirectstream(control_connection_t *conn, uint32_t len, + const char *body) +{ + entry_connection_t *ap_conn = NULL; + char *new_addr = NULL; + uint16_t new_port = 0; + smartlist_t *args; + (void) len; + + args = getargs_helper("REDIRECTSTREAM", conn, body, 2, -1); + if (!args) + return 0; + + if (!(ap_conn = get_stream(smartlist_get(args, 0))) + || !ap_conn->socks_request) { + connection_printf_to_buf(conn, "552 Unknown stream \"%s\"\r\n", + (char*)smartlist_get(args, 0)); + } else { + int ok = 1; + if (smartlist_len(args) > 2) { /* they included a port too */ + new_port = (uint16_t) tor_parse_ulong(smartlist_get(args, 2), + 10, 1, 65535, &ok, NULL); + } + if (!ok) { + connection_printf_to_buf(conn, "512 Cannot parse port \"%s\"\r\n", + (char*)smartlist_get(args, 2)); + } else { + new_addr = tor_strdup(smartlist_get(args, 1)); + } + } + + SMARTLIST_FOREACH(args, char *, cp, tor_free(cp)); + smartlist_free(args); + if (!new_addr) + return 0; + + strlcpy(ap_conn->socks_request->address, new_addr, + sizeof(ap_conn->socks_request->address)); + if (new_port) + ap_conn->socks_request->port = new_port; + tor_free(new_addr); + send_control_done(conn); + return 0; +} + +/** Called when we get a CLOSESTREAM command; try to close the named stream + * and report success or failure. */ +static int +handle_control_closestream(control_connection_t *conn, uint32_t len, + const char *body) +{ + entry_connection_t *ap_conn=NULL; + uint8_t reason=0; + smartlist_t *args; + int ok; + (void) len; + + args = getargs_helper("CLOSESTREAM", conn, body, 2, -1); + if (!args) + return 0; + + else if (!(ap_conn = get_stream(smartlist_get(args, 0)))) + connection_printf_to_buf(conn, "552 Unknown stream \"%s\"\r\n", + (char*)smartlist_get(args, 0)); + else { + reason = (uint8_t) tor_parse_ulong(smartlist_get(args,1), 10, 0, 255, + &ok, NULL); + if (!ok) { + connection_printf_to_buf(conn, "552 Unrecognized reason \"%s\"\r\n", + (char*)smartlist_get(args, 1)); + ap_conn = NULL; + } + } + SMARTLIST_FOREACH(args, char *, cp, tor_free(cp)); + smartlist_free(args); + if (!ap_conn) + return 0; + + connection_mark_unattached_ap(ap_conn, reason); + send_control_done(conn); + return 0; +} + +/** Called when we get a CLOSECIRCUIT command; try to close the named circuit + * and report success or failure. */ +static int +handle_control_closecircuit(control_connection_t *conn, uint32_t len, + const char *body) +{ + origin_circuit_t *circ = NULL; + int safe = 0; + smartlist_t *args; + (void) len; + + args = getargs_helper("CLOSECIRCUIT", conn, body, 1, -1); + if (!args) + return 0; + + if (!(circ=get_circ(smartlist_get(args, 0)))) + connection_printf_to_buf(conn, "552 Unknown circuit \"%s\"\r\n", + (char*)smartlist_get(args, 0)); + else { + int i; + for (i=1; i < smartlist_len(args); ++i) { + if (!strcasecmp(smartlist_get(args, i), "IfUnused")) + safe = 1; + else + log_info(LD_CONTROL, "Skipping unknown option %s", + (char*)smartlist_get(args,i)); + } + } + SMARTLIST_FOREACH(args, char *, cp, tor_free(cp)); + smartlist_free(args); + if (!circ) + return 0; + + if (!safe || !circ->p_streams) { + circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_REQUESTED); + } + + send_control_done(conn); + return 0; +} + +/** Called when we get a RESOLVE command: start trying to resolve + * the listed addresses. */ +static int +handle_control_resolve(control_connection_t *conn, uint32_t len, + const char *body) +{ + smartlist_t *args, *failed; + int is_reverse = 0; + (void) len; /* body is nul-terminated; it's safe to ignore the length */ + + if (!(conn->event_mask & (((event_mask_t)1)<have_sent_protocolinfo = 1; + args = smartlist_new(); + smartlist_split_string(args, body, " ", + SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); + SMARTLIST_FOREACH(args, const char *, arg, { + int ok; + tor_parse_long(arg, 10, 0, LONG_MAX, &ok, NULL); + if (!ok) { + bad_arg = arg; + break; + } + }); + if (bad_arg) { + connection_printf_to_buf(conn, "513 No such version %s\r\n", + escaped(bad_arg)); + /* Don't tolerate bad arguments when not authenticated. */ + if (!STATE_IS_OPEN(TO_CONN(conn)->state)) + connection_mark_for_close(TO_CONN(conn)); + goto done; + } else { + const or_options_t *options = get_options(); + int cookies = options->CookieAuthentication; + char *cfile = get_controller_cookie_file_name(); + char *abs_cfile; + char *esc_cfile; + char *methods; + abs_cfile = make_path_absolute(cfile); + esc_cfile = esc_for_log(abs_cfile); + { + int passwd = (options->HashedControlPassword != NULL || + options->HashedControlSessionPassword != NULL); + smartlist_t *mlist = smartlist_new(); + if (cookies) { + smartlist_add(mlist, (char*)"COOKIE"); + smartlist_add(mlist, (char*)"SAFECOOKIE"); + } + if (passwd) + smartlist_add(mlist, (char*)"HASHEDPASSWORD"); + if (!cookies && !passwd) + smartlist_add(mlist, (char*)"NULL"); + methods = smartlist_join_strings(mlist, ",", 0, NULL); + smartlist_free(mlist); + } + + connection_printf_to_buf(conn, + "250-PROTOCOLINFO 1\r\n" + "250-AUTH METHODS=%s%s%s\r\n" + "250-VERSION Tor=%s\r\n" + "250 OK\r\n", + methods, + cookies?" COOKIEFILE=":"", + cookies?esc_cfile:"", + escaped(VERSION)); + tor_free(methods); + tor_free(cfile); + tor_free(abs_cfile); + tor_free(esc_cfile); + } + done: + SMARTLIST_FOREACH(args, char *, cp, tor_free(cp)); + smartlist_free(args); + return 0; +} + +/** Called when we get a USEFEATURE command: parse the feature list, and + * set up the control_connection's options properly. */ +static int +handle_control_usefeature(control_connection_t *conn, + uint32_t len, + const char *body) +{ + smartlist_t *args; + int bad = 0; + (void) len; /* body is nul-terminated; it's safe to ignore the length */ + args = smartlist_new(); + smartlist_split_string(args, body, " ", + SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); + SMARTLIST_FOREACH_BEGIN(args, const char *, arg) { + if (!strcasecmp(arg, "VERBOSE_NAMES")) + ; + else if (!strcasecmp(arg, "EXTENDED_EVENTS")) + ; + else { + connection_printf_to_buf(conn, "552 Unrecognized feature \"%s\"\r\n", + arg); + bad = 1; + break; + } + } SMARTLIST_FOREACH_END(arg); + + if (!bad) { + send_control_done(conn); + } + + SMARTLIST_FOREACH(args, char *, cp, tor_free(cp)); + smartlist_free(args); + return 0; +} + +/** Implementation for the DROPGUARDS command. */ +static int +handle_control_dropguards(control_connection_t *conn, + uint32_t len, + const char *body) +{ + smartlist_t *args; + (void) len; /* body is nul-terminated; it's safe to ignore the length */ + args = smartlist_new(); + smartlist_split_string(args, body, " ", + SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); + + static int have_warned = 0; + if (! have_warned) { + log_warn(LD_CONTROL, "DROPGUARDS is dangerous; make sure you understand " + "the risks before using it. It may be removed in a future " + "version of Tor."); + have_warned = 1; + } + + if (smartlist_len(args)) { + connection_printf_to_buf(conn, "512 Too many arguments to DROPGUARDS\r\n"); + } else { + remove_all_entry_guards(); + send_control_done(conn); + } + + SMARTLIST_FOREACH(args, char *, cp, tor_free(cp)); + smartlist_free(args); + return 0; +} + +/** Implementation for the HSFETCH command. */ +static int +handle_control_hsfetch(control_connection_t *conn, uint32_t len, + const char *body) +{ + int i; + char digest[DIGEST_LEN], *hsaddress = NULL, *arg1 = NULL, *desc_id = NULL; + smartlist_t *args = NULL, *hsdirs = NULL; + (void) len; /* body is nul-terminated; it's safe to ignore the length */ + static const char *hsfetch_command = "HSFETCH"; + static const char *v2_str = "v2-"; + const size_t v2_str_len = strlen(v2_str); + rend_data_t *rend_query = NULL; + ed25519_public_key_t v3_pk; + uint32_t version; + + /* Make sure we have at least one argument, the HSAddress. */ + args = getargs_helper(hsfetch_command, conn, body, 1, -1); + if (!args) { + goto exit; + } + + /* Extract the first argument (either HSAddress or DescID). */ + arg1 = smartlist_get(args, 0); + /* Test if it's an HS address without the .onion part. */ + if (rend_valid_v2_service_id(arg1)) { + hsaddress = arg1; + version = HS_VERSION_TWO; + } else if (strcmpstart(arg1, v2_str) == 0 && + rend_valid_descriptor_id(arg1 + v2_str_len) && + base32_decode(digest, sizeof(digest), arg1 + v2_str_len, + REND_DESC_ID_V2_LEN_BASE32) == + REND_DESC_ID_V2_LEN_BASE32) { + /* We have a well formed version 2 descriptor ID. Keep the decoded value + * of the id. */ + desc_id = digest; + version = HS_VERSION_TWO; + } else if (hs_address_is_valid(arg1)) { + hsaddress = arg1; + version = HS_VERSION_THREE; + hs_parse_address(hsaddress, &v3_pk, NULL, NULL); + } else { + connection_printf_to_buf(conn, "513 Invalid argument \"%s\"\r\n", + arg1); + goto done; + } + + static const char *opt_server = "SERVER="; + + /* Skip first argument because it's the HSAddress or DescID. */ + for (i = 1; i < smartlist_len(args); ++i) { + const char *arg = smartlist_get(args, i); + const node_t *node; + + if (!strcasecmpstart(arg, opt_server)) { + const char *server; + + server = arg + strlen(opt_server); + node = node_get_by_hex_id(server, 0); + if (!node) { + connection_printf_to_buf(conn, "552 Server \"%s\" not found\r\n", + server); + goto done; + } + if (!hsdirs) { + /* Stores routerstatus_t object for each specified server. */ + hsdirs = smartlist_new(); + } + /* Valid server, add it to our local list. */ + smartlist_add(hsdirs, node->rs); + } else { + connection_printf_to_buf(conn, "513 Unexpected argument \"%s\"\r\n", + arg); + goto done; + } + } + + if (version == HS_VERSION_TWO) { + rend_query = rend_data_client_create(hsaddress, desc_id, NULL, + REND_NO_AUTH); + if (rend_query == NULL) { + connection_printf_to_buf(conn, "551 Error creating the HS query\r\n"); + goto done; + } + } + + /* Using a descriptor ID, we force the user to provide at least one + * hsdir server using the SERVER= option. */ + if (desc_id && (!hsdirs || !smartlist_len(hsdirs))) { + connection_printf_to_buf(conn, "512 %s option is required\r\n", + opt_server); + goto done; + } + + /* We are about to trigger HSDir fetch so send the OK now because after + * that 650 event(s) are possible so better to have the 250 OK before them + * to avoid out of order replies. */ + send_control_done(conn); + + /* Trigger the fetch using the built rend query and possibly a list of HS + * directory to use. This function ignores the client cache thus this will + * always send a fetch command. */ + if (version == HS_VERSION_TWO) { + rend_client_fetch_v2_desc(rend_query, hsdirs); + } else if (version == HS_VERSION_THREE) { + hs_control_hsfetch_command(&v3_pk, hsdirs); + } + + done: + SMARTLIST_FOREACH(args, char *, cp, tor_free(cp)); + smartlist_free(args); + /* Contains data pointer that we don't own thus no cleanup. */ + smartlist_free(hsdirs); + rend_data_free(rend_query); + exit: + return 0; +} + +/** Implementation for the HSPOST command. */ +static int +handle_control_hspost(control_connection_t *conn, + uint32_t len, + const char *body) +{ + static const char *opt_server = "SERVER="; + static const char *opt_hsaddress = "HSADDRESS="; + smartlist_t *hs_dirs = NULL; + const char *encoded_desc = body; + size_t encoded_desc_len = len; + const char *onion_address = NULL; + + char *cp = memchr(body, '\n', len); + if (cp == NULL) { + connection_printf_to_buf(conn, "251 Empty body\r\n"); + return 0; + } + char *argline = tor_strndup(body, cp-body); + + smartlist_t *args = smartlist_new(); + + /* If any SERVER= or HSADDRESS= options were specified, try to parse + * the options line. */ + if (!strcasecmpstart(argline, opt_server) || + !strcasecmpstart(argline, opt_hsaddress)) { + /* encoded_desc begins after a newline character */ + cp = cp + 1; + encoded_desc = cp; + encoded_desc_len = len-(cp-body); + + smartlist_split_string(args, argline, " ", + SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); + SMARTLIST_FOREACH_BEGIN(args, const char *, arg) { + if (!strcasecmpstart(arg, opt_server)) { + const char *server = arg + strlen(opt_server); + const node_t *node = node_get_by_hex_id(server, 0); + + if (!node || !node->rs) { + connection_printf_to_buf(conn, "552 Server \"%s\" not found\r\n", + server); + goto done; + } + /* Valid server, add it to our local list. */ + if (!hs_dirs) + hs_dirs = smartlist_new(); + smartlist_add(hs_dirs, node->rs); + } else if (!strcasecmpstart(arg, opt_hsaddress)) { + const char *address = arg + strlen(opt_hsaddress); + if (!hs_address_is_valid(address)) { + connection_printf_to_buf(conn, "512 Malformed onion address\r\n"); + goto done; + } + onion_address = address; + } else { + connection_printf_to_buf(conn, "512 Unexpected argument \"%s\"\r\n", + arg); + goto done; + } + } SMARTLIST_FOREACH_END(arg); + } + + /* Handle the v3 case. */ + if (onion_address) { + char *desc_str = NULL; + read_escaped_data(encoded_desc, encoded_desc_len, &desc_str); + if (hs_control_hspost_command(desc_str, onion_address, hs_dirs) < 0) { + connection_printf_to_buf(conn, "554 Invalid descriptor\r\n"); + } else { + send_control_done(conn); + } + tor_free(desc_str); + goto done; + } + + /* From this point on, it is only v2. */ + + /* Read the dot encoded descriptor, and parse it. */ + rend_encoded_v2_service_descriptor_t *desc = + tor_malloc_zero(sizeof(rend_encoded_v2_service_descriptor_t)); + read_escaped_data(encoded_desc, encoded_desc_len, &desc->desc_str); + + rend_service_descriptor_t *parsed = NULL; + char *intro_content = NULL; + size_t intro_size; + size_t encoded_size; + const char *next_desc; + if (!rend_parse_v2_service_descriptor(&parsed, desc->desc_id, &intro_content, + &intro_size, &encoded_size, + &next_desc, desc->desc_str, 1)) { + /* Post the descriptor. */ + char serviceid[REND_SERVICE_ID_LEN_BASE32+1]; + if (!rend_get_service_id(parsed->pk, serviceid)) { + smartlist_t *descs = smartlist_new(); + smartlist_add(descs, desc); + + /* We are about to trigger HS descriptor upload so send the OK now + * because after that 650 event(s) are possible so better to have the + * 250 OK before them to avoid out of order replies. */ + send_control_done(conn); + + /* Trigger the descriptor upload */ + directory_post_to_hs_dir(parsed, descs, hs_dirs, serviceid, 0); + smartlist_free(descs); + } + + rend_service_descriptor_free(parsed); + } else { + connection_printf_to_buf(conn, "554 Invalid descriptor\r\n"); + } + + tor_free(intro_content); + rend_encoded_v2_service_descriptor_free(desc); + done: + tor_free(argline); + smartlist_free(hs_dirs); /* Contents belong to the rend service code. */ + SMARTLIST_FOREACH(args, char *, arg, tor_free(arg)); + smartlist_free(args); + return 0; +} + +/* Helper function for ADD_ONION that adds an ephemeral service depending on + * the given hs_version. + * + * The secret key in pk depends on the hs_version. The ownership of the key + * used in pk is given to the HS subsystem so the caller must stop accessing + * it after. + * + * The port_cfgs is a list of service port. Ownership transferred to service. + * The max_streams refers to the MaxStreams= key. + * The max_streams_close_circuit refers to the MaxStreamsCloseCircuit key. + * The auth_type is the authentication type of the clients in auth_clients. + * The ownership of that list is transferred to the service. + * + * On success (RSAE_OKAY), the address_out points to a newly allocated string + * containing the onion address without the .onion part. On error, address_out + * is untouched. */ +static hs_service_add_ephemeral_status_t +add_onion_helper_add_service(int hs_version, + add_onion_secret_key_t *pk, + smartlist_t *port_cfgs, int max_streams, + int max_streams_close_circuit, int auth_type, + smartlist_t *auth_clients, char **address_out) +{ + hs_service_add_ephemeral_status_t ret; + + tor_assert(pk); + tor_assert(port_cfgs); + tor_assert(address_out); + + switch (hs_version) { + case HS_VERSION_TWO: + ret = rend_service_add_ephemeral(pk->v2, port_cfgs, max_streams, + max_streams_close_circuit, auth_type, + auth_clients, address_out); + break; + case HS_VERSION_THREE: + ret = hs_service_add_ephemeral(pk->v3, port_cfgs, max_streams, + max_streams_close_circuit, address_out); + break; + default: + tor_assert_unreached(); + } + + return ret; +} + +/** The list of onion services that have been added via ADD_ONION that do not + * belong to any particular control connection. + */ +static smartlist_t *detached_onion_services = NULL; + +/** + * Return a list of detached onion services, or NULL if none exist. + **/ +smartlist_t * +get_detached_onion_services(void) +{ + return detached_onion_services; +} + +/** Called when we get a ADD_ONION command; parse the body, and set up + * the new ephemeral Onion Service. */ +static int +handle_control_add_onion(control_connection_t *conn, + uint32_t len, + const char *body) +{ + smartlist_t *args; + int arg_len; + (void) len; /* body is nul-terminated; it's safe to ignore the length */ + args = getargs_helper("ADD_ONION", conn, body, 2, -1); + if (!args) + return 0; + arg_len = smartlist_len(args); + + /* Parse all of the arguments that do not involve handling cryptographic + * material first, since there's no reason to touch that at all if any of + * the other arguments are malformed. + */ + smartlist_t *port_cfgs = smartlist_new(); + smartlist_t *auth_clients = NULL; + smartlist_t *auth_created_clients = NULL; + int discard_pk = 0; + int detach = 0; + int max_streams = 0; + int max_streams_close_circuit = 0; + rend_auth_type_t auth_type = REND_NO_AUTH; + /* Default to adding an anonymous hidden service if no flag is given */ + int non_anonymous = 0; + for (int i = 1; i < arg_len; i++) { + static const char *port_prefix = "Port="; + static const char *flags_prefix = "Flags="; + static const char *max_s_prefix = "MaxStreams="; + static const char *auth_prefix = "ClientAuth="; + + const char *arg = smartlist_get(args, (int)i); + if (!strcasecmpstart(arg, port_prefix)) { + /* "Port=VIRTPORT[,TARGET]". */ + const char *port_str = arg + strlen(port_prefix); + + rend_service_port_config_t *cfg = + rend_service_parse_port_config(port_str, ",", NULL); + if (!cfg) { + connection_printf_to_buf(conn, "512 Invalid VIRTPORT/TARGET\r\n"); + goto out; + } + smartlist_add(port_cfgs, cfg); + } else if (!strcasecmpstart(arg, max_s_prefix)) { + /* "MaxStreams=[0..65535]". */ + const char *max_s_str = arg + strlen(max_s_prefix); + int ok = 0; + max_streams = (int)tor_parse_long(max_s_str, 10, 0, 65535, &ok, NULL); + if (!ok) { + connection_printf_to_buf(conn, "512 Invalid MaxStreams\r\n"); + goto out; + } + } else if (!strcasecmpstart(arg, flags_prefix)) { + /* "Flags=Flag[,Flag]", where Flag can be: + * * 'DiscardPK' - If tor generates the keypair, do not include it in + * the response. + * * 'Detach' - Do not tie this onion service to any particular control + * connection. + * * 'MaxStreamsCloseCircuit' - Close the circuit if MaxStreams is + * exceeded. + * * 'BasicAuth' - Client authorization using the 'basic' method. + * * 'NonAnonymous' - Add a non-anonymous Single Onion Service. If this + * flag is present, tor must be in non-anonymous + * hidden service mode. If this flag is absent, + * tor must be in anonymous hidden service mode. + */ + static const char *discard_flag = "DiscardPK"; + static const char *detach_flag = "Detach"; + static const char *max_s_close_flag = "MaxStreamsCloseCircuit"; + static const char *basicauth_flag = "BasicAuth"; + static const char *non_anonymous_flag = "NonAnonymous"; + + smartlist_t *flags = smartlist_new(); + int bad = 0; + + smartlist_split_string(flags, arg + strlen(flags_prefix), ",", + SPLIT_IGNORE_BLANK, 0); + if (smartlist_len(flags) < 1) { + connection_printf_to_buf(conn, "512 Invalid 'Flags' argument\r\n"); + bad = 1; + } + SMARTLIST_FOREACH_BEGIN(flags, const char *, flag) + { + if (!strcasecmp(flag, discard_flag)) { + discard_pk = 1; + } else if (!strcasecmp(flag, detach_flag)) { + detach = 1; + } else if (!strcasecmp(flag, max_s_close_flag)) { + max_streams_close_circuit = 1; + } else if (!strcasecmp(flag, basicauth_flag)) { + auth_type = REND_BASIC_AUTH; + } else if (!strcasecmp(flag, non_anonymous_flag)) { + non_anonymous = 1; + } else { + connection_printf_to_buf(conn, + "512 Invalid 'Flags' argument: %s\r\n", + escaped(flag)); + bad = 1; + break; + } + } SMARTLIST_FOREACH_END(flag); + SMARTLIST_FOREACH(flags, char *, cp, tor_free(cp)); + smartlist_free(flags); + if (bad) + goto out; + } else if (!strcasecmpstart(arg, auth_prefix)) { + char *err_msg = NULL; + int created = 0; + rend_authorized_client_t *client = + add_onion_helper_clientauth(arg + strlen(auth_prefix), + &created, &err_msg); + if (!client) { + if (err_msg) { + connection_write_str_to_buf(err_msg, conn); + tor_free(err_msg); + } + goto out; + } + + if (auth_clients != NULL) { + int bad = 0; + SMARTLIST_FOREACH_BEGIN(auth_clients, rend_authorized_client_t *, ac) { + if (strcmp(ac->client_name, client->client_name) == 0) { + bad = 1; + break; + } + } SMARTLIST_FOREACH_END(ac); + if (bad) { + connection_printf_to_buf(conn, + "512 Duplicate name in ClientAuth\r\n"); + rend_authorized_client_free(client); + goto out; + } + } else { + auth_clients = smartlist_new(); + auth_created_clients = smartlist_new(); + } + smartlist_add(auth_clients, client); + if (created) { + smartlist_add(auth_created_clients, client); + } + } else { + connection_printf_to_buf(conn, "513 Invalid argument\r\n"); + goto out; + } + } + if (smartlist_len(port_cfgs) == 0) { + connection_printf_to_buf(conn, "512 Missing 'Port' argument\r\n"); + goto out; + } else if (auth_type == REND_NO_AUTH && auth_clients != NULL) { + connection_printf_to_buf(conn, "512 No auth type specified\r\n"); + goto out; + } else if (auth_type != REND_NO_AUTH && auth_clients == NULL) { + connection_printf_to_buf(conn, "512 No auth clients specified\r\n"); + goto out; + } else if ((auth_type == REND_BASIC_AUTH && + smartlist_len(auth_clients) > 512) || + (auth_type == REND_STEALTH_AUTH && + smartlist_len(auth_clients) > 16)) { + connection_printf_to_buf(conn, "512 Too many auth clients\r\n"); + goto out; + } else if (non_anonymous != rend_service_non_anonymous_mode_enabled( + get_options())) { + /* If we failed, and the non-anonymous flag is set, Tor must be in + * anonymous hidden service mode. + * The error message changes based on the current Tor config: + * 512 Tor is in anonymous hidden service mode + * 512 Tor is in non-anonymous hidden service mode + * (I've deliberately written them out in full here to aid searchability.) + */ + connection_printf_to_buf(conn, "512 Tor is in %sanonymous hidden service " + "mode\r\n", + non_anonymous ? "" : "non-"); + goto out; + } + + /* Parse the "keytype:keyblob" argument. */ + int hs_version = 0; + add_onion_secret_key_t pk = { NULL }; + const char *key_new_alg = NULL; + char *key_new_blob = NULL; + char *err_msg = NULL; + + if (add_onion_helper_keyarg(smartlist_get(args, 0), discard_pk, + &key_new_alg, &key_new_blob, &pk, &hs_version, + &err_msg) < 0) { + if (err_msg) { + connection_write_str_to_buf(err_msg, conn); + tor_free(err_msg); + } + goto out; + } + tor_assert(!err_msg); + + /* Hidden service version 3 don't have client authentication support so if + * ClientAuth was given, send back an error. */ + if (hs_version == HS_VERSION_THREE && auth_clients) { + connection_printf_to_buf(conn, "513 ClientAuth not supported\r\n"); + goto out; + } + + /* Create the HS, using private key pk, client authentication auth_type, + * the list of auth_clients, and port config port_cfg. + * rend_service_add_ephemeral() will take ownership of pk and port_cfg, + * regardless of success/failure. + */ + char *service_id = NULL; + int ret = add_onion_helper_add_service(hs_version, &pk, port_cfgs, + max_streams, + max_streams_close_circuit, auth_type, + auth_clients, &service_id); + port_cfgs = NULL; /* port_cfgs is now owned by the rendservice code. */ + auth_clients = NULL; /* so is auth_clients */ + switch (ret) { + case RSAE_OKAY: + { + if (detach) { + if (!detached_onion_services) + detached_onion_services = smartlist_new(); + smartlist_add(detached_onion_services, service_id); + } else { + if (!conn->ephemeral_onion_services) + conn->ephemeral_onion_services = smartlist_new(); + smartlist_add(conn->ephemeral_onion_services, service_id); + } + + tor_assert(service_id); + connection_printf_to_buf(conn, "250-ServiceID=%s\r\n", service_id); + if (key_new_alg) { + tor_assert(key_new_blob); + connection_printf_to_buf(conn, "250-PrivateKey=%s:%s\r\n", + key_new_alg, key_new_blob); + } + if (auth_created_clients) { + SMARTLIST_FOREACH(auth_created_clients, rend_authorized_client_t *, ac, { + char *encoded = rend_auth_encode_cookie(ac->descriptor_cookie, + auth_type); + tor_assert(encoded); + connection_printf_to_buf(conn, "250-ClientAuth=%s:%s\r\n", + ac->client_name, encoded); + memwipe(encoded, 0, strlen(encoded)); + tor_free(encoded); + }); + } + + connection_printf_to_buf(conn, "250 OK\r\n"); + break; + } + case RSAE_BADPRIVKEY: + connection_printf_to_buf(conn, "551 Failed to generate onion address\r\n"); + break; + case RSAE_ADDREXISTS: + connection_printf_to_buf(conn, "550 Onion address collision\r\n"); + break; + case RSAE_BADVIRTPORT: + connection_printf_to_buf(conn, "512 Invalid VIRTPORT/TARGET\r\n"); + break; + case RSAE_BADAUTH: + connection_printf_to_buf(conn, "512 Invalid client authorization\r\n"); + break; + case RSAE_INTERNAL: /* FALLSTHROUGH */ + default: + connection_printf_to_buf(conn, "551 Failed to add Onion Service\r\n"); + } + if (key_new_blob) { + memwipe(key_new_blob, 0, strlen(key_new_blob)); + tor_free(key_new_blob); + } + + out: + if (port_cfgs) { + SMARTLIST_FOREACH(port_cfgs, rend_service_port_config_t*, p, + rend_service_port_config_free(p)); + smartlist_free(port_cfgs); + } + + if (auth_clients) { + SMARTLIST_FOREACH(auth_clients, rend_authorized_client_t *, ac, + rend_authorized_client_free(ac)); + smartlist_free(auth_clients); + } + if (auth_created_clients) { + // Do not free entries; they are the same as auth_clients + smartlist_free(auth_created_clients); + } + + SMARTLIST_FOREACH(args, char *, cp, { + memwipe(cp, 0, strlen(cp)); + tor_free(cp); + }); + smartlist_free(args); + return 0; +} + +/** Helper function to handle parsing the KeyType:KeyBlob argument to the + * ADD_ONION command. Return a new crypto_pk_t and if a new key was generated + * and the private key not discarded, the algorithm and serialized private key, + * or NULL and an optional control protocol error message on failure. The + * caller is responsible for freeing the returned key_new_blob and err_msg. + * + * Note: The error messages returned are deliberately vague to avoid echoing + * key material. + */ +STATIC int +add_onion_helper_keyarg(const char *arg, int discard_pk, + const char **key_new_alg_out, char **key_new_blob_out, + add_onion_secret_key_t *decoded_key, int *hs_version, + char **err_msg_out) +{ + smartlist_t *key_args = smartlist_new(); + crypto_pk_t *pk = NULL; + const char *key_new_alg = NULL; + char *key_new_blob = NULL; + char *err_msg = NULL; + int ret = -1; + + smartlist_split_string(key_args, arg, ":", SPLIT_IGNORE_BLANK, 0); + if (smartlist_len(key_args) != 2) { + err_msg = tor_strdup("512 Invalid key type/blob\r\n"); + goto err; + } + + /* The format is "KeyType:KeyBlob". */ + static const char *key_type_new = "NEW"; + static const char *key_type_best = "BEST"; + static const char *key_type_rsa1024 = "RSA1024"; + static const char *key_type_ed25519_v3 = "ED25519-V3"; + + const char *key_type = smartlist_get(key_args, 0); + const char *key_blob = smartlist_get(key_args, 1); + + if (!strcasecmp(key_type_rsa1024, key_type)) { + /* "RSA:" - Loading a pre-existing RSA1024 key. */ + pk = crypto_pk_base64_decode_private(key_blob, strlen(key_blob)); + if (!pk) { + err_msg = tor_strdup("512 Failed to decode RSA key\r\n"); + goto err; + } + if (crypto_pk_num_bits(pk) != PK_BYTES*8) { + crypto_pk_free(pk); + err_msg = tor_strdup("512 Invalid RSA key size\r\n"); + goto err; + } + decoded_key->v2 = pk; + *hs_version = HS_VERSION_TWO; + } else if (!strcasecmp(key_type_ed25519_v3, key_type)) { + /* "ED25519-V3:" - Loading a pre-existing ed25519 key. */ + ed25519_secret_key_t *sk = tor_malloc_zero(sizeof(*sk)); + if (base64_decode((char *) sk->seckey, sizeof(sk->seckey), key_blob, + strlen(key_blob)) != sizeof(sk->seckey)) { + tor_free(sk); + err_msg = tor_strdup("512 Failed to decode ED25519-V3 key\r\n"); + goto err; + } + decoded_key->v3 = sk; + *hs_version = HS_VERSION_THREE; + } else if (!strcasecmp(key_type_new, key_type)) { + /* "NEW:" - Generating a new key, blob as algorithm. */ + if (!strcasecmp(key_type_rsa1024, key_blob) || + !strcasecmp(key_type_best, key_blob)) { + /* "RSA1024", RSA 1024 bit, also currently "BEST" by default. */ + pk = crypto_pk_new(); + if (crypto_pk_generate_key(pk)) { + tor_asprintf(&err_msg, "551 Failed to generate %s key\r\n", + key_type_rsa1024); + goto err; + } + if (!discard_pk) { + if (crypto_pk_base64_encode_private(pk, &key_new_blob)) { + crypto_pk_free(pk); + tor_asprintf(&err_msg, "551 Failed to encode %s key\r\n", + key_type_rsa1024); + goto err; + } + key_new_alg = key_type_rsa1024; + } + decoded_key->v2 = pk; + *hs_version = HS_VERSION_TWO; + } else if (!strcasecmp(key_type_ed25519_v3, key_blob)) { + ed25519_secret_key_t *sk = tor_malloc_zero(sizeof(*sk)); + if (ed25519_secret_key_generate(sk, 1) < 0) { + tor_free(sk); + tor_asprintf(&err_msg, "551 Failed to generate %s key\r\n", + key_type_ed25519_v3); + goto err; + } + if (!discard_pk) { + ssize_t len = base64_encode_size(sizeof(sk->seckey), 0) + 1; + key_new_blob = tor_malloc_zero(len); + if (base64_encode(key_new_blob, len, (const char *) sk->seckey, + sizeof(sk->seckey), 0) != (len - 1)) { + tor_free(sk); + tor_free(key_new_blob); + tor_asprintf(&err_msg, "551 Failed to encode %s key\r\n", + key_type_ed25519_v3); + goto err; + } + key_new_alg = key_type_ed25519_v3; + } + decoded_key->v3 = sk; + *hs_version = HS_VERSION_THREE; + } else { + err_msg = tor_strdup("513 Invalid key type\r\n"); + goto err; + } + } else { + err_msg = tor_strdup("513 Invalid key type\r\n"); + goto err; + } + + /* Succeeded in loading or generating a private key. */ + ret = 0; + + err: + SMARTLIST_FOREACH(key_args, char *, cp, { + memwipe(cp, 0, strlen(cp)); + tor_free(cp); + }); + smartlist_free(key_args); + + if (err_msg_out) { + *err_msg_out = err_msg; + } else { + tor_free(err_msg); + } + *key_new_alg_out = key_new_alg; + *key_new_blob_out = key_new_blob; + + return ret; +} + +/** Helper function to handle parsing a ClientAuth argument to the + * ADD_ONION command. Return a new rend_authorized_client_t, or NULL + * and an optional control protocol error message on failure. The + * caller is responsible for freeing the returned auth_client and err_msg. + * + * If 'created' is specified, it will be set to 1 when a new cookie has + * been generated. + */ +STATIC rend_authorized_client_t * +add_onion_helper_clientauth(const char *arg, int *created, char **err_msg) +{ + int ok = 0; + + tor_assert(arg); + tor_assert(created); + tor_assert(err_msg); + *err_msg = NULL; + + smartlist_t *auth_args = smartlist_new(); + rend_authorized_client_t *client = + tor_malloc_zero(sizeof(rend_authorized_client_t)); + smartlist_split_string(auth_args, arg, ":", 0, 0); + if (smartlist_len(auth_args) < 1 || smartlist_len(auth_args) > 2) { + *err_msg = tor_strdup("512 Invalid ClientAuth syntax\r\n"); + goto err; + } + client->client_name = tor_strdup(smartlist_get(auth_args, 0)); + if (smartlist_len(auth_args) == 2) { + char *decode_err_msg = NULL; + if (rend_auth_decode_cookie(smartlist_get(auth_args, 1), + client->descriptor_cookie, + NULL, &decode_err_msg) < 0) { + tor_assert(decode_err_msg); + tor_asprintf(err_msg, "512 %s\r\n", decode_err_msg); + tor_free(decode_err_msg); + goto err; + } + *created = 0; + } else { + crypto_rand((char *) client->descriptor_cookie, REND_DESC_COOKIE_LEN); + *created = 1; + } + + if (!rend_valid_client_name(client->client_name)) { + *err_msg = tor_strdup("512 Invalid name in ClientAuth\r\n"); + goto err; + } + + ok = 1; + err: + SMARTLIST_FOREACH(auth_args, char *, item, tor_free(item)); + smartlist_free(auth_args); + if (!ok) { + rend_authorized_client_free(client); + client = NULL; + } + return client; +} + +/** Called when we get a DEL_ONION command; parse the body, and remove + * the existing ephemeral Onion Service. */ +static int +handle_control_del_onion(control_connection_t *conn, + uint32_t len, + const char *body) +{ + int hs_version = 0; + smartlist_t *args; + (void) len; /* body is nul-terminated; it's safe to ignore the length */ + args = getargs_helper("DEL_ONION", conn, body, 1, 1); + if (!args) + return 0; + + const char *service_id = smartlist_get(args, 0); + if (rend_valid_v2_service_id(service_id)) { + hs_version = HS_VERSION_TWO; + } else if (hs_address_is_valid(service_id)) { + hs_version = HS_VERSION_THREE; + } else { + connection_printf_to_buf(conn, "512 Malformed Onion Service id\r\n"); + goto out; + } + + /* Determine if the onion service belongs to this particular control + * connection, or if it is in the global list of detached services. If it + * is in neither, either the service ID is invalid in some way, or it + * explicitly belongs to a different control connection, and an error + * should be returned. + */ + smartlist_t *services[2] = { + conn->ephemeral_onion_services, + detached_onion_services + }; + smartlist_t *onion_services = NULL; + int idx = -1; + for (size_t i = 0; i < ARRAY_LENGTH(services); i++) { + idx = smartlist_string_pos(services[i], service_id); + if (idx != -1) { + onion_services = services[i]; + break; + } + } + if (onion_services == NULL) { + connection_printf_to_buf(conn, "552 Unknown Onion Service id\r\n"); + } else { + int ret = -1; + switch (hs_version) { + case HS_VERSION_TWO: + ret = rend_service_del_ephemeral(service_id); + break; + case HS_VERSION_THREE: + ret = hs_service_del_ephemeral(service_id); + break; + default: + /* The ret value will be -1 thus hitting the warning below. This should + * never happen because of the check at the start of the function. */ + break; + } + if (ret < 0) { + /* This should *NEVER* fail, since the service is on either the + * per-control connection list, or the global one. + */ + log_warn(LD_BUG, "Failed to remove Onion Service %s.", + escaped(service_id)); + tor_fragile_assert(); + } + + /* Remove/scrub the service_id from the appropriate list. */ + char *cp = smartlist_get(onion_services, idx); + smartlist_del(onion_services, idx); + memwipe(cp, 0, strlen(cp)); + tor_free(cp); + + send_control_done(conn); + } + + out: + SMARTLIST_FOREACH(args, char *, cp, { + memwipe(cp, 0, strlen(cp)); + tor_free(cp); + }); + smartlist_free(args); + return 0; +} + +int +handle_control_command(control_connection_t *conn, + uint32_t cmd_data_len, + char *args) +{ + /* XXXX Why is this not implemented as a table like the GETINFO + * items are? Even handling the plus signs at the beginnings of + * commands wouldn't be very hard with proper macros. */ + + if (!strcasecmp(conn->incoming_cmd, "SETCONF")) { + if (handle_control_setconf(conn, cmd_data_len, args)) + return -1; + } else if (!strcasecmp(conn->incoming_cmd, "RESETCONF")) { + if (handle_control_resetconf(conn, cmd_data_len, args)) + return -1; + } else if (!strcasecmp(conn->incoming_cmd, "GETCONF")) { + if (handle_control_getconf(conn, cmd_data_len, args)) + return -1; + } else if (!strcasecmp(conn->incoming_cmd, "+LOADCONF")) { + if (handle_control_loadconf(conn, cmd_data_len, args)) + return -1; + } else if (!strcasecmp(conn->incoming_cmd, "SETEVENTS")) { + if (handle_control_setevents(conn, cmd_data_len, args)) + return -1; + } else if (!strcasecmp(conn->incoming_cmd, "AUTHENTICATE")) { + if (handle_control_authenticate(conn, cmd_data_len, args)) + return -1; + } else if (!strcasecmp(conn->incoming_cmd, "SAVECONF")) { + if (handle_control_saveconf(conn, cmd_data_len, args)) + return -1; + } else if (!strcasecmp(conn->incoming_cmd, "SIGNAL")) { + if (handle_control_signal(conn, cmd_data_len, args)) + return -1; + } else if (!strcasecmp(conn->incoming_cmd, "TAKEOWNERSHIP")) { + if (handle_control_takeownership(conn, cmd_data_len, args)) + return -1; + } else if (!strcasecmp(conn->incoming_cmd, "DROPOWNERSHIP")) { + if (handle_control_dropownership(conn, cmd_data_len, args)) + return -1; + } else if (!strcasecmp(conn->incoming_cmd, "MAPADDRESS")) { + if (handle_control_mapaddress(conn, cmd_data_len, args)) + return -1; + } else if (!strcasecmp(conn->incoming_cmd, "GETINFO")) { + if (handle_control_getinfo(conn, cmd_data_len, args)) + return -1; + } else if (!strcasecmp(conn->incoming_cmd, "EXTENDCIRCUIT")) { + if (handle_control_extendcircuit(conn, cmd_data_len, args)) + return -1; + } else if (!strcasecmp(conn->incoming_cmd, "SETCIRCUITPURPOSE")) { + if (handle_control_setcircuitpurpose(conn, cmd_data_len, args)) + return -1; + } else if (!strcasecmp(conn->incoming_cmd, "SETROUTERPURPOSE")) { + connection_write_str_to_buf("511 SETROUTERPURPOSE is obsolete.\r\n", conn); + } else if (!strcasecmp(conn->incoming_cmd, "ATTACHSTREAM")) { + if (handle_control_attachstream(conn, cmd_data_len, args)) + return -1; + } else if (!strcasecmp(conn->incoming_cmd, "+POSTDESCRIPTOR")) { + if (handle_control_postdescriptor(conn, cmd_data_len, args)) + return -1; + } else if (!strcasecmp(conn->incoming_cmd, "REDIRECTSTREAM")) { + if (handle_control_redirectstream(conn, cmd_data_len, args)) + return -1; + } else if (!strcasecmp(conn->incoming_cmd, "CLOSESTREAM")) { + if (handle_control_closestream(conn, cmd_data_len, args)) + return -1; + } else if (!strcasecmp(conn->incoming_cmd, "CLOSECIRCUIT")) { + if (handle_control_closecircuit(conn, cmd_data_len, args)) + return -1; + } else if (!strcasecmp(conn->incoming_cmd, "USEFEATURE")) { + if (handle_control_usefeature(conn, cmd_data_len, args)) + return -1; + } else if (!strcasecmp(conn->incoming_cmd, "RESOLVE")) { + if (handle_control_resolve(conn, cmd_data_len, args)) + return -1; + } else if (!strcasecmp(conn->incoming_cmd, "PROTOCOLINFO")) { + if (handle_control_protocolinfo(conn, cmd_data_len, args)) + return -1; + } else if (!strcasecmp(conn->incoming_cmd, "AUTHCHALLENGE")) { + if (handle_control_authchallenge(conn, cmd_data_len, args)) + return -1; + } else if (!strcasecmp(conn->incoming_cmd, "DROPGUARDS")) { + if (handle_control_dropguards(conn, cmd_data_len, args)) + return -1; + } else if (!strcasecmp(conn->incoming_cmd, "HSFETCH")) { + if (handle_control_hsfetch(conn, cmd_data_len, args)) + return -1; + } else if (!strcasecmp(conn->incoming_cmd, "+HSPOST")) { + if (handle_control_hspost(conn, cmd_data_len, args)) + return -1; + } else if (!strcasecmp(conn->incoming_cmd, "ADD_ONION")) { + int ret = handle_control_add_onion(conn, cmd_data_len, args); + memwipe(args, 0, cmd_data_len); /* Scrub the private key. */ + if (ret) + return -1; + } else if (!strcasecmp(conn->incoming_cmd, "DEL_ONION")) { + int ret = handle_control_del_onion(conn, cmd_data_len, args); + memwipe(args, 0, cmd_data_len); /* Scrub the service id/pk. */ + if (ret) + return -1; + } else { + connection_printf_to_buf(conn, "510 Unrecognized command \"%s\"\r\n", + conn->incoming_cmd); + } + + return 0; +} + +void +control_cmd_free_all(void) +{ + if (detached_onion_services) { /* Free the detached onion services */ + SMARTLIST_FOREACH(detached_onion_services, char *, cp, tor_free(cp)); + smartlist_free(detached_onion_services); + } +} diff --git a/src/feature/control/control_cmd.h b/src/feature/control/control_cmd.h new file mode 100644 index 0000000000..a417e10da3 --- /dev/null +++ b/src/feature/control/control_cmd.h @@ -0,0 +1,48 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file control_cmd.h + * \brief Header file for control_cmd.c. + **/ + +#ifndef TOR_CONTROL_CMD_H +#define TOR_CONTROL_CMD_H + +int handle_control_command(control_connection_t *conn, + uint32_t cmd_data_len, + char *args); +void control_cmd_free_all(void); + +#ifdef CONTROL_CMD_PRIVATE +#include "lib/crypt_ops/crypto_ed25519.h" + +/* ADD_ONION secret key to create an ephemeral service. The command supports + * multiple versions so this union stores the key and passes it to the HS + * subsystem depending on the requested version. */ +typedef union add_onion_secret_key_t { + /* Hidden service v2 secret key. */ + crypto_pk_t *v2; + /* Hidden service v3 secret key. */ + ed25519_secret_key_t *v3; +} add_onion_secret_key_t; + +STATIC int add_onion_helper_keyarg(const char *arg, int discard_pk, + const char **key_new_alg_out, + char **key_new_blob_out, + add_onion_secret_key_t *decoded_key, + int *hs_version, char **err_msg_out); + +STATIC rend_authorized_client_t *add_onion_helper_clientauth(const char *arg, + int *created, char **err_msg_out); + +#endif /* defined(CONTROL_CMD_PRIVATE) */ + +#ifdef CONTROL_MODULE_PRIVATE +smartlist_t * get_detached_onion_services(void); +#endif /* defined(CONTROL_MODULE_PRIVATE) */ + +#endif /* !defined(TOR_CONTROL_CMD_H) */ diff --git a/src/feature/control/control_fmt.c b/src/feature/control/control_fmt.c index c87a224ac5..290ea61450 100644 --- a/src/feature/control/control_fmt.c +++ b/src/feature/control/control_fmt.c @@ -297,3 +297,98 @@ read_escaped_data(const char *data, size_t len, char **out) *outp = '\0'; return outp - *out; } + +/** Send a "DONE" message down the control connection conn. */ +void +send_control_done(control_connection_t *conn) +{ + connection_write_str_to_buf("250 OK\r\n", conn); +} + +/** If the first in_len_max characters in start contain a + * double-quoted string with escaped characters, return the length of that + * string (as encoded, including quotes). Otherwise return -1. */ +static inline int +get_escaped_string_length(const char *start, size_t in_len_max, + int *chars_out) +{ + const char *cp, *end; + int chars = 0; + + if (*start != '\"') + return -1; + + cp = start+1; + end = start+in_len_max; + + /* Calculate length. */ + while (1) { + if (cp >= end) { + return -1; /* Too long. */ + } else if (*cp == '\\') { + if (++cp == end) + return -1; /* Can't escape EOS. */ + ++cp; + ++chars; + } else if (*cp == '\"') { + break; + } else { + ++cp; + ++chars; + } + } + if (chars_out) + *chars_out = chars; + return (int)(cp - start+1); +} + +/** As decode_escaped_string, but does not decode the string: copies the + * entire thing, including quotation marks. */ +const char * +extract_escaped_string(const char *start, size_t in_len_max, + char **out, size_t *out_len) +{ + int length = get_escaped_string_length(start, in_len_max, NULL); + if (length<0) + return NULL; + *out_len = length; + *out = tor_strndup(start, *out_len); + return start+length; +} + +/** Given a pointer to a string starting at start containing + * in_len_max characters, decode a string beginning with one double + * quote, containing any number of non-quote characters or characters escaped + * with a backslash, and ending with a final double quote. Place the resulting + * string (unquoted, unescaped) into a newly allocated string in *out; + * store its length in out_len. On success, return a pointer to the + * character immediately following the escaped string. On failure, return + * NULL. */ +const char * +decode_escaped_string(const char *start, size_t in_len_max, + char **out, size_t *out_len) +{ + const char *cp, *end; + char *outp; + int len, n_chars = 0; + + len = get_escaped_string_length(start, in_len_max, &n_chars); + if (len<0) + return NULL; + + end = start+len-1; /* Index of last quote. */ + tor_assert(*end == '\"'); + outp = *out = tor_malloc(len+1); + *out_len = n_chars; + + cp = start+1; + while (cp < end) { + if (*cp == '\\') + ++cp; + *outp++ = *cp++; + } + *outp = '\0'; + tor_assert((outp - *out) == (int)*out_len); + + return end+1; +} diff --git a/src/feature/control/control_fmt.h b/src/feature/control/control_fmt.h index 1b4534cc3b..e7ab6608a4 100644 --- a/src/feature/control/control_fmt.h +++ b/src/feature/control/control_fmt.h @@ -25,5 +25,10 @@ char *circuit_describe_status_for_controller(origin_circuit_t *circ); size_t write_escaped_data(const char *data, size_t len, char **out); size_t read_escaped_data(const char *data, size_t len, char **out); +const char *extract_escaped_string(const char *start, size_t in_len_max, + char **out, size_t *out_len); +const char *decode_escaped_string(const char *start, size_t in_len_max, + char **out, size_t *out_len); +void send_control_done(control_connection_t *conn); #endif /* !defined(TOR_CONTROL_FMT_H) */ diff --git a/src/feature/control/control_getinfo.c b/src/feature/control/control_getinfo.c index 67a6e27316..a7a85f2fdf 100644 --- a/src/feature/control/control_getinfo.c +++ b/src/feature/control/control_getinfo.c @@ -24,6 +24,7 @@ #include "feature/client/bridges.h" #include "feature/client/entrynodes.h" #include "feature/control/control.h" +#include "feature/control/control_cmd.h" #include "feature/control/control_events.h" #include "feature/control/control_fmt.h" #include "feature/control/control_getinfo.h" diff --git a/src/test/test_controller.c b/src/test/test_controller.c index 82922d2028..f3af6d2ec0 100644 --- a/src/test/test_controller.c +++ b/src/test/test_controller.c @@ -1,12 +1,13 @@ /* Copyright (c) 2015-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ -#define CONTROL_PRIVATE +#define CONTROL_CMD_PRIVATE #define CONTROL_GETINFO_PRIVATE #include "core/or/or.h" #include "lib/crypt_ops/crypto_ed25519.h" #include "feature/client/bridges.h" #include "feature/control/control.h" +#include "feature/control/control_cmd.h" #include "feature/control/control_getinfo.h" #include "feature/client/entrynodes.h" #include "feature/hs/hs_common.h" diff --git a/src/test/test_pt.c b/src/test/test_pt.c index 612006906d..87e3ba356c 100644 --- a/src/test/test_pt.c +++ b/src/test/test_pt.c @@ -7,7 +7,6 @@ #define PT_PRIVATE #define UTIL_PRIVATE #define STATEFILE_PRIVATE -//#define CONTROL_PRIVATE #define CONTROL_EVENTS_PRIVATE #define PROCESS_PRIVATE #include "core/or/or.h" diff --git a/src/test/test_util.c b/src/test/test_util.c index 4875a7c943..b4e702e1d7 100644 --- a/src/test/test_util.c +++ b/src/test/test_util.c @@ -6,7 +6,6 @@ #include "orconfig.h" #define COMPAT_PRIVATE #define COMPAT_TIME_PRIVATE -#define CONTROL_PRIVATE #define UTIL_PRIVATE #define UTIL_MALLOC_PRIVATE #define SOCKET_PRIVATE From 61cebb2035b8cfb2f84ab5e198742d0b29b41192 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 25 Mar 2019 14:14:56 -0400 Subject: [PATCH 0636/2557] Minimize the includes in control.c --- scripts/maint/practracker/exceptions.txt | 2 - src/feature/control/control.c | 92 +----------------------- src/feature/control/control.h | 1 - src/feature/control/control_fmt.c | 15 ++++ src/feature/control/control_fmt.h | 2 + src/test/test_hs.c | 1 + src/test/test_hs_control.c | 1 + 7 files changed, 20 insertions(+), 94 deletions(-) diff --git a/scripts/maint/practracker/exceptions.txt b/scripts/maint/practracker/exceptions.txt index fe91e0c345..a6726ef570 100644 --- a/scripts/maint/practracker/exceptions.txt +++ b/scripts/maint/practracker/exceptions.txt @@ -109,8 +109,6 @@ problem function-size /src/tools/tor-gencert.c:parse_commandline() 111 problem function-size /src/feature/keymgt/loadkey.c:ed_key_init_from_file() 333 problem function-size /src/feature/dircommon/consdiff.c:gen_ed_diff() 204 problem function-size /src/feature/dircommon/consdiff.c:apply_ed_diff() 159 -problem file-size /src/feature/control/control.c 7592 -problem include-count /src/feature/control/control.c 90 problem function-size /src/feature/control/control_auth.c:handle_control_authenticate() 188 problem function-size /src/feature/control/control_auth.c:handle_control_authchallenge() 115 problem function-size /src/feature/control/control_getinfo.c:getinfo_helper_misc() 109 diff --git a/src/feature/control/control.c b/src/feature/control/control.c index 70d70bedac..41e21c0a14 100644 --- a/src/feature/control/control.c +++ b/src/feature/control/control.c @@ -33,91 +33,25 @@ **/ #define CONTROL_MODULE_PRIVATE -#define CONTROL_EVENTS_PRIVATE -#define OCIRC_EVENT_PRIVATE #include "core/or/or.h" #include "app/config/config.h" -#include "app/config/confparse.h" #include "app/main/main.h" #include "core/mainloop/connection.h" #include "core/mainloop/mainloop.h" -#include "core/or/channel.h" -#include "core/or/channeltls.h" -#include "core/or/circuitbuild.h" -#include "core/or/circuitlist.h" -#include "core/or/circuitstats.h" -#include "core/or/circuituse.h" -#include "core/or/command.h" -#include "core/or/connection_edge.h" #include "core/or/connection_or.h" -#include "core/or/ocirc_event.h" -#include "core/or/policies.h" -#include "core/or/reasons.h" -#include "core/or/versions.h" #include "core/proto/proto_control0.h" #include "core/proto/proto_http.h" -#include "feature/client/addressmap.h" -#include "feature/client/bridges.h" -#include "feature/client/dnsserv.h" -#include "feature/client/entrynodes.h" #include "feature/control/control.h" #include "feature/control/control_auth.h" #include "feature/control/control_cmd.h" #include "feature/control/control_events.h" #include "feature/control/control_fmt.h" -#include "feature/control/control_getinfo.h" -#include "feature/control/fmt_serverstatus.h" -#include "feature/control/getinfo_geoip.h" -#include "feature/dircache/dirserv.h" -#include "feature/dirclient/dirclient.h" -#include "feature/dirclient/dlstatus.h" -#include "feature/dircommon/directory.h" -#include "feature/hibernate/hibernate.h" -#include "feature/hs/hs_cache.h" -#include "feature/hs/hs_common.h" -#include "feature/hs/hs_control.h" -#include "feature/hs_common/shared_random_client.h" -#include "feature/nodelist/authcert.h" -#include "feature/nodelist/dirlist.h" -#include "feature/nodelist/microdesc.h" -#include "feature/nodelist/networkstatus.h" -#include "feature/nodelist/nodelist.h" -#include "feature/nodelist/routerinfo.h" -#include "feature/nodelist/routerlist.h" -#include "feature/relay/router.h" -#include "feature/relay/routermode.h" -#include "feature/relay/selftest.h" -#include "feature/rend/rendclient.h" #include "feature/rend/rendcommon.h" -#include "feature/rend/rendparse.h" #include "feature/rend/rendservice.h" -#include "feature/stats/geoip_stats.h" -#include "feature/stats/predict_ports.h" -#include "lib/buf/buffers.h" -#include "lib/crypt_ops/crypto_rand.h" -#include "lib/crypt_ops/crypto_util.h" -#include "lib/encoding/confline.h" -#include "lib/evloop/compat_libevent.h" -#include "lib/version/torversion.h" +#include "lib/evloop/procmon.h" -#include "feature/dircache/cached_dir_st.h" #include "feature/control/control_connection_st.h" -#include "core/or/cpath_build_state_st.h" -#include "core/or/entry_connection_st.h" -#include "feature/nodelist/extrainfo_st.h" -#include "feature/nodelist/networkstatus_st.h" -#include "feature/nodelist/node_st.h" -#include "core/or/or_connection_st.h" -#include "core/or/or_circuit_st.h" -#include "core/or/origin_circuit_st.h" -#include "feature/nodelist/microdesc_st.h" -#include "feature/rend/rend_authorized_client_st.h" -#include "feature/rend/rend_encoded_v2_service_descriptor_st.h" -#include "feature/rend/rend_service_descriptor_st.h" -#include "feature/nodelist/routerinfo_st.h" -#include "feature/nodelist/routerlist_st.h" -#include "core/or/socks_request_st.h" #ifdef HAVE_UNISTD_H #include @@ -126,15 +60,6 @@ #include #endif -#ifndef _WIN32 -#include -#include -#endif - -#include "lib/crypt_ops/crypto_s2k.h" -#include "lib/evloop/procmon.h" -#include "lib/evloop/compat_libevent.h" - /** Convert a connection_t* to an control_connection_t*; assert if the cast is * invalid. */ control_connection_t * @@ -610,21 +535,6 @@ monitor_owning_controller_process(const char *process_spec) } } -/** Return a longname the node whose identity is id_digest. If - * node_get_by_id() returns NULL, base 16 encoding of id_digest is - * returned instead. - * - * This function is not thread-safe. Each call to this function invalidates - * previous values returned by this function. - */ -MOCK_IMPL(const char *, -node_describe_longname_by_id,(const char *id_digest)) -{ - static char longname[MAX_VERBOSE_NICKNAME_LEN+1]; - node_get_verbose_nickname_by_id(id_digest, longname); - return longname; -} - /** Free any leftover allocated memory of the control.c subsystem. */ void control_free_all(void) diff --git a/src/feature/control/control.h b/src/feature/control/control.h index d20bc6fa2b..3083837931 100644 --- a/src/feature/control/control.h +++ b/src/feature/control/control.h @@ -48,7 +48,6 @@ void enable_control_logging(void); void monitor_owning_controller_process(const char *process_spec); const char *rend_auth_type_to_string(rend_auth_type_t auth_type); -MOCK_DECL(const char *, node_describe_longname_by_id,(const char *id_digest)); void control_free_all(void); #ifdef CONTROL_MODULE_PRIVATE diff --git a/src/feature/control/control_fmt.c b/src/feature/control/control_fmt.c index 290ea61450..71f9d82163 100644 --- a/src/feature/control/control_fmt.c +++ b/src/feature/control/control_fmt.c @@ -392,3 +392,18 @@ decode_escaped_string(const char *start, size_t in_len_max, return end+1; } + +/** Return a longname the node whose identity is id_digest. If + * node_get_by_id() returns NULL, base 16 encoding of id_digest is + * returned instead. + * + * This function is not thread-safe. Each call to this function invalidates + * previous values returned by this function. + */ +MOCK_IMPL(const char *, +node_describe_longname_by_id,(const char *id_digest)) +{ + static char longname[MAX_VERBOSE_NICKNAME_LEN+1]; + node_get_verbose_nickname_by_id(id_digest, longname); + return longname; +} diff --git a/src/feature/control/control_fmt.h b/src/feature/control/control_fmt.h index e7ab6608a4..74545eb309 100644 --- a/src/feature/control/control_fmt.h +++ b/src/feature/control/control_fmt.h @@ -31,4 +31,6 @@ const char *decode_escaped_string(const char *start, size_t in_len_max, char **out, size_t *out_len); void send_control_done(control_connection_t *conn); +MOCK_DECL(const char *, node_describe_longname_by_id,(const char *id_digest)); + #endif /* !defined(TOR_CONTROL_FMT_H) */ diff --git a/src/test/test_hs.c b/src/test/test_hs.c index de10a10d81..aeb3387471 100644 --- a/src/test/test_hs.c +++ b/src/test/test_hs.c @@ -16,6 +16,7 @@ #include "test/test.h" #include "feature/control/control.h" #include "feature/control/control_events.h" +#include "feature/control/control_fmt.h" #include "app/config/config.h" #include "feature/hs/hs_common.h" #include "feature/rend/rendcommon.h" diff --git a/src/test/test_hs_control.c b/src/test/test_hs_control.c index d23d31954b..481ef1eb39 100644 --- a/src/test/test_hs_control.c +++ b/src/test/test_hs_control.c @@ -12,6 +12,7 @@ #include "test/test.h" #include "feature/control/control.h" #include "feature/control/control_events.h" +#include "feature/control/control_fmt.h" #include "app/config/config.h" #include "feature/hs/hs_common.h" #include "feature/hs/hs_control.h" From b71febcea809f154f65e6e00825724b3fbeccb10 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 25 Mar 2019 15:08:36 -0400 Subject: [PATCH 0637/2557] changes file for control.c splitup --- changes/ticket29894 | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 changes/ticket29894 diff --git a/changes/ticket29894 b/changes/ticket29894 new file mode 100644 index 0000000000..6392598ec6 --- /dev/null +++ b/changes/ticket29894 @@ -0,0 +1,4 @@ + o Code simplification and refactoring: + - Split up the control.c file into several submodules, in preparation + for distributing its current responsibilities throughout the codebase. + Closes ticket 29894. From 301e3f22eff5021b0855a70cf98a1ccee69a19a8 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 25 Mar 2019 15:51:48 -0400 Subject: [PATCH 0638/2557] Practracker: add a string explaining the excptions file. --- scripts/maint/practracker/practracker.py | 32 ++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/scripts/maint/practracker/practracker.py b/scripts/maint/practracker/practracker.py index a6e6d0b607..aec7c8b295 100755 --- a/scripts/maint/practracker/practracker.py +++ b/scripts/maint/practracker/practracker.py @@ -113,6 +113,38 @@ def consider_metrics_for_file(fname, f): return found_new_issues +HEADER="""\ +# Welcome to the exceptions file for Tor's best-practices tracker! +# +# Each line of this file represents a single violation of Tor's best +# practices -- ideally one that was grandfathered in from before practracker.py +# existed. +# +# There are three kinds of problems that we recognize right now: +# function-size -- a function of more than {MAX_FUNCTION_SIZE} lines. +# file-size -- a file of more than {MAX_FILE_SIZE} lines. +# include-count -- a file with more than {MAX_INCLUDE_COUNT} #includes. +# +# Each line below represents a single exception that practracker should +# _ignore_. Each line has four parts: +# 1. The word "problem". +# 2. The kind of problem. +# 3. The location of the problem: either a filename, or a +# filename:functionname pair. +# 4. The magnitude of the problem to ignore. +# +# So for example, consider this line: +# problem file-size /src/core/or/connection_or.c 3200 +# +# It tells practracker to allow the mentioned file to be up to 3200 lines +# long, even though ordinarily it would warn about any file with more than +# {MAX_FILE_SIZE} lines. +# +# You can either edit this file by hand, or regenerate it completely by +# running `make practracker-regen`. + +""".format(**globals()) + def main(): if (len(sys.argv) != 2): print("Usage:\n\t$ practracker.py \n\t(e.g. $ practracker.py ~/tor/)") From 0260e0f6fc46cd196c59d3d2c4551be2f15a7547 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 25 Mar 2019 15:52:43 -0400 Subject: [PATCH 0639/2557] practracker: pass sys.argv to main() as an argument --- scripts/maint/practracker/practracker.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/maint/practracker/practracker.py b/scripts/maint/practracker/practracker.py index aec7c8b295..9f87415218 100755 --- a/scripts/maint/practracker/practracker.py +++ b/scripts/maint/practracker/practracker.py @@ -145,8 +145,8 @@ HEADER="""\ """.format(**globals()) -def main(): - if (len(sys.argv) != 2): +def main(argv): + if (len(argv) != 2): print("Usage:\n\t$ practracker.py \n\t(e.g. $ practracker.py ~/tor/)") return @@ -180,4 +180,4 @@ See doc/HACKING/HelpfulTools.md for more information on using practracker.\ sys.exit(found_new_issues) if __name__ == '__main__': - main() + main(sys.argv) From c2643842a9a93f131f7b1244cce253be657acc8f Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 25 Mar 2019 16:06:26 -0400 Subject: [PATCH 0640/2557] practracker: add ability to regenerate exceptions file. Also add a useful argument parser. --- scripts/maint/practracker/practracker.py | 44 +++++++++++++++++++----- scripts/maint/practracker/problem.py | 5 ++- 2 files changed, 39 insertions(+), 10 deletions(-) diff --git a/scripts/maint/practracker/practracker.py b/scripts/maint/practracker/practracker.py index 9f87415218..4be1a31b51 100755 --- a/scripts/maint/practracker/practracker.py +++ b/scripts/maint/practracker/practracker.py @@ -117,8 +117,8 @@ HEADER="""\ # Welcome to the exceptions file for Tor's best-practices tracker! # # Each line of this file represents a single violation of Tor's best -# practices -- ideally one that was grandfathered in from before practracker.py -# existed. +# practices -- typically, a violation that we had before practracker.py +# first existed. # # There are three kinds of problems that we recognize right now: # function-size -- a function of more than {MAX_FUNCTION_SIZE} lines. @@ -142,17 +142,30 @@ HEADER="""\ # # You can either edit this file by hand, or regenerate it completely by # running `make practracker-regen`. +# +# Remember: It is better to fix the problem than to add a new exception! """.format(**globals()) def main(argv): - if (len(argv) != 2): - print("Usage:\n\t$ practracker.py \n\t(e.g. $ practracker.py ~/tor/)") - return + import argparse + + progname = argv[0] + parser = argparse.ArgumentParser(prog=progname) + parser.add_argument("--regen", action="store_true", + help="Regenerate the exceptions file") + parser.add_argument("--exceptions", + help="Override the location for the exceptions file") + parser.add_argument("topdir", default=".", nargs="?", + help="Top-level directory for the tor source") + args = parser.parse_args(argv[1:]) global TOR_TOPDIR - TOR_TOPDIR = sys.argv[1] - exceptions_file = os.path.join(TOR_TOPDIR, "scripts/maint/practracker", EXCEPTIONS_FNAME) + TOR_TOPDIR = args.topdir + if args.exceptions: + exceptions_file = args.exceptions + else: + exceptions_file = os.path.join(TOR_TOPDIR, "scripts/maint/practracker", EXCEPTIONS_FNAME) # 1) Get all the .c files we care about files_list = util.get_tor_c_files(TOR_TOPDIR) @@ -160,13 +173,26 @@ def main(argv): # 2) Initialize problem vault and load an optional exceptions file so that # we don't warn about the past global ProblemVault - ProblemVault = problem.ProblemVault(exceptions_file) + + if args.regen: + tmpname = exceptions_file + ".tmp" + tmpfile = open(tmpname, "w") + sys.stdout = tmpfile + sys.stdout.write(HEADER) + ProblemVault = problem.ProblemVault() + else: + ProblemVault = problem.ProblemVault(exceptions_file) # 3) Go through all the files and report problems if they are not exceptions found_new_issues = consider_all_metrics(files_list) + if args.regen: + tmpfile.close() + os.rename(tmpname, exceptions_file) + sys.exit(0) + # If new issues were found, try to give out some advice to the developer on how to resolve it. - if (found_new_issues): + if found_new_issues and not args.regen: new_issues_str = """\ FAILURE: practracker found new problems in the code: see warnings above. diff --git a/scripts/maint/practracker/problem.py b/scripts/maint/practracker/problem.py index d5ebedd17f..c82c5db572 100644 --- a/scripts/maint/practracker/problem.py +++ b/scripts/maint/practracker/problem.py @@ -19,10 +19,13 @@ class ProblemVault(object): found in the code, and also the old problems we read from the exception file. """ - def __init__(self, exception_fname): + def __init__(self, exception_fname=None): # Exception dictionary: { problem.key() : Problem object } self.exceptions = {} + if exception_fname == None: + return + try: with open(exception_fname, 'r') as exception_f: self.register_exceptions(exception_f) From 2c8af79de52d091f98aa94615d9e6dd69faba0c7 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 25 Mar 2019 16:07:58 -0400 Subject: [PATCH 0641/2557] Add a practracker-regen make target --- Makefile.am | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Makefile.am b/Makefile.am index 415c2f2b4e..eea7c3a12f 100644 --- a/Makefile.am +++ b/Makefile.am @@ -352,6 +352,9 @@ if USEPYTHON $(PYTHON) $(top_srcdir)/scripts/maint/practracker/practracker.py $(top_srcdir) endif +practracker-regen: + $(PYTHON) $(top_srcdir)/scripts/maint/practracker/practracker.py --regen $(top_srcdir) + check-docs: all $(PERL) $(top_builddir)/scripts/maint/checkOptionDocs.pl From dfd7a7f5b63eb968caebf6823395689e6b77b654 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 25 Oct 2018 12:04:19 -0400 Subject: [PATCH 0642/2557] Add a type to map names to short identifiers We'll be using this for four kinds of identifier in dispatch.c --- src/lib/container/include.am | 3 + src/lib/container/namemap.c | 184 +++++++++++++++++++++++++++++++++ src/lib/container/namemap.h | 35 +++++++ src/lib/container/namemap_st.h | 34 ++++++ src/test/include.am | 1 + src/test/test.c | 1 + src/test/test.h | 1 + src/test/test_namemap.c | 154 +++++++++++++++++++++++++++ 8 files changed, 413 insertions(+) create mode 100644 src/lib/container/namemap.c create mode 100644 src/lib/container/namemap.h create mode 100644 src/lib/container/namemap_st.h create mode 100644 src/test/test_namemap.c diff --git a/src/lib/container/include.am b/src/lib/container/include.am index 032e4033da..50d35e749b 100644 --- a/src/lib/container/include.am +++ b/src/lib/container/include.am @@ -8,6 +8,7 @@ endif src_lib_libtor_container_a_SOURCES = \ src/lib/container/bloomfilt.c \ src/lib/container/map.c \ + src/lib/container/namemap.c \ src/lib/container/order.c \ src/lib/container/smartlist.c @@ -21,5 +22,7 @@ noinst_HEADERS += \ src/lib/container/bloomfilt.h \ src/lib/container/handles.h \ src/lib/container/map.h \ + src/lib/container/namemap.h \ + src/lib/container/namemap_st.h \ src/lib/container/order.h \ src/lib/container/smartlist.h diff --git a/src/lib/container/namemap.c b/src/lib/container/namemap.c new file mode 100644 index 0000000000..a90057b32c --- /dev/null +++ b/src/lib/container/namemap.c @@ -0,0 +1,184 @@ +/* Copyright (c) 2003-2004, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "orconfig.h" +#include "lib/container/smartlist.h" +#include "lib/container/namemap.h" +#include "lib/container/namemap_st.h" +#include "lib/log/util_bug.h" +#include "lib/malloc/malloc.h" +#include "lib/string/printf.h" + +#include "ext/siphash.h" + +#include + +/** Helper for namemap hashtable implementation: compare two entries. */ +static inline int +mapped_name_eq(const mapped_name_t *a, const mapped_name_t *b) +{ + return !strcmp(a->name, b->name); +} + +/** Helper for namemap hashtable implementation: hash an entry. */ +static inline unsigned +mapped_name_hash(const mapped_name_t *a) +{ + return (unsigned) siphash24g(a->name, strlen(a->name)); +} + +HT_PROTOTYPE(namemap_ht, mapped_name_t, node, mapped_name_hash, + mapped_name_eq) +HT_GENERATE2(namemap_ht, mapped_name_t, node, mapped_name_hash, + mapped_name_eq, 0.6, tor_reallocarray_, tor_free_) + +/** Set up an uninitialized map. */ +void +namemap_init(namemap_t *map) +{ + memset(map, 0, sizeof(*map)); + HT_INIT(namemap_ht, &map->ht); + map->names = smartlist_new(); +} + +/** Return the name that map associates with a given id, or + * NULL if there is no such name. */ +const char * +namemap_get_name(const namemap_t *map, unsigned id) +{ + if (map->names && id < (unsigned)smartlist_len(map->names)) { + mapped_name_t *name = smartlist_get(map->names, (int)id); + return name->name; + } else { + return NULL; + } +} + +/** + * Return the name that map associates with a given id, or a + * pointer to a statically allocated string describing the value of id + * if no such name exists. + **/ +const char * +namemap_fmt_name(const namemap_t *map, unsigned id) +{ + static char buf[32]; + + const char *name = namemap_get_name(map, id); + if (name) + return name; + + tor_snprintf(buf, sizeof(buf), "{%u}", id); + + return buf; +} + +/** + * Helper: As namemap_get_id(), but requires that name is + * namelen charaters long, and that namelen is no more than + * MAX_NAMEMAP_NAME_LEN. + */ +static unsigned +namemap_get_id_unchecked(const namemap_t *map, + const char *name, + size_t namelen) +{ + union { + mapped_name_t n; + char storage[MAX_NAMEMAP_NAME_LEN + sizeof(mapped_name_t) + 1]; + } u; + memcpy(u.n.name, name, namelen); + u.n.name[namelen] = 0; + const mapped_name_t *found = HT_FIND(namemap_ht, &map->ht, &u.n); + if (found) { + tor_assert(map->names); + tor_assert(smartlist_get(map->names, found->intval) == found); + return found->intval; + } + + return NAMEMAP_ERR; +} + +/** + * Return the identifier currently associated by map with the name + * name, or NAMEMAP_ERR if no such identifier exists. + **/ +unsigned +namemap_get_id(const namemap_t *map, + const char *name) +{ + size_t namelen = strlen(name); + if (namelen > MAX_NAMEMAP_NAME_LEN) { + return NAMEMAP_ERR; + } + + return namemap_get_id_unchecked(map, name, namelen); +} + +/** + * Return the identifier associated by map with the name + * name, allocating a new identifier in map if none exists. + * + * Return NAMEMAP_ERR if name is too long, or if there are no more + * identifiers we can allocate. + **/ +unsigned +namemap_get_or_create_id(namemap_t *map, + const char *name) +{ + size_t namelen = strlen(name); + if (namelen > MAX_NAMEMAP_NAME_LEN) { + return NAMEMAP_ERR; + } + + if (PREDICT_UNLIKELY(map->names == NULL)) + map->names = smartlist_new(); + + unsigned found = namemap_get_id_unchecked(map, name, namelen); + if (found != NAMEMAP_ERR) + return found; + + unsigned new_id = (unsigned)smartlist_len(map->names); + if (new_id == NAMEMAP_ERR) + return NAMEMAP_ERR; /* Can't allocate any more. */ + + mapped_name_t *insert = tor_malloc_zero( + offsetof(mapped_name_t, name) + namelen + 1); + memcpy(insert->name, name, namelen+1); + insert->intval = new_id; + + HT_INSERT(namemap_ht, &map->ht, insert); + smartlist_add(map->names, insert); + + return new_id; +} + +/** Return the number of entries in 'names' */ +size_t +namemap_get_size(const namemap_t *map) +{ + if (PREDICT_UNLIKELY(map->names == NULL)) + return 0; + + return smartlist_len(map->names); +} + +/** + * Release all storage held in map. + */ +void +namemap_clear(namemap_t *map) +{ + if (!map) + return; + + HT_CLEAR(namemap_ht, &map->ht); + if (map->names) { + SMARTLIST_FOREACH(map->names, mapped_name_t *, n, + tor_free(n)); + smartlist_free(map->names); + } + memset(map, 0, sizeof(*map)); +} diff --git a/src/lib/container/namemap.h b/src/lib/container/namemap.h new file mode 100644 index 0000000000..97792e13ba --- /dev/null +++ b/src/lib/container/namemap.h @@ -0,0 +1,35 @@ +/* Copyright (c) 2003-2004, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef TOR_NAMEMAP_H +#define TOR_NAMEMAP_H + +/** + * \file namemap.h + * + * \brief Header for namemap.c + **/ + +#include "lib/cc/compat_compiler.h" +#include "ext/ht.h" + +#include + +typedef struct namemap_t namemap_t; + +/** Returned in place of an identifier when an error occurs. */ +#define NAMEMAP_ERR UINT_MAX + +void namemap_init(namemap_t *map); +const char *namemap_get_name(const namemap_t *map, unsigned id); +const char *namemap_fmt_name(const namemap_t *map, unsigned id); +unsigned namemap_get_id(const namemap_t *map, + const char *name); +unsigned namemap_get_or_create_id(namemap_t *map, + const char *name); +size_t namemap_get_size(const namemap_t *map); +void namemap_clear(namemap_t *map); + +#endif diff --git a/src/lib/container/namemap_st.h b/src/lib/container/namemap_st.h new file mode 100644 index 0000000000..5717352fa2 --- /dev/null +++ b/src/lib/container/namemap_st.h @@ -0,0 +1,34 @@ +/* Copyright (c) 2003-2004, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef NAMEMAP_ST_H +#define NAMEMAP_ST_H + +#include "lib/cc/compat_compiler.h" +#include "ext/ht.h" + +struct smartlist_t; + +/** Longest allowed name that's allowed in a namemap_t. */ +#define MAX_NAMEMAP_NAME_LEN 128 + +/** An entry inside a namemap_t. Maps a string to a numeric identifier. */ +typedef struct mapped_name_t { + HT_ENTRY(mapped_name_t) node; + unsigned intval; + char name[FLEXIBLE_ARRAY_MEMBER]; +} mapped_name_t; + +/** A structure that allocates small numeric identifiers for names and maps + * back and forth between them. */ +struct namemap_t { + HT_HEAD(namemap_ht, mapped_name_t) ht; + struct smartlist_t *names; +}; + +/** Macro to initialize a namemap. */ +#define NAMEMAP_INIT() { HT_INITIALIZER(), NULL } + +#endif diff --git a/src/test/include.am b/src/test/include.am index b276500fd5..c3827d3eb9 100644 --- a/src/test/include.am +++ b/src/test/include.am @@ -148,6 +148,7 @@ src_test_test_SOURCES += \ src/test/test_logging.c \ src/test/test_mainloop.c \ src/test/test_microdesc.c \ + src/test/test_namemap.c \ src/test/test_netinfo.c \ src/test/test_nodelist.c \ src/test/test_oom.c \ diff --git a/src/test/test.c b/src/test/test.c index 902565dfbe..1230b632a5 100644 --- a/src/test/test.c +++ b/src/test/test.c @@ -857,6 +857,7 @@ struct testgroup_t testgroups[] = { { "consdiff/", consdiff_tests }, { "consdiffmgr/", consdiffmgr_tests }, { "container/", container_tests }, + { "container/namemap/", namemap_tests }, { "control/", controller_tests }, { "control/btrack/", btrack_tests }, { "control/event/", controller_event_tests }, diff --git a/src/test/test.h b/src/test/test.h index 39953e9f7e..7a3a4d8fdc 100644 --- a/src/test/test.h +++ b/src/test/test.h @@ -234,6 +234,7 @@ extern struct testcase_t link_handshake_tests[]; extern struct testcase_t logging_tests[]; extern struct testcase_t mainloop_tests[]; extern struct testcase_t microdesc_tests[]; +extern struct testcase_t namemap_tests[]; extern struct testcase_t netinfo_tests[]; extern struct testcase_t nodelist_tests[]; extern struct testcase_t oom_tests[]; diff --git a/src/test/test_namemap.c b/src/test/test_namemap.c new file mode 100644 index 0000000000..5134e1451b --- /dev/null +++ b/src/test/test_namemap.c @@ -0,0 +1,154 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "test/test.h" + +#include "lib/cc/torint.h" +#include "lib/container/namemap.h" +#include "lib/container/namemap_st.h" +#include "lib/malloc/malloc.h" + +#include +#include + +static void +test_namemap_empty(void *arg) +{ + (void)arg; + + namemap_t m; + namemap_init(&m); + namemap_t m2 = NAMEMAP_INIT(); + + tt_uint_op(0, OP_EQ, namemap_get_size(&m)); + tt_uint_op(NAMEMAP_ERR, OP_EQ, namemap_get_id(&m, "hello")); + tt_uint_op(NAMEMAP_ERR, OP_EQ, namemap_get_id(&m, "hello")); + tt_uint_op(NAMEMAP_ERR, OP_EQ, namemap_get_id(&m, "hello128")); + tt_uint_op(NAMEMAP_ERR, OP_EQ, namemap_get_id(&m, "")); + tt_uint_op(0, OP_EQ, namemap_get_size(&m)); + + tt_uint_op(0, OP_EQ, namemap_get_size(&m2)); + tt_uint_op(NAMEMAP_ERR, OP_EQ, namemap_get_id(&m2, "hello")); + tt_uint_op(NAMEMAP_ERR, OP_EQ, namemap_get_id(&m2, "hello")); + tt_uint_op(NAMEMAP_ERR, OP_EQ, namemap_get_id(&m2, "hello128")); + tt_uint_op(NAMEMAP_ERR, OP_EQ, namemap_get_id(&m2, "")); + tt_uint_op(0, OP_EQ, namemap_get_size(&m)); + + done: + namemap_clear(&m); + namemap_clear(&m2); +} + +static void +test_namemap_toolong(void *arg) +{ + (void)arg; + namemap_t m; + char *ok = NULL; + char *toolong = NULL; + namemap_init(&m); + + ok = tor_malloc_zero(MAX_NAMEMAP_NAME_LEN+1); + memset(ok, 'x', MAX_NAMEMAP_NAME_LEN); + + toolong = tor_malloc_zero(MAX_NAMEMAP_NAME_LEN+2); + memset(toolong, 'x', MAX_NAMEMAP_NAME_LEN+1); + + tt_uint_op(NAMEMAP_ERR, OP_EQ, namemap_get_id(&m, ok)); + tt_uint_op(NAMEMAP_ERR, OP_EQ, namemap_get_id(&m, toolong)); + unsigned u1 = namemap_get_or_create_id(&m, toolong); + unsigned u2 = namemap_get_or_create_id(&m, ok); + tt_uint_op(u1, OP_EQ, NAMEMAP_ERR); + tt_uint_op(u2, OP_NE, NAMEMAP_ERR); + tt_uint_op(u2, OP_EQ, namemap_get_id(&m, ok)); + tt_uint_op(NAMEMAP_ERR, OP_EQ, namemap_get_id(&m, toolong)); + + tt_str_op(ok, OP_EQ, namemap_get_name(&m, u2)); + tt_ptr_op(NULL, OP_EQ, namemap_get_name(&m, u1)); + + done: + tor_free(ok); + tor_free(toolong); + namemap_clear(&m); +} + +static void +test_namemap_blackbox(void *arg) +{ + (void)arg; + + namemap_t m1, m2; + namemap_init(&m1); + namemap_init(&m2); + + unsigned u1 = namemap_get_or_create_id(&m1, "hello"); + unsigned u2 = namemap_get_or_create_id(&m1, "world"); + tt_uint_op(u1, OP_NE, NAMEMAP_ERR); + tt_uint_op(u2, OP_NE, NAMEMAP_ERR); + tt_uint_op(u1, OP_NE, u2); + + tt_uint_op(u1, OP_EQ, namemap_get_id(&m1, "hello")); + tt_uint_op(u1, OP_EQ, namemap_get_or_create_id(&m1, "hello")); + tt_uint_op(u2, OP_EQ, namemap_get_id(&m1, "world")); + tt_uint_op(u2, OP_EQ, namemap_get_or_create_id(&m1, "world")); + + tt_uint_op(NAMEMAP_ERR, OP_EQ, namemap_get_id(&m1, "HELLO")); + tt_uint_op(NAMEMAP_ERR, OP_EQ, namemap_get_id(&m2, "hello")); + + unsigned u3 = namemap_get_or_create_id(&m2, "hola"); + tt_uint_op(u3, OP_NE, NAMEMAP_ERR); + tt_uint_op(NAMEMAP_ERR, OP_EQ, namemap_get_id(&m1, "hola")); + tt_uint_op(u3, OP_EQ, namemap_get_or_create_id(&m2, "hola")); + tt_uint_op(u3, OP_EQ, namemap_get_id(&m2, "hola")); + + unsigned int u4 = namemap_get_or_create_id(&m1, "hola"); + tt_uint_op(u4, OP_NE, NAMEMAP_ERR); + tt_uint_op(u4, OP_EQ, namemap_get_id(&m1, "hola")); + tt_uint_op(u3, OP_EQ, namemap_get_id(&m2, "hola")); + + tt_str_op("hello", OP_EQ, namemap_get_name(&m1, u1)); + tt_str_op("world", OP_EQ, namemap_get_name(&m1, u2)); + tt_str_op("hola", OP_EQ, namemap_get_name(&m2, u3)); + tt_str_op("hola", OP_EQ, namemap_get_name(&m1, u4)); + + tt_ptr_op(NULL, OP_EQ, namemap_get_name(&m2, u3 + 10)); + + done: + namemap_clear(&m1); + namemap_clear(&m2); +} + +static void +test_namemap_internals(void *arg) +{ + (void)arg; + // This test actually assumes know something about the identity layout. + namemap_t m; + namemap_init(&m); + + tt_uint_op(0, OP_EQ, namemap_get_or_create_id(&m, "that")); + tt_uint_op(0, OP_EQ, namemap_get_or_create_id(&m, "that")); + tt_uint_op(1, OP_EQ, namemap_get_or_create_id(&m, "is")); + tt_uint_op(1, OP_EQ, namemap_get_or_create_id(&m, "is")); + + tt_uint_op(0, OP_EQ, namemap_get_id(&m, "that")); + tt_uint_op(0, OP_EQ, namemap_get_id(&m, "that")); + tt_uint_op(1, OP_EQ, namemap_get_id(&m, "is")); + tt_uint_op(2, OP_EQ, namemap_get_or_create_id(&m, "not")); + tt_uint_op(1, OP_EQ, namemap_get_or_create_id(&m, "is")); + tt_uint_op(2, OP_EQ, namemap_get_or_create_id(&m, "not")); + + done: + namemap_clear(&m); +} + +#define T(name) \ + { #name, test_namemap_ ## name , 0, NULL, NULL } + +struct testcase_t namemap_tests[] = { + T(empty), + T(toolong), + T(blackbox), + T(internals), + END_OF_TESTCASES +}; From 0944500a8e2651854c948e91c178ac2a57c8ae41 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 26 Oct 2018 16:22:34 -0400 Subject: [PATCH 0643/2557] Add MESG as a new log domain. --- doc/tor.1.txt | 2 +- src/lib/log/log.c | 6 +++++- src/lib/log/log.h | 5 +++-- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/doc/tor.1.txt b/doc/tor.1.txt index 92355dfb54..568303713a 100644 --- a/doc/tor.1.txt +++ b/doc/tor.1.txt @@ -679,7 +679,7 @@ GENERAL OPTIONS The currently recognized domains are: general, crypto, net, config, fs, protocol, mm, http, app, control, circ, rend, bug, dir, dirserv, or, edge, acct, hist, handshake, heartbeat, channel, sched, guard, consdiff, dos, - process, pt, and btrack. + process, pt, btrack, and mesg. Domain names are case-insensitive. + + For example, "`Log [handshake]debug [~net,~mm]info notice stdout`" sends diff --git a/src/lib/log/log.c b/src/lib/log/log.c index d032f57add..fd3081443e 100644 --- a/src/lib/log/log.c +++ b/src/lib/log/log.c @@ -49,6 +49,7 @@ #include "lib/wallclock/approx_time.h" #include "lib/wallclock/time_to_tm.h" #include "lib/fdio/fdio.h" +#include "lib/cc/ctassert.h" #ifdef HAVE_ANDROID_LOG_H #include @@ -1268,9 +1269,12 @@ static const char *domain_list[] = { "GENERAL", "CRYPTO", "NET", "CONFIG", "FS", "PROTOCOL", "MM", "HTTP", "APP", "CONTROL", "CIRC", "REND", "BUG", "DIR", "DIRSERV", "OR", "EDGE", "ACCT", "HIST", "HANDSHAKE", "HEARTBEAT", "CHANNEL", - "SCHED", "GUARD", "CONSDIFF", "DOS", "PROCESS", "PT", "BTRACK", NULL + "SCHED", "GUARD", "CONSDIFF", "DOS", "PROCESS", "PT", "BTRACK", "MESG", + NULL }; +CTASSERT(ARRAY_LENGTH(domain_list) == N_LOGGING_DOMAINS + 1); + /** Return a bitmask for the log domain for which domain is the name, * or 0 if there is no such name. */ static log_domain_mask_t diff --git a/src/lib/log/log.h b/src/lib/log/log.h index 423e58e11b..45e01f3391 100644 --- a/src/lib/log/log.h +++ b/src/lib/log/log.h @@ -113,8 +113,9 @@ #define LD_PT (1u<<27) /** Bootstrap tracker. */ #define LD_BTRACK (1u<<28) -/** Number of logging domains in the code. */ -#define N_LOGGING_DOMAINS 29 +/** Message-passing backend. */ +#define LD_MESG (1u<<29) +#define N_LOGGING_DOMAINS 30 /** This log message is not safe to send to a callback-based logger * immediately. Used as a flag, not a log domain. */ From 56bda7464f372d65d5564784266320994f605e1c Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 10 Jan 2019 10:30:52 -0500 Subject: [PATCH 0644/2557] Add a macro for the fairly common "eat next semicolon" syntax trick You use this when you're defining a macro to be used at file scope, and you want to require a semicolon afterwards. --- src/lib/cc/compat_compiler.h | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/lib/cc/compat_compiler.h b/src/lib/cc/compat_compiler.h index d45316b241..83fa5dc33c 100644 --- a/src/lib/cc/compat_compiler.h +++ b/src/lib/cc/compat_compiler.h @@ -217,4 +217,16 @@ /** Macro: Yields the number of elements in array x. */ #define ARRAY_LENGTH(x) ((sizeof(x)) / sizeof(x[0])) +/** + * "Eat" a semicolon that somebody puts at the end of a top-level macro. + * + * Frequently, we want to declare a macro that people will use at file scope, + * and we want to allow people to put a semicolon after the macro. + * + * This declaration of a struct can be repeated any number of times, and takes + * a trailing semicolon afterwards. + **/ +#define EAT_SEMICOLON \ + struct dummy_semicolon_eater__ + #endif /* !defined(TOR_COMPAT_H) */ From 253fea84cf9504a733db6979e2fc140a8c702615 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 10 Jan 2019 13:56:22 -0500 Subject: [PATCH 0645/2557] Add a smartlist_grow() function to expand a smartlist Tests included. --- src/lib/smartlist_core/smartlist_core.c | 24 ++++++++++ src/lib/smartlist_core/smartlist_core.h | 1 + src/test/test_containers.c | 61 +++++++++++++++++++++++++ 3 files changed, 86 insertions(+) diff --git a/src/lib/smartlist_core/smartlist_core.c b/src/lib/smartlist_core/smartlist_core.c index 8364a8180b..d6b86e1e36 100644 --- a/src/lib/smartlist_core/smartlist_core.c +++ b/src/lib/smartlist_core/smartlist_core.c @@ -88,6 +88,30 @@ smartlist_ensure_capacity(smartlist_t *sl, size_t size) #undef MAX_CAPACITY } +/** Expand sl so that its length is at least new_size, + * filling in previously unused entries with NULL> + * + * Do nothing if sl already had at least new_size elements. + */ +void +smartlist_grow(smartlist_t *sl, size_t new_size) +{ + smartlist_ensure_capacity(sl, new_size); + + if (new_size > (size_t)sl->num_used) { + /* This memset() should be a no-op: everything else in the smartlist code + * tries to make sure that unused entries are always NULL. Still, that is + * meant as a safety mechanism, so let's clear the memory here. + */ + memset(sl->list + sl->num_used, 0, + sizeof(void *) * (new_size - sl->num_used)); + + /* This cast is safe, since we already asserted that we were below + * MAX_CAPACITY in smartlist_ensure_capacity(). */ + sl->num_used = (int)new_size; + } +} + /** Append element to the end of the list. */ void smartlist_add(smartlist_t *sl, void *element) diff --git a/src/lib/smartlist_core/smartlist_core.h b/src/lib/smartlist_core/smartlist_core.h index 974fb01758..7d7b7f07c6 100644 --- a/src/lib/smartlist_core/smartlist_core.h +++ b/src/lib/smartlist_core/smartlist_core.h @@ -43,6 +43,7 @@ void smartlist_clear(smartlist_t *sl); void smartlist_add(smartlist_t *sl, void *element); void smartlist_add_all(smartlist_t *sl, const smartlist_t *s2); void smartlist_add_strdup(struct smartlist_t *sl, const char *string); +void smartlist_grow(smartlist_t *sl, size_t new_size); void smartlist_remove(smartlist_t *sl, const void *element); void smartlist_remove_keeporder(smartlist_t *sl, const void *element); diff --git a/src/test/test_containers.c b/src/test/test_containers.c index ad0edf4aa3..4ca9542dc7 100644 --- a/src/test/test_containers.c +++ b/src/test/test_containers.c @@ -606,6 +606,66 @@ test_container_smartlist_ints_eq(void *arg) smartlist_free(sl2); } +static void +test_container_smartlist_grow(void *arg) +{ + (void)arg; + smartlist_t *sl = smartlist_new(); + int i; + const char *s[] = { "first", "2nd", "3rd" }; + + /* case 1: starting from empty. */ + smartlist_grow(sl, 10); + tt_int_op(10, OP_EQ, smartlist_len(sl)); + for (i = 0; i < 10; ++i) { + tt_ptr_op(smartlist_get(sl, i), OP_EQ, NULL); + } + + /* case 2: starting with a few elements, probably not reallocating. */ + smartlist_free(sl); + sl = smartlist_new(); + smartlist_add(sl, (char*)s[0]); + smartlist_add(sl, (char*)s[1]); + smartlist_add(sl, (char*)s[2]); + smartlist_grow(sl, 5); + tt_int_op(5, OP_EQ, smartlist_len(sl)); + for (i = 0; i < 3; ++i) { + tt_ptr_op(smartlist_get(sl, i), OP_EQ, s[i]); + } + tt_ptr_op(smartlist_get(sl, 3), OP_EQ, NULL); + tt_ptr_op(smartlist_get(sl, 4), OP_EQ, NULL); + + /* case 3: starting with a few elements, but reallocating. */ + smartlist_free(sl); + sl = smartlist_new(); + smartlist_add(sl, (char*)s[0]); + smartlist_add(sl, (char*)s[1]); + smartlist_add(sl, (char*)s[2]); + smartlist_grow(sl, 100); + tt_int_op(100, OP_EQ, smartlist_len(sl)); + for (i = 0; i < 3; ++i) { + tt_ptr_op(smartlist_get(sl, i), OP_EQ, s[i]); + } + for (i = 3; i < 100; ++i) { + tt_ptr_op(smartlist_get(sl, i), OP_EQ, NULL); + } + + /* case 4: shrinking doesn't happen. */ + smartlist_free(sl); + sl = smartlist_new(); + smartlist_add(sl, (char*)s[0]); + smartlist_add(sl, (char*)s[1]); + smartlist_add(sl, (char*)s[2]); + smartlist_grow(sl, 1); + tt_int_op(3, OP_EQ, smartlist_len(sl)); + for (i = 0; i < 3; ++i) { + tt_ptr_op(smartlist_get(sl, i), OP_EQ, s[i]); + } + + done: + smartlist_free(sl); +} + /** Run unit tests for bitarray code */ static void test_container_bitarray(void *arg) @@ -1312,6 +1372,7 @@ struct testcase_t container_tests[] = { CONTAINER_LEGACY(smartlist_pos), CONTAINER(smartlist_remove, 0), CONTAINER(smartlist_ints_eq, 0), + CONTAINER(smartlist_grow, 0), CONTAINER_LEGACY(bitarray), CONTAINER_LEGACY(digestset), CONTAINER_LEGACY(strmap), From a62ac1719887f0756ceb516ce3b12cd2aee18191 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Sun, 13 Jan 2019 15:45:06 -0500 Subject: [PATCH 0646/2557] Add a new inline function to check whether debug logging is on We already do this in our log_debug() macro, but there are times when we'd like to avoid allocating or precomputing something that we are only going to log if debugging is on. --- src/lib/log/log.h | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/src/lib/log/log.h b/src/lib/log/log.h index 45e01f3391..fda984e78f 100644 --- a/src/lib/log/log.h +++ b/src/lib/log/log.h @@ -193,6 +193,15 @@ void tor_log_get_logfile_names(struct smartlist_t *out); extern int log_global_min_severity_; +static inline bool debug_logging_enabled(void); +/** + * Return true iff debug logging is enabled for at least one domain. + */ +static inline bool debug_logging_enabled(void) +{ + return PREDICT_UNLIKELY(log_global_min_severity_ == LOG_DEBUG); +} + void log_fn_(int severity, log_domain_mask_t domain, const char *funcname, const char *format, ...) CHECK_PRINTF(4,5); @@ -222,8 +231,8 @@ void tor_log_string(int severity, log_domain_mask_t domain, log_fn_ratelim_(ratelim, severity, domain, __FUNCTION__, args) #define log_debug(domain, args...) \ STMT_BEGIN \ - if (PREDICT_UNLIKELY(log_global_min_severity_ == LOG_DEBUG)) \ - log_fn_(LOG_DEBUG, domain, __FUNCTION__, args); \ + if (debug_logging_enabled()) \ + log_fn_(LOG_DEBUG, domain, __FUNCTION__, args); \ STMT_END #define log_info(domain, args...) \ log_fn_(LOG_INFO, domain, __FUNCTION__, args) @@ -240,8 +249,8 @@ void tor_log_string(int severity, log_domain_mask_t domain, #define log_debug(domain, args, ...) \ STMT_BEGIN \ - if (PREDICT_UNLIKELY(log_global_min_severity_ == LOG_DEBUG)) \ - log_fn_(LOG_DEBUG, domain, __FUNCTION__, args, ##__VA_ARGS__); \ + if (debug_logging_enabled()) \ + log_fn_(LOG_DEBUG, domain, __FUNCTION__, args, ##__VA_ARGS__); \ STMT_END #define log_info(domain, args,...) \ log_fn_(LOG_INFO, domain, __FUNCTION__, args, ##__VA_ARGS__) From e4d3098d4d23686320013b80b6305fbd52863f76 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 11 Jan 2019 20:17:04 -0500 Subject: [PATCH 0647/2557] Low-level dispatch module for publish-subscribe mechanism This module implements a way to send messages from one module to another, with associated data types. It does not yet do anything to ensure that messages are correct, that types match, or that other forms of consistency are preserved. --- .gitignore | 2 + Makefile.am | 2 + src/include.am | 1 + src/lib/dispatch/.may_include | 9 ++ src/lib/dispatch/dispatch.h | 114 ++++++++++++++ src/lib/dispatch/dispatch_cfg.c | 138 +++++++++++++++++ src/lib/dispatch/dispatch_cfg.h | 39 +++++ src/lib/dispatch/dispatch_cfg_st.h | 25 +++ src/lib/dispatch/dispatch_core.c | 234 +++++++++++++++++++++++++++++ src/lib/dispatch/dispatch_new.c | 170 +++++++++++++++++++++ src/lib/dispatch/dispatch_st.h | 108 +++++++++++++ src/lib/dispatch/include.am | 23 +++ src/lib/dispatch/msgtypes.h | 80 ++++++++++ src/test/include.am | 1 + src/test/test.c | 1 + src/test/test.h | 1 + src/test/test_dispatch.c | 181 ++++++++++++++++++++++ 17 files changed, 1129 insertions(+) create mode 100644 src/lib/dispatch/.may_include create mode 100644 src/lib/dispatch/dispatch.h create mode 100644 src/lib/dispatch/dispatch_cfg.c create mode 100644 src/lib/dispatch/dispatch_cfg.h create mode 100644 src/lib/dispatch/dispatch_cfg_st.h create mode 100644 src/lib/dispatch/dispatch_core.c create mode 100644 src/lib/dispatch/dispatch_new.c create mode 100644 src/lib/dispatch/dispatch_st.h create mode 100644 src/lib/dispatch/include.am create mode 100644 src/lib/dispatch/msgtypes.h create mode 100644 src/test/test_dispatch.c diff --git a/.gitignore b/.gitignore index 6a49285b8a..f4f6dacbb0 100644 --- a/.gitignore +++ b/.gitignore @@ -168,6 +168,8 @@ uptime-*.json /src/lib/libtor-crypt-ops-testing.a /src/lib/libtor-ctime.a /src/lib/libtor-ctime-testing.a +/src/lib/libtor-dispatch.a +/src/lib/libtor-dispatch-testing.a /src/lib/libtor-encoding.a /src/lib/libtor-encoding-testing.a /src/lib/libtor-evloop.a diff --git a/Makefile.am b/Makefile.am index a5086b3035..36d9725f38 100644 --- a/Makefile.am +++ b/Makefile.am @@ -41,6 +41,7 @@ TOR_UTIL_LIBS = \ src/lib/libtor-geoip.a \ src/lib/libtor-process.a \ src/lib/libtor-buf.a \ + src/lib/libtor-dispatch.a \ src/lib/libtor-time.a \ src/lib/libtor-fs.a \ src/lib/libtor-encoding.a \ @@ -72,6 +73,7 @@ TOR_UTIL_TESTING_LIBS = \ src/lib/libtor-geoip-testing.a \ src/lib/libtor-process-testing.a \ src/lib/libtor-buf-testing.a \ + src/lib/libtor-dispatch-testing.a \ src/lib/libtor-time-testing.a \ src/lib/libtor-fs-testing.a \ src/lib/libtor-encoding-testing.a \ diff --git a/src/include.am b/src/include.am index 9070a69a03..c6c351c806 100644 --- a/src/include.am +++ b/src/include.am @@ -8,6 +8,7 @@ include src/lib/compress/include.am include src/lib/container/include.am include src/lib/crypt_ops/include.am include src/lib/defs/include.am +include src/lib/dispatch/include.am include src/lib/encoding/include.am include src/lib/evloop/include.am include src/lib/fdio/include.am diff --git a/src/lib/dispatch/.may_include b/src/lib/dispatch/.may_include new file mode 100644 index 0000000000..9b5373907e --- /dev/null +++ b/src/lib/dispatch/.may_include @@ -0,0 +1,9 @@ +orconfig.h + +ext/tor_queue.h + +lib/container/*.h +lib/dispatch/*.h +lib/intmath/*.h +lib/log/*.h +lib/malloc/*.h diff --git a/src/lib/dispatch/dispatch.h b/src/lib/dispatch/dispatch.h new file mode 100644 index 0000000000..8e62e8f168 --- /dev/null +++ b/src/lib/dispatch/dispatch.h @@ -0,0 +1,114 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef TOR_DISPATCH_H +#define TOR_DISPATCH_H + +#include "lib/dispatch/msgtypes.h" + +/** + * \file dispatch.h + * \brief Low-level APIs for message-passing system. + * + * This module implements message dispatch based on a set of short integer + * identifiers. For a higher-level interface, see pubsub.h. + * + * Each message is represented as a generic msg_t object, and is discriminated + * by its message_id_t. Messages are delivered by a dispatch_t object, which + * delivers each message to its recipients by a configured "channel". + * + * A "channel" is a means of delivering messages. Every message_id_t must + * be associated with exactly one channel, identified by channel_id_t. + * When a channel receives messages, a callback is invoked to either process + * the messages immediately, or to cause them to be processed later. + * + * Every message_id_t has zero or more associated receiver functions set up in + * the dispatch_t object. Once the dispatch_t object is created, receivers + * can be enabled or disabled [TODO], but not added or removed. + * + * Every message_id_t has an associated datatype, identified by a + * msg_type_id_t. These datatypes can be associated with functions to + * (for example) free them, or format them for debugging. + * + * To setup a dispatch_t object, first create a dispatch_cfg_t object, and + * configure messages with their types, channels, and receivers. Then, use + * dispatch_new() with that dispatch_cfg_t to create the dispatch_t object. + * + * (We use a two-phase contruction procedure here to enable better static + * reasoning about publish/subscribe relationships.) + * + * Once you have a dispatch_t, you can queue messages on it with + * dispatch_send*(), and cause those messages to be delivered with + * dispatch_flush(). + **/ + +/** + * A "dispatcher" is the highest-level object; it handles making sure that + * messages are received and delivered properly. Only the mainloop + * should handle this type directly. + */ +typedef struct dispatch_t dispatch_t; + +struct dispatch_cfg_t; + +dispatch_t *dispatch_new(const struct dispatch_cfg_t *cfg); + +/** + * Free a dispatcher. Tor does this at exit. + */ +#define dispatch_free(d) \ + FREE_AND_NULL(dispatch_t, dispatch_free_, (d)) + +void dispatch_free_(dispatch_t *); + +int dispatch_send(dispatch_t *d, + subsys_id_t sender, + channel_id_t channel, + message_id_t msg, + msg_type_id_t type, + msg_aux_data_t auxdata); + +int dispatch_send_msg(dispatch_t *d, msg_t *m); + +int dispatch_send_msg_unchecked(dispatch_t *d, msg_t *m); + +/* Flush up to max_msgs currently pending messages from the + * dispatcher. Messages that are not pending when this function are + * called, are not flushed by this call. Return 0 on success, -1 on + * unrecoverable error. + */ +int dispatch_flush(dispatch_t *, channel_id_t chan, int max_msgs); + +/** + * Function callback type used to alert some other module when a channel's + * queue changes from empty to nonempty. + * + * Ex 1: To cause messages to be processed immediately on-stack, this callback + * should invoke dispatch_flush() directly. + * + * Ex 2: To cause messages to be processed very soon, from the event queue, + * this callback should schedule an event callback to run dispatch_flush(). + * + * Ex 3: To cause messages to be processed periodically, this function should + * do nothing, and a periodic event should invoke dispatch_flush(). + **/ +typedef void (*dispatch_alertfn_t)(struct dispatch_t *, + channel_id_t, void *); + +int dispatch_set_alert_fn(dispatch_t *d, channel_id_t chan, + dispatch_alertfn_t fn, void *userdata); + +#define dispatch_free_msg(d,msg) \ + STMT_BEGIN { \ + msg_t **msg_tmp_ptr__ = &(msg); \ + dispatch_free_msg_((d), *msg_tmp_ptr__); \ + *msg_tmp_ptr__= NULL; \ + } STMT_END +void dispatch_free_msg_(const dispatch_t *d, msg_t *msg); + +char *dispatch_fmt_msg_data(const dispatch_t *d, const msg_t *msg); + +#endif diff --git a/src/lib/dispatch/dispatch_cfg.c b/src/lib/dispatch/dispatch_cfg.c new file mode 100644 index 0000000000..26e37f4694 --- /dev/null +++ b/src/lib/dispatch/dispatch_cfg.c @@ -0,0 +1,138 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file dispatch_cfg.c + * \brief Create and configure a dispatch_cfg_t. + * + * A dispatch_cfg_t object is used to configure a set of messages and + * associated information before creating a dispatch_t. + */ + +#define DISPATCH_PRIVATE + +#include "orconfig.h" +#include "lib/dispatch/dispatch_cfg.h" +#include "lib/dispatch/dispatch_cfg_st.h" +#include "lib/dispatch/dispatch.h" +#include "lib/dispatch/dispatch_st.h" + +#include "lib/container/smartlist.h" +#include "lib/malloc/malloc.h" + +/** + * Create and return a new dispatch_cfg_t. + **/ +dispatch_cfg_t * +dcfg_new(void) +{ + dispatch_cfg_t *cfg = tor_malloc(sizeof(dispatch_cfg_t)); + cfg->type_by_msg = smartlist_new(); + cfg->chan_by_msg = smartlist_new(); + cfg->fns_by_type = smartlist_new(); + cfg->recv_by_msg = smartlist_new(); + return cfg; +} + +/** DOCDOC */ +#define ID_TO_VOID(id) ((void*)(uintptr_t)(id)) + +/** + * Associate a message with a datatype. Return 0 on success, -1 if a + * different type was previously associated with the message ID. + **/ +int +dcfg_msg_set_type(dispatch_cfg_t *cfg, message_id_t msg, + msg_type_id_t type) +{ + smartlist_grow(cfg->type_by_msg, msg+1); + void *oldval = smartlist_get(cfg->type_by_msg, msg); + if (oldval != NULL && oldval != ID_TO_VOID(type)) { + return -1; + } + smartlist_set(cfg->type_by_msg, msg, ID_TO_VOID(type)); + return 0; +} + +/** + * Associate a message with a channel. Return 0 on success, -1 if a + * different channel was previously associated with the message ID. + **/ +int +dcfg_msg_set_chan(dispatch_cfg_t *cfg, message_id_t msg, + channel_id_t chan) +{ + smartlist_grow(cfg->chan_by_msg, msg+1); + void *oldval = smartlist_get(cfg->chan_by_msg, msg); + if (oldval != NULL && oldval != ID_TO_VOID(chan)) { + return -1; + } + smartlist_set(cfg->chan_by_msg, msg, ID_TO_VOID(chan)); + return 0; +} + +/** + * Associate a set of functions with a datatype. Return 0 on success, -1 if + * different functions were previously associated with the type. + **/ +int +dcfg_type_set_fns(dispatch_cfg_t *cfg, msg_type_id_t type, + const dispatch_typefns_t *fns) +{ + smartlist_grow(cfg->fns_by_type, type+1); + dispatch_typefns_t *oldfns = smartlist_get(cfg->fns_by_type, type); + if (oldfns && (oldfns->free_fn != fns->free_fn || + oldfns->fmt_fn != fns->fmt_fn)) + return -1; + smartlist_set(cfg->fns_by_type, type, (dispatch_typefns_t *) fns); + return 0; +} + +/** + * Associate a receiver with a message ID. Multiple receivers may be + * associated with a single messasge ID. + * + * Return 0 on success, on failure. + **/ +int +dcfg_add_recv(dispatch_cfg_t *cfg, message_id_t msg, + subsys_id_t sys, recv_fn_t fn) +{ + smartlist_grow(cfg->recv_by_msg, msg+1); + smartlist_t *receivers = smartlist_get(cfg->recv_by_msg, msg); + if (!receivers) { + receivers = smartlist_new(); + smartlist_set(cfg->recv_by_msg, msg, receivers); + } + + dispatch_rcv_t *rcv = tor_malloc(sizeof(dispatch_rcv_t)); + rcv->sys = sys; + rcv->enabled = true; + rcv->fn = fn; + smartlist_add(receivers, (void*)rcv); + return 0; +} + +/** Helper: release all storage held by cfg. */ +void +dcfg_free_(dispatch_cfg_t *cfg) +{ + if (!cfg) + return; + + smartlist_free(cfg->type_by_msg); + smartlist_free(cfg->chan_by_msg); + smartlist_free(cfg->fns_by_type); + SMARTLIST_FOREACH_BEGIN(cfg->recv_by_msg, smartlist_t *, receivers) { + if (!receivers) + continue; + SMARTLIST_FOREACH(receivers, dispatch_rcv_t *, rcv, tor_free(rcv)); + smartlist_free(receivers); + } SMARTLIST_FOREACH_END(receivers); + smartlist_free(cfg->recv_by_msg); + + tor_free(cfg); +} diff --git a/src/lib/dispatch/dispatch_cfg.h b/src/lib/dispatch/dispatch_cfg.h new file mode 100644 index 0000000000..2c755e39bc --- /dev/null +++ b/src/lib/dispatch/dispatch_cfg.h @@ -0,0 +1,39 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef TOR_DISPATCH_CFG_H +#define TOR_DISPATCH_CFG_H + +#include "lib/dispatch/msgtypes.h" + +/** + * A "dispatch_cfg" is the configuration used to set up a dispatcher. + * It is created and accessed with a set of dcfg_* functions, and then + * used with dispatcher_new() to make the dispatcher. + */ +typedef struct dispatch_cfg_t dispatch_cfg_t; + +dispatch_cfg_t *dcfg_new(void); + +int dcfg_msg_set_type(dispatch_cfg_t *cfg, message_id_t msg, + msg_type_id_t type); + +int dcfg_msg_set_chan(dispatch_cfg_t *cfg, message_id_t msg, + channel_id_t chan); + +int dcfg_type_set_fns(dispatch_cfg_t *cfg, msg_type_id_t type, + const dispatch_typefns_t *fns); + +int dcfg_add_recv(dispatch_cfg_t *cfg, message_id_t msg, + subsys_id_t sys, recv_fn_t fn); + +/** Free a dispatch_cfg_t. */ +#define dcfg_free(cfg) \ + FREE_AND_NULL(dispatch_cfg_t, dcfg_free_, (cfg)) + +void dcfg_free_(dispatch_cfg_t *cfg); + +#endif diff --git a/src/lib/dispatch/dispatch_cfg_st.h b/src/lib/dispatch/dispatch_cfg_st.h new file mode 100644 index 0000000000..d004fe5934 --- /dev/null +++ b/src/lib/dispatch/dispatch_cfg_st.h @@ -0,0 +1,25 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef TOR_DISPATCH_CFG_ST_H +#define TOR_DISPATCH_CFG_ST_H + +struct smartlist_t; + +/* Information needed to create a dispatcher, but in a less efficient, more + * mutable format. */ +struct dispatch_cfg_t { + /** A list of msg_type_id_t (cast to void*), indexed by msg_t. */ + struct smartlist_t *type_by_msg; + /** A list of channel_id_t (cast to void*), indexed by msg_t. */ + struct smartlist_t *chan_by_msg; + /** A list of dispatch_rcv_t, indexed by msg_type_id_t. */ + struct smartlist_t *fns_by_type; + /** A list of dispatch_typefns_t, indexed by msg_t. */ + struct smartlist_t *recv_by_msg; +}; + +#endif diff --git a/src/lib/dispatch/dispatch_core.c b/src/lib/dispatch/dispatch_core.c new file mode 100644 index 0000000000..24dfc649a1 --- /dev/null +++ b/src/lib/dispatch/dispatch_core.c @@ -0,0 +1,234 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file dispatch_core.c + * \brief Core module for sending and receiving messages. + */ + +#define DISPATCH_PRIVATE +#include "orconfig.h" + +#include "lib/dispatch/dispatch.h" +#include "lib/dispatch/dispatch_st.h" + +#include "lib/malloc/malloc.h" +#include "lib/log/util_bug.h" + +#include + +/** + * Use d to drop all storage held for msg. + * + * (We need the dispatcher so we know how to free the auxiliary data.) + **/ +void +dispatch_free_msg_(const dispatch_t *d, msg_t *msg) +{ + if (!msg) + return; + + d->typefns[msg->type].free_fn(msg->aux_data__); + tor_free(msg); +} + +/** + * Format the auxiliary data held by msg. + **/ +char * +dispatch_fmt_msg_data(const dispatch_t *d, const msg_t *msg) +{ + if (!msg) + return NULL; + + return d->typefns[msg->type].fmt_fn(msg->aux_data__); +} + +/** + * Release all storage held by d. + **/ +void +dispatch_free_(dispatch_t *d) +{ + if (d == NULL) + return; + + size_t n_queues = d->n_queues; + for (size_t i = 0; i < n_queues; ++i) { + msg_t *m, *mtmp; + TOR_SIMPLEQ_FOREACH_SAFE(m, &d->queues[i].queue, next, mtmp) { + dispatch_free_msg(d, m); + } + } + + size_t n_msgs = d->n_msgs; + + for (size_t i = 0; i < n_msgs; ++i) { + tor_free(d->table[i]); + } + tor_free(d->table); + tor_free(d->typefns); + tor_free(d->queues); + + // This is the only time we will treat d->cfg as non-const. + //dispatch_cfg_free_((dispatch_items_t *) d->cfg); + + tor_free(d); +} + +/** + * Tell the dispatcher to call fn with userdata whenever + * chan becomes nonempty. Return 0 on success, -1 on error. + **/ +int +dispatch_set_alert_fn(dispatch_t *d, channel_id_t chan, + dispatch_alertfn_t fn, void *userdata) +{ + if (BUG(chan >= d->n_queues)) + return -1; + + dqueue_t *q = &d->queues[chan]; + q->alert_fn = fn; + q->alert_fn_arg = userdata; + return 0; +} + +/** + * Send a message on the appropriate channel notifying that channel if + * necessary. + * + * This function takes ownership of the auxiliary data; it can't be static or + * stack-allocated, and the caller is not allowed to use it afterwards. + * + * This function does not check the various vields of the message object for + * consistency. + **/ +int +dispatch_send(dispatch_t *d, + subsys_id_t sender, + channel_id_t channel, + message_id_t msg, + msg_type_id_t type, + msg_aux_data_t auxdata) +{ + if (!d->table[msg]) { + /* Fast path: nobody wants this data. */ + + d->typefns[type].free_fn(auxdata); + return 0; + } + + msg_t *m = tor_malloc(sizeof(msg_t)); + + m->sender = sender; + m->channel = channel; + m->msg = msg; + m->type = type; + memcpy(&m->aux_data__, &auxdata, sizeof(msg_aux_data_t)); + + return dispatch_send_msg(d, m); +} + +int +dispatch_send_msg(dispatch_t *d, msg_t *m) +{ + if (BUG(!d)) + goto err; + if (BUG(!m)) + goto err; + if (BUG(m->channel >= d->n_queues)) + goto err; + if (BUG(m->msg >= d->n_msgs)) + goto err; + + dtbl_entry_t *ent = d->table[m->msg]; + if (ent) { + if (BUG(m->type != ent->type)) + goto err; + if (BUG(m->channel != ent->channel)) + goto err; + } + + return dispatch_send_msg_unchecked(d, m); + err: + /* Probably it isn't safe to free m, since type could be wrong. */ + return -1; +} + +/** + * Send a message on the appropriate queue, notifying that queue if necessary. + * + * This function takes ownership of the message object and its auxiliary data; + * it can't be static or stack-allocated, and the caller isn't allowed to use + * it afterwards. + * + * This function does not check the various fields of the message object for + * consistency, and can crash if they are out of range. Only functions that + * have already constructed the message in a safe way, or checked it for + * correctness themselves, should call this function. + **/ +int +dispatch_send_msg_unchecked(dispatch_t *d, msg_t *m) +{ + /* Find the right queue. */ + dqueue_t *q = &d->queues[m->channel]; + bool was_empty = TOR_SIMPLEQ_EMPTY(&q->queue); + + /* Append the message. */ + TOR_SIMPLEQ_INSERT_TAIL(&q->queue, m, next); + + /* If we just made the queue nonempty for the first time, call the alert + * function. */ + if (was_empty) { + q->alert_fn(d, m->channel, q->alert_fn_arg); + } + + return 0; +} + +/** + * Run all of the callbacks on d associated with m. + **/ +static void +dispatcher_run_msg_cbs(const dispatch_t *d, msg_t *m) +{ + tor_assert(m->msg <= d->n_msgs); + dtbl_entry_t *ent = d->table[m->msg]; + int n_fns = ent->n_fns; + + int i; + for (i=0; i < n_fns; ++i) { + if (ent->rcv[i].enabled) + ent->rcv[i].fn(m); + } +} + +/** + * Run up to max_msgs callbacks for messages on the channel ch + * on the given dispatcher. Return 0 on success or recoverable failure, + * -1 on unrecoverable error. + **/ +int +dispatch_flush(dispatch_t *d, channel_id_t ch, int max_msgs) +{ + if (BUG(ch >= d->n_queues)) + return 0; + + int n_flushed = 0; + dqueue_t *q = &d->queues[ch]; + + while (n_flushed < max_msgs) { + msg_t *m = TOR_SIMPLEQ_FIRST(&q->queue); + if (!m) + break; + TOR_SIMPLEQ_REMOVE_HEAD(&q->queue, next); + dispatcher_run_msg_cbs(d, m); + dispatch_free_msg(d, m); + ++n_flushed; + } + + return 0; +} diff --git a/src/lib/dispatch/dispatch_new.c b/src/lib/dispatch/dispatch_new.c new file mode 100644 index 0000000000..a2879016a7 --- /dev/null +++ b/src/lib/dispatch/dispatch_new.c @@ -0,0 +1,170 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file dispatch_new.c + * \brief Code to construct a dispatch_t from a dispatch_cfg_t. + **/ + +#define DISPATCH_PRIVATE +#include "orconfig.h" + +#include "lib/dispatch/dispatch.h" +#include "lib/dispatch/dispatch_st.h" +#include "lib/dispatch/dispatch_cfg.h" +#include "lib/dispatch/dispatch_cfg_st.h" + +#include "lib/intmath/cmp.h" +#include "lib/malloc/malloc.h" +#include "lib/log/util_bug.h" + +#include + +/** Convert a void* in a smartlist to the corresponding integer. */ +#define VOID_TO_ID(p) ((intptr_t)(p)) + +/** Given a smartlist full of void* fields encoding intptr_t values, + * return the largest intptr_t, or dflt if the list is empty. */ +static intptr_t +max_in_sl(const smartlist_t *sl, intptr_t dflt) +{ + if (!smartlist_len(sl)) + return dflt; + void *as_ptr = smartlist_get(sl, 0); + intptr_t max = VOID_TO_ID(as_ptr); + SMARTLIST_FOREACH_BEGIN(sl, void *, p) { + intptr_t i = VOID_TO_ID(p); + if (i > max) + max = i; + } SMARTLIST_FOREACH_END(p); + return max; +} + +/** Helper: Format an unformattable message auxiliary data item: just return a +* copy of the string <>. */ +static char * +type_fmt_nop(msg_aux_data_t arg) +{ + (void)arg; + return tor_strdup("<>"); +} + +/** Helper: Free an unfreeable message auxiliary data item: do nothing. */ +static void +type_free_nop(msg_aux_data_t arg) +{ + (void)arg; +} + +/** Type functions to use when no type functions are provided. */ +static dispatch_typefns_t nop_typefns = { + .free_fn = type_free_nop, + .fmt_fn = type_fmt_nop +}; + +/** + * Alert function to use when none is configured: do nothing. + **/ +static void +alert_fn_nop(dispatch_t *d, channel_id_t ch, void *arg) +{ + (void)d; + (void)ch; + (void)arg; +} + +/** + * Given a list of recvfn_t, create and return a new dtbl_entry_t mapping + * to each of those functions. + **/ +static dtbl_entry_t * +dtbl_entry_from_lst(smartlist_t *receivers) +{ + if (!receivers) + return NULL; + + size_t n_recv = smartlist_len(receivers); + dtbl_entry_t *ent; + ent = tor_malloc_zero(offsetof(dtbl_entry_t, rcv) + + sizeof(dispatch_rcv_t) * n_recv); + + ent->n_fns = n_recv; + + SMARTLIST_FOREACH_BEGIN(receivers, const dispatch_rcv_t *, rcv) { + memcpy(&ent->rcv[rcv_sl_idx], rcv, sizeof(*rcv)); + if (rcv->enabled) { + ++ent->n_enabled; + } + } SMARTLIST_FOREACH_END(rcv); + + return ent; +} + +/** Create and return a new dispatcher from a given dispatch_cfg_t. */ +dispatch_t * +dispatch_new(const dispatch_cfg_t *cfg) +{ + dispatch_t *d = tor_malloc_zero(sizeof(dispatch_t)); + + /* Any message that has a type or a receiver counts towards our messages */ + const size_t n_msgs = MAX(smartlist_len(cfg->type_by_msg), + smartlist_len(cfg->recv_by_msg)) + 1; + + /* Any channel that any message has counts towards the number of channels. */ + const size_t n_chans = (size_t) MAX(1, max_in_sl(cfg->chan_by_msg,0)) + 1; + + /* Any type that a message has, or that has functions, counts towards + * the number of types. */ + const size_t n_types = (size_t) MAX(max_in_sl(cfg->type_by_msg,0), + smartlist_len(cfg->fns_by_type)) + 1; + + d->n_msgs = n_msgs; + d->n_queues = n_chans; + d->n_types = n_types; + + /* Initialize the array of type-functions. */ + d->typefns = tor_calloc(n_types, sizeof(dispatch_typefns_t)); + for (size_t i = 0; i < n_types; ++i) { + /* Default to no-op for everything... */ + memcpy(&d->typefns[i], &nop_typefns, sizeof(dispatch_typefns_t)); + } + SMARTLIST_FOREACH_BEGIN(cfg->fns_by_type, dispatch_typefns_t *, fns) { + /* Set the functions if they are provided. */ + if (fns) { + if (fns->free_fn) + d->typefns[fns_sl_idx].free_fn = fns->free_fn; + if (fns->fmt_fn) + d->typefns[fns_sl_idx].fmt_fn = fns->fmt_fn; + } + } SMARTLIST_FOREACH_END(fns); + + /* Initialize the message queues: one for each channel. */ + d->queues = tor_calloc(d->n_queues, sizeof(dqueue_t)); + for (size_t i = 0; i < d->n_queues; ++i) { + TOR_SIMPLEQ_INIT(&d->queues[i].queue); + d->queues[i].alert_fn = alert_fn_nop; + } + + /* Build the dispatch tables mapping message IDs to receivers. */ + d->table = tor_calloc(d->n_msgs, sizeof(dtbl_entry_t *)); + SMARTLIST_FOREACH_BEGIN(cfg->recv_by_msg, smartlist_t *, rcv) { + d->table[rcv_sl_idx] = dtbl_entry_from_lst(rcv); + } SMARTLIST_FOREACH_END(rcv); + + /* Fill in the empty entries in the dispatch tables: + * types and channels for each message. */ + SMARTLIST_FOREACH_BEGIN(cfg->type_by_msg, smartlist_t *, type) { + if (d->table[type_sl_idx]) + d->table[type_sl_idx]->type = VOID_TO_ID(type); + } SMARTLIST_FOREACH_END(type); + + SMARTLIST_FOREACH_BEGIN(cfg->chan_by_msg, smartlist_t *, chan) { + if (d->table[chan_sl_idx]) + d->table[chan_sl_idx]->channel = VOID_TO_ID(chan); + } SMARTLIST_FOREACH_END(chan); + + return d; +} diff --git a/src/lib/dispatch/dispatch_st.h b/src/lib/dispatch/dispatch_st.h new file mode 100644 index 0000000000..568107b700 --- /dev/null +++ b/src/lib/dispatch/dispatch_st.h @@ -0,0 +1,108 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file dispatch_st.h + * + * \brief private structures used for the dispatcher module + */ + +#ifndef TOR_DISPATCH_ST_H +#define TOR_DISPATCH_ST_H + +#ifdef DISPATCH_PRIVATE + +#include "lib/container/smartlist.h" + +/** + * Information about the recipient of a message. + **/ +typedef struct dispatch_rcv_t { + /** The subsystem receiving a message. */ + subsys_id_t sys; + /** True iff this recipient is enabled. */ + bool enabled; + /** The function that will handle the message. */ + recv_fn_t fn; +} dispatch_rcv_t; + +/** + * Information used by a dispatcher to handle and dispatch a single message + * ID. It maps that message ID to its type, channel, and list of receiver + * functions. + * + * This structure is used when the dispatcher is running. + **/ +typedef struct dtbl_entry_t { + /** The number of enabled non-stub subscribers for this message. + * + * Note that for now, this will be the same as n_fns, since there is + * no way to turn these subscribers on an off yet. */ + uint16_t n_enabled; + /** The channel that handles this message. */ + channel_id_t channel; + /** The associated C type for this message. */ + msg_type_id_t type; + /** + * The number of functions pointers for subscribers that receive this + * message, in rcv. */ + uint16_t n_fns; + /** + * The recipients for this message. + */ + dispatch_rcv_t rcv[FLEXIBLE_ARRAY_MEMBER]; +} dtbl_entry_t; + +/** + * A queue of messages for a given channel, used by a live dispatcher. + */ +typedef struct dqueue_t { + /** The queue of messages itself. */ + TOR_SIMPLEQ_HEAD( , msg_t) queue; + /** A function to be called when the queue becomes nonempty. */ + dispatch_alertfn_t alert_fn; + /** An argument for the alert_fn. */ + void *alert_fn_arg; +} dqueue_t ; + +/** + * A single dispatcher for cross-module messages. + */ +struct dispatch_t { + /** + * The length of table: the number of message IDs that this + * dispatcher can handle. + */ + size_t n_msgs; + /** + * The length of queues: the number of channels that this dispatcher + * has configured. + */ + size_t n_queues; + /** + * The length of typefns: the number of C type IDs that this + * dispatcher has configured. + */ + size_t n_types; + /** + * An array of message queues, indexed by channel ID. + */ + dqueue_t *queues; + /** + * An array of entries about how to handle particular message types, indexed + * by message ID. + */ + dtbl_entry_t **table; + /** + * An array of function tables for manipulating types, index by message + * type ID. + **/ + dispatch_typefns_t *typefns; +}; + +#endif + +#endif diff --git a/src/lib/dispatch/include.am b/src/lib/dispatch/include.am new file mode 100644 index 0000000000..c4aa170dbc --- /dev/null +++ b/src/lib/dispatch/include.am @@ -0,0 +1,23 @@ + +noinst_LIBRARIES += src/lib/libtor-dispatch.a + +if UNITTESTS_ENABLED +noinst_LIBRARIES += src/lib/libtor-dispatch-testing.a +endif + +src_lib_libtor_dispatch_a_SOURCES = \ + src/lib/dispatch/dispatch_cfg.c \ + src/lib/dispatch/dispatch_core.c \ + src/lib/dispatch/dispatch_new.c + +src_lib_libtor_dispatch_testing_a_SOURCES = \ + $(src_lib_libtor_dispatch_a_SOURCES) +src_lib_libtor_dispatch_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) +src_lib_libtor_dispatch_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) + +noinst_HEADERS += \ + src/lib/dispatch/dispatch.h \ + src/lib/dispatch/dispatch_cfg.h \ + src/lib/dispatch/dispatch_cfg_st.h \ + src/lib/dispatch/dispatch_st.h \ + src/lib/dispatch/msgtypes.h diff --git a/src/lib/dispatch/msgtypes.h b/src/lib/dispatch/msgtypes.h new file mode 100644 index 0000000000..4e79e592a6 --- /dev/null +++ b/src/lib/dispatch/msgtypes.h @@ -0,0 +1,80 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file msgtypes.h + * \brief Types used for messages in the dispatcher code. + **/ + +#ifndef TOR_DISPATCH_MSGTYPES_H +#define TOR_DISPATCH_MSGTYPES_H + +#include + +#include "ext/tor_queue.h" + +/** + * These types are aliases for subsystems, channels, and message IDs. + **/ +typedef uint16_t subsys_id_t; +typedef uint16_t channel_id_t; +typedef uint16_t message_id_t; + +/** + * This identifies a C type that can be sent along with a message. + **/ +typedef uint16_t msg_type_id_t; + +/** + * An ID value returned for *_type_t when none exists. + */ +#define ERROR_ID 65535 + +/** + * Auxiliary (untyped) data sent along with a message. + * + * We define this as a union of a pointer and a u64, so that the integer + * types will have the same range across platforms. + **/ +typedef union { + void *ptr; + uint64_t u64; +} msg_aux_data_t; + +/** + * Structure of a received message. + **/ +typedef struct msg_t { + TOR_SIMPLEQ_ENTRY(msg_t) next; + subsys_id_t sender; + channel_id_t channel; + message_id_t msg; + /** We could omit this field, since it is implicit in the message type, but + * IMO let's leave it in for safety. */ + msg_type_id_t type; + /** Untyped auxiliary data. You shouldn't have to mess with this + * directly. */ + msg_aux_data_t aux_data__; +} msg_t; + +/** + * A function that a subscriber uses to receive a message. + **/ +typedef void (*recv_fn_t)(const msg_t *m); + +/** + * Table of functions to use for a given C type. Any omitted (NULL) functions + * will be treated as no-ops. + **/ +typedef struct dispatch_typefns_t { + /** Release storage held for the auxiliary data of this type. */ + void (*free_fn)(msg_aux_data_t); + /** Format and return a newly allocated string describing the contents + * of this data element. */ + char *(*fmt_fn)(msg_aux_data_t); +} dispatch_typefns_t; + +#endif diff --git a/src/test/include.am b/src/test/include.am index c3827d3eb9..d4e8eb4e8c 100644 --- a/src/test/include.am +++ b/src/test/include.am @@ -124,6 +124,7 @@ src_test_test_SOURCES += \ src/test/test_dir.c \ src/test/test_dir_common.c \ src/test/test_dir_handle_get.c \ + src/test/test_dispatch.c \ src/test/test_dos.c \ src/test/test_entryconn.c \ src/test/test_entrynodes.c \ diff --git a/src/test/test.c b/src/test/test.c index 1230b632a5..4c6d9775b4 100644 --- a/src/test/test.c +++ b/src/test/test.c @@ -872,6 +872,7 @@ struct testgroup_t testgroups[] = { { "dir/voting/flags/", voting_flags_tests }, { "dir/voting/schedule/", voting_schedule_tests }, { "dir_handle_get/", dir_handle_get_tests }, + { "dispatch/", dispatch_tests, }, { "dns/", dns_tests }, { "dos/", dos_tests }, { "entryconn/", entryconn_tests }, diff --git a/src/test/test.h b/src/test/test.h index 7a3a4d8fdc..cd0ce4f6df 100644 --- a/src/test/test.h +++ b/src/test/test.h @@ -209,6 +209,7 @@ extern struct testcase_t crypto_openssl_tests[]; extern struct testcase_t crypto_tests[]; extern struct testcase_t dir_handle_get_tests[]; extern struct testcase_t dir_tests[]; +extern struct testcase_t dispatch_tests[]; extern struct testcase_t dns_tests[]; extern struct testcase_t dos_tests[]; extern struct testcase_t entryconn_tests[]; diff --git a/src/test/test_dispatch.c b/src/test/test_dispatch.c new file mode 100644 index 0000000000..ec704c042a --- /dev/null +++ b/src/test/test_dispatch.c @@ -0,0 +1,181 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#define DISPATCH_PRIVATE + +#include "test/test.h" + +#include "lib/dispatch/dispatch.h" +#include "lib/dispatch/dispatch_cfg.h" +#include "lib/dispatch/dispatch_st.h" +#include "lib/dispatch/msgtypes.h" + +#include "lib/log/escape.h" +#include "lib/malloc/malloc.h" +#include "lib/string/printf.h" + +#include +#include + +/* Construct an empty dispatch_t. */ +static void +test_dispatch_empty(void *arg) +{ + (void)arg; + + dispatch_t *d=NULL; + dispatch_cfg_t *cfg=NULL; + + cfg = dcfg_new(); + d = dispatch_new(cfg); + tt_assert(d); + + done: + dispatch_free(d); + dcfg_free(cfg); +} + +static int total_recv1_simple = 0; +static int total_recv2_simple = 0; + +static void +simple_recv1(const msg_t *m) +{ + total_recv1_simple += m->aux_data__.u64; +} + +static void +simple_recv2(const msg_t *m) +{ + total_recv2_simple += m->aux_data__.u64*10; +} + +/* Construct a dispatch_t with two messages, make sure that they both get + * delivered. */ +static void +test_dispatch_simple(void *arg) +{ + (void)arg; + + dispatch_t *d=NULL; + dispatch_cfg_t *cfg=NULL; + int r; + + cfg = dcfg_new(); + r = dcfg_msg_set_type(cfg,0,0); + r += dcfg_msg_set_chan(cfg,0,0); + r += dcfg_add_recv(cfg,0,1,simple_recv1); + r += dcfg_msg_set_type(cfg,1,0); + r += dcfg_msg_set_chan(cfg,1,0); + r += dcfg_add_recv(cfg,1,1,simple_recv2); + r += dcfg_add_recv(cfg,1,1,simple_recv2); /* second copy */ + tt_int_op(r, OP_EQ, 0); + + d = dispatch_new(cfg); + tt_assert(d); + + msg_aux_data_t data = {.u64 = 7}; + r = dispatch_send(d, 99, 0, 0, 0, data); + tt_int_op(r, OP_EQ, 0); + tt_int_op(total_recv1_simple, OP_EQ, 0); + + r = dispatch_flush(d, 0, INT_MAX); + tt_int_op(r, OP_EQ, 0); + tt_int_op(total_recv1_simple, OP_EQ, 7); + tt_int_op(total_recv2_simple, OP_EQ, 0); + + total_recv1_simple = 0; + r = dispatch_send(d, 99, 0, 1, 0, data); + tt_int_op(r, OP_EQ, 0); + r = dispatch_flush(d, 0, INT_MAX); + tt_int_op(total_recv1_simple, OP_EQ, 0); + tt_int_op(total_recv2_simple, OP_EQ, 140); + + done: + dispatch_free(d); + dcfg_free(cfg); +} + +struct coord { int x; int y; }; +static void +free_coord(msg_aux_data_t d) +{ + tor_free(d.ptr); +} +static char * +fmt_coord(msg_aux_data_t d) +{ + char *v; + struct coord *c = d.ptr; + tor_asprintf(&v, "[%d, %d]", c->x, c->y); + return v; +} +static dispatch_typefns_t coord_fns = { + .fmt_fn = fmt_coord, + .free_fn = free_coord, +}; +static void +alert_run_immediate(dispatch_t *d, channel_id_t ch, void *arg) +{ + (void)arg; + dispatch_flush(d, ch, INT_MAX); +} + +static dispatch_t *dispatcher_in_use=NULL; +static char *received_data=NULL; + +static void +recv_typed_data(const msg_t *m) +{ + tor_free(received_data); + received_data = dispatch_fmt_msg_data(dispatcher_in_use, m); +} + +static void +test_dispatch_with_types(void *arg) +{ + (void)arg; + + dispatch_t *d=NULL; + dispatch_cfg_t *cfg=NULL; + int r; + + cfg = dcfg_new(); + r = dcfg_msg_set_type(cfg,5,3); + r += dcfg_msg_set_chan(cfg,5,2); + r += dcfg_add_recv(cfg,5,0,recv_typed_data); + r += dcfg_type_set_fns(cfg,3,&coord_fns); + tt_int_op(r, OP_EQ, 0); + + d = dispatch_new(cfg); + tt_assert(d); + dispatcher_in_use = d; + + /* Make this message get run immediately. */ + r = dispatch_set_alert_fn(d, 2, alert_run_immediate, NULL); + tt_int_op(r, OP_EQ, 0); + + struct coord *xy = tor_malloc(sizeof(*xy)); + xy->x = 13; + xy->y = 37; + msg_aux_data_t data = {.ptr = xy}; + r = dispatch_send(d, 99/*sender*/, 2/*channel*/, 5/*msg*/, 3/*type*/, data); + tt_int_op(r, OP_EQ, 0); + tt_str_op(received_data, OP_EQ, "[13, 37]"); + + done: + dispatch_free(d); + dcfg_free(cfg); + tor_free(received_data); + dispatcher_in_use = NULL; +} + +#define T(name) \ + { #name, test_dispatch_ ## name, TT_FORK, NULL, NULL } + +struct testcase_t dispatch_tests[] = { + T(empty), + T(simple), + T(with_types), + END_OF_TESTCASES +}; From f5683d90be693ecf0561fe90803f5a54c7ed264d Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Sun, 13 Jan 2019 14:43:13 -0500 Subject: [PATCH 0648/2557] Add a naming system for IDs in dispatch. --- src/lib/dispatch/.may_include | 1 + src/lib/dispatch/dispatch_naming.c | 63 ++++++++++++++++++++++++++++++ src/lib/dispatch/dispatch_naming.h | 46 ++++++++++++++++++++++ src/lib/dispatch/include.am | 2 + 4 files changed, 112 insertions(+) create mode 100644 src/lib/dispatch/dispatch_naming.c create mode 100644 src/lib/dispatch/dispatch_naming.h diff --git a/src/lib/dispatch/.may_include b/src/lib/dispatch/.may_include index 9b5373907e..7f2df5859f 100644 --- a/src/lib/dispatch/.may_include +++ b/src/lib/dispatch/.may_include @@ -2,6 +2,7 @@ orconfig.h ext/tor_queue.h +lib/cc/*.h lib/container/*.h lib/dispatch/*.h lib/intmath/*.h diff --git a/src/lib/dispatch/dispatch_naming.c b/src/lib/dispatch/dispatch_naming.c new file mode 100644 index 0000000000..83d9a2d604 --- /dev/null +++ b/src/lib/dispatch/dispatch_naming.c @@ -0,0 +1,63 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "orconfig.h" + +#include "lib/cc/compat_compiler.h" + +#include "lib/dispatch/dispatch_naming.h" +#include "lib/dispatch/msgtypes.h" + +#include "lib/container/namemap.h" +#include "lib/container/namemap_st.h" + +#include "lib/log/util_bug.h" +#include "lib/log/log.h" + +#include + +/** Global namemap for message IDs. */ +static namemap_t message_id_map = NAMEMAP_INIT(); +/** Global namemap for subsystem IDs. */ +static namemap_t subsys_id_map = NAMEMAP_INIT(); +/** Global namemap for channel IDs. */ +static namemap_t channel_id_map = NAMEMAP_INIT(); +/** Global namemap for message type IDs. */ +static namemap_t msg_type_id_map = NAMEMAP_INIT(); + +void +dispatch_naming_init(void) +{ +} + +/* Helper macro: declare functions to map IDs to and from names for a given + * type in a namemap_t. + */ +#define DECLARE_ID_MAP_FNS(type) \ + type##_id_t \ + get_##type##_id(const char *name) \ + { \ + unsigned u = namemap_get_or_create_id(&type##_id_map, name); \ + tor_assert(u != NAMEMAP_ERR); \ + tor_assert(u != ERROR_ID); \ + return (type##_id_t) u; \ + } \ + const char * \ + get_##type##_id_name(type##_id_t id) \ + { \ + return namemap_fmt_name(&type##_id_map, id); \ + } \ + size_t \ + get_num_##type##_ids(void) \ + { \ + return namemap_get_size(&type##_id_map); \ + } \ + EAT_SEMICOLON + +DECLARE_ID_MAP_FNS(message); +DECLARE_ID_MAP_FNS(channel); +DECLARE_ID_MAP_FNS(subsys); +DECLARE_ID_MAP_FNS(msg_type); diff --git a/src/lib/dispatch/dispatch_naming.h b/src/lib/dispatch/dispatch_naming.h new file mode 100644 index 0000000000..c116d2184d --- /dev/null +++ b/src/lib/dispatch/dispatch_naming.h @@ -0,0 +1,46 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef TOR_DISPATCH_NAMING_H +#define TOR_DISPATCH_NAMING_H + +#include "lib/dispatch/msgtypes.h" +#include + +/** + * Return an existing channel ID by name, allocating the channel ID if + * if necessary. Returns ERROR_ID if we have run out of + * channels + */ +channel_id_t get_channel_id(const char *); +/** + * Return the name corresponding to a given channel ID. + **/ +const char *get_channel_id_name(channel_id_t); +/** + * Return the total number of _named_ channel IDs. + **/ +size_t get_num_channel_ids(void); + +/* As above, but for messages. */ +message_id_t get_message_id(const char *); +const char *get_message_id_name(message_id_t); +size_t get_num_message_ids(void); + +/* As above, but for subsystems */ +subsys_id_t get_subsys_id(const char *); +const char *get_subsys_id_name(subsys_id_t); +size_t get_num_subsys_ids(void); + +/* As above, but for types. Note that types additionally must be + * "defined", if any message is to use them. */ +msg_type_id_t get_msg_type_id(const char *); +const char *get_msg_type_id_name(msg_type_id_t); +size_t get_num_msg_type_ids(void); + +void dispatch_naming_init(void); + +#endif diff --git a/src/lib/dispatch/include.am b/src/lib/dispatch/include.am index c4aa170dbc..4ec5b75cd1 100644 --- a/src/lib/dispatch/include.am +++ b/src/lib/dispatch/include.am @@ -8,6 +8,7 @@ endif src_lib_libtor_dispatch_a_SOURCES = \ src/lib/dispatch/dispatch_cfg.c \ src/lib/dispatch/dispatch_core.c \ + src/lib/dispatch/dispatch_naming.c \ src/lib/dispatch/dispatch_new.c src_lib_libtor_dispatch_testing_a_SOURCES = \ @@ -19,5 +20,6 @@ noinst_HEADERS += \ src/lib/dispatch/dispatch.h \ src/lib/dispatch/dispatch_cfg.h \ src/lib/dispatch/dispatch_cfg_st.h \ + src/lib/dispatch/dispatch_naming.h \ src/lib/dispatch/dispatch_st.h \ src/lib/dispatch/msgtypes.h From 24b945f713a713bba0ec4f0d8297b49cbc45c5a1 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Sun, 13 Jan 2019 16:01:44 -0500 Subject: [PATCH 0649/2557] Debug logs to record all messages sent and delivered. --- src/lib/dispatch/dispatch_core.c | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/src/lib/dispatch/dispatch_core.c b/src/lib/dispatch/dispatch_core.c index 24dfc649a1..da54f9b437 100644 --- a/src/lib/dispatch/dispatch_core.c +++ b/src/lib/dispatch/dispatch_core.c @@ -14,6 +14,7 @@ #include "lib/dispatch/dispatch.h" #include "lib/dispatch/dispatch_st.h" +#include "lib/dispatch/dispatch_naming.h" #include "lib/malloc/malloc.h" #include "lib/log/util_bug.h" @@ -180,6 +181,17 @@ dispatch_send_msg_unchecked(dispatch_t *d, msg_t *m) /* Append the message. */ TOR_SIMPLEQ_INSERT_TAIL(&q->queue, m, next); + if (debug_logging_enabled()) { + char *arg = dispatch_fmt_msg_data(d, m); + log_debug(LD_MESG, + "Queued: %s (%s) from %s, on %s.", + get_message_id_name(m->msg), + arg, + get_subsys_id_name(m->sender), + get_channel_id_name(m->channel)); + tor_free(arg); + } + /* If we just made the queue nonempty for the first time, call the alert * function. */ if (was_empty) { @@ -199,10 +211,24 @@ dispatcher_run_msg_cbs(const dispatch_t *d, msg_t *m) dtbl_entry_t *ent = d->table[m->msg]; int n_fns = ent->n_fns; + if (debug_logging_enabled()) { + char *arg = dispatch_fmt_msg_data(d, m); + log_debug(LD_MESG, + "Delivering: %s (%s) from %s, on %s:", + get_message_id_name(m->msg), + arg, + get_subsys_id_name(m->sender), + get_channel_id_name(m->channel)); + tor_free(arg); + } + int i; for (i=0; i < n_fns; ++i) { - if (ent->rcv[i].enabled) + if (ent->rcv[i].enabled) { + log_debug(LD_MESG, " Delivering to %s.", + get_subsys_id_name(ent->rcv[i].sys)); ent->rcv[i].fn(m); + } } } From 9e60482b8073f2d43187c36c9159fd4367d7140a Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 15 Jan 2019 09:01:20 -0500 Subject: [PATCH 0650/2557] Pubsub: an OO layer on top of lib/dispatch This "publish/subscribe" layer sits on top of lib/dispatch, and tries to provide more type-safety and cross-checking for the lower-level layer. Even with this commit, we're still not done: more checking will come in the next commit, and a set of usability/typesafety macros will come after. --- src/lib/pubsub/.may_include | 10 + src/lib/pubsub/include.am | 24 +++ src/lib/pubsub/pub_binding_st.h | 36 ++++ src/lib/pubsub/pubsub.h | 85 ++++++++ src/lib/pubsub/pubsub_build.c | 286 +++++++++++++++++++++++++++ src/lib/pubsub/pubsub_build.h | 87 ++++++++ src/lib/pubsub/pubsub_builder_st.h | 161 +++++++++++++++ src/lib/pubsub/pubsub_connect.h | 47 +++++ src/lib/pubsub/pubsub_flags.h | 32 +++ src/lib/pubsub/pubsub_publish.c | 70 +++++++ src/lib/pubsub/pubsub_publish.h | 15 ++ src/test/include.am | 1 + src/test/test.c | 1 + src/test/test.h | 1 + src/test/test_pubsub_msg.c | 305 +++++++++++++++++++++++++++++ 15 files changed, 1161 insertions(+) create mode 100644 src/lib/pubsub/.may_include create mode 100644 src/lib/pubsub/include.am create mode 100644 src/lib/pubsub/pub_binding_st.h create mode 100644 src/lib/pubsub/pubsub.h create mode 100644 src/lib/pubsub/pubsub_build.c create mode 100644 src/lib/pubsub/pubsub_build.h create mode 100644 src/lib/pubsub/pubsub_builder_st.h create mode 100644 src/lib/pubsub/pubsub_connect.h create mode 100644 src/lib/pubsub/pubsub_flags.h create mode 100644 src/lib/pubsub/pubsub_publish.c create mode 100644 src/lib/pubsub/pubsub_publish.h create mode 100644 src/test/test_pubsub_msg.c diff --git a/src/lib/pubsub/.may_include b/src/lib/pubsub/.may_include new file mode 100644 index 0000000000..5623492f00 --- /dev/null +++ b/src/lib/pubsub/.may_include @@ -0,0 +1,10 @@ +orconfig.h + +lib/cc/*.h +lib/container/*.h +lib/dispatch/*.h +lib/intmath/*.h +lib/log/*.h +lib/malloc/*.h +lib/pubsub/*.h +lib/string/*.h diff --git a/src/lib/pubsub/include.am b/src/lib/pubsub/include.am new file mode 100644 index 0000000000..9856c94a5d --- /dev/null +++ b/src/lib/pubsub/include.am @@ -0,0 +1,24 @@ + +noinst_LIBRARIES += src/lib/libtor-pubsub.a + +if UNITTESTS_ENABLED +noinst_LIBRARIES += src/lib/libtor-pubsub-testing.a +endif + +src_lib_libtor_pubsub_a_SOURCES = \ + src/lib/pubsub/pubsub_build.c \ + src/lib/pubsub/pubsub_publish.c + +src_lib_libtor_pubsub_testing_a_SOURCES = \ + $(src_lib_libtor_pubsub_a_SOURCES) +src_lib_libtor_pubsub_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) +src_lib_libtor_pubsub_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) + +noinst_HEADERS += \ + src/lib/pubsub/pub_binding_st.h \ + src/lib/pubsub/pubsub.h \ + src/lib/pubsub/pubsub_build.h \ + src/lib/pubsub/pubsub_builder_st.h \ + src/lib/pubsub/pubsub_connect.h \ + src/lib/pubsub/pubsub_flags.h \ + src/lib/pubsub/pubsub_publish.h diff --git a/src/lib/pubsub/pub_binding_st.h b/src/lib/pubsub/pub_binding_st.h new file mode 100644 index 0000000000..e8c0d47efd --- /dev/null +++ b/src/lib/pubsub/pub_binding_st.h @@ -0,0 +1,36 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file pubsub_build.h + * @brief Declaration of pub_binding_t. + */ + +#ifndef TOR_PUB_BINDING_ST_H +#define TOR_PUB_BINDING_ST_H + +#include "lib/dispatch/msgtypes.h" +struct dispatch_t; + +/** + * A pub_binding_t is an opaque object that subsystems use to publish + * messages. The DISPATCH_ADD_PUB*() macros set it up. + **/ +typedef struct pub_binding_t { + /** + * A pointer to a configured dispatch_t object. This is filled in + * when the dispatch_t is finally constructed. + **/ + struct dispatch_t *dispatch_ptr; + /** + * A template for the msg_t fields that are filled in for this message. + * This is copied into outgoing messages, ensuring that their fields are set + * corretly. + **/ + msg_t msg_template; +} pub_binding_t; + +#endif diff --git a/src/lib/pubsub/pubsub.h b/src/lib/pubsub/pubsub.h new file mode 100644 index 0000000000..303b36ad5b --- /dev/null +++ b/src/lib/pubsub/pubsub.h @@ -0,0 +1,85 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file pubsub.h + * @brief Header for OO publish-subscribe functionality. + * + * This module provides a wrapper around the "dispatch" module, + * ensuring type-safety and allowing us to do static analysis on + * publication and subscriptions. + * + * With this module, we enforce: + *
    + *
  • that every message has (potential) publishers and subscribers; + *
  • that every message is published and subscribed from the correct + * channels, with the correct type ID, every time it is published. + *
  • that type IDs correspond to a single C type, and that the C types are + * used correctly. + *
  • that when a message is published or subscribed, it is done with + * a correct subsystem identifier + *
+ * + * We do this by making "publication requests" and "subscription requests" + * into objects, and doing some computation on them before we create + * a dispatch_t with them. + * + * Rather than using the dispatch module directly, a publishing module + * receives a "binding" object that it uses to send messages with the right + * settings. + */ + +/* + * + * Overview: Messages are sent over channels. Before sending a message on a + * channel, or receiving a message on a channel, a subsystem needs to register + * that it publishes, or subscribes, to that message, on that channel. + * + * Messages, channels, and subsystems are represented internally as short + * integers, though they are associated with human-readable strings for + * initialization and debugging. + * + * When registering for a message, a subsystem must say whether it is an + * exclusive publisher/subscriber to that message type, or whether other + * subsystems may also publish/subscribe to it. + * + * All messages and their publishers/subscribers must be registered early in + * the initialization process. + * + * By default, it is an error for a message type to have publishers and no + * subscribers on a channel, or subscribers and no publishers on a channel. + * + * A subsystem may register for a message with a note that delivery or + * production is disabled -- for example, because the subsystem is + * disabled at compile-time. It is not an error for a message type to + * have all of its publishers or subscribers disabled. + * + * After a message is sent, it is delivered to every recipient. This + * delivery happens from the top level of the event loop; it may be + * interleaved with network events, timers, etc. + * + * Messages may have associated data. This data is typed, and is owned + * by the message. Strings, byte-arrays, and integers have built-in + * support. Other types may be added. If objects are to be sent, + * they should be identified by handle. If an object requires cleanup, + * it should be declared with an associated free function. + * + * Semantically, if two subsystems communicate only by this kind of + * message passing, neither is considered to depend on the other, though + * both are considered to have a dependency on the message and on any + * types it contains. + * + * (Or generational index?) + */ +#ifndef TOR_PUBSUB_PUBSUB_H +#define TOR_PUBSUB_PUBSUB_H + +#include "lib/pubsub/pub_binding_st.h" +#include "lib/pubsub/pubsub_connect.h" +#include "lib/pubsub/pubsub_flags.h" +#include "lib/pubsub/pubsub_publish.h" + +#endif diff --git a/src/lib/pubsub/pubsub_build.c b/src/lib/pubsub/pubsub_build.c new file mode 100644 index 0000000000..72f2eacea8 --- /dev/null +++ b/src/lib/pubsub/pubsub_build.c @@ -0,0 +1,286 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file pubsub_build.c + * @brief Construct a dispatch_t in safer, more OO way. + **/ + +#define PUBSUB_PRIVATE + +#include "lib/dispatch/dispatch.h" +#include "lib/dispatch/dispatch_cfg.h" +#include "lib/dispatch/dispatch_naming.h" +#include "lib/dispatch/msgtypes.h" +#include "lib/pubsub/pubsub_flags.h" +#include "lib/pubsub/pub_binding_st.h" +#include "lib/pubsub/pubsub_build.h" +#include "lib/pubsub/pubsub_builder_st.h" +#include "lib/pubsub/pubsub_connect.h" + +#include "lib/container/smartlist.h" +#include "lib/log/util_bug.h" +#include "lib/malloc/malloc.h" + + #include + +/** Construct and return a new empty pubsub_items_t. */ +static pubsub_items_t * +pubsub_items_new(void) +{ + pubsub_items_t *cfg = tor_malloc_zero(sizeof(*cfg)); + cfg->items = smartlist_new(); + cfg->type_items = smartlist_new(); + return cfg; +} + +/** Release all storage held in a pubsub_items_t. */ +void +pubsub_items_free_(pubsub_items_t *cfg) +{ + if (! cfg) + return; + SMARTLIST_FOREACH(cfg->items, pubsub_cfg_t *, item, tor_free(item)); + SMARTLIST_FOREACH(cfg->type_items, + pubsub_type_cfg_t *, item, tor_free(item)); + smartlist_free(cfg->items); + smartlist_free(cfg->type_items); + tor_free(cfg); +} + +/** Construct and return a new pubsub_builder_t. */ +pubsub_builder_t * +pubsub_builder_new(void) +{ + dispatch_naming_init(); + + pubsub_builder_t *pb = tor_malloc_zero(sizeof(*pb)); + pb->cfg = dcfg_new(); + pb->items = pubsub_items_new(); + return pb; +} + +/** + * Release all storage held by a pubsub_builder_t. + * + * You'll (mostly) only want to call this function on an error case: if you're + * constructing a dispatch_t instead, you should call + * pubsub_builder_finalize() to consume the pubsub_builder_t. + */ +void +pubsub_builder_free_(pubsub_builder_t *pb) +{ + if (pb == NULL) + return; + pubsub_items_free(pb->items); + dcfg_free(pb->cfg); + tor_free(pb); +} + +/** + * Create and return a pubsub_connector_t for the subsystem with ID + * subsys to use in adding publications, subscriptions, and types to + * builder. + **/ +pubsub_connector_t * +pubsub_connector_for_subsystem(pubsub_builder_t *builder, + subsys_id_t subsys) +{ + tor_assert(builder); + ++builder->n_connectors; + + pubsub_connector_t *con = tor_malloc_zero(sizeof(*con)); + + con->builder = builder; + con->subsys_id = subsys; + + return con; +} + +/** + * Release all storage held by a pubsub_connector_t. + **/ +void +pubsub_connector_free_(pubsub_connector_t *con) +{ + if (!con) + return; + + if (con->builder) { + --con->builder->n_connectors; + tor_assert(con->builder->n_connectors >= 0); + } + tor_free(con); +} + +/** + * Use con to add a request for being able to publish messages of type + * msg with auxiliary data of type on channel. + **/ +int +pubsub_add_pub_(pubsub_connector_t *con, + pub_binding_t *out, + channel_id_t channel, + message_id_t msg, + msg_type_id_t type, + unsigned flags, + const char *file, + unsigned line) +{ + pubsub_cfg_t *cfg = tor_malloc_zero(sizeof(*cfg)); + + memset(out, 0, sizeof(*out)); + cfg->is_publish = true; + + out->msg_template.sender = cfg->subsys = con->subsys_id; + out->msg_template.channel = cfg->channel = channel; + out->msg_template.msg = cfg->msg = msg; + out->msg_template.type = cfg->type = type; + + cfg->flags = flags; + cfg->added_by_file = file; + cfg->added_by_line = line; + + /* We're grabbing a pointer to the pub_binding_t so we can tell it about + * the dispatcher later on. + */ + cfg->pub_binding = out; + + smartlist_add(con->builder->items->items, cfg); + + if (dcfg_msg_set_type(con->builder->cfg, msg, type) < 0) + goto err; + if (dcfg_msg_set_chan(con->builder->cfg, msg, channel) < 0) + goto err; + + return 0; + err: + ++con->builder->n_errors; + return -1; +} + +/** + * Use con to add a request for being able to publish messages of type + * msg with auxiliary data of type on channel, + * passing them to the callback in recv_fn. + **/ +int +pubsub_add_sub_(pubsub_connector_t *con, + recv_fn_t recv_fn, + channel_id_t channel, + message_id_t msg, + msg_type_id_t type, + unsigned flags, + const char *file, + unsigned line) +{ + pubsub_cfg_t *cfg = tor_malloc_zero(sizeof(*cfg)); + + cfg->is_publish = false; + cfg->subsys = con->subsys_id; + cfg->channel = channel; + cfg->msg = msg; + cfg->type = type; + cfg->flags = flags; + cfg->added_by_file = file; + cfg->added_by_line = line; + + cfg->recv_fn = recv_fn; + + smartlist_add(con->builder->items->items, cfg); + + if (dcfg_msg_set_type(con->builder->cfg, msg, type) < 0) + goto err; + if (dcfg_msg_set_chan(con->builder->cfg, msg, channel) < 0) + goto err; + if (! (flags & DISP_FLAG_STUB)) { + if (dcfg_add_recv(con->builder->cfg, msg, cfg->subsys, recv_fn) < 0) + goto err; + } + + return 0; + err: + ++con->builder->n_errors; + return -1; +} + +/** + * Use con to define a the functions to use for manipulating the type + * type. Any function pointers left as NULL will be implemented as + * no-ops. + **/ +int +pubsub_connector_define_type_(pubsub_connector_t *con, + msg_type_id_t type, + dispatch_typefns_t *fns, + const char *file, + unsigned line) +{ + pubsub_type_cfg_t *cfg = tor_malloc_zero(sizeof(*cfg)); + cfg->type = type; + memcpy(&cfg->fns, fns, sizeof(*fns)); + cfg->subsys = con->subsys_id; + cfg->added_by_file = file; + cfg->added_by_line = line; + + smartlist_add(con->builder->items->type_items, cfg); + + if (dcfg_type_set_fns(con->builder->cfg, type, fns) < 0) + goto err; + + return 0; + err: + ++con->builder->n_errors; + return -1; +} + +/** + * Initialize the dispatch_ptr field in every relevant publish binding + * for d. + */ +static void +dispatch_fill_pub_binding_backptrs(pubsub_builder_t *builder, + dispatch_t *d) +{ + SMARTLIST_FOREACH_BEGIN(builder->items->items, pubsub_cfg_t *, cfg) { + if (cfg->pub_binding) { + // XXXX we could skip this for STUB publishers, and for any publishers + // XXXX where all subscribers are STUB. + cfg->pub_binding->dispatch_ptr = d; + } + } SMARTLIST_FOREACH_END(cfg); +} + +/** + * Create a new dispatcher as configured in a pubsub_builder_t. + * + * Consumes and frees its input. + **/ +dispatch_t * +pubsub_builder_finalize(pubsub_builder_t *builder) +{ + dispatch_t *dispatcher = NULL; + tor_assert_nonfatal(builder->n_connectors == 0); + + if (builder->n_errors) + goto err; + + /* Coming in the next commit. + if (pubsub_builder_check(builder) < 0) + goto err; + */ + + dispatcher = dispatch_new(builder->cfg); + + if (!dispatcher) + goto err; + + dispatch_fill_pub_binding_backptrs(builder, dispatcher); + + err: + pubsub_builder_free(builder); + return dispatcher; +} diff --git a/src/lib/pubsub/pubsub_build.h b/src/lib/pubsub/pubsub_build.h new file mode 100644 index 0000000000..199eab219a --- /dev/null +++ b/src/lib/pubsub/pubsub_build.h @@ -0,0 +1,87 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file pubsub_build.h + * @brief Header used for constructing the OO publish-subscribe facility. + * + * (See pubsub.h for more general information on this API.) + **/ + +#ifndef TOR_PUBSUB_BUILD_H +#define TOR_PUBSUB_BUILD_H + +struct dispatch_t; + +/** + * A "dispatch builder" is an incomplete dispatcher, used when + * registering messages. It does not have the same integrity guarantees + * as a dispatcher. It cannot actually handle messages itself: once all + * subsystems have registered, it is converted into a dispatch_t. + **/ +typedef struct pubsub_builder_t pubsub_builder_t; + +/** + * A "dispatch connector" is a view of the dispatcher that a subsystem + * uses while initializing itself. It is specific to the subsystem, and + * ensures that each subsystem doesn't need to identify itself + * repeatedly while registering its messages. + **/ +typedef struct pubsub_connector_t pubsub_connector_t; + +/** + * Create a new pubsub_builder. This should only happen in the + * main-init code. + */ +pubsub_builder_t *pubsub_builder_new(void); + +/** DOCDOC */ +int pubsub_builder_check(pubsub_builder_t *); + +/** + * Free a pubsub builder. This should only happen on error paths, where + * we have decided not to construct a dispatcher for some reason. + */ +#define pubsub_builder_free(db) \ + FREE_AND_NULL(pubsub_builder_t, pubsub_builder_free_, (db)) + +/** Internal implementation of pubsub_builder_free(). */ +void pubsub_builder_free_(pubsub_builder_t *); + +/** + * Create a pubsub connector that a single subsystem will use to + * register its messages. The main-init code does this during susbsystem + * initialization. + */ +pubsub_connector_t *pubsub_connector_for_subsystem(pubsub_builder_t *, + subsys_id_t); + +/** + * The main-init code does this after subsystem initialization. + */ +#define pubsub_connector_free(c) \ + FREE_AND_NULL(pubsub_connector_t, pubsub_connector_free_, (c)) + +void pubsub_connector_free_(pubsub_connector_t *); + +/** + * Constructs a dispatcher from a dispatch_builder, after checking that the + * invariances on the messages, channels, and connections have been + * respected. + * + * This should happen after every subsystem has initialized, and before + * entering the mainloop. + */ +struct dispatch_t *pubsub_builder_finalize(pubsub_builder_t *); + +#ifdef PUBSUB_PRIVATE +struct pubsub_items_t; +#define pubsub_items_free(cfg) \ + FREE_AND_NULL(pubsub_items_t, pubsub_items_free_, (cfg)) +void pubsub_items_free_(struct pubsub_items_t *cfg); +#endif + +#endif diff --git a/src/lib/pubsub/pubsub_builder_st.h b/src/lib/pubsub/pubsub_builder_st.h new file mode 100644 index 0000000000..a1cc6e718b --- /dev/null +++ b/src/lib/pubsub/pubsub_builder_st.h @@ -0,0 +1,161 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file pubsub_builder_st.h + * + * @brief private structures used for configuring dispatchers and messages. + */ + +#ifndef TOR_PUBSUB_BUILDER_ST_H +#define TOR_PUBSUB_BUILDER_ST_H + +#ifdef PUBSUB_PRIVATE + +#include +#include + +struct dispatch_cfg_t; +struct smartlist_t; +struct pub_binding_t; + +/** + * Configuration for a single publication or subscription request. + * + * These can be stored while the dispatcher is in use, but are only used for + * setup, teardown, and debugging. + * + * There are various fields in this request describing the message; all of + * them must match other descriptions of the message, or a bug has occurred. + **/ +typedef struct pubsub_cfg_t { + /** True if this is a publishing request; false for a subscribing request. */ + bool is_publish; + /** The system making this request. */ + subsys_id_t subsys; + /** The channel on which the message is to be sent. */ + channel_id_t channel; + /** The message ID to be sent or received. */ + message_id_t msg; + /** The C type associated with the message. */ + msg_type_id_t type; + /** One or more DISP_FLAGS_* items, combined with bitwise OR. */ + unsigned flags; + + /** + * Publishing only: a pub_binding object that will receive the binding for + * this request. We will finish filling this in when the dispatcher is + * constructed, so that the subsystem can publish then and not before. + */ + struct pub_binding_t *pub_binding; + + /** + * Subscribing only: a function to receive message objects for this request. + */ + recv_fn_t recv_fn; + + /** The file from which this message was configured */ + const char *added_by_file; + /** The line at which this message was configured */ + unsigned added_by_line; +} pubsub_cfg_t; + +/** + * Configuration request for a single C type. + * + * These are stored while the dispatcher is in use, but are only used for + * setup, teardown, and debugging. + **/ +typedef struct pubsub_type_cfg_t { + /** + * The identifier for this type. + */ + msg_type_id_t type; + /** + * Functions to use when manipulating the type. + */ + dispatch_typefns_t fns; + + /** The subsystem that configured this type. */ + subsys_id_t subsys; + /** The file from which this type was configured */ + const char *added_by_file; + /** The line at which this type was configured */ + unsigned added_by_line; +} pubsub_type_cfg_t; + +/** + * The set of configuration requests for a dispatcher, as made by various + * subsystems. + **/ +typedef struct pubsub_items_t { + /** List of pubsub_cfg_t. */ + struct smartlist_t *items; + /** List of pubsub_type_cfg_t. */ + struct smartlist_t *type_items; +} pubsub_items_t; + +/** + * Type used to construct a dispatcher. We use this type to build up the + * configuration for a dispatcher, and then pass ownership of that + * configuration to the newly constructed dispatcher. + **/ +struct pubsub_builder_t { + /** Number of outstanding pubsub_connector_t objects pointing to this + * pubsub_builder_t. */ + int n_connectors; + /** Number of errors encountered while constructing this object so far. */ + int n_errors; + /** In-progress configuration that we're constructing, as a list of the + * requests that have been made. */ + pubsub_items_t *items; + /** In-progress configuration that we're constructing, in a form that can + * be converted to a dispatch_t. */ + struct dispatch_cfg_t *cfg; +}; + +/** + * Type given to a subsystem when adding connections to a pubsub_builder_t. + * We use this type to force each subsystem to get blamed for the + * publications, subscriptions, and types that it adds. + **/ +struct pubsub_connector_t { + /** The pubsub_builder that this connector refers to. */ + struct pubsub_builder_t *builder; + /** The subsystem that has been given this connector. */ + subsys_id_t subsys_id; +}; + +/** + * Helper structure used when constructing a dispatcher that sorts the + * pubsub_cfg_t objects in various ways. + **/ +typedef struct pubsub_adjmap_t { + /* XXXX The next three fields are currently constructed but not yet + * XXXX used. I believe we'll want them in the future, though. -nickm + */ + /** Number of subsystems; length of the *_by_subsys arrays. */ + size_t n_subsystems; + /** Array of lists of publisher pubsub_cfg_t objects, indexed by + * subsystem. */ + struct smartlist_t **pub_by_subsys; + /** Array of lists of subscriber pubsub_cfg_t objects, indexed by + * subsystem. */ + struct smartlist_t **sub_by_subsys; + + /** Number of message IDs; length of the *_by_msg arrays. */ + size_t n_msgs; + /** Array of lists of publisher pubsub_cfg_t objects, indexed by + * message ID. */ + struct smartlist_t **pub_by_msg; + /** Array of lists of subscriber pubsub_cfg_t objects, indexed by + * message ID. */ + struct smartlist_t **sub_by_msg; +} pubsub_adjmap_t; + +#endif + +#endif diff --git a/src/lib/pubsub/pubsub_connect.h b/src/lib/pubsub/pubsub_connect.h new file mode 100644 index 0000000000..b63f9dc438 --- /dev/null +++ b/src/lib/pubsub/pubsub_connect.h @@ -0,0 +1,47 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file pubsub_connect.h + * @brief Header for functions that add relationships to a pubsub builder. + * + * These functions are used by modules that need to add publication and + * subscription requests. + **/ + +#ifndef TOR_PUBSUB_CONNECT_H +#define TOR_PUBSUB_CONNECT_H + +#include "lib/dispatch/msgtypes.h" + +struct pub_binding_t; +struct pubsub_connector_t; + +int pubsub_add_pub_(struct pubsub_connector_t *con, + struct pub_binding_t *out, + channel_id_t channel, + message_id_t msg, + msg_type_id_t type, + unsigned flags, + const char *file, + unsigned line); + +int pubsub_add_sub_(struct pubsub_connector_t *con, + recv_fn_t recv_fn, + channel_id_t channel, + message_id_t msg, + msg_type_id_t type, + unsigned flags, + const char *file, + unsigned line); + +int pubsub_connector_define_type_(struct pubsub_connector_t *, + msg_type_id_t, + dispatch_typefns_t *, + const char *file, + unsigned line); + +#endif diff --git a/src/lib/pubsub/pubsub_flags.h b/src/lib/pubsub/pubsub_flags.h new file mode 100644 index 0000000000..d07a06be7b --- /dev/null +++ b/src/lib/pubsub/pubsub_flags.h @@ -0,0 +1,32 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file pubsub_flags.h + * @brief Flags that can be set on publish/subscribe messages. + **/ + +#ifndef TOR_PUBSUB_FLAGS_H +#define TOR_PUBSUB_FLAGS_H + +/** + * Flag for registering a message: declare that no other module is allowed to + * publish this message if we are publishing it, or subscribe to it if we are + * subscribing to it. + */ +#define DISP_FLAG_EXCL (1u<<0) + +/** + * Flag for registering a message: declare that this message is a stub, and we + * will not actually publish/subscribe it, but that the dispatcher should + * treat us as if we did when typechecking. + * + * We use this so that messages aren't treated as "dangling" if they are + * potentially used by some other build of Tor. + */ +#define DISP_FLAG_STUB (1u<<1) + +#endif diff --git a/src/lib/pubsub/pubsub_publish.c b/src/lib/pubsub/pubsub_publish.c new file mode 100644 index 0000000000..8c469e8add --- /dev/null +++ b/src/lib/pubsub/pubsub_publish.c @@ -0,0 +1,70 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file pubsub_publish.h + * @brief Header for functions to publish using a pub_binding_t. + **/ + +#define PUBSUB_PRIVATE +#define DISPATCH_PRIVATE +#include "orconfig.h" + +#include "lib/dispatch/dispatch.h" +#include "lib/dispatch/dispatch_st.h" + +#include "lib/pubsub/pub_binding_st.h" +#include "lib/pubsub/pubsub_publish.h" + +#include "lib/malloc/malloc.h" +#include "lib/log/util_bug.h" + +#include + +/** + * Publish a message from the publication binding pub using the + * auxiliary data auxdata. + * + * Return 0 on success, -1 on failure. + **/ +int +pubsub_pub_(const pub_binding_t *pub, msg_aux_data_t auxdata) +{ + dispatch_t *d = pub->dispatch_ptr; + if (BUG(! d)) { + /* Tried to publish a message before the dispatcher was configured. */ + /* (Without a dispatcher, we don't know how to free auxdata.) */ + return -1; + } + + if (BUG(pub->msg_template.type >= d->n_types)) { + /* The type associated with this message is not known to the dispatcher. */ + /* (Without a correct type, we don't know how to free auxdata.) */ + return -1; + } + + if (BUG(pub->msg_template.msg >= d->n_msgs) || + BUG(pub->msg_template.channel >= d->n_queues)) { + /* The message ID or channel ID was out of bounds. */ + d->typefns[pub->msg_template.type].free_fn(auxdata); + return -1; + } + + if (! d->table[pub->msg_template.msg]) { + /* Fast path: nobody wants this data. */ + + // XXXX Faster path: we could store this in the pub_binding_t. + d->typefns[pub->msg_template.type].free_fn(auxdata); + return 0; + } + + /* Construct the message object */ + msg_t *m = tor_malloc(sizeof(msg_t)); + memcpy(m, &pub->msg_template, sizeof(msg_t)); + m->aux_data__ = auxdata; + + return dispatch_send_msg_unchecked(d, m); +} diff --git a/src/lib/pubsub/pubsub_publish.h b/src/lib/pubsub/pubsub_publish.h new file mode 100644 index 0000000000..0250fd0760 --- /dev/null +++ b/src/lib/pubsub/pubsub_publish.h @@ -0,0 +1,15 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef TOR_PUBSUB_PUBLISH_H +#define TOR_PUBSUB_PUBLISH_H + +#include "lib/dispatch/msgtypes.h" +struct pub_binding_t; + +int pubsub_pub_(const struct pub_binding_t *pub, msg_aux_data_t auxdata); + +#endif diff --git a/src/test/include.am b/src/test/include.am index d4e8eb4e8c..7734b73de8 100644 --- a/src/test/include.am +++ b/src/test/include.am @@ -165,6 +165,7 @@ src_test_test_SOURCES += \ src/test/test_proto_misc.c \ src/test/test_protover.c \ src/test/test_pt.c \ + src/test/test_pubsub_msg.c \ src/test/test_relay.c \ src/test/test_relaycell.c \ src/test/test_relaycrypt.c \ diff --git a/src/test/test.c b/src/test/test.c index 4c6d9775b4..da0be41332 100644 --- a/src/test/test.c +++ b/src/test/test.c @@ -910,6 +910,7 @@ struct testgroup_t testgroups[] = { { "proto/misc/", proto_misc_tests }, { "protover/", protover_tests }, { "pt/", pt_tests }, + { "pubsub/msg/", pubsub_msg_tests }, { "relay/" , relay_tests }, { "relaycell/", relaycell_tests }, { "relaycrypt/", relaycrypt_tests }, diff --git a/src/test/test.h b/src/test/test.h index cd0ce4f6df..fb417124ce 100644 --- a/src/test/test.h +++ b/src/test/test.h @@ -253,6 +253,7 @@ extern struct testcase_t proto_http_tests[]; extern struct testcase_t proto_misc_tests[]; extern struct testcase_t protover_tests[]; extern struct testcase_t pt_tests[]; +extern struct testcase_t pubsub_msg_tests[]; extern struct testcase_t relay_tests[]; extern struct testcase_t relaycell_tests[]; extern struct testcase_t relaycrypt_tests[]; diff --git a/src/test/test_pubsub_msg.c b/src/test/test_pubsub_msg.c new file mode 100644 index 0000000000..5b771d45a5 --- /dev/null +++ b/src/test/test_pubsub_msg.c @@ -0,0 +1,305 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#define DISPATCH_PRIVATE + +#include "test/test.h" + +#include "lib/dispatch/dispatch.h" +#include "lib/dispatch/dispatch_naming.h" +#include "lib/dispatch/dispatch_st.h" +#include "lib/dispatch/msgtypes.h" +#include "lib/pubsub/pubsub_flags.h" +#include "lib/pubsub/pub_binding_st.h" +#include "lib/pubsub/pubsub_build.h" +#include "lib/pubsub/pubsub_builder_st.h" +#include "lib/pubsub/pubsub_connect.h" +#include "lib/pubsub/pubsub_publish.h" + +#include "lib/log/escape.h" +#include "lib/malloc/malloc.h" +#include "lib/string/printf.h" + +#include +#include + +static char * +ex_str_fmt(msg_aux_data_t aux) +{ + return esc_for_log(aux.ptr); +} +static void +ex_str_free(msg_aux_data_t aux) +{ + tor_free_(aux.ptr); +} +static dispatch_typefns_t stringfns = { + .free_fn = ex_str_free, + .fmt_fn = ex_str_fmt +}; + +// We're using the lowest-level publish/subscribe logic here, to avoid the +// pubsub_macros.h macros and just test the dispatch core. We'll use a string +// type for everything. + +#define DECLARE_MESSAGE(suffix) \ + static pub_binding_t pub_binding_##suffix; \ + static int msg_received_##suffix = 0; \ + static void recv_msg_##suffix(const msg_t *m) { \ + (void)m; \ + ++msg_received_##suffix; \ + } \ + EAT_SEMICOLON + +#define ADD_PUBLISH(binding_suffix, subsys, channel, msg, flags) \ + STMT_BEGIN { \ + con = pubsub_connector_for_subsystem(builder, \ + get_subsys_id(#subsys)); \ + pubsub_add_pub_(con, &pub_binding_##binding_suffix, \ + get_channel_id(#channel), \ + get_message_id(#msg), get_msg_type_id("string"), \ + (flags), __FILE__, __LINE__); \ + pubsub_connector_free(con); \ + } STMT_END + +#define ADD_SUBSCRIBE(hook_suffix, subsys, channel, msg, flags) \ + STMT_BEGIN { \ + con = pubsub_connector_for_subsystem(builder, \ + get_subsys_id(#subsys)); \ + pubsub_add_sub_(con, recv_msg_##hook_suffix, \ + get_channel_id(#channel), \ + get_message_id(#msg), get_msg_type_id("string"), \ + (flags), __FILE__, __LINE__); \ + pubsub_connector_free(con); \ + } STMT_END + +#define SEND(binding_suffix, val) \ + STMT_BEGIN { \ + msg_aux_data_t data_; \ + data_.ptr = tor_strdup(val); \ + pubsub_pub_(&pub_binding_##binding_suffix, data_); \ + } STMT_END + +DECLARE_MESSAGE(msg1); +DECLARE_MESSAGE(msg2); +DECLARE_MESSAGE(msg3); +DECLARE_MESSAGE(msg4); +DECLARE_MESSAGE(msg5); + +static smartlist_t *strings_received = NULL; +static void +recv_msg_copy_string(const msg_t *m) +{ + const char *s = m->aux_data__.ptr; + smartlist_add(strings_received, tor_strdup(s)); +} + +static void * +setup_dispatcher(const struct testcase_t *testcase) +{ + (void)testcase; + pubsub_builder_t *builder = pubsub_builder_new(); + pubsub_connector_t *con; + + { + con = pubsub_connector_for_subsystem(builder, get_subsys_id("types")); + pubsub_connector_define_type_(con, + get_msg_type_id("string"), + &stringfns, + "nowhere.c", 99); + pubsub_connector_free(con); + } + // message1 has one publisher and one subscriber. + ADD_PUBLISH(msg1, sys1, main, message1, 0); + ADD_SUBSCRIBE(msg1, sys2, main, message1, 0); + + // message2 has a publisher and a stub subscriber. + ADD_PUBLISH(msg2, sys1, main, message2, 0); + ADD_SUBSCRIBE(msg2, sys2, main, message2, DISP_FLAG_STUB); + + // message3 has a publisher and three subscribers. + ADD_PUBLISH(msg3, sys1, main, message3, 0); + ADD_SUBSCRIBE(msg3, sys2, main, message3, 0); + ADD_SUBSCRIBE(msg3, sys3, main, message3, 0); + ADD_SUBSCRIBE(msg3, sys4, main, message3, 0); + + // message4 has one publisher and two subscribers, but it's on another + // channel. + ADD_PUBLISH(msg4, sys2, other, message4, 0); + ADD_SUBSCRIBE(msg4, sys1, other, message4, 0); + ADD_SUBSCRIBE(msg4, sys3, other, message4, 0); + + // message5 has a huge number of recipients. + ADD_PUBLISH(msg5, sys3, main, message5, 0); + ADD_SUBSCRIBE(msg5, sys4, main, message5, 0); + ADD_SUBSCRIBE(msg5, sys5, main, message5, 0); + ADD_SUBSCRIBE(msg5, sys6, main, message5, 0); + ADD_SUBSCRIBE(msg5, sys7, main, message5, 0); + ADD_SUBSCRIBE(msg5, sys8, main, message5, 0); + for (int i = 0; i < 1000-5; ++i) { + char *sys; + tor_asprintf(&sys, "xsys-%d", i); + con = pubsub_connector_for_subsystem(builder, get_subsys_id(sys)); + pubsub_add_sub_(con, recv_msg_copy_string, + get_channel_id("main"), + get_message_id("message5"), + get_msg_type_id("string"), 0, "here", 100); + pubsub_connector_free(con); + tor_free(sys); + } + + return pubsub_builder_finalize(builder); +} + +static int +cleanup_dispatcher(const struct testcase_t *testcase, void *dispatcher_) +{ + (void)testcase; + dispatch_t *dispatcher = dispatcher_; + dispatch_free(dispatcher); + return 1; +} + +static const struct testcase_setup_t dispatcher_setup = { + setup_dispatcher, cleanup_dispatcher +}; + +static void +test_pubsub_msg_minimal(void *arg) +{ + dispatch_t *d = arg; + + tt_int_op(0, OP_EQ, msg_received_msg1); + SEND(msg1, "hello world"); + tt_int_op(0, OP_EQ, msg_received_msg1); // hasn't actually arrived yet. + + tt_int_op(0, OP_EQ, dispatch_flush(d, get_channel_id("main"), 1000)); + tt_int_op(1, OP_EQ, msg_received_msg1); // we got the message! + + done: + ; +} + +static void +test_pubsub_msg_send_to_stub(void *arg) +{ + dispatch_t *d = arg; + + tt_int_op(0, OP_EQ, msg_received_msg2); + SEND(msg2, "hello silence"); + tt_int_op(0, OP_EQ, msg_received_msg2); // hasn't actually arrived yet. + + tt_int_op(0, OP_EQ, dispatch_flush(d, get_channel_id("main"), 1000)); + tt_int_op(0, OP_EQ, msg_received_msg2); // doesn't arrive -- stub hook. + + done: + ; +} + +static void +test_pubsub_msg_cancel_msgs(void *arg) +{ + dispatch_t *d = arg; + + tt_int_op(0, OP_EQ, msg_received_msg1); + for (int i = 0; i < 100; ++i) { + SEND(msg1, "hello world"); + } + tt_int_op(0, OP_EQ, msg_received_msg1); // hasn't actually arrived yet. + + tt_int_op(0, OP_EQ, dispatch_flush(d, get_channel_id("main"), 10)); + tt_int_op(10, OP_EQ, msg_received_msg1); // we got the message 10 times. + + // At this point, the dispatcher will be freed with queued, undelivered + // messages. + done: + ; +} + +struct alertfn_target { + dispatch_t *d; + channel_id_t ch; + int count; +}; +static void +alertfn_generic(dispatch_t *d, channel_id_t ch, void *arg) +{ + struct alertfn_target *t = arg; + tt_ptr_op(d, OP_EQ, t->d); + tt_int_op(ch, OP_EQ, t->ch); + ++t->count; + done: + ; +} + +static void +test_pubsub_msg_alertfns(void *arg) +{ + dispatch_t *d = arg; + struct alertfn_target ch1_a = { d, get_channel_id("main"), 0 }; + struct alertfn_target ch2_a = { d, get_channel_id("other"), 0 }; + + tt_int_op(0, OP_EQ, + dispatch_set_alert_fn(d, get_channel_id("main"), + alertfn_generic, &ch1_a)); + tt_int_op(0, OP_EQ, + dispatch_set_alert_fn(d, get_channel_id("other"), + alertfn_generic, &ch2_a)); + + SEND(msg3, "hello"); + tt_int_op(ch1_a.count, OP_EQ, 1); + SEND(msg3, "world"); + tt_int_op(ch1_a.count, OP_EQ, 1); // only the first message sends an alert + tt_int_op(ch2_a.count, OP_EQ, 0); // no alert for 'other' + + SEND(msg4, "worse things happen in C"); + tt_int_op(ch2_a.count, OP_EQ, 1); + + // flush the first (main) channel... + tt_int_op(0, OP_EQ, dispatch_flush(d, get_channel_id("main"), 1000)); + tt_int_op(6, OP_EQ, msg_received_msg3); // 3 subscribers, 2 instances. + + // now that the main channel is flushed, sending another message on it + // starts another alert. + tt_int_op(ch1_a.count, OP_EQ, 1); + SEND(msg1, "plover"); + tt_int_op(ch1_a.count, OP_EQ, 2); + tt_int_op(ch2_a.count, OP_EQ, 1); + + done: + ; +} + +/* try more than N_FAST_FNS hooks on msg5 */ +static void +test_pubsub_msg_many_hooks(void *arg) +{ + dispatch_t *d = arg; + strings_received = smartlist_new(); + + tt_int_op(0, OP_EQ, msg_received_msg5); + SEND(msg5, "hello world"); + tt_int_op(0, OP_EQ, msg_received_msg5); + tt_int_op(0, OP_EQ, smartlist_len(strings_received)); + + tt_int_op(0, OP_EQ, dispatch_flush(d, get_channel_id("main"), 100000)); + tt_int_op(5, OP_EQ, msg_received_msg5); + tt_int_op(995, OP_EQ, smartlist_len(strings_received)); + + done: + SMARTLIST_FOREACH(strings_received, char *, s, tor_free(s)); + smartlist_free(strings_received); +} + +#define T(name) \ + { #name, test_pubsub_msg_ ## name , TT_FORK, \ + &dispatcher_setup, NULL } + +struct testcase_t pubsub_msg_tests[] = { + T(minimal), + T(send_to_stub), + T(cancel_msgs), + T(alertfns), + T(many_hooks), + END_OF_TESTCASES +}; From 271a67182211a954dba1082739f8fe7daf421f6c Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 15 Jan 2019 09:01:41 -0500 Subject: [PATCH 0651/2557] pubsub: relationship checking functionality This code tries to prevent a large number of possible errors by enforcing different restrictions on the messages that different modules publish and subscribe to. Some of these rules are probably too strict, and some too lax: we should feel free to change them as needed as we move forward and learn more. --- .gitignore | 2 + Makefile.am | 2 + src/include.am | 1 + src/lib/pubsub/include.am | 1 + src/lib/pubsub/pubsub_build.c | 2 - src/lib/pubsub/pubsub_check.c | 368 ++++++++++++++++++++++++++++++++++ 6 files changed, 374 insertions(+), 2 deletions(-) create mode 100644 src/lib/pubsub/pubsub_check.c diff --git a/.gitignore b/.gitignore index f4f6dacbb0..47b23c6470 100644 --- a/.gitignore +++ b/.gitignore @@ -202,6 +202,8 @@ uptime-*.json /src/lib/libtor-osinfo-testing.a /src/lib/libtor-process.a /src/lib/libtor-process-testing.a +/src/lib/libtor-pubsub.a +/src/lib/libtor-pubsub-testing.a /src/lib/libtor-sandbox.a /src/lib/libtor-sandbox-testing.a /src/lib/libtor-string.a diff --git a/Makefile.am b/Makefile.am index 36d9725f38..44cfe17877 100644 --- a/Makefile.am +++ b/Makefile.am @@ -41,6 +41,7 @@ TOR_UTIL_LIBS = \ src/lib/libtor-geoip.a \ src/lib/libtor-process.a \ src/lib/libtor-buf.a \ + src/lib/libtor-pubsub.a \ src/lib/libtor-dispatch.a \ src/lib/libtor-time.a \ src/lib/libtor-fs.a \ @@ -73,6 +74,7 @@ TOR_UTIL_TESTING_LIBS = \ src/lib/libtor-geoip-testing.a \ src/lib/libtor-process-testing.a \ src/lib/libtor-buf-testing.a \ + src/lib/libtor-pubsub-testing.a \ src/lib/libtor-dispatch-testing.a \ src/lib/libtor-time-testing.a \ src/lib/libtor-fs-testing.a \ diff --git a/src/include.am b/src/include.am index c6c351c806..77c126ba45 100644 --- a/src/include.am +++ b/src/include.am @@ -25,6 +25,7 @@ include src/lib/malloc/include.am include src/lib/net/include.am include src/lib/osinfo/include.am include src/lib/process/include.am +include src/lib/pubsub/include.am include src/lib/sandbox/include.am include src/lib/string/include.am include src/lib/subsys/include.am diff --git a/src/lib/pubsub/include.am b/src/lib/pubsub/include.am index 9856c94a5d..0ab2fd7b33 100644 --- a/src/lib/pubsub/include.am +++ b/src/lib/pubsub/include.am @@ -7,6 +7,7 @@ endif src_lib_libtor_pubsub_a_SOURCES = \ src/lib/pubsub/pubsub_build.c \ + src/lib/pubsub/pubsub_check.c \ src/lib/pubsub/pubsub_publish.c src_lib_libtor_pubsub_testing_a_SOURCES = \ diff --git a/src/lib/pubsub/pubsub_build.c b/src/lib/pubsub/pubsub_build.c index 72f2eacea8..7e4ab5ba8f 100644 --- a/src/lib/pubsub/pubsub_build.c +++ b/src/lib/pubsub/pubsub_build.c @@ -268,10 +268,8 @@ pubsub_builder_finalize(pubsub_builder_t *builder) if (builder->n_errors) goto err; - /* Coming in the next commit. if (pubsub_builder_check(builder) < 0) goto err; - */ dispatcher = dispatch_new(builder->cfg); diff --git a/src/lib/pubsub/pubsub_check.c b/src/lib/pubsub/pubsub_check.c new file mode 100644 index 0000000000..1b3853d8bb --- /dev/null +++ b/src/lib/pubsub/pubsub_check.c @@ -0,0 +1,368 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file pubsub_check.c + * @brief Enforce various requirements on a pubsub_builder. + **/ + +#define PUBSUB_PRIVATE + +#include "lib/dispatch/dispatch_naming.h" +#include "lib/dispatch/msgtypes.h" +#include "lib/pubsub/pubsub_flags.h" +#include "lib/pubsub/pubsub_builder_st.h" +#include "lib/pubsub/pubsub_build.h" + +#include "lib/container/bitarray.h" +#include "lib/container/smartlist.h" +#include "lib/log/util_bug.h" +#include "lib/malloc/malloc.h" +#include "lib/string/compat_string.h" + +#include + +static void pubsub_adjmap_add(pubsub_adjmap_t *map, + const pubsub_cfg_t *item); + +/** + * Helper: contruct and return a new pubsub_adjacency_map from cfg. + * Return NULL on error. + **/ +static pubsub_adjmap_t * +pubsub_build_adjacency_map(const pubsub_items_t *cfg) +{ + pubsub_adjmap_t *map = tor_malloc_zero(sizeof(*map)); + const size_t n_subsystems = get_num_subsys_ids(); + const size_t n_msgs = get_num_message_ids(); + + map->n_subsystems = n_subsystems; + map->n_msgs = n_msgs; + + map->pub_by_subsys = tor_calloc(n_subsystems, sizeof(smartlist_t*)); + map->sub_by_subsys = tor_calloc(n_subsystems, sizeof(smartlist_t*)); + map->pub_by_msg = tor_calloc(n_msgs, sizeof(smartlist_t*)); + map->sub_by_msg = tor_calloc(n_msgs, sizeof(smartlist_t*)); + + SMARTLIST_FOREACH_BEGIN(cfg->items, const pubsub_cfg_t *, item) { + pubsub_adjmap_add(map, item); + } SMARTLIST_FOREACH_END(item); + + return map; +} + +/** + * Helper: add a single pubsub_cfg_t to an adjacency map. + **/ +static void +pubsub_adjmap_add(pubsub_adjmap_t *map, + const pubsub_cfg_t *item) +{ + smartlist_t **by_subsys; + smartlist_t **by_msg; + + tor_assert(item->subsys < map->n_subsystems); + tor_assert(item->msg < map->n_msgs); + + if (item->is_publish) { + by_subsys = &map->pub_by_subsys[item->subsys]; + by_msg = &map->pub_by_msg[item->msg]; + } else { + by_subsys = &map->sub_by_subsys[item->subsys]; + by_msg = &map->sub_by_msg[item->msg]; + } + + if (! *by_subsys) + *by_subsys = smartlist_new(); + if (! *by_msg) + *by_msg = smartlist_new(); + smartlist_add(*by_subsys, (void*) item); + smartlist_add(*by_msg, (void *) item); +} + +/** + * Release all storage held by m and set m to NULL. + **/ +#define pubsub_adjmap_free(m) \ + FREE_AND_NULL(pubsub_adjmap_t, pubsub_adjmap_free_, m) + +/** + * Free every element of an n-element array of smartlists, then + * free the array itself. + **/ +static void +pubsub_adjmap_free_helper(smartlist_t **lsts, size_t n) +{ + if (!lsts) + return; + + for (unsigned i = 0; i < n; ++i) { + smartlist_free(lsts[i]); + } + tor_free(lsts); +} + +/** + * Release all storage held by map. + **/ +static void +pubsub_adjmap_free_(pubsub_adjmap_t *map) +{ + if (!map) + return; + pubsub_adjmap_free_helper(map->pub_by_subsys, map->n_subsystems); + pubsub_adjmap_free_helper(map->sub_by_subsys, map->n_subsystems); + pubsub_adjmap_free_helper(map->pub_by_msg, map->n_msgs); + pubsub_adjmap_free_helper(map->sub_by_msg, map->n_msgs); + tor_free(map); +} + +/** + * Helper: return the length of sl, or 0 if sl is NULL. + **/ +static int +smartlist_len_opt(const smartlist_t *sl) +{ + if (sl) + return smartlist_len(sl); + else + return 0; +} + +/** Return a pointer to a statically allocated string encoding the + * dispatcher flags in flags. */ +static const char * +format_flags(unsigned flags) +{ + static char buf[32]; + buf[0] = 0; + if (flags & DISP_FLAG_EXCL) { + strlcat(buf, " EXCL", sizeof(buf)); + } + if (flags & DISP_FLAG_STUB) { + strlcat(buf, " STUB", sizeof(buf)); + } + return buf[0] ? buf+1 : buf; +} + +/** + * Log a message containing a description of cfg at severity, prefixed + * by the string prefix. + */ +static void +pubsub_cfg_dump(const pubsub_cfg_t *cfg, int severity, const char *prefix) +{ + if (!prefix) + prefix = 0; + + tor_log(severity, LD_MESG, + "%s%s %s: %s{%s} on %s (%s) <%u %u %u %u %x> [%s:%d]", + prefix, + get_subsys_id_name(cfg->subsys), + cfg->is_publish ? "PUB" : "SUB", + get_message_id_name(cfg->msg), + get_msg_type_id_name(cfg->type), + get_channel_id_name(cfg->channel), + format_flags(cfg->flags), + cfg->subsys, cfg->msg, cfg->type, cfg->channel, cfg->flags, + cfg->added_by_file, cfg->added_by_line); +} + +/** + * Check whether there are any errors or inconsistencies for the message + * described by msg in map. If there are problems, log about + * them, and return -1. Otherwise return 0. + **/ +static int +lint_message(const pubsub_adjmap_t *map, message_id_t msg) +{ + /* NOTE: Some of the checks in this function are maybe over-zealous, and we + * might not want to have them forever. I've marked them with [?] below. + */ + if (BUG(msg >= map->n_msgs)) + return 0; // LCOV_EXCL_LINE + + const smartlist_t *pub = map->pub_by_msg[msg]; + const smartlist_t *sub = map->sub_by_msg[msg]; + + const size_t n_pub = smartlist_len_opt(pub); + const size_t n_sub = smartlist_len_opt(sub); + + if (n_pub == 0 && n_sub == 0) { + log_info(LD_MESG, "Nobody is publishing or subscribing to message %u " + "(%s).", + msg, get_message_id_name(msg)); + return 0; // No publishers or subscribers: nothing to do. + } + + /* We'll set this to false if there are any problems. */ + bool ok = true; + + /* First make sure that if there are publishers, there are subscribers. */ + if (n_pub == 0) { + log_warn(LD_MESG|LD_BUG, + "Message %u (%s) has subscribers, but no publishers.", + msg, get_message_id_name(msg)); + ok = false; + } else if (n_sub == 0) { + log_warn(LD_MESG|LD_BUG, + "Message %u (%s) has publishers, but no subscribers.", + msg, get_message_id_name(msg)); + ok = false; + } + + /* The 'all' list has the publishers and the subscribers. */ + smartlist_t *all = smartlist_new(); + if (pub) + smartlist_add_all(all, pub); + if (sub) + smartlist_add_all(all, sub); + const pubsub_cfg_t *item0 = smartlist_get(all, 0); + + /* Indicates which subsystems we've found publishing/subscribing here. */ + bitarray_t *published_by = bitarray_init_zero((unsigned)map->n_subsystems); + bitarray_t *subscribed_by = bitarray_init_zero((unsigned)map->n_subsystems); + bool pub_excl = false, sub_excl = false, chan_same = true, type_same = true; + + /* Make sure that the messages all have the same channel and type; + * check whether the DISP_FLAG_EXCL flag is used; + * and if any subsystem is publishing or subscribing to it twice [??]. + */ + SMARTLIST_FOREACH_BEGIN(all, const pubsub_cfg_t *, cfg) { + if (cfg->channel != item0->channel) { + chan_same = false; + } + if (cfg->type != item0->type) { + type_same = false; + } + if (cfg->flags & DISP_FLAG_EXCL) { + if (cfg->is_publish) + pub_excl = true; + else + sub_excl = true; + } + if (cfg->is_publish) { + if (bitarray_is_set(published_by, cfg->subsys)) { + log_warn(LD_MESG|LD_BUG, + "Message %u (%s) is configured to be published by subsystem " + "%u (%s) more than once.", + msg, get_message_id_name(msg), + cfg->subsys, get_subsys_id_name(cfg->subsys)); + ok = false; + } + bitarray_set(published_by, cfg->subsys); + } else { + if (bitarray_is_set(subscribed_by, cfg->subsys)) { + log_warn(LD_MESG|LD_BUG, + "Message %u (%s) is configured to be subscribed by subsystem " + "%u (%s) more than once.", + msg, get_message_id_name(msg), + cfg->subsys, get_subsys_id_name(cfg->subsys)); + ok = false; + } + bitarray_set(subscribed_by, cfg->subsys); + } + } SMARTLIST_FOREACH_END(cfg); + + /* Check whether any subsystem is publishing and subscribing the same + * message. [??] + */ + for (unsigned i = 0; i < map->n_subsystems; ++i) { + if (bitarray_is_set(published_by, i) && + bitarray_is_set(subscribed_by, i)) { + log_warn(LD_MESG|LD_BUG, + "Message %u (%s) is published and subscribed by the same " + "subsystem %u (%s)", + msg, get_message_id_name(msg), + i, get_subsys_id_name(i)); + ok = false; + } + } + + if (! chan_same) { + log_warn(LD_MESG|LD_BUG, + "Message %u (%s) is associated with multiple inconsistent " + "channels.", + msg, get_message_id_name(msg)); + ok = false; + } + if (! type_same) { + log_warn(LD_MESG|LD_BUG, + "Message %u (%s) is associated with multiple inconsistent " + "message types.", + msg, get_message_id_name(msg)); + ok = false; + } + + /* Enforce exclusive-ness for publishers and subscribers that have asked for + * it. + */ + if (pub_excl && smartlist_len(pub) > 1) { + log_warn(LD_MESG|LD_BUG, + "Message %u (%s) has multiple publishers, but at least one is " + "marked as exclusive.", + msg, get_message_id_name(msg)); + ok = false; + } + if (sub_excl && smartlist_len(sub) > 1) { + log_warn(LD_MESG|LD_BUG, + "Message %u (%s) has multiple subscribers, but at least one is " + "marked as exclusive.", + msg, get_message_id_name(msg)); + ok = false; + } + + if (!ok) { + /* There was a problem -- let's log all the publishers and subscribers on + * this message */ + SMARTLIST_FOREACH(all, pubsub_cfg_t *, cfg, + pubsub_cfg_dump(cfg, LOG_WARN, " ")); + } + + smartlist_free(all); + bitarray_free(published_by); + bitarray_free(subscribed_by); + + return ok ? 0 : -1; +} + +/** + * Check all the messages in map for consistency. Return 0 on success, + * -1 on problems. + **/ +static int +pubsub_adjmap_check(const pubsub_adjmap_t *map) +{ + bool all_ok = true; + for (unsigned i = 0; i < map->n_msgs; ++i) { + if (lint_message(map, i) < 0) { + all_ok = false; + } + } + return all_ok ? 0 : -1; +} + +/** + * Check builder for consistency and various constraints. Return 0 on success, + * -1 on failure. + **/ +int +pubsub_builder_check(pubsub_builder_t *builder) +{ + pubsub_adjmap_t *map = pubsub_build_adjacency_map(builder->items); + int rv = -1; + + if (!map) + goto err; // should be impossible + + if (pubsub_adjmap_check(map) < 0) + goto err; + + rv = 0; + err: + pubsub_adjmap_free(map); + return rv; +} From a7681525ab670c2b7a783f9e1285bf9a8e97d1ea Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 15 Jan 2019 10:46:01 -0500 Subject: [PATCH 0652/2557] Add function to clear publish bindings. When we clean up, we'd like to clear all the bindings that refer to a dispatch_t, so that they don't have dangling pointers to it. --- src/lib/pubsub/pubsub_build.c | 30 +++++++++++++++++++++++++----- src/lib/pubsub/pubsub_build.h | 20 +++++++++++++++----- src/lib/pubsub/pubsub_builder_st.h | 6 +++--- src/test/test_pubsub_msg.c | 2 +- 4 files changed, 44 insertions(+), 14 deletions(-) diff --git a/src/lib/pubsub/pubsub_build.c b/src/lib/pubsub/pubsub_build.c index 7e4ab5ba8f..64cdcc1d57 100644 --- a/src/lib/pubsub/pubsub_build.c +++ b/src/lib/pubsub/pubsub_build.c @@ -242,10 +242,10 @@ pubsub_connector_define_type_(pubsub_connector_t *con, * for d. */ static void -dispatch_fill_pub_binding_backptrs(pubsub_builder_t *builder, - dispatch_t *d) +pubsub_items_install_bindings(pubsub_items_t *items, + dispatch_t *d) { - SMARTLIST_FOREACH_BEGIN(builder->items->items, pubsub_cfg_t *, cfg) { + SMARTLIST_FOREACH_BEGIN(items->items, pubsub_cfg_t *, cfg) { if (cfg->pub_binding) { // XXXX we could skip this for STUB publishers, and for any publishers // XXXX where all subscribers are STUB. @@ -254,13 +254,29 @@ dispatch_fill_pub_binding_backptrs(pubsub_builder_t *builder, } SMARTLIST_FOREACH_END(cfg); } +/** + * Remove the dispatch_ptr fields for all the relevant publish bindings + * in items. The prevents subsequent dispatch_pub_() calls from + * sending messages to a dispatcher that has been freed. + **/ +void +pubsub_items_clear_bindings(pubsub_items_t *items) +{ + SMARTLIST_FOREACH_BEGIN(items->items, pubsub_cfg_t *, cfg) { + if (cfg->pub_binding) { + cfg->pub_binding->dispatch_ptr = NULL; + } + } SMARTLIST_FOREACH_END(cfg); +} + /** * Create a new dispatcher as configured in a pubsub_builder_t. * * Consumes and frees its input. **/ dispatch_t * -pubsub_builder_finalize(pubsub_builder_t *builder) +pubsub_builder_finalize(pubsub_builder_t *builder, + pubsub_items_t **items_out) { dispatch_t *dispatcher = NULL; tor_assert_nonfatal(builder->n_connectors == 0); @@ -276,7 +292,11 @@ pubsub_builder_finalize(pubsub_builder_t *builder) if (!dispatcher) goto err; - dispatch_fill_pub_binding_backptrs(builder, dispatcher); + pubsub_items_install_bindings(builder->items, dispatcher); + if (items_out) { + *items_out = builder->items; + builder->items = NULL; /* Prevent free */ + } err: pubsub_builder_free(builder); diff --git a/src/lib/pubsub/pubsub_build.h b/src/lib/pubsub/pubsub_build.h index 199eab219a..d2920e0216 100644 --- a/src/lib/pubsub/pubsub_build.h +++ b/src/lib/pubsub/pubsub_build.h @@ -32,6 +32,13 @@ typedef struct pubsub_builder_t pubsub_builder_t; **/ typedef struct pubsub_connector_t pubsub_connector_t; +/** + * A "pubsub items" holds the configuration items used to configure a + * pubsub_builder. After the builder is finalized, this field is extracted, + * and used later to tear down pointers that enable publishing. + **/ +typedef struct pubsub_items_t pubsub_items_t; + /** * Create a new pubsub_builder. This should only happen in the * main-init code. @@ -75,13 +82,16 @@ void pubsub_connector_free_(pubsub_connector_t *); * This should happen after every subsystem has initialized, and before * entering the mainloop. */ -struct dispatch_t *pubsub_builder_finalize(pubsub_builder_t *); +struct dispatch_t *pubsub_builder_finalize(pubsub_builder_t *, + pubsub_items_t **items_out); + +/** + * Clear all pub_binding_t backpointers in items. + **/ +void pubsub_items_clear_bindings(pubsub_items_t *items); -#ifdef PUBSUB_PRIVATE -struct pubsub_items_t; #define pubsub_items_free(cfg) \ FREE_AND_NULL(pubsub_items_t, pubsub_items_free_, (cfg)) -void pubsub_items_free_(struct pubsub_items_t *cfg); -#endif +void pubsub_items_free_(pubsub_items_t *cfg); #endif diff --git a/src/lib/pubsub/pubsub_builder_st.h b/src/lib/pubsub/pubsub_builder_st.h index a1cc6e718b..cedeb02b16 100644 --- a/src/lib/pubsub/pubsub_builder_st.h +++ b/src/lib/pubsub/pubsub_builder_st.h @@ -91,12 +91,12 @@ typedef struct pubsub_type_cfg_t { * The set of configuration requests for a dispatcher, as made by various * subsystems. **/ -typedef struct pubsub_items_t { +struct pubsub_items_t { /** List of pubsub_cfg_t. */ struct smartlist_t *items; /** List of pubsub_type_cfg_t. */ struct smartlist_t *type_items; -} pubsub_items_t; +}; /** * Type used to construct a dispatcher. We use this type to build up the @@ -111,7 +111,7 @@ struct pubsub_builder_t { int n_errors; /** In-progress configuration that we're constructing, as a list of the * requests that have been made. */ - pubsub_items_t *items; + struct pubsub_items_t *items; /** In-progress configuration that we're constructing, in a form that can * be converted to a dispatch_t. */ struct dispatch_cfg_t *cfg; diff --git a/src/test/test_pubsub_msg.c b/src/test/test_pubsub_msg.c index 5b771d45a5..41a8a25ad6 100644 --- a/src/test/test_pubsub_msg.c +++ b/src/test/test_pubsub_msg.c @@ -148,7 +148,7 @@ setup_dispatcher(const struct testcase_t *testcase) tor_free(sys); } - return pubsub_builder_finalize(builder); + return pubsub_builder_finalize(builder, NULL); } static int From 24df14eb096e73438d6045ff3b2840499a9af9b5 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 14 Jan 2019 11:29:21 -0500 Subject: [PATCH 0653/2557] Pubsub: macros for ease-of-use and typesafety. --- src/lib/pubsub/include.am | 1 + src/lib/pubsub/pubsub.h | 1 + src/lib/pubsub/pubsub_macros.h | 350 +++++++++++++++++++ src/test/include.am | 1 + src/test/test.c | 1 + src/test/test.h | 1 + src/test/test_pubsub_build.c | 596 +++++++++++++++++++++++++++++++++ 7 files changed, 951 insertions(+) create mode 100644 src/lib/pubsub/pubsub_macros.h create mode 100644 src/test/test_pubsub_build.c diff --git a/src/lib/pubsub/include.am b/src/lib/pubsub/include.am index 0ab2fd7b33..c0ec13d039 100644 --- a/src/lib/pubsub/include.am +++ b/src/lib/pubsub/include.am @@ -22,4 +22,5 @@ noinst_HEADERS += \ src/lib/pubsub/pubsub_builder_st.h \ src/lib/pubsub/pubsub_connect.h \ src/lib/pubsub/pubsub_flags.h \ + src/lib/pubsub/pubsub_macros.h \ src/lib/pubsub/pubsub_publish.h diff --git a/src/lib/pubsub/pubsub.h b/src/lib/pubsub/pubsub.h index 303b36ad5b..1c51f7a78a 100644 --- a/src/lib/pubsub/pubsub.h +++ b/src/lib/pubsub/pubsub.h @@ -80,6 +80,7 @@ #include "lib/pubsub/pub_binding_st.h" #include "lib/pubsub/pubsub_connect.h" #include "lib/pubsub/pubsub_flags.h" +#include "lib/pubsub/pubsub_macros.h" #include "lib/pubsub/pubsub_publish.h" #endif diff --git a/src/lib/pubsub/pubsub_macros.h b/src/lib/pubsub/pubsub_macros.h new file mode 100644 index 0000000000..aed9c51282 --- /dev/null +++ b/src/lib/pubsub/pubsub_macros.h @@ -0,0 +1,350 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file msg.h + * \brief Macros to help with the publish/subscribe dispatch API. + * + * The dispatch API allows different subsystems of Tor to communicate with + * another asynchronously via a shared "message" system. Some subsystems + * declare that they publish a given message, and others declare that they + * subscribe to it. Both subsystems depend on the message, but not upon one + * another. + * + * To declare a message, use DECLARE_MESSAGE() (for messages that take their + * data as a pointer) or DECLARE_MESSAGE_INT() (for messages that take their + * data as an integer. For example, you might say + * + * DECLARE_MESSAGE(new_circuit, circ, circuit_handle_t *); + * or + * DECLARE_MESSAGE_INT(shutdown_requested, boolean, bool); + * + * Every message has a unique name, a "type name" that the dispatch system + * uses to manage associated data, and a C type name. You'll want to put + * these declarations in a header, to be included by all publishers and all + * subscribers. + * + * When a subsystem wants to publish a message, it uses DECLARE_PUBLISH() at + * file scope to create necessary static functions. Then, in its subsystem + * initialization (in the "bind to dispatcher" callback) (TODO: name this + * properly!), it calls DISPATCH_ADD_PUB() to tell the dispatcher about its + * intent to publish. When it actually wants to publish, it uses the + * PUBLISH() macro. For example: + * + * // At file scope + * DECLARE_PUBLISH(shutdown_requested); + * + * static void bind_to_dispatcher(pubsub_connector_t *con) + * { + * DISPATCH_ADD_PUB(con, mainchannel, shutdown_requested); + * } + * + * // somewhere in a function + * { + * PUBLISH(shutdown_requested, true); + * } + * + * When a subsystem wants to subscribe to a message, it uses + * DECLARE_SUBSCRIBE() at file scope to declare static functions. It must + * declare a hook function that receives the message type. Then, in its "bind + * to dispatcher" function, it calls DISPATCHER_ADD_SUB() to tell the + * dispatcher about its intent to subscribe. When another module publishes + * the message, the dispatcher will call the provided hook function. + * + * // At file scope. The first argument is the message that you're + * // subscribing to; the second argument is the hook function to declare. + * DECLARE_SUBSCRIBE(shutdown_requested, on_shutdown_req_cb); + * + * // You need to declare this function. + * static void on_shutdown_req_cb(const msg_t *msg, + * bool value) + * { + * // (do something here.) + * } + * + * static void bind_to_dispatcher(pubsub_connector_t *con) + * { + * DISPATCH_ADD_SUB(con, mainchannel, shutdown_requested); + * } + * + * Where did these types come from? Somewhere in the code, you need to call + * DISPATCH_DEFINE_TYPE() to make sure that the dispatcher can manage the + * message auxiliary data. It associates a vtbl-like structure with the + * type name, so that the dispatcher knows how to manipulate the type you're + * giving it. + * + * For example, the "boolean" type we're using above could be defined as: + * + * static char *boolean_fmt(msg_aux_data_t d) + * { + * // This is used for debugging and dumping messages. + * if (d.u64) + * return tor_strdup("true"); + * else + * return tor_strdup("false"); + * } + * + * static void boolean_free(msg_aux_data_t d) + * { + * // We don't actually need to do anything to free a boolean. + * // We could use "NULL" instead of this function, but I'm including + * // it as an example. + * } + * + * static void bind_to_dispatcher(pubsub_connector_t *con) + * { + * dispatch_typefns_t boolean_fns = { + * .fmt_fn = boolean_fmt, + * .free_fn = boolean_free, + * }; + * DISPATCH_DEFINE_TYPE(con, boolean, &boolean_fns); + * } + * + * + * + * So, how does this all work? (You can stop reading here, unless you're + * debugging something.) + * + * When you declare a message in a header with DECLARE_MESSAGE() or + * DECLARE_MESSAGE_INT(), it creates five things: + * + * * two typedefs for the message argument (constant and non-constant + * variants). + * * a constant string to hold the declared message type name + * * two inline functions, to coerce the message argument type to and from + * a "msg_aux_data_t" union. + * + * All of these declarations have names based on the message name. + * + * Later, when you say DECLARE_PUBLISH() or DECLARE_SUBSCRIBE(), we use the + * elements defined by DECLARE_MESSAGE() to make sure that the publish + * function takes the correct argument type, and that the subscription hook is + * declared with the right argument type. + **/ + +#ifndef TOR_DISPATCH_MSG_H +#define TOR_DISPATCH_MSG_H + +#include "lib/cc/compat_compiler.h" +#include "lib/dispatch/dispatch_naming.h" +#include "lib/pubsub/pub_binding_st.h" +#include "lib/pubsub/pubsub_connect.h" +#include "lib/pubsub/pubsub_flags.h" +#include "lib/pubsub/pubsub_publish.h" + +/* Implemenation notes: + * + * For a messagename "foo", the DECLARE_MESSAGE*() macros must declare: + * + * msg_arg_type__foo -- a typedef for the argument type of the foo message. + * msg_arg_consttype__foo -- a typedef for the const argument type of the + * foo message. + * msg_arg_name__foo[] -- a static string constant holding the unique + * identifier for the type of the foo message. + * msg_arg_get__foo() -- an inline function taking a msg_aux_data_t and + * returning the C data type. + * msg_arg_set__foo() -- an inline function taking a msg_aux_data_t and + * the C type, setting the msg_aux_data_t to hold the C type. + * + * For a messagename "foo", the DECLARE_PUBLISH() macro must declare: + * + * pub_binding__foo -- A static pub_binding_t object used to send messages + * from this module. + * publish_fn__foo -- A function taking an argument of the appropriate + * C type, to be invoked by PUBLISH(). + * + * For a messagename "foo", the DECLARE_SUBSCRIBE() macro must declare: + * + * hookfn -- A user-provided function name, with the correct signature. + * recv_fn__foo -- A wrapper callback that takes a msg_t *, and calls + * hookfn with the appropriate arguments. + */ + +/* Macro to declare common elements shared by DECLARE_MESSAGE and + * DECLARE_MESSAGE_INT. Don't call this directly. + */ +#define DECLARE_MESSAGE_COMMON__(messagename, typename, c_type) \ + typedef c_type msg_arg_type__ ##messagename; \ + typedef const c_type msg_arg_consttype__ ##messagename; \ + ATTR_UNUSED static const char msg_arg_name__ ##messagename[] = # typename; + +/** + * Use this macro in a header to declare the existence of a given message, + * taking a pointer as auxiliary data. + * + * "messagename" is a unique identifier for the message. + * + * "typename" is a unique identifier for the type of the auxiliary data. + * + * "c_type" is a C pointer type (like "char *" or "struct foo *"). + */ +#define DECLARE_MESSAGE(messagename, typename, c_type) \ + DECLARE_MESSAGE_COMMON__(messagename, typename, c_type) \ + ATTR_UNUSED static inline c_type \ + msg_arg_get__ ##messagename(msg_aux_data_t m) \ + { \ + return m.ptr; \ + } \ + ATTR_UNUSED static inline void \ + msg_arg_set__ ##messagename(msg_aux_data_t *m, c_type v) \ + { \ + m->ptr = v; \ + } \ + EAT_SEMICOLON + +/** + * Use this macro in a header to declare the existence of a given message, + * taking an integer as auxiliary data. + * + * "messagename" is a unique identifier for the message. + * + * "typename" is a unique identifier for the type of the auxiliary data. + * + * "c_type" is a C integer type, like "int" or "bool". It needs to fit inside + * a uint64_t. + */ +#define DECLARE_MESSAGE_INT(messagename, typename, c_type) \ + DECLARE_MESSAGE_COMMON__(messagename, typename, c_type) \ + ATTR_UNUSED static inline c_type \ + msg_arg_get__ ##messagename(msg_aux_data_t m) \ + { \ + return (c_type)m.u64; \ + } \ + ATTR_UNUSED static inline void \ + msg_arg_set__ ##messagename(msg_aux_data_t *m, c_type v) \ + { \ + m->u64 = (uint64_t)v; \ + } \ + EAT_SEMICOLON + +/** + * Use this macro inside a C module declare that we'll be publishing a given + * message type from within this module. + * + * It creates necessary functions and wrappers to publish a message whose + * unique identifier is "messagename". + * + * Before you use this, you need to include the header where DECLARE_MESSAGE*() + * was used for this message. + */ +#define DECLARE_PUBLISH(messagename) \ + static pub_binding_t pub_binding__ ##messagename; \ + static void \ + publish_fn__ ##messagename(msg_arg_type__ ##messagename arg) \ + { \ + msg_aux_data_t data; \ + msg_arg_set__ ##messagename(&data, arg); \ + pubsub_pub_(&pub_binding__ ##messagename, data); \ + } \ + EAT_SEMICOLON + +/** + * Use this macro inside a C file to declare that we're subscribing to a + * given message and associating it with a given "hook function". It + * declares the hook function static, and helps with strong typing. + * + * Before you use this, you need to include the header where + * DECLARE_MESSAGE*() was used for the message whose unique identifier is + * "messagename". + * + * You will need to define a function with the name that you provide for + * "hookfn". The type of this function will be: + * static void hookfn(const msg_t *, const c_type) + * where c_type is the c type that you declared in the header. + */ +#define DECLARE_SUBSCRIBE(messagename, hookfn) \ + static void hookfn(const msg_t *, \ + const msg_arg_consttype__ ##messagename); \ + static void recv_fn__ ## messagename(const msg_t *m) \ + { \ + msg_arg_type__ ## messagename arg; \ + arg = msg_arg_get__ ##messagename(m->aux_data__); \ + hookfn(m, arg); \ + } \ + EAT_SEMICOLON + +/* + * This macro is for internal use. It backs DISPATCH_ADD_PUB*() + */ +#define DISPATCH_ADD_PUB_(connector, channel, messagename, flags) \ + ( \ + ((void)publish_fn__ ##messagename), \ + pubsub_add_pub_((connector), \ + &pub_binding__ ##messagename, \ + get_channel_id(# channel), \ + get_message_id(# messagename), \ + get_msg_type_id(msg_arg_name__ ## messagename), \ + (flags), \ + __FILE__, \ + __LINE__) \ + ) + +/** + * Use a given connector and channel name to declare that this subsystem will + * publish a given message type. + * + * Call this macro from within the add_subscriptions() function of a module. + */ +#define DISPATCH_ADD_PUB(connector, channel, messagename) \ + DISPATCH_ADD_PUB_(connector, channel, messagename, 0) + +/** + * Use a given connector and channel name to declare that this subsystem will + * publish a given message type, and that no other subsystem is allowed to. + * + * Call this macro from within the add_subscriptions() function of a module. + */ +#define DISPATCH_ADD_PUB_EXCL(connector, channel, messagename) \ + DISPATCH_ADD_PUB_(connector, channel, messagename, DISP_FLAG_EXCL) + +/* + * This macro is for internal use. It backs DISPATCH_ADD_SUB*() + */ +#define DISPATCH_ADD_SUB_(connector, channel, messagename, flags) \ + pubsub_add_sub_((connector), \ + recv_fn__ ##messagename, \ + get_channel_id(#channel), \ + get_message_id(# messagename), \ + get_msg_type_id(msg_arg_name__ ##messagename), \ + (flags), \ + __FILE__, \ + __LINE__) +/* + * Use a given connector and channel name to declare that this subsystem will + * receive a given message type. + * + * Call this macro from within the add_subscriptions() function of a module. + */ +#define DISPATCH_ADD_SUB(connector, channel, messagename) \ + DISPATCH_ADD_SUB_(connector, channel, messagename, 0) +/** + * Use a given connector and channel name to declare that this subsystem will + * receive a given message type, and that no other subsystem is allowed to do + * so. + * + * Call this macro from within the add_subscriptions() function of a module. + */ +#define DISPATCH_ADD_SUB_EXCL(connector, channel, messagename) \ + DISPATCH_ADD_SUB_(connector, channel, messagename, DISP_FLAG_EXCL) + +/** + * Publish a given message with a given argument. + */ +#define PUBLISH(messagename, arg) \ + publish_fn__ ##messagename(arg) + +/** + * Use a given connector to declare that the functions to be used to manipuate + * a certain C type. + **/ +#define DISPATCH_DEFINE_TYPE(con, type, fns) \ + pubsub_connector_define_type_((con), \ + get_msg_type_id(#type), \ + (fns), \ + __FILE__, \ + __LINE__) + +#endif diff --git a/src/test/include.am b/src/test/include.am index 7734b73de8..2f3bcd59cc 100644 --- a/src/test/include.am +++ b/src/test/include.am @@ -165,6 +165,7 @@ src_test_test_SOURCES += \ src/test/test_proto_misc.c \ src/test/test_protover.c \ src/test/test_pt.c \ + src/test/test_pubsub_build.c \ src/test/test_pubsub_msg.c \ src/test/test_relay.c \ src/test/test_relaycell.c \ diff --git a/src/test/test.c b/src/test/test.c index da0be41332..2309da208a 100644 --- a/src/test/test.c +++ b/src/test/test.c @@ -910,6 +910,7 @@ struct testgroup_t testgroups[] = { { "proto/misc/", proto_misc_tests }, { "protover/", protover_tests }, { "pt/", pt_tests }, + { "pubsub/build/", pubsub_build_tests }, { "pubsub/msg/", pubsub_msg_tests }, { "relay/" , relay_tests }, { "relaycell/", relaycell_tests }, diff --git a/src/test/test.h b/src/test/test.h index fb417124ce..96b22a9a05 100644 --- a/src/test/test.h +++ b/src/test/test.h @@ -253,6 +253,7 @@ extern struct testcase_t proto_http_tests[]; extern struct testcase_t proto_misc_tests[]; extern struct testcase_t protover_tests[]; extern struct testcase_t pt_tests[]; +extern struct testcase_t pubsub_build_tests[]; extern struct testcase_t pubsub_msg_tests[]; extern struct testcase_t relay_tests[]; extern struct testcase_t relaycell_tests[]; diff --git a/src/test/test_pubsub_build.c b/src/test/test_pubsub_build.c new file mode 100644 index 0000000000..86b5f763a4 --- /dev/null +++ b/src/test/test_pubsub_build.c @@ -0,0 +1,596 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#define DISPATCH_PRIVATE + +#include "test/test.h" + +#include "lib/cc/torint.h" +#include "lib/dispatch/dispatch.h" +#include "lib/dispatch/dispatch_naming.h" +#include "lib/dispatch/dispatch_st.h" +#include "lib/dispatch/msgtypes.h" +#include "lib/pubsub/pubsub_macros.h" +#include "lib/pubsub/pubsub_build.h" +#include "lib/pubsub/pubsub_builder_st.h" + +#include "lib/log/escape.h" +#include "lib/malloc/malloc.h" +#include "lib/string/printf.h" + +#include "test/log_test_helpers.h" + +#include +#include + +static char * +ex_int_fmt(msg_aux_data_t aux) +{ + int val = (int) aux.u64; + char *r=NULL; + tor_asprintf(&r, "%d", val); + return r; +} + +static char * +ex_str_fmt(msg_aux_data_t aux) +{ + return esc_for_log(aux.ptr); +} + +static void +ex_str_free(msg_aux_data_t aux) +{ + tor_free_(aux.ptr); +} + +static dispatch_typefns_t intfns = { + .fmt_fn = ex_int_fmt +}; + +static dispatch_typefns_t stringfns = { + .free_fn = ex_str_free, + .fmt_fn = ex_str_fmt +}; + +DECLARE_MESSAGE_INT(bunch_of_coconuts, int, int); +DECLARE_PUBLISH(bunch_of_coconuts); +DECLARE_SUBSCRIBE(bunch_of_coconuts, coconut_recipient_cb); + +DECLARE_MESSAGE(yes_we_have_no, string, char *); +DECLARE_PUBLISH(yes_we_have_no); +DECLARE_SUBSCRIBE(yes_we_have_no, absent_item_cb); + +static void +coconut_recipient_cb(const msg_t *m, int n_coconuts) +{ + (void)m; + (void)n_coconuts; +} + +static void +absent_item_cb(const msg_t *m, const char *fruitname) +{ + (void)m; + (void)fruitname; +} + +#define FLAG_SKIP 99999 + +static void +seed_dispatch_builder(pubsub_builder_t *b, + unsigned fl1, unsigned fl2, unsigned fl3, unsigned fl4) +{ + pubsub_connector_t *c = NULL; + + { + c = pubsub_connector_for_subsystem(b, get_subsys_id("sys1")); + DISPATCH_DEFINE_TYPE(c, int, &intfns); + if (fl1 != FLAG_SKIP) + DISPATCH_ADD_PUB_(c, main, bunch_of_coconuts, fl1); + if (fl2 != FLAG_SKIP) + DISPATCH_ADD_SUB_(c, main, yes_we_have_no, fl2); + pubsub_connector_free(c); + } + + { + c = pubsub_connector_for_subsystem(b, get_subsys_id("sys2")); + DISPATCH_DEFINE_TYPE(c, string, &stringfns); + if (fl3 != FLAG_SKIP) + DISPATCH_ADD_PUB_(c, main, yes_we_have_no, fl3); + if (fl4 != FLAG_SKIP) + DISPATCH_ADD_SUB_(c, main, bunch_of_coconuts, fl4); + pubsub_connector_free(c); + } +} + +static void +seed_pubsub_builder_basic(pubsub_builder_t *b) +{ + seed_dispatch_builder(b, 0, 0, 0, 0); +} + +/* Regular builder with valid types and messages. + */ +static void +test_pubsub_build_types_ok(void *arg) +{ + (void)arg; + pubsub_builder_t *b = NULL; + dispatch_t *dispatcher = NULL; + pubsub_connector_t *c = NULL; + + b = pubsub_builder_new(); + seed_pubsub_builder_basic(b); + + dispatcher = pubsub_builder_finalize(b, NULL); + b = NULL; + tt_assert(dispatcher); + + tt_int_op(dispatcher->n_types, OP_GE, 2); + tt_assert(dispatcher->typefns); + + tt_assert(dispatcher->typefns[get_msg_type_id("int")].fmt_fn == ex_int_fmt); + tt_assert(dispatcher->typefns[get_msg_type_id("string")].fmt_fn == + ex_str_fmt); + + done: + pubsub_connector_free(c); + pubsub_builder_free(b); + dispatch_free(dispatcher); +} + +/* We fail if the same type is defined in two places with different functions. + */ +static void +test_pubsub_build_types_decls_conflict(void *arg) +{ + (void)arg; + pubsub_builder_t *b = NULL; + dispatch_t *dispatcher = NULL; + pubsub_connector_t *c = NULL; + + b = pubsub_builder_new(); + seed_pubsub_builder_basic(b); + { + c = pubsub_connector_for_subsystem(b, get_subsys_id("sys3")); + // Extra declaration of int: we don't allow this. + DISPATCH_DEFINE_TYPE(c, int, &stringfns); + pubsub_connector_free(c); + } + + setup_full_capture_of_logs(LOG_WARN); + dispatcher = pubsub_builder_finalize(b, NULL); + b = NULL; + tt_assert(dispatcher == NULL); + // expect_log_msg_containing("(int) declared twice"); // XXXX + + done: + pubsub_connector_free(c); + pubsub_builder_free(b); + dispatch_free(dispatcher); + teardown_capture_of_logs(); +} + +/* If a message ID exists but nobody is publishing or subscribing to it, + * that's okay. */ +static void +test_pubsub_build_unused_message(void *arg) +{ + (void)arg; + pubsub_builder_t *b = NULL; + dispatch_t *dispatcher = NULL; + + b = pubsub_builder_new(); + seed_pubsub_builder_basic(b); + + // This message isn't actually generated by anyone, but that will be fine: + // we just log it at info. + get_message_id("unused"); + setup_capture_of_logs(LOG_INFO); + + dispatcher = pubsub_builder_finalize(b, NULL); + b = NULL; + tt_assert(dispatcher); + expect_log_msg_containing( + "Nobody is publishing or subscribing to message"); + + done: + pubsub_builder_free(b); + dispatch_free(dispatcher); + teardown_capture_of_logs(); +} + +/* Publishing or subscribing to a message with no subscribers / publishers + * should fail and warn. */ +static void +test_pubsub_build_missing_pubsub(void *arg) +{ + (void)arg; + pubsub_builder_t *b = NULL; + dispatch_t *dispatcher = NULL; + + b = pubsub_builder_new(); + seed_dispatch_builder(b, 0, 0, FLAG_SKIP, FLAG_SKIP); + + setup_full_capture_of_logs(LOG_WARN); + dispatcher = pubsub_builder_finalize(b, NULL); + b = NULL; + tt_assert(dispatcher == NULL); + + expect_log_msg_containing( + "Message 0 (bunch_of_coconuts) has publishers, but no subscribers."); + expect_log_msg_containing( + "Message 1 (yes_we_have_no) has subscribers, but no publishers."); + + done: + pubsub_builder_free(b); + dispatch_free(dispatcher); + teardown_capture_of_logs(); +} + +/* Make sure that a stub publisher or subscriber prevents an error from + * happening even if there are no other publishers/subscribers for a message + */ +static void +test_pubsub_build_stub_pubsub(void *arg) +{ + (void)arg; + pubsub_builder_t *b = NULL; + dispatch_t *dispatcher = NULL; + + b = pubsub_builder_new(); + seed_dispatch_builder(b, 0, 0, DISP_FLAG_STUB, DISP_FLAG_STUB); + + dispatcher = pubsub_builder_finalize(b, NULL); + b = NULL; + tt_assert(dispatcher); + + // 1 subscriber. + tt_int_op(1, OP_EQ, + dispatcher->table[get_message_id("yes_we_have_no")]->n_enabled); + // no subscribers + tt_ptr_op(NULL, OP_EQ, + dispatcher->table[get_message_id("bunch_of_coconuts")]); + + done: + pubsub_builder_free(b); + dispatch_free(dispatcher); +} + +/* Only one channel per msg id. */ +static void +test_pubsub_build_channels_conflict(void *arg) +{ + (void)arg; + pubsub_builder_t *b = NULL; + dispatch_t *dispatcher = NULL; + pubsub_connector_t *c = NULL; + + b = pubsub_builder_new(); + seed_pubsub_builder_basic(b); + pub_binding_t btmp; + + { + c = pubsub_connector_for_subsystem(b, get_subsys_id("problems")); + /* Usually the DISPATCH_ADD_PUB macro would keep us from using + * the wrong channel */ + pubsub_add_pub_(c, &btmp, get_channel_id("hithere"), + get_message_id("bunch_of_coconuts"), + get_msg_type_id("int"), + 0 /* flags */, + "somewhere.c", 22); + pubsub_connector_free(c); + }; + + setup_full_capture_of_logs(LOG_WARN); + dispatcher = pubsub_builder_finalize(b, NULL); + b = NULL; + tt_assert(dispatcher == NULL); + + expect_log_msg_containing("Message 0 (bunch_of_coconuts) is associated " + "with multiple inconsistent channels."); + + done: + pubsub_builder_free(b); + dispatch_free(dispatcher); + teardown_capture_of_logs(); +} + +/* Only one type per msg id. */ +static void +test_pubsub_build_types_conflict(void *arg) +{ + (void)arg; + pubsub_builder_t *b = NULL; + dispatch_t *dispatcher = NULL; + pubsub_connector_t *c = NULL; + + b = pubsub_builder_new(); + seed_pubsub_builder_basic(b); + pub_binding_t btmp; + + { + c = pubsub_connector_for_subsystem(b, get_subsys_id("problems")); + /* Usually the DISPATCH_ADD_PUB macro would keep us from using + * the wrong channel */ + pubsub_add_pub_(c, &btmp, get_channel_id("hithere"), + get_message_id("bunch_of_coconuts"), + get_msg_type_id("string"), + 0 /* flags */, + "somewhere.c", 22); + pubsub_connector_free(c); + }; + + setup_full_capture_of_logs(LOG_WARN); + dispatcher = pubsub_builder_finalize(b, NULL); + b = NULL; + tt_assert(dispatcher == NULL); + + expect_log_msg_containing("Message 0 (bunch_of_coconuts) is associated " + "with multiple inconsistent message types."); + + done: + pubsub_builder_free(b); + dispatch_free(dispatcher); + teardown_capture_of_logs(); +} + +/* The same module can't publish and subscribe the same message */ +static void +test_pubsub_build_pubsub_same(void *arg) +{ + (void)arg; + pubsub_builder_t *b = NULL; + dispatch_t *dispatcher = NULL; + pubsub_connector_t *c = NULL; + + b = pubsub_builder_new(); + seed_pubsub_builder_basic(b); + + { + c = pubsub_connector_for_subsystem(b, get_subsys_id("sys1")); + // already publishing this. + DISPATCH_ADD_SUB(c, main, bunch_of_coconuts); + pubsub_connector_free(c); + }; + + setup_full_capture_of_logs(LOG_WARN); + dispatcher = pubsub_builder_finalize(b, NULL); + b = NULL; + tt_assert(dispatcher == NULL); + + expect_log_msg_containing("Message 0 (bunch_of_coconuts) is published " + "and subscribed by the same subsystem 0 (sys1)"); + + done: + pubsub_builder_free(b); + dispatch_free(dispatcher); + teardown_capture_of_logs(); +} + +/* More than one subsystem may publish or subscribe, and that's okay. */ +static void +test_pubsub_build_pubsub_multi(void *arg) +{ + (void)arg; + pubsub_builder_t *b = NULL; + dispatch_t *dispatcher = NULL; + pubsub_connector_t *c = NULL; + + b = pubsub_builder_new(); + seed_pubsub_builder_basic(b); + pub_binding_t btmp; + + { + c = pubsub_connector_for_subsystem(b, get_subsys_id("sys3")); + DISPATCH_ADD_SUB(c, main, bunch_of_coconuts); + pubsub_add_pub_(c, &btmp, get_channel_id("main"), + get_message_id("yes_we_have_no"), + get_msg_type_id("string"), + 0 /* flags */, + "somewhere.c", 22); + pubsub_connector_free(c); + }; + + dispatcher = pubsub_builder_finalize(b, NULL); + b = NULL; + tt_assert(dispatcher); + + // 1 subscribers + tt_int_op(1, OP_EQ, + dispatcher->table[get_message_id("yes_we_have_no")]->n_enabled); + // 2 subscribers. + dtbl_entry_t *ent = + dispatcher->table[get_message_id("bunch_of_coconuts")]; + tt_int_op(2, OP_EQ, ent->n_enabled); + tt_int_op(2, OP_EQ, ent->n_fns); + tt_ptr_op(ent->rcv[0].fn, OP_EQ, recv_fn__bunch_of_coconuts); + tt_ptr_op(ent->rcv[1].fn, OP_EQ, recv_fn__bunch_of_coconuts); + + done: + pubsub_builder_free(b); + dispatch_free(dispatcher); +} + +static void +some_other_coconut_hook(const msg_t *m) +{ + (void)m; +} + +/* Subscribe hooks should be build correctly when there are a bunch of + * them. */ +static void +test_pubsub_build_sub_many(void *arg) +{ + (void)arg; + pubsub_builder_t *b = NULL; + dispatch_t *dispatcher = NULL; + pubsub_connector_t *c = NULL; + char *sysname = NULL; + b = pubsub_builder_new(); + seed_pubsub_builder_basic(b); + + int i; + for (i = 1; i < 100; ++i) { + tor_asprintf(&sysname, "system%d",i); + c = pubsub_connector_for_subsystem(b, get_subsys_id(sysname)); + if (i % 7) { + DISPATCH_ADD_SUB(c, main, bunch_of_coconuts); + } else { + pubsub_add_sub_(c, some_other_coconut_hook, + get_channel_id("main"), + get_message_id("bunch_of_coconuts"), + get_msg_type_id("int"), + 0 /* flags */, + "somewhere.c", 22); + } + pubsub_connector_free(c); + tor_free(sysname); + }; + + dispatcher = pubsub_builder_finalize(b, NULL); + b = NULL; + tt_assert(dispatcher); + + dtbl_entry_t *ent = + dispatcher->table[get_message_id("bunch_of_coconuts")]; + tt_int_op(100, OP_EQ, ent->n_enabled); + tt_int_op(100, OP_EQ, ent->n_fns); + tt_ptr_op(ent->rcv[0].fn, OP_EQ, recv_fn__bunch_of_coconuts); + tt_ptr_op(ent->rcv[1].fn, OP_EQ, recv_fn__bunch_of_coconuts); + tt_ptr_op(ent->rcv[76].fn, OP_EQ, recv_fn__bunch_of_coconuts); + tt_ptr_op(ent->rcv[77].fn, OP_EQ, some_other_coconut_hook); + tt_ptr_op(ent->rcv[78].fn, OP_EQ, recv_fn__bunch_of_coconuts); + + done: + pubsub_builder_free(b); + dispatch_free(dispatcher); + tor_free(sysname); +} + +/* The same subsystem can only declare one publish or subscribe. */ +static void +test_pubsub_build_pubsub_redundant(void *arg) +{ + (void)arg; + pubsub_builder_t *b = NULL; + dispatch_t *dispatcher = NULL; + pubsub_connector_t *c = NULL; + + b = pubsub_builder_new(); + seed_pubsub_builder_basic(b); + pub_binding_t btmp; + + { + c = pubsub_connector_for_subsystem(b, get_subsys_id("sys2")); + DISPATCH_ADD_SUB(c, main, bunch_of_coconuts); + pubsub_add_pub_(c, &btmp, get_channel_id("main"), + get_message_id("yes_we_have_no"), + get_msg_type_id("string"), + 0 /* flags */, + "somewhere.c", 22); + pubsub_connector_free(c); + }; + + setup_full_capture_of_logs(LOG_WARN); + dispatcher = pubsub_builder_finalize(b, NULL); + b = NULL; + tt_assert(dispatcher == NULL); + + expect_log_msg_containing( + "is configured to be published by subsystem 1 (sys2) more than once"); + expect_log_msg_containing( + "is configured to be subscribed by subsystem 1 (sys2) more than once"); + + done: + pubsub_builder_free(b); + dispatch_free(dispatcher); + teardown_capture_of_logs(); +} + +/* It's fine to declare the excl flag. */ +static void +test_pubsub_build_excl_ok(void *arg) +{ + (void)arg; + pubsub_builder_t *b = NULL; + dispatch_t *dispatcher = NULL; + + b = pubsub_builder_new(); + // Try one excl/excl pair and one excl/non pair. + seed_dispatch_builder(b, DISP_FLAG_EXCL, 0, + DISP_FLAG_EXCL, DISP_FLAG_EXCL); + + dispatcher = pubsub_builder_finalize(b, NULL); + b = NULL; + tt_assert(dispatcher); + + // 1 subscribers + tt_int_op(1, OP_EQ, + dispatcher->table[get_message_id("yes_we_have_no")]->n_enabled); + // 1 subscriber. + tt_int_op(1, OP_EQ, + dispatcher->table[get_message_id("bunch_of_coconuts")]->n_enabled); + + done: + pubsub_builder_free(b); + dispatch_free(dispatcher); +} + +/* but if you declare the excl flag, you need to mean it. */ +static void +test_pubsub_build_excl_bad(void *arg) +{ + (void)arg; + pubsub_builder_t *b = NULL; + dispatch_t *dispatcher = NULL; + pubsub_connector_t *c = NULL; + + b = pubsub_builder_new(); + seed_dispatch_builder(b, DISP_FLAG_EXCL, DISP_FLAG_EXCL, + 0, 0); + + { + c = pubsub_connector_for_subsystem(b, get_subsys_id("sys3")); + DISPATCH_ADD_PUB_(c, main, bunch_of_coconuts, 0); + DISPATCH_ADD_SUB_(c, main, yes_we_have_no, 0); + pubsub_connector_free(c); + }; + + setup_full_capture_of_logs(LOG_WARN); + dispatcher = pubsub_builder_finalize(b, NULL); + b = NULL; + tt_assert(dispatcher == NULL); + + expect_log_msg_containing("has multiple publishers, but at least one is " + "marked as exclusive."); + expect_log_msg_containing("has multiple subscribers, but at least one is " + "marked as exclusive."); + + done: + pubsub_builder_free(b); + dispatch_free(dispatcher); + teardown_capture_of_logs(); +} + +#define T(name, flags) \ + { #name, test_pubsub_build_ ## name , (flags), NULL, NULL } + +struct testcase_t pubsub_build_tests[] = { + T(types_ok, TT_FORK), + T(types_decls_conflict, TT_FORK), + T(unused_message, TT_FORK), + T(missing_pubsub, TT_FORK), + T(stub_pubsub, TT_FORK), + T(channels_conflict, TT_FORK), + T(types_conflict, TT_FORK), + T(pubsub_same, TT_FORK), + T(pubsub_multi, TT_FORK), + T(sub_many, TT_FORK), + T(pubsub_redundant, TT_FORK), + T(excl_ok, TT_FORK), + T(excl_bad, TT_FORK), + END_OF_TESTCASES +}; From bdeaf7d4b2929609c4d3f2ce9adfd973361ef578 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 15 Jan 2019 10:27:39 -0500 Subject: [PATCH 0654/2557] Code to manage publish/subscribe setup via subsystem interface. This commit has the necessary logic to run the publish/subscribe system from the mainloop, and to initialize it on startup and tear it down later. --- src/app/main/subsysmgr.c | 52 +++++++++- src/app/main/subsysmgr.h | 5 + src/core/include.am | 2 + src/core/mainloop/mainloop_pubsub.c | 149 ++++++++++++++++++++++++++++ src/core/mainloop/mainloop_pubsub.h | 23 +++++ src/lib/subsys/subsys.h | 4 +- 6 files changed, 232 insertions(+), 3 deletions(-) create mode 100644 src/core/mainloop/mainloop_pubsub.c create mode 100644 src/core/mainloop/mainloop_pubsub.h diff --git a/src/app/main/subsysmgr.c b/src/app/main/subsysmgr.c index abd2edd10b..91a567ce0b 100644 --- a/src/app/main/subsysmgr.c +++ b/src/app/main/subsysmgr.c @@ -5,9 +5,14 @@ #include "orconfig.h" #include "app/main/subsysmgr.h" -#include "lib/err/torerr.h" +#include "lib/dispatch/dispatch_naming.h" +#include "lib/dispatch/msgtypes.h" +#include "lib/err/torerr.h" #include "lib/log/log.h" +#include "lib/malloc/malloc.h" +#include "lib/pubsub/pubsub_build.h" +#include "lib/pubsub/pubsub_connect.h" #include #include @@ -105,6 +110,51 @@ subsystems_init_upto(int target_level) return 0; } +/** + * Add publish/subscribe relationships to builder for all + * initialized subsystems of level no more than target_level. + **/ +int +subsystems_add_pubsub_upto(pubsub_builder_t *builder, + int target_level) +{ + for (unsigned i = 0; i < n_tor_subsystems; ++i) { + const subsys_fns_t *sys = tor_subsystems[i]; + if (!sys->supported) + continue; + if (sys->level > target_level) + break; + if (! sys_initialized[i]) + continue; + int r = 0; + if (sys->add_pubsub) { + subsys_id_t sysid = get_subsys_id(sys->name); + raw_assert(sysid != ERROR_ID); + pubsub_connector_t *connector; + connector = pubsub_connector_for_subsystem(builder, sysid); + r = sys->add_pubsub(connector); + pubsub_connector_free(connector); + } + if (r < 0) { + fprintf(stderr, "BUG: subsystem %s (at %u) could not connect to " + "publish/subscribe system.", sys->name, sys->level); + raw_assert_unreached_msg("A subsystem couldn't be connected."); + } + } + + return 0; +} + +/** + * Add publish/subscribe relationships to builder for all + * initialized subsystems. + **/ +int +subsystems_add_pubsub(pubsub_builder_t *builder) +{ + return subsystems_add_pubsub_upto(builder, MAX_SUBSYS_LEVEL); +} + /** * Shut down all the subsystems. **/ diff --git a/src/app/main/subsysmgr.h b/src/app/main/subsysmgr.h index 4b3cad62ad..4878cf8c36 100644 --- a/src/app/main/subsysmgr.h +++ b/src/app/main/subsysmgr.h @@ -14,6 +14,11 @@ extern const unsigned n_tor_subsystems; int subsystems_init(void); int subsystems_init_upto(int level); +struct pubsub_builder_t; +int subsystems_add_pubsub_upto(struct pubsub_builder_t *builder, + int target_level); +int subsystems_add_pubsub(struct pubsub_builder_t *builder); + void subsystems_shutdown(void); void subsystems_shutdown_downto(int level); diff --git a/src/core/include.am b/src/core/include.am index ae47c75e09..3a0e907edb 100644 --- a/src/core/include.am +++ b/src/core/include.am @@ -22,6 +22,7 @@ LIBTOR_APP_A_SOURCES = \ src/core/mainloop/connection.c \ src/core/mainloop/cpuworker.c \ src/core/mainloop/mainloop.c \ + src/core/mainloop/mainloop_pubsub.c \ src/core/mainloop/netstatus.c \ src/core/mainloop/periodic.c \ src/core/or/address_set.c \ @@ -213,6 +214,7 @@ noinst_HEADERS += \ src/core/mainloop/connection.h \ src/core/mainloop/cpuworker.h \ src/core/mainloop/mainloop.h \ + src/core/mainloop/mainloop_pubsub.h \ src/core/mainloop/netstatus.h \ src/core/mainloop/periodic.h \ src/core/or/addr_policy_st.h \ diff --git a/src/core/mainloop/mainloop_pubsub.c b/src/core/mainloop/mainloop_pubsub.c new file mode 100644 index 0000000000..ab3614ae00 --- /dev/null +++ b/src/core/mainloop/mainloop_pubsub.c @@ -0,0 +1,149 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "orconfig.h" + +#include "src/core/or/or.h" +#include "src/core/mainloop/mainloop.h" +#include "src/core/mainloop/mainloop_pubsub.h" + +#include "lib/container/smartlist.h" +#include "lib/dispatch/dispatch.h" +#include "lib/dispatch/dispatch_naming.h" +#include "lib/evloop/compat_libevent.h" +#include "lib/pubsub/pubsub.h" +#include "lib/pubsub/pubsub_build.h" + +/** + * Dispatcher to use for delivering messages. + **/ +static dispatch_t *the_dispatcher = NULL; +static pubsub_items_t *the_pubsub_items = NULL; +/** + * A list of mainloop_event_t, indexed by channel ID, to flush the messages + * on a channel. + **/ +static smartlist_t *alert_events = NULL; + +/** + * Mainloop event callback: flush all the messages in a channel. + * + * The channel is encoded as a pointer, and passed via arg. + **/ +static void +flush_channel_event(mainloop_event_t *ev, void *arg) +{ + (void)ev; + if (!the_dispatcher) + return; + + channel_id_t chan = (channel_id_t)(uintptr_t)(arg); + dispatch_flush(the_dispatcher, chan, INT_MAX); +} + +int +tor_mainloop_connect_pubsub(struct pubsub_builder_t *builder) +{ + int rv = -1; + tor_mainloop_disconnect_pubsub(); + + the_dispatcher = pubsub_builder_finalize(builder, &the_pubsub_items); + if (! the_dispatcher) + goto err; + + const size_t num_channels = get_num_channel_ids(); + alert_events = smartlist_new(); + for (size_t i = 0; i < num_channels; ++i) { + smartlist_add(alert_events, + mainloop_event_postloop_new(flush_channel_event, + (void*)(uintptr_t)(i))); + } + + rv = 0; + err: + tor_mainloop_disconnect_pubsub(); + return rv; +} + +/** + * Dispatch alertfn callback: do nothing. Implements DELIV_NEVER. + **/ +static void +alertfn_never(dispatch_t *d, channel_id_t chan, void *arg) +{ + (void)d; + (void)chan; + (void)arg; +} + +/** + * Dispatch alertfn callback: activate a mainloop event. Implements + * DELIV_PROMPT. + **/ +static void +alertfn_prompt(dispatch_t *d, channel_id_t chan, void *arg) +{ + (void)d; + (void)chan; + mainloop_event_t *event = arg; + mainloop_event_activate(event); +} + +/** + * Dispatch alertfn callback: flush all messages right now. Implements + * DELIV_IMMEDIATE. + **/ +static void +alertfn_immediate(dispatch_t *d, channel_id_t chan, void *arg) +{ + (void) arg; + dispatch_flush(d, chan, INT_MAX); +} + +/** + * Set the strategy to be used for delivering messages on the named channel. + **/ +int +tor_mainloop_set_delivery_strategy(const char *msg_channel_name, + deliv_strategy_t strategy) +{ + channel_id_t chan = get_channel_id(msg_channel_name); + if (BUG(chan == ERROR_ID) || + BUG(chan >= smartlist_len(alert_events))) + return -1; + + switch (strategy) { + case DELIV_NEVER: + dispatch_set_alert_fn(the_dispatcher, chan, alertfn_never, NULL); + break; + case DELIV_PROMPT: + dispatch_set_alert_fn(the_dispatcher, chan, alertfn_prompt, + smartlist_get(alert_events, chan)); + break; + case DELIV_IMMEDIATE: + dispatch_set_alert_fn(the_dispatcher, chan, alertfn_immediate, NULL); + break; + } + return 0; +} + +/** + * Remove all pubsub dispatchers and events from the mainloop. + **/ +void +tor_mainloop_disconnect_pubsub(void) +{ + if (the_pubsub_items) { + pubsub_items_clear_bindings(the_pubsub_items); + pubsub_items_free(the_pubsub_items); + } + if (alert_events) { + SMARTLIST_FOREACH(alert_events, mainloop_event_t *, ev, + mainloop_event_free(ev)); + smartlist_free(alert_events); + } + dispatch_free(the_dispatcher); +} diff --git a/src/core/mainloop/mainloop_pubsub.h b/src/core/mainloop/mainloop_pubsub.h new file mode 100644 index 0000000000..6eff778420 --- /dev/null +++ b/src/core/mainloop/mainloop_pubsub.h @@ -0,0 +1,23 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef TOR_MAINLOOP_PUBSUB_H +#define TOR_MAINLOOP_PUBSUB_H + +struct pubsub_builder_t; + +typedef enum { + DELIV_NEVER=0, + DELIV_PROMPT, + DELIV_IMMEDIATE, +} deliv_strategy_t; + +int tor_mainloop_connect_pubsub(struct pubsub_builder_t *builder); +int tor_mainloop_set_delivery_strategy(const char *msg_channel_name, + deliv_strategy_t strategy); +void tor_mainloop_disconnect_pubsub(void); + +#endif diff --git a/src/lib/subsys/subsys.h b/src/lib/subsys/subsys.h index 2452ec6e2f..6f1710c719 100644 --- a/src/lib/subsys/subsys.h +++ b/src/lib/subsys/subsys.h @@ -8,7 +8,7 @@ #include -struct dispatch_connector_t; +struct pubsub_connector_t; /** * A subsystem is a part of Tor that is initialized, shut down, configured, @@ -58,7 +58,7 @@ typedef struct subsys_fns_t { /** * Connect a subsystem to the message dispatch system. **/ - int (*add_pubsub)(struct dispatch_connector_t *); + int (*add_pubsub)(struct pubsub_connector_t *); /** * Perform any necessary pre-fork cleanup. This function may not fail. From 02e0a39d396ad4f86d27770a5cf6f46553a01b0c Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 11 Mar 2019 15:56:13 -0400 Subject: [PATCH 0655/2557] Add msgtypes.h include to pubsub_build.h (The header won't compile without it.) --- src/lib/pubsub/pubsub_build.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/lib/pubsub/pubsub_build.h b/src/lib/pubsub/pubsub_build.h index d2920e0216..3fad254534 100644 --- a/src/lib/pubsub/pubsub_build.h +++ b/src/lib/pubsub/pubsub_build.h @@ -14,6 +14,8 @@ #ifndef TOR_PUBSUB_BUILD_H #define TOR_PUBSUB_BUILD_H +#include "lib/dispatch/msgtypes.h" + struct dispatch_t; /** From 6d1abd37e27761bc3c92c398dbc03711fcf9e5c8 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 11 Mar 2019 16:02:38 -0400 Subject: [PATCH 0656/2557] Connect the mainloop pubsub dispatcher on startup; free it on shutdown. --- src/app/main/main.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/app/main/main.c b/src/app/main/main.c index ba2dfebd77..7bf9d3fe2b 100644 --- a/src/app/main/main.c +++ b/src/app/main/main.c @@ -19,6 +19,7 @@ #include "core/mainloop/connection.h" #include "core/mainloop/cpuworker.h" #include "core/mainloop/mainloop.h" +#include "core/mainloop/mainloop_pubsub.h" #include "core/mainloop/netstatus.h" #include "core/or/channel.h" #include "core/or/channelpadding.h" @@ -75,6 +76,7 @@ #include "lib/net/resolve.h" #include "lib/process/waitpid.h" +#include "lib/pubsub/pubsub_build.h" #include "lib/meminfo/meminfo.h" #include "lib/osinfo/uname.h" @@ -807,6 +809,7 @@ tor_free_all(int postfork) } /* stuff in main.c */ + tor_mainloop_disconnect_pubsub(); tor_mainloop_free_all(); if (!postfork) { @@ -1407,6 +1410,15 @@ tor_run_main(const tor_main_configuration_t *tor_cfg) } } #endif /* defined(NT_SERVICE) */ + + { + pubsub_builder_t *builder = pubsub_builder_new(); + int r = subsystems_add_pubsub(builder); + tor_assert(r == 0); + r = tor_mainloop_connect_pubsub(builder); // consumes builder + tor_assert(r == 0); + } + { int init_rv = tor_init(argc, argv); if (init_rv) { From b4f28b9df8188af82a0140b1831ee4b50c6e4f6d Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 11 Mar 2019 16:57:33 -0400 Subject: [PATCH 0657/2557] pubsub: install libevent events separately from the_dispatcher. Also, add documentation, and fix a free-on-error bug. --- src/app/main/main.c | 7 +++++++ src/core/mainloop/mainloop_pubsub.c | 28 +++++++++++++++++++++++----- src/core/mainloop/mainloop_pubsub.h | 1 + 3 files changed, 31 insertions(+), 5 deletions(-) diff --git a/src/app/main/main.c b/src/app/main/main.c index 7bf9d3fe2b..e0f6352879 100644 --- a/src/app/main/main.c +++ b/src/app/main/main.c @@ -1428,6 +1428,13 @@ tor_run_main(const tor_main_configuration_t *tor_cfg) } } + if (get_options()->command == CMD_RUN_TOR) { + tor_mainloop_connect_pubsub_events(); + /* XXXX For each pubsub channel, its delivery strategy should be set at + * this XXXX point, using tor_mainloop_set_delivery_strategy(). + */ + } + if (get_options()->Sandbox && get_options()->command == CMD_RUN_TOR) { sandbox_cfg_t* cfg = sandbox_init_filter(); diff --git a/src/core/mainloop/mainloop_pubsub.c b/src/core/mainloop/mainloop_pubsub.c index ab3614ae00..aac607126d 100644 --- a/src/core/mainloop/mainloop_pubsub.c +++ b/src/core/mainloop/mainloop_pubsub.c @@ -44,6 +44,9 @@ flush_channel_event(mainloop_event_t *ev, void *arg) dispatch_flush(the_dispatcher, chan, INT_MAX); } +/** + * Construct our global pubsub object from builder. Return 0 on + * success, -1 on failure. */ int tor_mainloop_connect_pubsub(struct pubsub_builder_t *builder) { @@ -54,6 +57,26 @@ tor_mainloop_connect_pubsub(struct pubsub_builder_t *builder) if (! the_dispatcher) goto err; + rv = 0; + goto done; + err: + tor_mainloop_disconnect_pubsub(); + done: + return rv; +} + +/** + * Install libevent events for all of the pubsub channels. + * + * Invoke this after tor_mainloop_connect_pubsub, and after libevent has been + * initialized. + */ +void +tor_mainloop_connect_pubsub_events(void) +{ + tor_assert(the_dispatcher); + tor_assert(! alert_events); + const size_t num_channels = get_num_channel_ids(); alert_events = smartlist_new(); for (size_t i = 0; i < num_channels; ++i) { @@ -61,11 +84,6 @@ tor_mainloop_connect_pubsub(struct pubsub_builder_t *builder) mainloop_event_postloop_new(flush_channel_event, (void*)(uintptr_t)(i))); } - - rv = 0; - err: - tor_mainloop_disconnect_pubsub(); - return rv; } /** diff --git a/src/core/mainloop/mainloop_pubsub.h b/src/core/mainloop/mainloop_pubsub.h index 6eff778420..a31b2b4ba7 100644 --- a/src/core/mainloop/mainloop_pubsub.h +++ b/src/core/mainloop/mainloop_pubsub.h @@ -16,6 +16,7 @@ typedef enum { } deliv_strategy_t; int tor_mainloop_connect_pubsub(struct pubsub_builder_t *builder); +void tor_mainloop_connect_pubsub_events(void); int tor_mainloop_set_delivery_strategy(const char *msg_channel_name, deliv_strategy_t strategy); void tor_mainloop_disconnect_pubsub(void); From 3d6bf7b36e419bbabd48b4bed82f12fca1a922b3 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 12 Mar 2019 18:53:24 -0400 Subject: [PATCH 0658/2557] Document several issues found by Taylor --- src/lib/pubsub/pub_binding_st.h | 4 +++- src/lib/pubsub/pubsub.h | 3 +++ src/lib/pubsub/pubsub_connect.h | 3 ++- src/lib/pubsub/pubsub_macros.h | 6 ++++-- 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/lib/pubsub/pub_binding_st.h b/src/lib/pubsub/pub_binding_st.h index e8c0d47efd..4f5df8ff38 100644 --- a/src/lib/pubsub/pub_binding_st.h +++ b/src/lib/pubsub/pub_binding_st.h @@ -5,8 +5,10 @@ /* See LICENSE for licensing information */ /** - * @file pubsub_build.h + * @file pub_binding_st.h * @brief Declaration of pub_binding_t. + * + * This is an internal type for the pubsub implementation. */ #ifndef TOR_PUB_BINDING_ST_H diff --git a/src/lib/pubsub/pubsub.h b/src/lib/pubsub/pubsub.h index 1c51f7a78a..08e3f42f7c 100644 --- a/src/lib/pubsub/pubsub.h +++ b/src/lib/pubsub/pubsub.h @@ -30,6 +30,9 @@ * Rather than using the dispatch module directly, a publishing module * receives a "binding" object that it uses to send messages with the right * settings. + * + * Most users of this module will want to use this header, and the + * pubsub_macros.h header for convenience. */ /* diff --git a/src/lib/pubsub/pubsub_connect.h b/src/lib/pubsub/pubsub_connect.h index b63f9dc438..f3de74c870 100644 --- a/src/lib/pubsub/pubsub_connect.h +++ b/src/lib/pubsub/pubsub_connect.h @@ -9,7 +9,8 @@ * @brief Header for functions that add relationships to a pubsub builder. * * These functions are used by modules that need to add publication and - * subscription requests. + * subscription requests. Most users will want to call these functions + * indirectly, via the macros in pubsub_macros.h. **/ #ifndef TOR_PUBSUB_CONNECT_H diff --git a/src/lib/pubsub/pubsub_macros.h b/src/lib/pubsub/pubsub_macros.h index aed9c51282..4d0638d1df 100644 --- a/src/lib/pubsub/pubsub_macros.h +++ b/src/lib/pubsub/pubsub_macros.h @@ -175,7 +175,8 @@ * Use this macro in a header to declare the existence of a given message, * taking a pointer as auxiliary data. * - * "messagename" is a unique identifier for the message. + * "messagename" is a unique identifier for the message; it must be a valid + * C identifier. * * "typename" is a unique identifier for the type of the auxiliary data. * @@ -199,7 +200,8 @@ * Use this macro in a header to declare the existence of a given message, * taking an integer as auxiliary data. * - * "messagename" is a unique identifier for the message. + * "messagename" is a unique identifier for the message; it must be a valid + * C identifier. * * "typename" is a unique identifier for the type of the auxiliary data. * From 22ad8658cd3bc0c0a261975f1eb67cf408fc47d5 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 12 Mar 2019 18:57:28 -0400 Subject: [PATCH 0659/2557] Correct doxygen @file directives --- src/lib/pubsub/pubsub_macros.h | 2 +- src/lib/pubsub/pubsub_publish.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/pubsub/pubsub_macros.h b/src/lib/pubsub/pubsub_macros.h index 4d0638d1df..e4fdd15aec 100644 --- a/src/lib/pubsub/pubsub_macros.h +++ b/src/lib/pubsub/pubsub_macros.h @@ -5,7 +5,7 @@ /* See LICENSE for licensing information */ /** - * \file msg.h + * \file pubsub_macros.h * \brief Macros to help with the publish/subscribe dispatch API. * * The dispatch API allows different subsystems of Tor to communicate with diff --git a/src/lib/pubsub/pubsub_publish.c b/src/lib/pubsub/pubsub_publish.c index 8c469e8add..7b070f35c5 100644 --- a/src/lib/pubsub/pubsub_publish.c +++ b/src/lib/pubsub/pubsub_publish.c @@ -5,7 +5,7 @@ /* See LICENSE for licensing information */ /** - * @file pubsub_publish.h + * @file pubsub_publish.c * @brief Header for functions to publish using a pub_binding_t. **/ From 94feec59cf2f4c28578ba9c75c7ece781ed6f19b Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 12 Mar 2019 19:11:52 -0400 Subject: [PATCH 0660/2557] move pubsub_connector_t typedef to pubsub_connect.h --- src/lib/pubsub/pubsub_build.h | 17 +++++------------ src/lib/pubsub/pubsub_connect.h | 8 +++++++- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/src/lib/pubsub/pubsub_build.h b/src/lib/pubsub/pubsub_build.h index 3fad254534..93aad50b28 100644 --- a/src/lib/pubsub/pubsub_build.h +++ b/src/lib/pubsub/pubsub_build.h @@ -17,6 +17,7 @@ #include "lib/dispatch/msgtypes.h" struct dispatch_t; +struct pubsub_connector_t; /** * A "dispatch builder" is an incomplete dispatcher, used when @@ -26,14 +27,6 @@ struct dispatch_t; **/ typedef struct pubsub_builder_t pubsub_builder_t; -/** - * A "dispatch connector" is a view of the dispatcher that a subsystem - * uses while initializing itself. It is specific to the subsystem, and - * ensures that each subsystem doesn't need to identify itself - * repeatedly while registering its messages. - **/ -typedef struct pubsub_connector_t pubsub_connector_t; - /** * A "pubsub items" holds the configuration items used to configure a * pubsub_builder. After the builder is finalized, this field is extracted, @@ -65,16 +58,16 @@ void pubsub_builder_free_(pubsub_builder_t *); * register its messages. The main-init code does this during susbsystem * initialization. */ -pubsub_connector_t *pubsub_connector_for_subsystem(pubsub_builder_t *, - subsys_id_t); +struct pubsub_connector_t *pubsub_connector_for_subsystem(pubsub_builder_t *, + subsys_id_t); /** * The main-init code does this after subsystem initialization. */ #define pubsub_connector_free(c) \ - FREE_AND_NULL(pubsub_connector_t, pubsub_connector_free_, (c)) + FREE_AND_NULL(struct pubsub_connector_t, pubsub_connector_free_, (c)) -void pubsub_connector_free_(pubsub_connector_t *); +void pubsub_connector_free_(struct pubsub_connector_t *); /** * Constructs a dispatcher from a dispatch_builder, after checking that the diff --git a/src/lib/pubsub/pubsub_connect.h b/src/lib/pubsub/pubsub_connect.h index f3de74c870..e4d8600056 100644 --- a/src/lib/pubsub/pubsub_connect.h +++ b/src/lib/pubsub/pubsub_connect.h @@ -19,7 +19,13 @@ #include "lib/dispatch/msgtypes.h" struct pub_binding_t; -struct pubsub_connector_t; +/** + * A "dispatch connector" is a view of the dispatcher that a subsystem + * uses while initializing itself. It is specific to the subsystem, and + * ensures that each subsystem doesn't need to identify itself + * repeatedly while registering its messages. + **/ +typedef struct pubsub_connector_t pubsub_connector_t; int pubsub_add_pub_(struct pubsub_connector_t *con, struct pub_binding_t *out, From beedadbeacc00222e1bdebec638c1e8cd48a8b14 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 12 Mar 2019 19:25:31 -0400 Subject: [PATCH 0661/2557] Try a different approach to making publish function seem used. We want the DISPATCH_ADD_PUB() macro to count as making a DECLARE_PUBLISH() invocation "used", so let's try a new approach that preserves that idea. The old one apparently did not work for some versions of osx clang. --- src/lib/pubsub/pubsub_macros.h | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/lib/pubsub/pubsub_macros.h b/src/lib/pubsub/pubsub_macros.h index e4fdd15aec..12cfa7d39e 100644 --- a/src/lib/pubsub/pubsub_macros.h +++ b/src/lib/pubsub/pubsub_macros.h @@ -268,12 +268,20 @@ } \ EAT_SEMICOLON +/** + * Add a fake use of the publish function for 'messagename', so that + * the compiler does not call it unused. + */ +#define DISPATCH__FAKE_USE_OF_PUBFN_(messagename) \ + ( 0 ? (publish_fn__ ##messagename((msg_arg_type__##messagename)0), 1) \ + : 1) + /* * This macro is for internal use. It backs DISPATCH_ADD_PUB*() */ #define DISPATCH_ADD_PUB_(connector, channel, messagename, flags) \ ( \ - ((void)publish_fn__ ##messagename), \ + DISPATCH__FAKE_USE_OF_PUBFN_(messagename), \ pubsub_add_pub_((connector), \ &pub_binding__ ##messagename, \ get_channel_id(# channel), \ From 9fb511526ae37cb7c7b5ed4714550f58882ef697 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 13 Mar 2019 15:41:36 -0400 Subject: [PATCH 0662/2557] pubsub test: add a test to make sure typefns can't be changed. --- src/test/test_dispatch.c | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/src/test/test_dispatch.c b/src/test/test_dispatch.c index ec704c042a..061e9df14a 100644 --- a/src/test/test_dispatch.c +++ b/src/test/test_dispatch.c @@ -170,6 +170,30 @@ test_dispatch_with_types(void *arg) dispatcher_in_use = NULL; } +static void +test_dispatch_bad_type_setup(void *arg) +{ + (void)arg; + static dispatch_typefns_t fns; + dispatch_cfg_t *cfg = dcfg_new(); + + tt_int_op(0, OP_EQ, dcfg_type_set_fns(cfg, 7, &coord_fns)); + + fns = coord_fns; + fns.fmt_fn = NULL; + tt_int_op(-1, OP_EQ, dcfg_type_set_fns(cfg, 7, &fns)); + + fns = coord_fns; + fns.free_fn = NULL; + tt_int_op(-1, OP_EQ, dcfg_type_set_fns(cfg, 7, &fns)); + + fns = coord_fns; + tt_int_op(0, OP_EQ, dcfg_type_set_fns(cfg, 7, &fns)); + + done: + dcfg_free(cfg); +} + #define T(name) \ { #name, test_dispatch_ ## name, TT_FORK, NULL, NULL } @@ -177,5 +201,6 @@ struct testcase_t dispatch_tests[] = { T(empty), T(simple), T(with_types), + T(bad_type_setup), END_OF_TESTCASES }; From f74301f8fdf168dbd9dbfe77362658656c70fe71 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 13 Mar 2019 15:48:45 -0400 Subject: [PATCH 0663/2557] Unit test for namemap_fmt_name() --- src/test/test_namemap.c | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/test/test_namemap.c b/src/test/test_namemap.c index 5134e1451b..df77d4e2de 100644 --- a/src/test/test_namemap.c +++ b/src/test/test_namemap.c @@ -142,6 +142,25 @@ test_namemap_internals(void *arg) namemap_clear(&m); } +static void +test_namemap_fmt(void *arg) +{ + (void)arg; + namemap_t m = NAMEMAP_INIT(); + + unsigned a = namemap_get_or_create_id(&m, "greetings"); + unsigned b = namemap_get_or_create_id(&m, "earthlings"); + + tt_str_op(namemap_fmt_name(&m, a), OP_EQ, "greetings"); + tt_str_op(namemap_fmt_name(&m, b), OP_EQ, "earthlings"); + tt_int_op(a, OP_NE, 100); + tt_int_op(b, OP_NE, 100); + tt_str_op(namemap_fmt_name(&m, 100), OP_EQ, "{100}"); + + done: + namemap_clear(&m); +} + #define T(name) \ { #name, test_namemap_ ## name , 0, NULL, NULL } @@ -150,5 +169,6 @@ struct testcase_t namemap_tests[] = { T(toolong), T(blackbox), T(internals), + T(fmt), END_OF_TESTCASES }; From c40bcab85df19feb4ad6503f160c07b23501dc85 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 13 Mar 2019 15:52:48 -0400 Subject: [PATCH 0664/2557] dispatch: Test behavior of formatting type with no set fmt function. --- src/test/test_dispatch.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/test/test_dispatch.c b/src/test/test_dispatch.c index 061e9df14a..886f45868e 100644 --- a/src/test/test_dispatch.c +++ b/src/test/test_dispatch.c @@ -17,6 +17,8 @@ #include #include +static dispatch_t *dispatcher_in_use=NULL; + /* Construct an empty dispatch_t. */ static void test_dispatch_empty(void *arg) @@ -44,9 +46,14 @@ simple_recv1(const msg_t *m) total_recv1_simple += m->aux_data__.u64; } +static char *recv2_received = NULL; + static void simple_recv2(const msg_t *m) { + tor_free(recv2_received); + recv2_received = dispatch_fmt_msg_data(dispatcher_in_use, m); + total_recv2_simple += m->aux_data__.u64*10; } @@ -73,6 +80,7 @@ test_dispatch_simple(void *arg) d = dispatch_new(cfg); tt_assert(d); + dispatcher_in_use = d; msg_aux_data_t data = {.u64 = 7}; r = dispatch_send(d, 99, 0, 0, 0, data); @@ -91,9 +99,12 @@ test_dispatch_simple(void *arg) tt_int_op(total_recv1_simple, OP_EQ, 0); tt_int_op(total_recv2_simple, OP_EQ, 140); + tt_str_op(recv2_received, OP_EQ, "<>"); // no format function was set. + done: dispatch_free(d); dcfg_free(cfg); + tor_free(recv2_received); } struct coord { int x; int y; }; @@ -121,7 +132,6 @@ alert_run_immediate(dispatch_t *d, channel_id_t ch, void *arg) dispatch_flush(d, ch, INT_MAX); } -static dispatch_t *dispatcher_in_use=NULL; static char *received_data=NULL; static void From 2e7f80d5f4e8b56f060516f71f13590593b1d876 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 13 Mar 2019 15:54:26 -0400 Subject: [PATCH 0665/2557] pubsub_check.c: Stop accepting NULL prefix, which we never send. (Our code to handle it was broken, too) --- src/lib/pubsub/pubsub_check.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/lib/pubsub/pubsub_check.c b/src/lib/pubsub/pubsub_check.c index 1b3853d8bb..e87c347c45 100644 --- a/src/lib/pubsub/pubsub_check.c +++ b/src/lib/pubsub/pubsub_check.c @@ -155,8 +155,7 @@ format_flags(unsigned flags) static void pubsub_cfg_dump(const pubsub_cfg_t *cfg, int severity, const char *prefix) { - if (!prefix) - prefix = 0; + tor_assert(prefix); tor_log(severity, LD_MESG, "%s%s %s: %s{%s} on %s (%s) <%u %u %u %u %x> [%s:%d]", From 3552cd69bdeefb5d4fbbc905b24120573cc26f81 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 13 Mar 2019 15:56:58 -0400 Subject: [PATCH 0666/2557] coverage: Exclude lines in pubsub_pub that can only be reached on bug --- src/lib/pubsub/pubsub_publish.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/lib/pubsub/pubsub_publish.c b/src/lib/pubsub/pubsub_publish.c index 7b070f35c5..454a335a78 100644 --- a/src/lib/pubsub/pubsub_publish.c +++ b/src/lib/pubsub/pubsub_publish.c @@ -49,8 +49,10 @@ pubsub_pub_(const pub_binding_t *pub, msg_aux_data_t auxdata) if (BUG(pub->msg_template.msg >= d->n_msgs) || BUG(pub->msg_template.channel >= d->n_queues)) { /* The message ID or channel ID was out of bounds. */ + // LCOV_EXCL_START d->typefns[pub->msg_template.type].free_fn(auxdata); return -1; + // LCOV_EXCL_STOP } if (! d->table[pub->msg_template.msg]) { From d976cda49f7346e8fab16c062973c38ec4da4f4b Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 13 Mar 2019 16:30:56 -0400 Subject: [PATCH 0667/2557] pubsub: add test for items_out in builder_finalize() --- src/test/test_pubsub_build.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/test/test_pubsub_build.c b/src/test/test_pubsub_build.c index 86b5f763a4..19aa1f0e0e 100644 --- a/src/test/test_pubsub_build.c +++ b/src/test/test_pubsub_build.c @@ -2,6 +2,7 @@ /* See LICENSE for licensing information */ #define DISPATCH_PRIVATE +#define PUBSUB_PRIVATE #include "test/test.h" @@ -119,13 +120,16 @@ test_pubsub_build_types_ok(void *arg) pubsub_builder_t *b = NULL; dispatch_t *dispatcher = NULL; pubsub_connector_t *c = NULL; + pubsub_items_t *items = NULL; b = pubsub_builder_new(); seed_pubsub_builder_basic(b); - dispatcher = pubsub_builder_finalize(b, NULL); + dispatcher = pubsub_builder_finalize(b, &items); b = NULL; tt_assert(dispatcher); + tt_assert(items); + tt_int_op(smartlist_len(items->items), OP_EQ, 4); tt_int_op(dispatcher->n_types, OP_GE, 2); tt_assert(dispatcher->typefns); @@ -138,6 +142,7 @@ test_pubsub_build_types_ok(void *arg) pubsub_connector_free(c); pubsub_builder_free(b); dispatch_free(dispatcher); + pubsub_items_free(items); } /* We fail if the same type is defined in two places with different functions. From 4bdff5e3e9d4ea11c7e8043e75d63c4f366558e8 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 14 Mar 2019 08:54:20 -0400 Subject: [PATCH 0668/2557] practracker compliance: split lint_message into more logical parts --- src/lib/pubsub/pubsub_check.c | 241 +++++++++++++++++++++------------- 1 file changed, 151 insertions(+), 90 deletions(-) diff --git a/src/lib/pubsub/pubsub_check.c b/src/lib/pubsub/pubsub_check.c index e87c347c45..94a55beebc 100644 --- a/src/lib/pubsub/pubsub_check.c +++ b/src/lib/pubsub/pubsub_check.c @@ -171,100 +171,61 @@ pubsub_cfg_dump(const pubsub_cfg_t *cfg, int severity, const char *prefix) } /** - * Check whether there are any errors or inconsistencies for the message - * described by msg in map. If there are problems, log about - * them, and return -1. Otherwise return 0. + * Helper: fill a bitarray out with entries corresponding to the + * subsystems listed in items. If any subsystem is listed more than + * once, log a warning. Return 0 on success, -1 on failure. **/ static int -lint_message(const pubsub_adjmap_t *map, message_id_t msg) +get_message_bitarray(const pubsub_adjmap_t *map, + message_id_t msg, + const smartlist_t *items, + const char *operation, + bitarray_t **out) { - /* NOTE: Some of the checks in this function are maybe over-zealous, and we - * might not want to have them forever. I've marked them with [?] below. - */ - if (BUG(msg >= map->n_msgs)) - return 0; // LCOV_EXCL_LINE + bool ok = true; + *out = bitarray_init_zero((unsigned)map->n_subsystems); + if (! items) + return 0; - const smartlist_t *pub = map->pub_by_msg[msg]; - const smartlist_t *sub = map->sub_by_msg[msg]; + SMARTLIST_FOREACH_BEGIN(items, const pubsub_cfg_t *, cfg) { + if (bitarray_is_set(*out, cfg->subsys)) { + log_warn(LD_MESG|LD_BUG, + "Message %u (%s) is configured to be %s by subsystem " + "%u (%s) more than once.", + msg, get_message_id_name(msg), operation, + cfg->subsys, get_subsys_id_name(cfg->subsys)); + ok = false; + } + bitarray_set(*out, cfg->subsys); + } SMARTLIST_FOREACH_END(cfg); - const size_t n_pub = smartlist_len_opt(pub); - const size_t n_sub = smartlist_len_opt(sub); + return ok ? 0 : -1; +} - if (n_pub == 0 && n_sub == 0) { - log_info(LD_MESG, "Nobody is publishing or subscribing to message %u " - "(%s).", - msg, get_message_id_name(msg)); - return 0; // No publishers or subscribers: nothing to do. - } - - /* We'll set this to false if there are any problems. */ +/** + * Helper for lint_message: check that all the pubsub_cfg_t items in the two + * respective smartlists obey our local graph topology rules. + * + * (Right now this is just a matter of "each subsystem only + * publishes/subscribes once; no subsystem is a publisher and subscriber for + * the same message.") + * + * Return 0 on success, -1 on failure. + **/ +static int +lint_message_graph(const pubsub_adjmap_t *map, + message_id_t msg, + const smartlist_t *pub, + const smartlist_t *sub) +{ + bitarray_t *published_by = NULL; + bitarray_t *subscribed_by = NULL; bool ok = true; - /* First make sure that if there are publishers, there are subscribers. */ - if (n_pub == 0) { - log_warn(LD_MESG|LD_BUG, - "Message %u (%s) has subscribers, but no publishers.", - msg, get_message_id_name(msg)); + if (get_message_bitarray(map, msg, pub, "published", &published_by) < 0) ok = false; - } else if (n_sub == 0) { - log_warn(LD_MESG|LD_BUG, - "Message %u (%s) has publishers, but no subscribers.", - msg, get_message_id_name(msg)); + if (get_message_bitarray(map, msg, sub, "subscribed", &subscribed_by) < 0) ok = false; - } - - /* The 'all' list has the publishers and the subscribers. */ - smartlist_t *all = smartlist_new(); - if (pub) - smartlist_add_all(all, pub); - if (sub) - smartlist_add_all(all, sub); - const pubsub_cfg_t *item0 = smartlist_get(all, 0); - - /* Indicates which subsystems we've found publishing/subscribing here. */ - bitarray_t *published_by = bitarray_init_zero((unsigned)map->n_subsystems); - bitarray_t *subscribed_by = bitarray_init_zero((unsigned)map->n_subsystems); - bool pub_excl = false, sub_excl = false, chan_same = true, type_same = true; - - /* Make sure that the messages all have the same channel and type; - * check whether the DISP_FLAG_EXCL flag is used; - * and if any subsystem is publishing or subscribing to it twice [??]. - */ - SMARTLIST_FOREACH_BEGIN(all, const pubsub_cfg_t *, cfg) { - if (cfg->channel != item0->channel) { - chan_same = false; - } - if (cfg->type != item0->type) { - type_same = false; - } - if (cfg->flags & DISP_FLAG_EXCL) { - if (cfg->is_publish) - pub_excl = true; - else - sub_excl = true; - } - if (cfg->is_publish) { - if (bitarray_is_set(published_by, cfg->subsys)) { - log_warn(LD_MESG|LD_BUG, - "Message %u (%s) is configured to be published by subsystem " - "%u (%s) more than once.", - msg, get_message_id_name(msg), - cfg->subsys, get_subsys_id_name(cfg->subsys)); - ok = false; - } - bitarray_set(published_by, cfg->subsys); - } else { - if (bitarray_is_set(subscribed_by, cfg->subsys)) { - log_warn(LD_MESG|LD_BUG, - "Message %u (%s) is configured to be subscribed by subsystem " - "%u (%s) more than once.", - msg, get_message_id_name(msg), - cfg->subsys, get_subsys_id_name(cfg->subsys)); - ok = false; - } - bitarray_set(subscribed_by, cfg->subsys); - } - } SMARTLIST_FOREACH_END(cfg); /* Check whether any subsystem is publishing and subscribing the same * message. [??] @@ -281,6 +242,49 @@ lint_message(const pubsub_adjmap_t *map, message_id_t msg) } } + bitarray_free(published_by); + bitarray_free(subscribed_by); + + return ok ? 0 : -1; +} + +/** + * Helper for lint_message: check that all the pubsub_cfg_t items in the two + * respective smartlists have compatible flags, channels, and types. + **/ +static int +lint_message_consistency(message_id_t msg, + const smartlist_t *pub, + const smartlist_t *sub) +{ + if (!smartlist_len_opt(pub) && !smartlist_len_opt(sub)) + return 0; // LCOV_EXCL_LINE -- this was already checked. + + /* The 'all' list has the publishers and the subscribers. */ + smartlist_t *all = smartlist_new(); + if (pub) + smartlist_add_all(all, pub); + if (sub) + smartlist_add_all(all, sub); + + const pubsub_cfg_t *item0 = smartlist_get(all, 0); + + /* Indicates which subsystems we've found publishing/subscribing here. */ + bool pub_excl = false, sub_excl = false, chan_same = true, type_same = true; + + /* Simple message consistency properties across messages. + */ + SMARTLIST_FOREACH_BEGIN(all, const pubsub_cfg_t *, cfg) { + chan_same &= (cfg->channel == item0->channel); + type_same &= (cfg->type == item0->type); + if (cfg->is_publish) + pub_excl |= (cfg->flags & DISP_FLAG_EXCL) != 0; + else + sub_excl |= (cfg->flags & DISP_FLAG_EXCL) != 0; + } SMARTLIST_FOREACH_END(cfg); + + bool ok = true; + if (! chan_same) { log_warn(LD_MESG|LD_BUG, "Message %u (%s) is associated with multiple inconsistent " @@ -314,17 +318,74 @@ lint_message(const pubsub_adjmap_t *map, message_id_t msg) ok = false; } + smartlist_free(all); + + return ok ? 0 : -1; +} + +/** + * Check whether there are any errors or inconsistencies for the message + * described by msg in map. If there are problems, log about + * them, and return -1. Otherwise return 0. + **/ +static int +lint_message(const pubsub_adjmap_t *map, message_id_t msg) +{ + /* NOTE: Some of the checks in this function are maybe over-zealous, and we + * might not want to have them forever. I've marked them with [?] below. + */ + if (BUG(msg >= map->n_msgs)) + return 0; // LCOV_EXCL_LINE + + const smartlist_t *pub = map->pub_by_msg[msg]; + const smartlist_t *sub = map->sub_by_msg[msg]; + + const size_t n_pub = smartlist_len_opt(pub); + const size_t n_sub = smartlist_len_opt(sub); + + if (n_pub == 0 && n_sub == 0) { + log_info(LD_MESG, "Nobody is publishing or subscribing to message %u " + "(%s).", + msg, get_message_id_name(msg)); + return 0; // No publishers or subscribers: nothing to do. + } + /* We'll set this to false if there are any problems. */ + bool ok = true; + + /* First make sure that if there are publishers, there are subscribers. */ + if (n_pub == 0) { + log_warn(LD_MESG|LD_BUG, + "Message %u (%s) has subscribers, but no publishers.", + msg, get_message_id_name(msg)); + ok = false; + } else if (n_sub == 0) { + log_warn(LD_MESG|LD_BUG, + "Message %u (%s) has publishers, but no subscribers.", + msg, get_message_id_name(msg)); + ok = false; + } + + /* Check the message graph topology. */ + if (lint_message_graph(map, msg, pub, sub) < 0) + ok = false; + + /* Check whether the messages have the same fields set on them. */ + if (lint_message_consistency(msg, pub, sub) < 0) + ok = false; + if (!ok) { /* There was a problem -- let's log all the publishers and subscribers on * this message */ - SMARTLIST_FOREACH(all, pubsub_cfg_t *, cfg, - pubsub_cfg_dump(cfg, LOG_WARN, " ")); + if (pub) { + SMARTLIST_FOREACH(pub, pubsub_cfg_t *, cfg, + pubsub_cfg_dump(cfg, LOG_WARN, " ")); + } + if (sub) { + SMARTLIST_FOREACH(sub, pubsub_cfg_t *, cfg, + pubsub_cfg_dump(cfg, LOG_WARN, " ")); + } } - smartlist_free(all); - bitarray_free(published_by); - bitarray_free(subscribed_by); - return ok ? 0 : -1; } From ab6ddc7a33f7ae6611ebaa00b6bc526d9c286b41 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 14 Mar 2019 09:37:44 -0400 Subject: [PATCH 0669/2557] practracker: split shutdown code out of main.c This is necessary to get the number of includes in main.c back under control. (In the future, we could just use the subsystem manager for this kind of stuff.) --- src/app/main/main.c | 188 ++++++--------------------------------- src/app/main/main.h | 3 - src/app/main/ntmain.c | 1 + src/app/main/shutdown.c | 192 ++++++++++++++++++++++++++++++++++++++++ src/app/main/shutdown.h | 18 ++++ src/core/include.am | 2 + 6 files changed, 240 insertions(+), 164 deletions(-) create mode 100644 src/app/main/shutdown.c create mode 100644 src/app/main/shutdown.h diff --git a/src/app/main/main.c b/src/app/main/main.c index e0f6352879..21e02f9efb 100644 --- a/src/app/main/main.c +++ b/src/app/main/main.c @@ -15,6 +15,7 @@ #include "app/config/statefile.h" #include "app/main/main.h" #include "app/main/ntmain.h" +#include "app/main/shutdown.h" #include "app/main/subsysmgr.h" #include "core/mainloop/connection.h" #include "core/mainloop/cpuworker.h" @@ -24,55 +25,35 @@ #include "core/or/channel.h" #include "core/or/channelpadding.h" #include "core/or/circuitpadding.h" -#include "core/or/channeltls.h" #include "core/or/circuitlist.h" -#include "core/or/circuitmux_ewma.h" #include "core/or/command.h" -#include "core/or/connection_edge.h" #include "core/or/connection_or.h" -#include "core/or/dos.h" -#include "core/or/policies.h" -#include "core/or/protover.h" #include "core/or/relay.h" -#include "core/or/scheduler.h" #include "core/or/status.h" -#include "core/or/versions.h" #include "feature/api/tor_api.h" #include "feature/api/tor_api_internal.h" #include "feature/client/addressmap.h" -#include "feature/client/bridges.h" -#include "feature/client/entrynodes.h" -#include "feature/client/transports.h" #include "feature/control/control.h" -#include "feature/dirauth/bwauth.h" #include "feature/dirauth/keypin.h" #include "feature/dirauth/process_descs.h" #include "feature/dircache/consdiffmgr.h" -#include "feature/dircache/dirserv.h" #include "feature/dirparse/routerparse.h" #include "feature/hibernate/hibernate.h" -#include "feature/hs/hs_cache.h" #include "feature/nodelist/authcert.h" -#include "feature/nodelist/microdesc.h" #include "feature/nodelist/networkstatus.h" -#include "feature/nodelist/nodelist.h" #include "feature/nodelist/routerlist.h" #include "feature/relay/dns.h" #include "feature/relay/ext_orport.h" -#include "feature/relay/onion_queue.h" #include "feature/relay/routerkeys.h" #include "feature/relay/routermode.h" #include "feature/rend/rendcache.h" -#include "feature/rend/rendclient.h" #include "feature/rend/rendservice.h" -#include "feature/stats/geoip_stats.h" #include "feature/stats/predict_ports.h" #include "feature/stats/rephist.h" #include "lib/compress/compress.h" #include "lib/buf/buffers.h" #include "lib/crypt_ops/crypto_rand.h" #include "lib/crypt_ops/crypto_s2k.h" -#include "lib/geoip/geoip.h" #include "lib/net/resolve.h" #include "lib/process/waitpid.h" @@ -92,7 +73,6 @@ #include -#include "feature/dirauth/dirvote.h" #include "feature/dirauth/authmode.h" #include "feature/dirauth/shared_random.h" @@ -113,8 +93,6 @@ #include #endif /* defined(HAVE_SYSTEMD) */ -void evdns_shutdown(int); - #ifdef HAVE_RUST // helper function defined in Rust to output a log message indicating if tor is // running with Rust enabled. See src/rust/tor_util @@ -744,87 +722,6 @@ release_lockfile(void) } } -/** Free all memory that we might have allocated somewhere. - * If postfork, we are a worker process and we want to free - * only the parts of memory that we won't touch. If !postfork, - * Tor is shutting down and we should free everything. - * - * Helps us find the real leaks with sanitizers and the like. Also valgrind - * should then report 0 reachable in its leak report (in an ideal world -- - * in practice libevent, SSL, libc etc never quite free everything). */ -void -tor_free_all(int postfork) -{ - if (!postfork) { - evdns_shutdown(1); - } - geoip_free_all(); - geoip_stats_free_all(); - dirvote_free_all(); - routerlist_free_all(); - networkstatus_free_all(); - addressmap_free_all(); - dirserv_free_fingerprint_list(); - dirserv_free_all(); - dirserv_clear_measured_bw_cache(); - rend_cache_free_all(); - rend_service_authorization_free_all(); - rep_hist_free_all(); - dns_free_all(); - clear_pending_onions(); - circuit_free_all(); - circpad_machines_free(); - entry_guards_free_all(); - pt_free_all(); - channel_tls_free_all(); - channel_free_all(); - connection_free_all(); - connection_edge_free_all(); - scheduler_free_all(); - nodelist_free_all(); - microdesc_free_all(); - routerparse_free_all(); - ext_orport_free_all(); - control_free_all(); - protover_free_all(); - bridges_free_all(); - consdiffmgr_free_all(); - hs_free_all(); - dos_free_all(); - circuitmux_ewma_free_all(); - accounting_free_all(); - protover_summary_cache_free_all(); - - if (!postfork) { - config_free_all(); - or_state_free_all(); - router_free_all(); - routerkeys_free_all(); - policies_free_all(); - } - if (!postfork) { -#ifndef _WIN32 - tor_getpwnam(NULL); -#endif - } - /* stuff in main.c */ - - tor_mainloop_disconnect_pubsub(); - tor_mainloop_free_all(); - - if (!postfork) { - release_lockfile(); - } - tor_libevent_free_all(); - - subsystems_shutdown(); - - /* Stuff in util.c and address.c*/ - if (!postfork) { - esc_router_info(NULL); - } -} - /** * Remove the specified file, and log a warning if the operation fails for * any reason other than the file not existing. Ignores NULL filenames. @@ -838,50 +735,6 @@ tor_remove_file(const char *filename) } } -/** Do whatever cleanup is necessary before shutting Tor down. */ -void -tor_cleanup(void) -{ - const or_options_t *options = get_options(); - if (options->command == CMD_RUN_TOR) { - time_t now = time(NULL); - /* Remove our pid file. We don't care if there was an error when we - * unlink, nothing we could do about it anyways. */ - tor_remove_file(options->PidFile); - /* Remove control port file */ - tor_remove_file(options->ControlPortWriteToFile); - /* Remove cookie authentication file */ - { - char *cookie_fname = get_controller_cookie_file_name(); - tor_remove_file(cookie_fname); - tor_free(cookie_fname); - } - /* Remove Extended ORPort cookie authentication file */ - { - char *cookie_fname = get_ext_or_auth_cookie_file_name(); - tor_remove_file(cookie_fname); - tor_free(cookie_fname); - } - if (accounting_is_enabled(options)) - accounting_record_bandwidth_usage(now, get_or_state()); - or_state_mark_dirty(get_or_state(), 0); /* force an immediate save. */ - or_state_save(now); - if (authdir_mode(options)) { - sr_save_and_cleanup(); - } - if (authdir_mode_tests_reachability(options)) - rep_hist_record_mtbf_data(now, 0); - keypin_close_journal(); - } - - timers_shutdown(); - - tor_free_all(0); /* We could move tor_free_all back into the ifdef below - later, if it makes shutdown unacceptably slow. But for - now, leave it here: it's helped us catch bugs in the - past. */ -} - /** Read/create keys as needed, and echo our fingerprint to stdout. */ static int do_list_fingerprint(void) @@ -1379,6 +1232,30 @@ run_tor_main_loop(void) return do_main_loop(); } +/** Install the publish/subscribe relationships for all the subsystems. */ +static void +pubsub_install(void) +{ + pubsub_builder_t *builder = pubsub_builder_new(); + int r = subsystems_add_pubsub(builder); + tor_assert(r == 0); + r = tor_mainloop_connect_pubsub(builder); // consumes builder + tor_assert(r == 0); +} + +/** Connect the mainloop to its publish/subscribe message delivery events if + * appropriate, and configure the global channels appropriately. */ +static void +pubsub_connect(void) +{ + if (get_options()->command == CMD_RUN_TOR) { + tor_mainloop_connect_pubsub_events(); + /* XXXX For each pubsub channel, its delivery strategy should be set at + * this XXXX point, using tor_mainloop_set_delivery_strategy(). + */ + } +} + /* Main entry point for the Tor process. Called from tor_main(), and by * anybody embedding Tor. */ int @@ -1411,13 +1288,7 @@ tor_run_main(const tor_main_configuration_t *tor_cfg) } #endif /* defined(NT_SERVICE) */ - { - pubsub_builder_t *builder = pubsub_builder_new(); - int r = subsystems_add_pubsub(builder); - tor_assert(r == 0); - r = tor_mainloop_connect_pubsub(builder); // consumes builder - tor_assert(r == 0); - } + pubsub_install(); { int init_rv = tor_init(argc, argv); @@ -1428,12 +1299,7 @@ tor_run_main(const tor_main_configuration_t *tor_cfg) } } - if (get_options()->command == CMD_RUN_TOR) { - tor_mainloop_connect_pubsub_events(); - /* XXXX For each pubsub channel, its delivery strategy should be set at - * this XXXX point, using tor_mainloop_set_delivery_strategy(). - */ - } + pubsub_connect(); if (get_options()->Sandbox && get_options()->command == CMD_RUN_TOR) { sandbox_cfg_t* cfg = sandbox_init_filter(); diff --git a/src/app/main/main.h b/src/app/main/main.h index 23a436703e..6910082090 100644 --- a/src/app/main/main.h +++ b/src/app/main/main.h @@ -21,9 +21,6 @@ void release_lockfile(void); void tor_remove_file(const char *filename); -void tor_cleanup(void); -void tor_free_all(int postfork); - int tor_init(int argc, char **argv); int run_tor_main_loop(void); diff --git a/src/app/main/ntmain.c b/src/app/main/ntmain.c index 8d2135a587..00afbf375e 100644 --- a/src/app/main/ntmain.c +++ b/src/app/main/ntmain.c @@ -24,6 +24,7 @@ #include "app/config/config.h" #include "app/main/main.h" #include "app/main/ntmain.h" +#include "app/main/shutdown.h" #include "core/mainloop/mainloop.h" #include "lib/evloop/compat_libevent.h" #include "lib/fs/winlib.h" diff --git a/src/app/main/shutdown.c b/src/app/main/shutdown.c new file mode 100644 index 0000000000..a4e6cb088e --- /dev/null +++ b/src/app/main/shutdown.c @@ -0,0 +1,192 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file shutdown.c + * @brief Code to free global resources used by Tor. + * + * In the future, this should all be handled by the subsystem manager. */ + +#include "core/or/or.h" + +#include "app/config/config.h" +#include "app/config/statefile.h" +#include "app/main/main.h" +#include "app/main/shutdown.h" +#include "app/main/subsysmgr.h" +#include "core/mainloop/connection.h" +#include "core/mainloop/mainloop.h" +#include "core/mainloop/mainloop_pubsub.h" +#include "core/or/channeltls.h" +#include "core/or/circuitlist.h" +#include "core/or/circuitmux_ewma.h" +#include "core/or/circuitpadding.h" +#include "core/or/connection_edge.h" +#include "core/or/dos.h" +#include "core/or/policies.h" +#include "core/or/protover.h" +#include "core/or/scheduler.h" +#include "core/or/versions.h" +#include "feature/client/addressmap.h" +#include "feature/client/bridges.h" +#include "feature/client/entrynodes.h" +#include "feature/client/transports.h" +#include "feature/control/control.h" +#include "feature/control/control.h" +#include "feature/dirauth/authmode.h" +#include "feature/dirauth/bwauth.h" +#include "feature/dirauth/dirvote.h" +#include "feature/dirauth/keypin.h" +#include "feature/dirauth/process_descs.h" +#include "feature/dirauth/shared_random.h" +#include "feature/dircache/consdiffmgr.h" +#include "feature/dircache/dirserv.h" +#include "feature/dirparse/routerparse.h" +#include "feature/hibernate/hibernate.h" +#include "feature/hs/hs_common.h" +#include "feature/nodelist/microdesc.h" +#include "feature/nodelist/networkstatus.h" +#include "feature/nodelist/nodelist.h" +#include "feature/nodelist/routerlist.h" +#include "feature/nodelist/routerlist.h" +#include "feature/relay/dns.h" +#include "feature/relay/ext_orport.h" +#include "feature/relay/onion_queue.h" +#include "feature/relay/routerkeys.h" +#include "feature/rend/rendcache.h" +#include "feature/rend/rendclient.h" +#include "feature/stats/geoip_stats.h" +#include "feature/stats/rephist.h" +#include "lib/evloop/compat_libevent.h" +#include "lib/geoip/geoip.h" +#include "src/feature/relay/router.h" + +void evdns_shutdown(int); + +/** Do whatever cleanup is necessary before shutting Tor down. */ +void +tor_cleanup(void) +{ + const or_options_t *options = get_options(); + if (options->command == CMD_RUN_TOR) { + time_t now = time(NULL); + /* Remove our pid file. We don't care if there was an error when we + * unlink, nothing we could do about it anyways. */ + tor_remove_file(options->PidFile); + /* Remove control port file */ + tor_remove_file(options->ControlPortWriteToFile); + /* Remove cookie authentication file */ + { + char *cookie_fname = get_controller_cookie_file_name(); + tor_remove_file(cookie_fname); + tor_free(cookie_fname); + } + /* Remove Extended ORPort cookie authentication file */ + { + char *cookie_fname = get_ext_or_auth_cookie_file_name(); + tor_remove_file(cookie_fname); + tor_free(cookie_fname); + } + if (accounting_is_enabled(options)) + accounting_record_bandwidth_usage(now, get_or_state()); + or_state_mark_dirty(get_or_state(), 0); /* force an immediate save. */ + or_state_save(now); + if (authdir_mode(options)) { + sr_save_and_cleanup(); + } + if (authdir_mode_tests_reachability(options)) + rep_hist_record_mtbf_data(now, 0); + keypin_close_journal(); + } + + timers_shutdown(); + + tor_free_all(0); /* We could move tor_free_all back into the ifdef below + later, if it makes shutdown unacceptably slow. But for + now, leave it here: it's helped us catch bugs in the + past. */ +} + +/** Free all memory that we might have allocated somewhere. + * If postfork, we are a worker process and we want to free + * only the parts of memory that we won't touch. If !postfork, + * Tor is shutting down and we should free everything. + * + * Helps us find the real leaks with sanitizers and the like. Also valgrind + * should then report 0 reachable in its leak report (in an ideal world -- + * in practice libevent, SSL, libc etc never quite free everything). */ +void +tor_free_all(int postfork) +{ + if (!postfork) { + evdns_shutdown(1); + } + geoip_free_all(); + geoip_stats_free_all(); + dirvote_free_all(); + routerlist_free_all(); + networkstatus_free_all(); + addressmap_free_all(); + dirserv_free_fingerprint_list(); + dirserv_free_all(); + dirserv_clear_measured_bw_cache(); + rend_cache_free_all(); + rend_service_authorization_free_all(); + rep_hist_free_all(); + dns_free_all(); + clear_pending_onions(); + circuit_free_all(); + circpad_machines_free(); + entry_guards_free_all(); + pt_free_all(); + channel_tls_free_all(); + channel_free_all(); + connection_free_all(); + connection_edge_free_all(); + scheduler_free_all(); + nodelist_free_all(); + microdesc_free_all(); + routerparse_free_all(); + ext_orport_free_all(); + control_free_all(); + protover_free_all(); + bridges_free_all(); + consdiffmgr_free_all(); + hs_free_all(); + dos_free_all(); + circuitmux_ewma_free_all(); + accounting_free_all(); + protover_summary_cache_free_all(); + + if (!postfork) { + config_free_all(); + or_state_free_all(); + router_free_all(); + routerkeys_free_all(); + policies_free_all(); + } + if (!postfork) { +#ifndef _WIN32 + tor_getpwnam(NULL); +#endif + } + /* stuff in main.c */ + + tor_mainloop_disconnect_pubsub(); + tor_mainloop_free_all(); + + if (!postfork) { + release_lockfile(); + } + tor_libevent_free_all(); + + subsystems_shutdown(); + + /* Stuff in util.c and address.c*/ + if (!postfork) { + esc_router_info(NULL); + } +} diff --git a/src/app/main/shutdown.h b/src/app/main/shutdown.h new file mode 100644 index 0000000000..1bca96a0aa --- /dev/null +++ b/src/app/main/shutdown.h @@ -0,0 +1,18 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file shutdown.h + * \brief Header file for shutdown.c. + **/ + +#ifndef TOR_SHUTDOWN_H +#define TOR_SHUTDOWN_H + +void tor_cleanup(void); +void tor_free_all(int postfork); + +#endif /* !defined(TOR_SHUTDOWN_H) */ diff --git a/src/core/include.am b/src/core/include.am index 3a0e907edb..7fb5c1d7a0 100644 --- a/src/core/include.am +++ b/src/core/include.am @@ -11,6 +11,7 @@ LIBTOR_APP_A_SOURCES = \ src/app/config/confparse.c \ src/app/config/statefile.c \ src/app/main/main.c \ + src/app/main/shutdown.c \ src/app/main/subsystem_list.c \ src/app/main/subsysmgr.c \ src/core/crypto/hs_ntor.c \ @@ -204,6 +205,7 @@ noinst_HEADERS += \ src/app/config/statefile.h \ src/app/main/main.h \ src/app/main/ntmain.h \ + src/app/main/shutdown.h \ src/app/main/subsysmgr.h \ src/core/crypto/hs_ntor.h \ src/core/crypto/onion_crypto.h \ From b11b4b7bb77c87d2673e816fde114e25a5482a6f Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 14 Mar 2019 11:50:45 -0400 Subject: [PATCH 0670/2557] Add test for dispatch_send() fast path. --- src/test/test_dispatch.c | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/src/test/test_dispatch.c b/src/test/test_dispatch.c index 886f45868e..d6fe7e781a 100644 --- a/src/test/test_dispatch.c +++ b/src/test/test_dispatch.c @@ -107,6 +107,38 @@ test_dispatch_simple(void *arg) tor_free(recv2_received); } +/* Construct a dispatch_t with a message and no reciever; make sure that it + * gets dropped properly. */ +static void +test_dispatch_no_recipient(void *arg) +{ + (void)arg; + + dispatch_t *d=NULL; + dispatch_cfg_t *cfg=NULL; + int r; + + cfg = dcfg_new(); + r = dcfg_msg_set_type(cfg,0,0); + r += dcfg_msg_set_chan(cfg,0,0); + tt_int_op(r, OP_EQ, 0); + + d = dispatch_new(cfg); + tt_assert(d); + dispatcher_in_use = d; + + msg_aux_data_t data = { .u64 = 7}; + r = dispatch_send(d, 99, 0, 0, 0, data); + tt_int_op(r, OP_EQ, 0); + + r = dispatch_flush(d, 0, INT_MAX); + tt_int_op(r, OP_EQ, 0); + + done: + dispatch_free(d); + dcfg_free(cfg); +} + struct coord { int x; int y; }; static void free_coord(msg_aux_data_t d) @@ -210,6 +242,7 @@ test_dispatch_bad_type_setup(void *arg) struct testcase_t dispatch_tests[] = { T(empty), T(simple), + T(no_recipient), T(with_types), T(bad_type_setup), END_OF_TESTCASES From 8d70f217175b69a7b8e5d35b564f50712c882d7e Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 14 Mar 2019 13:23:58 -0400 Subject: [PATCH 0671/2557] Add a test for pubsub_items_clear_bindings() --- src/test/test_pubsub_build.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/test/test_pubsub_build.c b/src/test/test_pubsub_build.c index 19aa1f0e0e..f6c6b81a2a 100644 --- a/src/test/test_pubsub_build.c +++ b/src/test/test_pubsub_build.c @@ -131,6 +131,14 @@ test_pubsub_build_types_ok(void *arg) tt_assert(items); tt_int_op(smartlist_len(items->items), OP_EQ, 4); + // Make sure that the bindings got build correctly. + SMARTLIST_FOREACH_BEGIN(items->items, pubsub_cfg_t *, item) { + if (item->is_publish) { + tt_assert(item->pub_binding); + tt_ptr_op(item->pub_binding->dispatch_ptr, OP_EQ, dispatcher); + } + } SMARTLIST_FOREACH_END(item); + tt_int_op(dispatcher->n_types, OP_GE, 2); tt_assert(dispatcher->typefns); @@ -138,6 +146,16 @@ test_pubsub_build_types_ok(void *arg) tt_assert(dispatcher->typefns[get_msg_type_id("string")].fmt_fn == ex_str_fmt); + // Now clear the bindings, like we would do before freeing the + // the dispatcher. + pubsub_items_clear_bindings(items); + SMARTLIST_FOREACH_BEGIN(items->items, pubsub_cfg_t *, item) { + if (item->is_publish) { + tt_assert(item->pub_binding); + tt_ptr_op(item->pub_binding->dispatch_ptr, OP_EQ, NULL); + } + } SMARTLIST_FOREACH_END(item); + done: pubsub_connector_free(c); pubsub_builder_free(b); From 47de9c7b0a828de7fb8129413db70bc4e4ecac6d Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 14 Mar 2019 15:15:03 -0400 Subject: [PATCH 0672/2557] Use actual pointers in dispatch_cfg.c. Previously, I had used integers encoded as pointers. This introduced a flaw: NULL represented both the integer zero, and the absence of a setting. This in turn made the checks in cfg_msg_set_{type,chan}() not actually check for an altered value if the previous value had been set to zero. Also, I had previously kept a pointer to a dispatch_fypefns_t rather than making a copy of it. This meant that if the dispatch_typefns_t were changed between defining the typefns and creating the dispatcher, we'd get the modified version. Found while investigating coverage in pubsub_add_{pub,sub}_() --- src/lib/dispatch/dispatch_cfg.c | 23 ++++++++++------- src/lib/dispatch/dispatch_new.c | 46 ++++++++++++++++++--------------- 2 files changed, 38 insertions(+), 31 deletions(-) diff --git a/src/lib/dispatch/dispatch_cfg.c b/src/lib/dispatch/dispatch_cfg.c index 26e37f4694..b3a72ec22f 100644 --- a/src/lib/dispatch/dispatch_cfg.c +++ b/src/lib/dispatch/dispatch_cfg.c @@ -37,9 +37,6 @@ dcfg_new(void) return cfg; } -/** DOCDOC */ -#define ID_TO_VOID(id) ((void*)(uintptr_t)(id)) - /** * Associate a message with a datatype. Return 0 on success, -1 if a * different type was previously associated with the message ID. @@ -49,11 +46,12 @@ dcfg_msg_set_type(dispatch_cfg_t *cfg, message_id_t msg, msg_type_id_t type) { smartlist_grow(cfg->type_by_msg, msg+1); - void *oldval = smartlist_get(cfg->type_by_msg, msg); - if (oldval != NULL && oldval != ID_TO_VOID(type)) { + msg_type_id_t *oldval = smartlist_get(cfg->type_by_msg, msg); + if (oldval != NULL && *oldval != type) { return -1; } - smartlist_set(cfg->type_by_msg, msg, ID_TO_VOID(type)); + if (!oldval) + smartlist_set(cfg->type_by_msg, msg, tor_memdup(&type, sizeof(type))); return 0; } @@ -66,11 +64,12 @@ dcfg_msg_set_chan(dispatch_cfg_t *cfg, message_id_t msg, channel_id_t chan) { smartlist_grow(cfg->chan_by_msg, msg+1); - void *oldval = smartlist_get(cfg->chan_by_msg, msg); - if (oldval != NULL && oldval != ID_TO_VOID(chan)) { + channel_id_t *oldval = smartlist_get(cfg->chan_by_msg, msg); + if (oldval != NULL && *oldval != chan) { return -1; } - smartlist_set(cfg->chan_by_msg, msg, ID_TO_VOID(chan)); + if (!oldval) + smartlist_set(cfg->chan_by_msg, msg, tor_memdup(&chan, sizeof(chan))); return 0; } @@ -87,7 +86,8 @@ dcfg_type_set_fns(dispatch_cfg_t *cfg, msg_type_id_t type, if (oldfns && (oldfns->free_fn != fns->free_fn || oldfns->fmt_fn != fns->fmt_fn)) return -1; - smartlist_set(cfg->fns_by_type, type, (dispatch_typefns_t *) fns); + if (!oldfns) + smartlist_set(cfg->fns_by_type, type, tor_memdup(fns, sizeof(*fns))); return 0; } @@ -123,6 +123,9 @@ dcfg_free_(dispatch_cfg_t *cfg) if (!cfg) return; + SMARTLIST_FOREACH(cfg->type_by_msg, msg_type_id_t *, id, tor_free(id)); + SMARTLIST_FOREACH(cfg->chan_by_msg, channel_id_t *, id, tor_free(id)); + SMARTLIST_FOREACH(cfg->fns_by_type, dispatch_typefns_t *, f, tor_free(f)); smartlist_free(cfg->type_by_msg); smartlist_free(cfg->chan_by_msg); smartlist_free(cfg->fns_by_type); diff --git a/src/lib/dispatch/dispatch_new.c b/src/lib/dispatch/dispatch_new.c index a2879016a7..b89ef43ea7 100644 --- a/src/lib/dispatch/dispatch_new.c +++ b/src/lib/dispatch/dispatch_new.c @@ -17,32 +17,36 @@ #include "lib/dispatch/dispatch_cfg.h" #include "lib/dispatch/dispatch_cfg_st.h" +#include "lib/cc/ctassert.h" #include "lib/intmath/cmp.h" #include "lib/malloc/malloc.h" #include "lib/log/util_bug.h" #include -/** Convert a void* in a smartlist to the corresponding integer. */ -#define VOID_TO_ID(p) ((intptr_t)(p)) - -/** Given a smartlist full of void* fields encoding intptr_t values, - * return the largest intptr_t, or dflt if the list is empty. */ -static intptr_t -max_in_sl(const smartlist_t *sl, intptr_t dflt) +/** Given a smartlist full of (possibly NULL) pointers to uint16_t values, + * return the largest value, or dflt if the list is empty. */ +static int +max_in_sl(const smartlist_t *sl, int dflt) { - if (!smartlist_len(sl)) - return dflt; - void *as_ptr = smartlist_get(sl, 0); - intptr_t max = VOID_TO_ID(as_ptr); - SMARTLIST_FOREACH_BEGIN(sl, void *, p) { - intptr_t i = VOID_TO_ID(p); - if (i > max) - max = i; - } SMARTLIST_FOREACH_END(p); - return max; + uint16_t *maxptr = NULL; + SMARTLIST_FOREACH_BEGIN(sl, uint16_t *, u) { + if (!maxptr) + maxptr = u; + else if (*u > *maxptr) + maxptr = u; + } SMARTLIST_FOREACH_END(u); + + return maxptr ? *maxptr : dflt; } +/* The above function is only safe to call if we are sure that channel_id_t + * and msg_type_id_t are really uint16_t. They should be so defined in + * msgtypes.h, but let's be extra cautious. + */ +CTASSERT(sizeof(uint16_t) == sizeof(msg_type_id_t)); +CTASSERT(sizeof(uint16_t) == sizeof(channel_id_t)); + /** Helper: Format an unformattable message auxiliary data item: just return a * copy of the string <>. */ static char * @@ -156,14 +160,14 @@ dispatch_new(const dispatch_cfg_t *cfg) /* Fill in the empty entries in the dispatch tables: * types and channels for each message. */ - SMARTLIST_FOREACH_BEGIN(cfg->type_by_msg, smartlist_t *, type) { + SMARTLIST_FOREACH_BEGIN(cfg->type_by_msg, msg_type_id_t *, type) { if (d->table[type_sl_idx]) - d->table[type_sl_idx]->type = VOID_TO_ID(type); + d->table[type_sl_idx]->type = *type; } SMARTLIST_FOREACH_END(type); - SMARTLIST_FOREACH_BEGIN(cfg->chan_by_msg, smartlist_t *, chan) { + SMARTLIST_FOREACH_BEGIN(cfg->chan_by_msg, channel_id_t *, chan) { if (d->table[chan_sl_idx]) - d->table[chan_sl_idx]->channel = VOID_TO_ID(chan); + d->table[chan_sl_idx]->channel = *chan; } SMARTLIST_FOREACH_END(chan); return d; From a8ca464cee14ad8d0782f626c7f36216dba2e400 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 14 Mar 2019 15:22:19 -0400 Subject: [PATCH 0673/2557] Log warning messages _before_ exiting because of earlier dcfg failure This helps diagnostics. --- src/lib/pubsub/pubsub_build.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/lib/pubsub/pubsub_build.c b/src/lib/pubsub/pubsub_build.c index 64cdcc1d57..1dc20f51f4 100644 --- a/src/lib/pubsub/pubsub_build.c +++ b/src/lib/pubsub/pubsub_build.c @@ -281,12 +281,15 @@ pubsub_builder_finalize(pubsub_builder_t *builder, dispatch_t *dispatcher = NULL; tor_assert_nonfatal(builder->n_connectors == 0); - if (builder->n_errors) - goto err; - if (pubsub_builder_check(builder) < 0) goto err; + if (builder->n_errors) { + log_warn(LD_GENERAL, "At least one error occurred previously when " + "configuring the dispatcher."); + goto err; + } + dispatcher = dispatch_new(builder->cfg); if (!dispatcher) From 28fd4996aca0c560d4af0c035b53323951c13476 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 14 Mar 2019 15:35:41 -0400 Subject: [PATCH 0674/2557] Various documentation notes and tweaks for pubsub --- src/core/mainloop/mainloop_pubsub.c | 3 +++ src/lib/pubsub/pubsub_macros.h | 16 ++++++++++++++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/src/core/mainloop/mainloop_pubsub.c b/src/core/mainloop/mainloop_pubsub.c index aac607126d..724a3115c8 100644 --- a/src/core/mainloop/mainloop_pubsub.c +++ b/src/core/mainloop/mainloop_pubsub.c @@ -123,6 +123,9 @@ alertfn_immediate(dispatch_t *d, channel_id_t chan, void *arg) /** * Set the strategy to be used for delivering messages on the named channel. + * + * This function needs to be called once globally for each channel, to + * set up how messages are delivered. **/ int tor_mainloop_set_delivery_strategy(const char *msg_channel_name, diff --git a/src/lib/pubsub/pubsub_macros.h b/src/lib/pubsub/pubsub_macros.h index 12cfa7d39e..11c7c63eca 100644 --- a/src/lib/pubsub/pubsub_macros.h +++ b/src/lib/pubsub/pubsub_macros.h @@ -165,6 +165,10 @@ /* Macro to declare common elements shared by DECLARE_MESSAGE and * DECLARE_MESSAGE_INT. Don't call this directly. + * + * Note that the "msg_arg_name" string constant is defined in each + * translation unit. This might be undesirable; we can tweak it in the + * future if need be. */ #define DECLARE_MESSAGE_COMMON__(messagename, typename, c_type) \ typedef c_type msg_arg_type__ ##messagename; \ @@ -179,6 +183,8 @@ * C identifier. * * "typename" is a unique identifier for the type of the auxiliary data. + * It needs to be defined somewhere in Tor, using + * "DISPATCH_DEFINE_TYPE." * * "c_type" is a C pointer type (like "char *" or "struct foo *"). */ @@ -203,7 +209,8 @@ * "messagename" is a unique identifier for the message; it must be a valid * C identifier. * - * "typename" is a unique identifier for the type of the auxiliary data. + * "typename" is a unique identifier for the type of the auxiliary data. It + * needs to be defined somewhere in Tor, using "DISPATCH_DEFINE_TYPE." * * "c_type" is a C integer type, like "int" or "bool". It needs to fit inside * a uint64_t. @@ -231,6 +238,8 @@ * * Before you use this, you need to include the header where DECLARE_MESSAGE*() * was used for this message. + * + * You can only use this once per message in each subsystem. */ #define DECLARE_PUBLISH(messagename) \ static pub_binding_t pub_binding__ ##messagename; \ @@ -256,6 +265,8 @@ * "hookfn". The type of this function will be: * static void hookfn(const msg_t *, const c_type) * where c_type is the c type that you declared in the header. + * + * You can only use this once per message in each subsystem. */ #define DECLARE_SUBSCRIBE(messagename, hookfn) \ static void hookfn(const msg_t *, \ @@ -341,7 +352,8 @@ DISPATCH_ADD_SUB_(connector, channel, messagename, DISP_FLAG_EXCL) /** - * Publish a given message with a given argument. + * Publish a given message with a given argument. (Takes ownership of the + * argument if it is a pointer.) */ #define PUBLISH(messagename, arg) \ publish_fn__ ##messagename(arg) From 3f0bfe1d29e571f1450286085e62246b01509b3b Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 14 Mar 2019 15:39:14 -0400 Subject: [PATCH 0675/2557] Rename DISPATCH_DEFINE_TYPE() to DISPATCH_REGISTER_TYPE() Also fix a grammar error in a comment. --- src/lib/pubsub/pubsub_build.c | 12 ++++++------ src/lib/pubsub/pubsub_connect.h | 10 +++++----- src/lib/pubsub/pubsub_macros.h | 12 ++++++------ src/test/test_pubsub_build.c | 6 +++--- src/test/test_pubsub_msg.c | 2 +- 5 files changed, 21 insertions(+), 21 deletions(-) diff --git a/src/lib/pubsub/pubsub_build.c b/src/lib/pubsub/pubsub_build.c index 1dc20f51f4..e44b7d76ec 100644 --- a/src/lib/pubsub/pubsub_build.c +++ b/src/lib/pubsub/pubsub_build.c @@ -208,16 +208,16 @@ pubsub_add_sub_(pubsub_connector_t *con, } /** - * Use con to define a the functions to use for manipulating the type + * Use con to define the functions to use for manipulating the type * type. Any function pointers left as NULL will be implemented as * no-ops. **/ int -pubsub_connector_define_type_(pubsub_connector_t *con, - msg_type_id_t type, - dispatch_typefns_t *fns, - const char *file, - unsigned line) +pubsub_connector_register_type_(pubsub_connector_t *con, + msg_type_id_t type, + dispatch_typefns_t *fns, + const char *file, + unsigned line) { pubsub_type_cfg_t *cfg = tor_malloc_zero(sizeof(*cfg)); cfg->type = type; diff --git a/src/lib/pubsub/pubsub_connect.h b/src/lib/pubsub/pubsub_connect.h index e4d8600056..bdcb33d2f5 100644 --- a/src/lib/pubsub/pubsub_connect.h +++ b/src/lib/pubsub/pubsub_connect.h @@ -45,10 +45,10 @@ int pubsub_add_sub_(struct pubsub_connector_t *con, const char *file, unsigned line); -int pubsub_connector_define_type_(struct pubsub_connector_t *, - msg_type_id_t, - dispatch_typefns_t *, - const char *file, - unsigned line); +int pubsub_connector_register_type_(struct pubsub_connector_t *, + msg_type_id_t, + dispatch_typefns_t *, + const char *file, + unsigned line); #endif diff --git a/src/lib/pubsub/pubsub_macros.h b/src/lib/pubsub/pubsub_macros.h index 11c7c63eca..f0429dbde4 100644 --- a/src/lib/pubsub/pubsub_macros.h +++ b/src/lib/pubsub/pubsub_macros.h @@ -71,7 +71,7 @@ * } * * Where did these types come from? Somewhere in the code, you need to call - * DISPATCH_DEFINE_TYPE() to make sure that the dispatcher can manage the + * DISPATCH_REGISTER_TYPE() to make sure that the dispatcher can manage the * message auxiliary data. It associates a vtbl-like structure with the * type name, so that the dispatcher knows how to manipulate the type you're * giving it. @@ -100,7 +100,7 @@ * .fmt_fn = boolean_fmt, * .free_fn = boolean_free, * }; - * DISPATCH_DEFINE_TYPE(con, boolean, &boolean_fns); + * DISPATCH_REGISTER_TYPE(con, boolean, &boolean_fns); * } * * @@ -184,7 +184,7 @@ * * "typename" is a unique identifier for the type of the auxiliary data. * It needs to be defined somewhere in Tor, using - * "DISPATCH_DEFINE_TYPE." + * "DISPATCH_REGISTER_TYPE." * * "c_type" is a C pointer type (like "char *" or "struct foo *"). */ @@ -210,7 +210,7 @@ * C identifier. * * "typename" is a unique identifier for the type of the auxiliary data. It - * needs to be defined somewhere in Tor, using "DISPATCH_DEFINE_TYPE." + * needs to be defined somewhere in Tor, using "DISPATCH_REGISTER_TYPE." * * "c_type" is a C integer type, like "int" or "bool". It needs to fit inside * a uint64_t. @@ -362,8 +362,8 @@ * Use a given connector to declare that the functions to be used to manipuate * a certain C type. **/ -#define DISPATCH_DEFINE_TYPE(con, type, fns) \ - pubsub_connector_define_type_((con), \ +#define DISPATCH_REGISTER_TYPE(con, type, fns) \ + pubsub_connector_register_type_((con), \ get_msg_type_id(#type), \ (fns), \ __FILE__, \ diff --git a/src/test/test_pubsub_build.c b/src/test/test_pubsub_build.c index f6c6b81a2a..9ffe428c0a 100644 --- a/src/test/test_pubsub_build.c +++ b/src/test/test_pubsub_build.c @@ -86,7 +86,7 @@ seed_dispatch_builder(pubsub_builder_t *b, { c = pubsub_connector_for_subsystem(b, get_subsys_id("sys1")); - DISPATCH_DEFINE_TYPE(c, int, &intfns); + DISPATCH_REGISTER_TYPE(c, int, &intfns); if (fl1 != FLAG_SKIP) DISPATCH_ADD_PUB_(c, main, bunch_of_coconuts, fl1); if (fl2 != FLAG_SKIP) @@ -96,7 +96,7 @@ seed_dispatch_builder(pubsub_builder_t *b, { c = pubsub_connector_for_subsystem(b, get_subsys_id("sys2")); - DISPATCH_DEFINE_TYPE(c, string, &stringfns); + DISPATCH_REGISTER_TYPE(c, string, &stringfns); if (fl3 != FLAG_SKIP) DISPATCH_ADD_PUB_(c, main, yes_we_have_no, fl3); if (fl4 != FLAG_SKIP) @@ -178,7 +178,7 @@ test_pubsub_build_types_decls_conflict(void *arg) { c = pubsub_connector_for_subsystem(b, get_subsys_id("sys3")); // Extra declaration of int: we don't allow this. - DISPATCH_DEFINE_TYPE(c, int, &stringfns); + DISPATCH_REGISTER_TYPE(c, int, &stringfns); pubsub_connector_free(c); } diff --git a/src/test/test_pubsub_msg.c b/src/test/test_pubsub_msg.c index 41a8a25ad6..73c7c9f540 100644 --- a/src/test/test_pubsub_msg.c +++ b/src/test/test_pubsub_msg.c @@ -103,7 +103,7 @@ setup_dispatcher(const struct testcase_t *testcase) { con = pubsub_connector_for_subsystem(builder, get_subsys_id("types")); - pubsub_connector_define_type_(con, + pubsub_connector_register_type_(con, get_msg_type_id("string"), &stringfns, "nowhere.c", 99); From ddb31dd58301b524564d6cc98da20e5717b05d4b Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 14 Mar 2019 15:41:45 -0400 Subject: [PATCH 0676/2557] Rename one case of c_type to c_ptr_type. --- src/lib/pubsub/pubsub_macros.h | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/lib/pubsub/pubsub_macros.h b/src/lib/pubsub/pubsub_macros.h index f0429dbde4..d091e40dfa 100644 --- a/src/lib/pubsub/pubsub_macros.h +++ b/src/lib/pubsub/pubsub_macros.h @@ -186,17 +186,18 @@ * It needs to be defined somewhere in Tor, using * "DISPATCH_REGISTER_TYPE." * - * "c_type" is a C pointer type (like "char *" or "struct foo *"). + * "c_ptr_type" is a C pointer type (like "char *" or "struct foo *"). + * The "*" needs to be included. */ -#define DECLARE_MESSAGE(messagename, typename, c_type) \ - DECLARE_MESSAGE_COMMON__(messagename, typename, c_type) \ - ATTR_UNUSED static inline c_type \ +#define DECLARE_MESSAGE(messagename, typename, c_ptr_type) \ + DECLARE_MESSAGE_COMMON__(messagename, typename, c_ptr_type) \ + ATTR_UNUSED static inline c_ptr_type \ msg_arg_get__ ##messagename(msg_aux_data_t m) \ { \ return m.ptr; \ } \ ATTR_UNUSED static inline void \ - msg_arg_set__ ##messagename(msg_aux_data_t *m, c_type v) \ + msg_arg_set__ ##messagename(msg_aux_data_t *m, c_ptr_type v) \ { \ m->ptr = v; \ } \ From 3af9a51118aadde538580f6d4f8043a0a9bba512 Mon Sep 17 00:00:00 2001 From: teor Date: Thu, 21 Mar 2019 23:35:56 +1000 Subject: [PATCH 0677/2557] test/dir: add a 4th argument to dirserv_read_measured_bandwidths() Part of 29806. --- src/test/test_dir.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/test/test_dir.c b/src/test/test_dir.c index b9a48c6506..07a2641c9f 100644 --- a/src/test/test_dir.c +++ b/src/test/test_dir.c @@ -2064,7 +2064,8 @@ test_dir_dirserv_read_measured_bandwidths(void *arg) /* Read the bandwidth file */ setup_full_capture_of_logs(LOG_DEBUG); - tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL, NULL)); + tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL, NULL, + NULL)); expect_log_msg_containing("Ignoring bandwidth file line"); teardown_capture_of_logs(); @@ -2082,7 +2083,8 @@ test_dir_dirserv_read_measured_bandwidths(void *arg) /* Read the bandwidth file */ setup_full_capture_of_logs(LOG_DEBUG); - tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL, NULL)); + tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL, NULL, + NULL)); expect_log_msg_not_containing("Ignoring bandwidth file line"); teardown_capture_of_logs(); From 3eacae42b2ade91bc5d0e3761dce1e785aad7621 Mon Sep 17 00:00:00 2001 From: juga0 Date: Fri, 2 Nov 2018 18:38:46 +0000 Subject: [PATCH 0678/2557] Serve bandwidth file used in the next vote When a directory authority is using a bandwidth file to obtain the bandwidth values that will be included in the next vote, serve this bandwidth file at /tor/status-vote/next/bandwidth.z. --- src/feature/dircache/dircache.c | 24 +++++++++++ src/test/test_dir_handle_get.c | 76 +++++++++++++++++++++++++++++++++ 2 files changed, 100 insertions(+) diff --git a/src/feature/dircache/dircache.c b/src/feature/dircache/dircache.c index e8cb284165..f373b74c85 100644 --- a/src/feature/dircache/dircache.c +++ b/src/feature/dircache/dircache.c @@ -357,12 +357,15 @@ static int handle_get_robots(dir_connection_t *conn, const get_handler_args_t *args); static int handle_get_networkstatus_bridges(dir_connection_t *conn, const get_handler_args_t *args); +static int handle_get_next_bandwidth(dir_connection_t *conn, + const get_handler_args_t *args); /** Table for handling GET requests. */ static const url_table_ent_t url_table[] = { { "/tor/", 0, handle_get_frontpage }, { "/tor/status-vote/current/consensus", 1, handle_get_current_consensus }, { "/tor/status-vote/current/", 1, handle_get_status_vote }, + { "/tor/status-vote/next/bandwidth", 0, handle_get_next_bandwidth }, { "/tor/status-vote/next/", 1, handle_get_status_vote }, { "/tor/micro/d/", 1, handle_get_microdesc }, { "/tor/server/", 1, handle_get_descriptor }, @@ -1438,6 +1441,27 @@ handle_get_networkstatus_bridges(dir_connection_t *conn, return 0; } +/** Helper function for GET the bandwidth file used for the next vote */ +static int +handle_get_next_bandwidth(dir_connection_t *conn, + const get_handler_args_t *args) +{ + (void)args; + log_debug(LD_DIR, "Getting next bandwidth."); + const or_options_t *options = get_options(); + if (options->V3BandwidthsFile) { + int lifetime = 60; + char *bandwidth = read_file_to_str(options->V3BandwidthsFile, 0, NULL); + size_t len = strlen(bandwidth); + write_http_response_header(conn, len, NO_METHOD, lifetime); + connection_buf_add(bandwidth, len, TO_CONN(conn)); + tor_free(bandwidth); + } else { + write_short_http_response(conn, 404, "Not found"); + } + return 0; +} + /** Helper function for GET robots.txt or /tor/robots.txt */ static int handle_get_robots(dir_connection_t *conn, const get_handler_args_t *args) diff --git a/src/test/test_dir_handle_get.c b/src/test/test_dir_handle_get.c index 90691fff94..8b2ee3ecac 100644 --- a/src/test/test_dir_handle_get.c +++ b/src/test/test_dir_handle_get.c @@ -2217,6 +2217,31 @@ test_dir_handle_get_status_vote_next_authority_not_found(void* data) tor_free(header); } +static void +test_dir_handle_get_status_vote_next_bandwidth_not_found(void* data) +{ + dir_connection_t *conn = NULL; + char *header = NULL; + (void) data; + + MOCK(connection_write_to_buf_impl_, connection_write_to_buf_mock); + + conn = new_dir_conn(); + + tt_int_op(0, OP_EQ, directory_handle_command_get(conn, + GET("/tor/status-vote/next/bandwdith"), NULL, 0)); + + fetch_from_buf_http(TO_CONN(conn)->outbuf, &header, MAX_HEADERS_SIZE, + NULL, NULL, 1, 0); + tt_assert(header); + tt_str_op(NOT_FOUND, OP_EQ, header); + + done: + UNMOCK(connection_write_to_buf_impl_); + connection_free_minimal(TO_CONN(conn)); + tor_free(header); +} + NS_DECL(const char*, dirvote_get_pending_consensus, (consensus_flavor_t flav)); @@ -2454,6 +2479,55 @@ test_dir_handle_get_status_vote_next_authority(void* data) dirvote_free_all(); } +static void +test_dir_handle_get_status_vote_next_bandwidth(void* data) +{ + dir_connection_t *conn = NULL; + char *header = NULL, *body = NULL; + size_t body_used = 0; + (void) data; + + const char *content = + "1541171221\n" + "node_id=$68A483E05A2ABDCA6DA5A3EF8DB5177638A27F80 " + "master_key_ed25519=YaqV4vbvPYKucElk297eVdNArDz9HtIwUoIeo0+cVIpQ " + "bw=760 nick=Test time=2018-05-08T16:13:26\n"; + + init_mock_options(); + MOCK(get_options, mock_get_options); + mock_options->V3BandwidthsFile = tor_strdup( + get_fname_rnd("V3BandwidthsFile") + ); + + write_str_to_file(mock_options->V3BandwidthsFile, content, 0); + + MOCK(connection_write_to_buf_impl_, connection_write_to_buf_mock); + + conn = new_dir_conn(); + tt_int_op(0, OP_EQ, directory_handle_command_get(conn, + GET("/tor/status-vote/next/bandwidth"), NULL, 0)); + + fetch_from_buf_http(TO_CONN(conn)->outbuf, &header, MAX_HEADERS_SIZE, + &body, &body_used, strlen(content)+1, 0); + + tt_assert(header); + tt_ptr_op(strstr(header, "HTTP/1.0 200 OK\r\n"), OP_EQ, header); + tt_assert(strstr(header, "Content-Type: text/plain\r\n")); + tt_assert(strstr(header, "Content-Encoding: identity\r\n")); + tt_assert(strstr(header, "Content-Length: 167\r\n")); + + tt_int_op(body_used, OP_EQ, strlen(body)); + tt_str_op(content, OP_EQ, body); + + done: + UNMOCK(get_options); + UNMOCK(connection_write_to_buf_impl_); + connection_free_minimal(TO_CONN(conn)); + tor_free(header); + tor_free(body); + or_options_free(mock_options); mock_options = NULL; +} + static void test_dir_handle_get_status_vote_current_authority(void* data) { @@ -2627,6 +2701,8 @@ struct testcase_t dir_handle_get_tests[] = { DIR_HANDLE_CMD(status_vote_current_authority, 0), DIR_HANDLE_CMD(status_vote_next_authority_not_found, 0), DIR_HANDLE_CMD(status_vote_next_authority, 0), + DIR_HANDLE_CMD(status_vote_next_bandwidth_not_found, 0), + DIR_HANDLE_CMD(status_vote_next_bandwidth, 0), DIR_HANDLE_CMD(status_vote_current_consensus_ns_not_enough_sigs, TT_FORK), DIR_HANDLE_CMD(status_vote_current_consensus_ns_not_found, TT_FORK), DIR_HANDLE_CMD(status_vote_current_consensus_too_old, TT_FORK), From ee09e5d7eacdfde0cd5e1bc4a18c72fd7e47c1a7 Mon Sep 17 00:00:00 2001 From: juga0 Date: Wed, 7 Nov 2018 08:49:51 +0000 Subject: [PATCH 0679/2557] bwauth: use flag to do not warn when file is missing Use flag to do not warn when the bandwidth file is missing trying to serve it by http. Also remove double space in the assignement. --- src/feature/dircache/dircache.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/feature/dircache/dircache.c b/src/feature/dircache/dircache.c index f373b74c85..2aaaf7be11 100644 --- a/src/feature/dircache/dircache.c +++ b/src/feature/dircache/dircache.c @@ -1451,7 +1451,8 @@ handle_get_next_bandwidth(dir_connection_t *conn, const or_options_t *options = get_options(); if (options->V3BandwidthsFile) { int lifetime = 60; - char *bandwidth = read_file_to_str(options->V3BandwidthsFile, 0, NULL); + char *bandwidth = read_file_to_str(options->V3BandwidthsFile, + RFTS_IGNORE_MISSING, NULL); size_t len = strlen(bandwidth); write_http_response_header(conn, len, NO_METHOD, lifetime); connection_buf_add(bandwidth, len, TO_CONN(conn)); From b75e2539f9994dc04ee7662445878e8486eb77b0 Mon Sep 17 00:00:00 2001 From: juga0 Date: Wed, 7 Nov 2018 12:04:09 +0000 Subject: [PATCH 0680/2557] bwauth: check if a bw file could be read Before serving it by HTTP. --- src/feature/dircache/dircache.c | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/feature/dircache/dircache.c b/src/feature/dircache/dircache.c index 2aaaf7be11..d9625ee698 100644 --- a/src/feature/dircache/dircache.c +++ b/src/feature/dircache/dircache.c @@ -1453,13 +1453,15 @@ handle_get_next_bandwidth(dir_connection_t *conn, int lifetime = 60; char *bandwidth = read_file_to_str(options->V3BandwidthsFile, RFTS_IGNORE_MISSING, NULL); - size_t len = strlen(bandwidth); - write_http_response_header(conn, len, NO_METHOD, lifetime); - connection_buf_add(bandwidth, len, TO_CONN(conn)); - tor_free(bandwidth); - } else { - write_short_http_response(conn, 404, "Not found"); + if (bandwidth != NULL) { + site_t len = strlen(bandwidth); + write_http_response_header(conn, len, NO_METHOD, lifetime); + connection_buf_add(bandwidth, len, TO_CONN(conn)); + tor_free(bandwidth); + return 0; + } } + write_short_http_response(conn, 404, "Not found"); return 0; } From 4d3502e45bc9b0da91b32c362a7995f33cee0539 Mon Sep 17 00:00:00 2001 From: juga0 Date: Wed, 7 Nov 2018 12:15:31 +0000 Subject: [PATCH 0681/2557] bwauth: check and use compression serving bw file --- src/feature/dircache/dircache.c | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/src/feature/dircache/dircache.c b/src/feature/dircache/dircache.c index d9625ee698..e5ea9f73b3 100644 --- a/src/feature/dircache/dircache.c +++ b/src/feature/dircache/dircache.c @@ -1449,14 +1449,28 @@ handle_get_next_bandwidth(dir_connection_t *conn, (void)args; log_debug(LD_DIR, "Getting next bandwidth."); const or_options_t *options = get_options(); + const compress_method_t compress_method = + find_best_compression_method(args->compression_supported, 1); + if (options->V3BandwidthsFile) { int lifetime = 60; char *bandwidth = read_file_to_str(options->V3BandwidthsFile, RFTS_IGNORE_MISSING, NULL); if (bandwidth != NULL) { - site_t len = strlen(bandwidth); - write_http_response_header(conn, len, NO_METHOD, lifetime); - connection_buf_add(bandwidth, len, TO_CONN(conn)); + ssize_t len = strlen(bandwidth); + write_http_response_header(conn, compress_method != NO_METHOD ? -1 : len, + compress_method, BANDWIDTH_CACHE_LIFETIME); + if (compress_method != NO_METHOD) { + conn->compress_state = tor_compress_new(1, compress_method, + choose_compression_level(len/2)); + log_debug(LD_DIR, "Compressing bandwidth file."); + connection_buf_add_compress(bandwidth, len, conn, 0); + /* Flush the compression state. */ + connection_buf_add_compress("", 0, conn, 1); + } else { + log_debug(LD_DIR, "Not compressing bandwidth file."); + connection_buf_add(bandwidth, len, TO_CONN(conn)); + } tor_free(bandwidth); return 0; } From 76271347438ff03796a9424019860332f50c5134 Mon Sep 17 00:00:00 2001 From: juga0 Date: Wed, 7 Nov 2018 12:19:20 +0000 Subject: [PATCH 0682/2557] bwauth: increment bw file cache lifetime Increment bw file cache lifetime when serving it by HTTP. And add a constant to define that lifetime. --- src/feature/dircache/dircache.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/feature/dircache/dircache.c b/src/feature/dircache/dircache.c index e5ea9f73b3..0f92dfaea1 100644 --- a/src/feature/dircache/dircache.c +++ b/src/feature/dircache/dircache.c @@ -49,7 +49,8 @@ #define ROUTERDESC_BY_DIGEST_CACHE_LIFETIME (48*60*60) #define ROBOTS_CACHE_LIFETIME (24*60*60) #define MICRODESC_CACHE_LIFETIME (48*60*60) - +/* Bandwidth files change every hour. */ +#define BANDWIDTH_CACHE_LIFETIME (30*60) /** Parse an HTTP request string headers of the form * \verbatim * "\%s [http[s]://]\%s HTTP/1..." @@ -1453,7 +1454,6 @@ handle_get_next_bandwidth(dir_connection_t *conn, find_best_compression_method(args->compression_supported, 1); if (options->V3BandwidthsFile) { - int lifetime = 60; char *bandwidth = read_file_to_str(options->V3BandwidthsFile, RFTS_IGNORE_MISSING, NULL); if (bandwidth != NULL) { From fb4a40c32c4a7e5b7175ea2c635687b6b24a29d0 Mon Sep 17 00:00:00 2001 From: juga0 Date: Wed, 7 Nov 2018 12:38:45 +0000 Subject: [PATCH 0683/2557] test: Check bw file cache lifetime --- src/test/test_dir_handle_get.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/test/test_dir_handle_get.c b/src/test/test_dir_handle_get.c index 8b2ee3ecac..6d478f6b49 100644 --- a/src/test/test_dir_handle_get.c +++ b/src/test/test_dir_handle_get.c @@ -2516,6 +2516,16 @@ test_dir_handle_get_status_vote_next_bandwidth(void* data) tt_assert(strstr(header, "Content-Encoding: identity\r\n")); tt_assert(strstr(header, "Content-Length: 167\r\n")); + /* Check cache lifetime */ + char expbuf[RFC1123_TIME_LEN+1]; + time_t now = time(NULL); + /* BANDWIDTH_CACHE_LIFETIME is defined in dircache.c. */ + format_rfc1123_time(expbuf, (time_t)(now + 30*60)); + char *expires = NULL; + /* Change to 'Cache-control: max-age=%d' if using http/1.1. */ + tor_asprintf(&expires, "Expires: %s\r\n", expbuf); + tt_assert(strstr(header, expires)); + tt_int_op(body_used, OP_EQ, strlen(body)); tt_str_op(content, OP_EQ, body); @@ -2525,6 +2535,7 @@ test_dir_handle_get_status_vote_next_bandwidth(void* data) connection_free_minimal(TO_CONN(conn)); tor_free(header); tor_free(body); + tor_free(expires); or_options_free(mock_options); mock_options = NULL; } From a4bf3be8bce662c77422fb88072e4685083e90b2 Mon Sep 17 00:00:00 2001 From: juga0 Date: Wed, 7 Nov 2018 12:40:51 +0000 Subject: [PATCH 0684/2557] test: check that .../bandwidth.z is compressed --- src/test/test_dir_handle_get.c | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/test/test_dir_handle_get.c b/src/test/test_dir_handle_get.c index 6d478f6b49..0cd7e1050a 100644 --- a/src/test/test_dir_handle_get.c +++ b/src/test/test_dir_handle_get.c @@ -2529,6 +2529,25 @@ test_dir_handle_get_status_vote_next_bandwidth(void* data) tt_int_op(body_used, OP_EQ, strlen(body)); tt_str_op(content, OP_EQ, body); + tor_free(header); + tor_free(body); + + /* Request the file using compression, the result should be the same. */ + tt_int_op(0, OP_EQ, directory_handle_command_get(conn, + GET("/tor/status-vote/next/bandwidth.z"), NULL, 0)); + + fetch_from_buf_http(TO_CONN(conn)->outbuf, &header, MAX_HEADERS_SIZE, + &body, &body_used, strlen(content)+1, 0); + + tt_assert(header); + tt_ptr_op(strstr(header, "HTTP/1.0 200 OK\r\n"), OP_EQ, header); + tt_assert(strstr(header, "Content-Encoding: deflate\r\n")); + + /* Since using connection_write_to_buf_mock instead of mocking + * connection_buf_add_compress, the content is not actually compressed. + * If it would, the size and content would be different than the original. + */ + done: UNMOCK(get_options); UNMOCK(connection_write_to_buf_impl_); From 892b918b6610d662a9536d29c4edf9f4eac5da88 Mon Sep 17 00:00:00 2001 From: juga0 Date: Thu, 8 Nov 2018 09:13:54 +0000 Subject: [PATCH 0685/2557] bwauth: remove declaring args, they are now in use --- src/feature/dircache/dircache.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/feature/dircache/dircache.c b/src/feature/dircache/dircache.c index 0f92dfaea1..14c6302c25 100644 --- a/src/feature/dircache/dircache.c +++ b/src/feature/dircache/dircache.c @@ -1447,7 +1447,6 @@ static int handle_get_next_bandwidth(dir_connection_t *conn, const get_handler_args_t *args) { - (void)args; log_debug(LD_DIR, "Getting next bandwidth."); const or_options_t *options = get_options(); const compress_method_t compress_method = From da7a8d7624b76053210bad6f830df7842a4b30f1 Mon Sep 17 00:00:00 2001 From: juga0 Date: Mon, 19 Nov 2018 17:27:41 +0000 Subject: [PATCH 0686/2557] dircache: fix identation and remove unneded goto --- src/test/test_dir_handle_get.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/test/test_dir_handle_get.c b/src/test/test_dir_handle_get.c index 0cd7e1050a..22f5a9b3da 100644 --- a/src/test/test_dir_handle_get.c +++ b/src/test/test_dir_handle_get.c @@ -2548,14 +2548,14 @@ test_dir_handle_get_status_vote_next_bandwidth(void* data) * If it would, the size and content would be different than the original. */ - done: - UNMOCK(get_options); - UNMOCK(connection_write_to_buf_impl_); - connection_free_minimal(TO_CONN(conn)); - tor_free(header); - tor_free(body); - tor_free(expires); - or_options_free(mock_options); mock_options = NULL; + done: + UNMOCK(get_options); + UNMOCK(connection_write_to_buf_impl_); + connection_free_minimal(TO_CONN(conn)); + tor_free(header); + tor_free(body); + tor_free(expires); + or_options_free(mock_options); } static void From 6ecf9590eae8cac125705d2f99b284de998d5649 Mon Sep 17 00:00:00 2001 From: juga0 Date: Fri, 21 Dec 2018 17:10:19 +0000 Subject: [PATCH 0687/2557] changes: add file for #21377 --- changes/ticket21377 | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 changes/ticket21377 diff --git a/changes/ticket21377 b/changes/ticket21377 new file mode 100644 index 0000000000..2bf5149a0a --- /dev/null +++ b/changes/ticket21377 @@ -0,0 +1,4 @@ + o Minor features (dircache): + - When a directory authority is using a bandwidth file to obtain the + bandwidth values that will be included in the next vote, serve this + bandwidth file at /tor/status-vote/next/bandwidth. Closes ticket 21377. \ No newline at end of file From 4e6ba575a66108f38a55cffdb197352757f77d78 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Tue, 26 Mar 2019 09:49:32 +0200 Subject: [PATCH 0688/2557] Add header guards to ptr_helpers.h --- src/test/ptr_helpers.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/test/ptr_helpers.h b/src/test/ptr_helpers.h index fe2c8c9705..67776b1006 100644 --- a/src/test/ptr_helpers.h +++ b/src/test/ptr_helpers.h @@ -3,6 +3,9 @@ * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ +#ifndef TOR_PTR_HELPERS_H +#define TOR_PTR_HELPERS_H + #include void * @@ -16,3 +19,5 @@ cast_uintptr_to_voidstar(uintptr_t x); uintptr_t cast_voidstar_to_uintptr(void *x); + +#endif From 39e449434421c8af5d8d6e89b4b4a1bccd78f284 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 26 Mar 2019 08:42:14 -0400 Subject: [PATCH 0689/2557] practracker: update usage note in docstring --- scripts/maint/practracker/practracker.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/maint/practracker/practracker.py b/scripts/maint/practracker/practracker.py index 4be1a31b51..c2efa61f2c 100755 --- a/scripts/maint/practracker/practracker.py +++ b/scripts/maint/practracker/practracker.py @@ -13,9 +13,9 @@ practracker.py should be run with its second argument pointing to the Tor top-level source directory like this: $ python3 ./scripts/maint/practracker/practracker.py . -The exceptions file is meant to be initialized once with the current state of -the source code and then get saved in the repository for ever after: - $ python3 ./scripts/maint/practracker/practracker.py . > ./scripts/maint/practracker/exceptions.txt +To regenerate the exceptions file so that it allows all current +problems in the Tor source, use the --regen flag: + $ python3 --regen ./scripts/maint/practracker/practracker.py . """ from __future__ import print_function From 08176c239651656bf691f9414d037ab803be0fc0 Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Thu, 21 Mar 2019 20:57:42 +0200 Subject: [PATCH 0690/2557] prob-distr: Silence some coverity warnings. --- changes/bug29805 | 3 +++ src/lib/math/prob_distr.h | 15 ++++++++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 changes/bug29805 diff --git a/changes/bug29805 b/changes/bug29805 new file mode 100644 index 0000000000..00c846e9af --- /dev/null +++ b/changes/bug29805 @@ -0,0 +1,3 @@ + o Minor bugfixes (probability distributions): + - Refactor and improve parts of the probability distribution code that made + Coverity complain. Fixes bug 29805; bugfix on 0.4.0.1-alpha. \ No newline at end of file diff --git a/src/lib/math/prob_distr.h b/src/lib/math/prob_distr.h index 66acb796fd..9b2ce41240 100644 --- a/src/lib/math/prob_distr.h +++ b/src/lib/math/prob_distr.h @@ -19,9 +19,22 @@ struct dist { const struct dist_ops *ops; }; +/** Assign the right ops to dist.dist_ops */ #define DIST_BASE(OPS) { .ops = (OPS) } + +/** A compile-time type-checking macro for use with DIST_BASE_TYPED. */ +#ifdef __COVERITY___ +/* Disable type-checking if coverity is enabled, since they don't like it */ +#define TYPE_CHECK_OBJ(OPS, OBJ, TYPE) 0 +#else +#define TYPE_CHECK_OBJ(OPS, OBJ, TYPE) \ + (0*sizeof(&(OBJ) - (const TYPE *)&(OBJ))) +#endif + +/** Macro to initialize a distribution with the right OPS, while making sure + * that OBJ is of the right TYPE */ #define DIST_BASE_TYPED(OPS, OBJ, TYPE) \ - DIST_BASE((OPS) + 0*sizeof(&(OBJ) - (const TYPE *)&(OBJ))) + DIST_BASE((OPS) + TYPE_CHECK_OBJ(OPS,OBJ,TYPE)) const char *dist_name(const struct dist *); double dist_sample(const struct dist *); From 27f24484d47cbf72bfad5f81280773128da9e3d2 Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Mon, 25 Mar 2019 15:27:03 +0200 Subject: [PATCH 0691/2557] prob-distr: Some more comments about the initializers. Based on patches and review comments by Riastradh and Catalyst. Co-authored-by: Taylor R Campbell Co-authored-by: Taylor Yu --- src/lib/math/prob_distr.h | 92 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 87 insertions(+), 5 deletions(-) diff --git a/src/lib/math/prob_distr.h b/src/lib/math/prob_distr.h index 9b2ce41240..2eb935e4a8 100644 --- a/src/lib/math/prob_distr.h +++ b/src/lib/math/prob_distr.h @@ -19,23 +19,100 @@ struct dist { const struct dist_ops *ops; }; -/** Assign the right ops to dist.dist_ops */ +/** + * Untyped initializer element for struct dist using the specified + * struct dist_ops pointer. Don't actually use this directly -- use + * the type-specific macro built out of DIST_BASE_TYPED below -- but if + * you did use this directly, it would be something like: + * + * struct weibull mydist = { + * DIST_BASE(&weibull_ops), + * .lambda = ..., + * .k = ..., + * }; + * + * Note there is NO COMPILER FEEDBACK if you accidentally do something + * like + * + * struct geometric mydist = { + * DIST_BASE(&weibull_ops), + * ... + * }; + */ #define DIST_BASE(OPS) { .ops = (OPS) } -/** A compile-time type-checking macro for use with DIST_BASE_TYPED. */ +/** A compile-time type-checking macro for use with DIST_BASE_TYPED. + * + * This macro works by checking that &OBJ is a pointer type that is the same + * type (except for qualifiers) as (const TYPE *)&OBJ. It's a C constraint + * violation (which requires a diagnostic) if two pointers are different types + * and are subtracted. The sizeof() forces compile-time evaluation, and the + * multiplication by zero is to discard the result of the sizeof() from the + * expression. + * + * We define this conditionally to suppress false positives from + * Coverity, which gets confused by the sizeof business. + */ #ifdef __COVERITY___ -/* Disable type-checking if coverity is enabled, since they don't like it */ #define TYPE_CHECK_OBJ(OPS, OBJ, TYPE) 0 #else #define TYPE_CHECK_OBJ(OPS, OBJ, TYPE) \ (0*sizeof(&(OBJ) - (const TYPE *)&(OBJ))) #endif -/** Macro to initialize a distribution with the right OPS, while making sure - * that OBJ is of the right TYPE */ +/** +* Typed initializer element for struct dist using the specified struct +* dist_ops pointer. Don't actually use this directly -- use a +* type-specific macro built out of it -- but if you did use this +* directly, it would be something like: +* +* struct weibull mydist = { +* DIST_BASE_TYPED(&weibull_ops, mydist, struct weibull), +* .lambda = ..., +* .k = ..., +* }; +* +* If you want to define a distribution type, define a canonical set of +* operations and define a type-specific initializer element like so: +* +* struct foo { +* struct dist base; +* int omega; +* double tau; +* double phi; +* }; +* +* struct dist_ops foo_ops = ...; +* +* #define FOO(OBJ) DIST_BASE_TYPED(&foo_ops, OBJ, struct foo) +* +* Then users can do: +* +* struct foo mydist = { +* FOO(mydist), +* .omega = ..., +* .tau = ..., +* .phi = ..., +* }; +* +* If you accidentally write +* +* struct bar mydist = { +* FOO(mydist), +* ... +* }; +* +* then the compiler will report a type mismatch in the sizeof +* expression, which otherwise evaporates at runtime. +*/ #define DIST_BASE_TYPED(OPS, OBJ, TYPE) \ DIST_BASE((OPS) + TYPE_CHECK_OBJ(OPS,OBJ,TYPE)) +/** + * Generic operations on distributions. These simply defer to the + * corresponding dist_ops function. In the parlance of C++, these call + * virtual member functions. + */ const char *dist_name(const struct dist *); double dist_sample(const struct dist *); double dist_cdf(const struct dist *, double x); @@ -43,6 +120,11 @@ double dist_sf(const struct dist *, double x); double dist_icdf(const struct dist *, double p); double dist_isf(const struct dist *, double p); +/** + * Set of operations on a potentially parametric family of + * distributions. In the parlance of C++, this would be called a + * `vtable' and the members are virtual member functions. + */ struct dist_ops { const char *name; double (*sample)(const struct dist *); From e028ec6bb7a97596496cc44f21c83ff56d8d55c9 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 26 Mar 2019 09:43:46 -0400 Subject: [PATCH 0692/2557] Add new exceptions.txt entries --- scripts/maint/practracker/exceptions.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/maint/practracker/exceptions.txt b/scripts/maint/practracker/exceptions.txt index fee22ec7fc..c9585a4e9d 100644 --- a/scripts/maint/practracker/exceptions.txt +++ b/scripts/maint/practracker/exceptions.txt @@ -132,11 +132,12 @@ problem function-size /src/feature/dircache/dircache.c:handle_get_current_consen problem function-size /src/feature/dircache/dircache.c:directory_handle_command_post() 120 problem function-size /src/feature/hibernate/hibernate.c:accounting_parse_options() 109 problem function-size /src/feature/relay/routerkeys.c:load_ed_keys() 294 -problem file-size /src/feature/relay/router.c 3221 +problem file-size /src/feature/relay/router.c 3412 problem include-count /src/feature/relay/router.c 56 problem function-size /src/feature/relay/router.c:init_keys() 252 problem function-size /src/feature/relay/router.c:get_my_declared_family() 114 problem function-size /src/feature/relay/router.c:router_build_fresh_descriptor() 190 +problem function-size /src/feature/relay/router.c:router_build_fresh_unsigned_routerinfo() 140 problem function-size /src/feature/relay/router.c:router_dump_router_to_string() 375 problem function-size /src/feature/relay/router.c:extrainfo_dump_to_string() 208 problem function-size /src/feature/relay/dns.c:dns_resolve_impl() 134 From 7502e5467b5b22bee118616d393ace5d67b3607f Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 26 Mar 2019 19:28:28 -0400 Subject: [PATCH 0693/2557] Regenerate practracker file from scratch. Closes ticket 29912. Since this is the first time that practracker has had a stable output order, this diff will be larger than usual. --- scripts/maint/practracker/exceptions.txt | 413 ++++++++++++----------- 1 file changed, 220 insertions(+), 193 deletions(-) diff --git a/scripts/maint/practracker/exceptions.txt b/scripts/maint/practracker/exceptions.txt index baccbef255..9e372728ce 100644 --- a/scripts/maint/practracker/exceptions.txt +++ b/scripts/maint/practracker/exceptions.txt @@ -1,84 +1,34 @@ -problem function-size /src/core/proto/proto_socks.c:parse_socks_client() 112 -problem file-size /src/core/or/channel.c 3476 -problem function-size /src/core/or/scheduler_kist.c:kist_scheduler_run() 171 -problem function-size /src/core/or/scheduler_vanilla.c:vanilla_scheduler_run() 109 -problem function-size /src/core/or/command.c:command_process_create_cell() 156 -problem function-size /src/core/or/command.c:command_process_relay_cell() 132 -problem function-size /src/core/or/protover.c:protover_all_supported() 116 -problem function-size /src/core/or/circuitmux.c:circuitmux_detach_all_circuits() 109 -problem function-size /src/core/or/circuitmux.c:circuitmux_set_policy() 110 -problem function-size /src/core/or/circuitmux.c:circuitmux_attach_circuit() 129 -problem include-count /src/core/or/circuitlist.c 54 -problem function-size /src/core/or/circuitlist.c:HT_PROTOTYPE() 128 -problem function-size /src/core/or/circuitlist.c:circuit_free_() 137 -problem function-size /src/core/or/circuitlist.c:circuit_find_to_cannibalize() 102 -problem function-size /src/core/or/circuitlist.c:circuit_about_to_free() 120 -problem function-size /src/core/or/circuitlist.c:circuits_handle_oom() 117 -problem file-size /src/core/or/connection_or.c 3111 -problem include-count /src/core/or/connection_or.c 51 -problem function-size /src/core/or/connection_or.c:connection_or_group_set_badness_() 105 -problem function-size /src/core/or/connection_or.c:connection_or_client_learned_peer_id() 144 -problem function-size /src/core/or/connection_or.c:connection_or_compute_authenticate_cell_body() 235 -problem file-size /src/core/or/policies.c 3163 -problem function-size /src/core/or/policies.c:policy_summarize() 107 -problem function-size /src/core/or/channeltls.c:channel_tls_handle_var_cell() 160 -problem function-size /src/core/or/channeltls.c:channel_tls_process_versions_cell() 170 -problem function-size /src/core/or/channeltls.c:channel_tls_process_netinfo_cell() 214 -problem function-size /src/core/or/channeltls.c:channel_tls_process_certs_cell() 246 -problem function-size /src/core/or/channeltls.c:channel_tls_process_authenticate_cell() 202 -problem file-size /src/core/or/circuituse.c 3150 -problem function-size /src/core/or/circuituse.c:circuit_is_acceptable() 129 -problem function-size /src/core/or/circuituse.c:circuit_expire_building() 394 -problem function-size /src/core/or/circuituse.c:circuit_log_ancient_one_hop_circuits() 126 -problem function-size /src/core/or/circuituse.c:circuit_build_failed() 149 -problem function-size /src/core/or/circuituse.c:circuit_launch_by_extend_info() 110 -problem function-size /src/core/or/circuituse.c:circuit_get_open_circ_or_launch() 354 -problem function-size /src/core/or/circuituse.c:connection_ap_handshake_attach_circuit() 244 -problem file-size /src/core/or/connection_edge.c 4550 -problem include-count /src/core/or/connection_edge.c 64 -problem function-size /src/core/or/connection_edge.c:connection_ap_expire_beginning() 117 -problem function-size /src/core/or/connection_edge.c:connection_ap_handshake_rewrite() 192 -problem function-size /src/core/or/connection_edge.c:connection_ap_handle_onion() 188 -problem function-size /src/core/or/connection_edge.c:connection_ap_handshake_rewrite_and_attach() 423 -problem function-size /src/core/or/connection_edge.c:connection_ap_handshake_send_begin() 111 -problem function-size /src/core/or/connection_edge.c:connection_ap_handshake_socks_resolved() 106 -problem function-size /src/core/or/connection_edge.c:connection_exit_begin_conn() 184 -problem function-size /src/core/or/connection_edge.c:connection_exit_connect() 102 -problem function-size /src/core/or/circuitstats.c:circuit_build_times_parse_state() 124 -problem function-size /src/core/or/versions.c:tor_version_parse() 104 -problem file-size /src/core/or/circuitbuild.c 3060 -problem include-count /src/core/or/circuitbuild.c 53 -problem function-size /src/core/or/circuitbuild.c:get_unique_circ_id_by_chan() 128 -problem function-size /src/core/or/circuitbuild.c:circuit_extend() 147 -problem function-size /src/core/or/circuitbuild.c:choose_good_exit_server_general() 206 -problem file-size /src/core/or/relay.c 3183 -problem function-size /src/core/or/relay.c:circuit_receive_relay_cell() 123 -problem function-size /src/core/or/relay.c:relay_send_command_from_edge_() 101 -problem function-size /src/core/or/relay.c:connection_ap_process_end_not_open() 194 -problem function-size /src/core/or/relay.c:connection_edge_process_relay_cell_not_open() 139 -problem function-size /src/core/or/relay.c:connection_edge_process_relay_cell() 520 -problem function-size /src/core/or/relay.c:connection_edge_package_raw_inbuf() 130 -problem function-size /src/core/or/relay.c:circuit_resume_edge_reading_helper() 148 -problem file-size /src/core/mainloop/mainloop.c 3051 -problem include-count /src/core/mainloop/mainloop.c 66 -problem function-size /src/core/mainloop/mainloop.c:conn_close_if_marked() 108 -problem function-size /src/core/mainloop/mainloop.c:run_connection_housekeeping() 123 -problem function-size /src/core/mainloop/mainloop.c:CALLBACK() 116 -problem file-size /src/core/mainloop/connection.c 5548 -problem include-count /src/core/mainloop/connection.c 61 -problem function-size /src/core/mainloop/connection.c:connection_free_minimal() 184 -problem function-size /src/core/mainloop/connection.c:connection_listener_new() 328 -problem function-size /src/core/mainloop/connection.c:connection_handle_listener_read() 161 -problem function-size /src/core/mainloop/connection.c:connection_connect_sockaddr() 103 -problem function-size /src/core/mainloop/connection.c:connection_proxy_connect() 148 -problem function-size /src/core/mainloop/connection.c:connection_read_proxy_handshake() 153 -problem function-size /src/core/mainloop/connection.c:retry_listener_ports() 116 -problem function-size /src/core/mainloop/connection.c:connection_handle_read_impl() 111 -problem function-size /src/core/mainloop/connection.c:connection_buf_read_from_socket() 177 -problem function-size /src/core/mainloop/connection.c:connection_handle_write_impl() 241 -problem function-size /src/core/mainloop/connection.c:assert_connection_ok() 143 -problem function-size /src/app/config/confparse.c:config_assign_value() 205 -problem function-size /src/app/config/confparse.c:config_get_assigned_option() 129 +# Welcome to the exceptions file for Tor's best-practices tracker! +# +# Each line of this file represents a single violation of Tor's best +# practices -- typically, a violation that we had before practracker.py +# first existed. +# +# There are three kinds of problems that we recognize right now: +# function-size -- a function of more than 100 lines. +# file-size -- a file of more than 3000 lines. +# include-count -- a file with more than 50 #includes. +# +# Each line below represents a single exception that practracker should +# _ignore_. Each line has four parts: +# 1. The word "problem". +# 2. The kind of problem. +# 3. The location of the problem: either a filename, or a +# filename:functionname pair. +# 4. The magnitude of the problem to ignore. +# +# So for example, consider this line: +# problem file-size /src/core/or/connection_or.c 3200 +# +# It tells practracker to allow the mentioned file to be up to 3200 lines +# long, even though ordinarily it would warn about any file with more than +# 3000 lines. +# +# You can either edit this file by hand, or regenerate it completely by +# running `make practracker-regen`. +# +# Remember: It is better to fix the problem than to add a new exception! + problem file-size /src/app/config/config.c 8490 problem include-count /src/app/config/config.c 86 problem function-size /src/app/config/config.c:options_act_reversible() 296 @@ -95,63 +45,117 @@ problem function-size /src/app/config/config.c:parse_dir_fallback_line() 102 problem function-size /src/app/config/config.c:parse_port_config() 452 problem function-size /src/app/config/config.c:parse_ports() 170 problem function-size /src/app/config/config.c:getinfo_helper_config() 116 -problem function-size /src/app/main/ntmain.c:nt_service_install() 125 +problem function-size /src/app/config/confparse.c:config_assign_value() 205 +problem function-size /src/app/config/confparse.c:config_get_assigned_option() 129 problem include-count /src/app/main/main.c 85 problem function-size /src/app/main/main.c:dumpstats() 102 problem function-size /src/app/main/main.c:tor_init() 136 problem function-size /src/app/main/main.c:sandbox_init_filter() 291 problem function-size /src/app/main/main.c:run_tor_main_loop() 105 -problem include-count /src/app/main/shutdown.c 55 -problem function-size /src/tools/tor-resolve.c:build_socks5_resolve_request() 104 -problem function-size /src/tools/tor-resolve.c:do_resolve() 173 -problem function-size /src/tools/tor-resolve.c:main() 112 -problem function-size /src/tools/tor-gencert.c:parse_commandline() 111 -problem function-size /src/feature/keymgt/loadkey.c:ed_key_init_from_file() 333 -problem function-size /src/feature/dircommon/consdiff.c:gen_ed_diff() 204 -problem function-size /src/feature/dircommon/consdiff.c:apply_ed_diff() 159 -problem function-size /src/feature/control/control_auth.c:handle_control_authenticate() 188 +problem function-size /src/app/main/ntmain.c:nt_service_install() 125 +problem file-size /src/core/mainloop/connection.c 5548 +problem include-count /src/core/mainloop/connection.c 61 +problem function-size /src/core/mainloop/connection.c:connection_free_minimal() 184 +problem function-size /src/core/mainloop/connection.c:connection_listener_new() 328 +problem function-size /src/core/mainloop/connection.c:connection_handle_listener_read() 161 +problem function-size /src/core/mainloop/connection.c:connection_connect_sockaddr() 103 +problem function-size /src/core/mainloop/connection.c:connection_proxy_connect() 148 +problem function-size /src/core/mainloop/connection.c:connection_read_proxy_handshake() 153 +problem function-size /src/core/mainloop/connection.c:retry_listener_ports() 116 +problem function-size /src/core/mainloop/connection.c:connection_handle_read_impl() 111 +problem function-size /src/core/mainloop/connection.c:connection_buf_read_from_socket() 177 +problem function-size /src/core/mainloop/connection.c:connection_handle_write_impl() 241 +problem function-size /src/core/mainloop/connection.c:assert_connection_ok() 143 +problem file-size /src/core/mainloop/mainloop.c 3051 +problem include-count /src/core/mainloop/mainloop.c 66 +problem function-size /src/core/mainloop/mainloop.c:conn_close_if_marked() 108 +problem function-size /src/core/mainloop/mainloop.c:run_connection_housekeeping() 123 +problem function-size /src/core/mainloop/mainloop.c:CALLBACK() 116 +problem file-size /src/core/or/channel.c 3476 +problem function-size /src/core/or/channeltls.c:channel_tls_handle_var_cell() 160 +problem function-size /src/core/or/channeltls.c:channel_tls_process_versions_cell() 170 +problem function-size /src/core/or/channeltls.c:channel_tls_process_netinfo_cell() 214 +problem function-size /src/core/or/channeltls.c:channel_tls_process_certs_cell() 246 +problem function-size /src/core/or/channeltls.c:channel_tls_process_authenticate_cell() 202 +problem file-size /src/core/or/circuitbuild.c 3060 +problem include-count /src/core/or/circuitbuild.c 53 +problem function-size /src/core/or/circuitbuild.c:get_unique_circ_id_by_chan() 128 +problem function-size /src/core/or/circuitbuild.c:circuit_extend() 147 +problem function-size /src/core/or/circuitbuild.c:choose_good_exit_server_general() 206 +problem include-count /src/core/or/circuitlist.c 54 +problem function-size /src/core/or/circuitlist.c:HT_PROTOTYPE() 128 +problem function-size /src/core/or/circuitlist.c:circuit_free_() 137 +problem function-size /src/core/or/circuitlist.c:circuit_find_to_cannibalize() 102 +problem function-size /src/core/or/circuitlist.c:circuit_about_to_free() 120 +problem function-size /src/core/or/circuitlist.c:circuits_handle_oom() 117 +problem function-size /src/core/or/circuitmux.c:circuitmux_set_policy() 110 +problem function-size /src/core/or/circuitmux.c:circuitmux_attach_circuit() 114 +problem function-size /src/core/or/circuitstats.c:circuit_build_times_parse_state() 124 +problem file-size /src/core/or/circuituse.c 3150 +problem function-size /src/core/or/circuituse.c:circuit_is_acceptable() 129 +problem function-size /src/core/or/circuituse.c:circuit_expire_building() 394 +problem function-size /src/core/or/circuituse.c:circuit_log_ancient_one_hop_circuits() 126 +problem function-size /src/core/or/circuituse.c:circuit_build_failed() 149 +problem function-size /src/core/or/circuituse.c:circuit_launch_by_extend_info() 110 +problem function-size /src/core/or/circuituse.c:circuit_get_open_circ_or_launch() 354 +problem function-size /src/core/or/circuituse.c:connection_ap_handshake_attach_circuit() 244 +problem function-size /src/core/or/command.c:command_process_create_cell() 156 +problem function-size /src/core/or/command.c:command_process_relay_cell() 132 +problem file-size /src/core/or/connection_edge.c 4550 +problem include-count /src/core/or/connection_edge.c 64 +problem function-size /src/core/or/connection_edge.c:connection_ap_expire_beginning() 117 +problem function-size /src/core/or/connection_edge.c:connection_ap_handshake_rewrite() 192 +problem function-size /src/core/or/connection_edge.c:connection_ap_handle_onion() 188 +problem function-size /src/core/or/connection_edge.c:connection_ap_handshake_rewrite_and_attach() 423 +problem function-size /src/core/or/connection_edge.c:connection_ap_handshake_send_begin() 111 +problem function-size /src/core/or/connection_edge.c:connection_ap_handshake_socks_resolved() 106 +problem function-size /src/core/or/connection_edge.c:connection_exit_begin_conn() 184 +problem function-size /src/core/or/connection_edge.c:connection_exit_connect() 102 +problem file-size /src/core/or/connection_or.c 3111 +problem include-count /src/core/or/connection_or.c 51 +problem function-size /src/core/or/connection_or.c:connection_or_group_set_badness_() 105 +problem function-size /src/core/or/connection_or.c:connection_or_client_learned_peer_id() 144 +problem function-size /src/core/or/connection_or.c:connection_or_compute_authenticate_cell_body() 235 +problem file-size /src/core/or/policies.c 3163 +problem function-size /src/core/or/policies.c:policy_summarize() 107 +problem function-size /src/core/or/protover.c:protover_all_supported() 116 +problem file-size /src/core/or/relay.c 3173 +problem function-size /src/core/or/relay.c:circuit_receive_relay_cell() 123 +problem function-size /src/core/or/relay.c:relay_send_command_from_edge_() 101 +problem function-size /src/core/or/relay.c:connection_ap_process_end_not_open() 194 +problem function-size /src/core/or/relay.c:connection_edge_process_relay_cell_not_open() 139 +problem function-size /src/core/or/relay.c:connection_edge_process_relay_cell() 520 +problem function-size /src/core/or/relay.c:connection_edge_package_raw_inbuf() 130 +problem function-size /src/core/or/relay.c:circuit_resume_edge_reading_helper() 148 +problem function-size /src/core/or/scheduler_kist.c:kist_scheduler_run() 171 +problem function-size /src/core/or/scheduler_vanilla.c:vanilla_scheduler_run() 109 +problem function-size /src/core/or/versions.c:tor_version_parse() 104 +problem function-size /src/core/proto/proto_socks.c:parse_socks_client() 112 +problem function-size /src/feature/client/addressmap.c:addressmap_rewrite() 112 +problem function-size /src/feature/client/bridges.c:rewrite_node_address_for_bridge() 126 +problem function-size /src/feature/client/circpathbias.c:pathbias_measure_close_rate() 108 +problem function-size /src/feature/client/dnsserv.c:evdns_server_callback() 153 +problem file-size /src/feature/client/entrynodes.c 3817 +problem function-size /src/feature/client/entrynodes.c:entry_guards_upgrade_waiting_circuits() 153 +problem function-size /src/feature/client/entrynodes.c:entry_guard_parse_from_state() 246 +problem function-size /src/feature/client/transports.c:handle_proxy_line() 108 +problem function-size /src/feature/client/transports.c:parse_method_line_helper() 112 +problem function-size /src/feature/client/transports.c:create_managed_proxy_environment() 109 +problem function-size /src/feature/control/control.c:connection_control_process_inbuf() 144 problem function-size /src/feature/control/control_auth.c:handle_control_authchallenge() 115 -problem function-size /src/feature/control/control_getinfo.c:getinfo_helper_misc() 109 -problem function-size /src/feature/control/control_getinfo.c:getinfo_helper_dir() 304 -problem function-size /src/feature/control/control_getinfo.c:getinfo_helper_events() 236 -problem function-size /src/feature/control/control_cmd.c:handle_control_command() 104 +problem function-size /src/feature/control/control_auth.c:handle_control_authenticate() 188 problem function-size /src/feature/control/control_cmd.c:handle_control_extendcircuit() 151 problem function-size /src/feature/control/control_cmd.c:handle_control_hsfetch() 114 problem function-size /src/feature/control/control_cmd.c:handle_control_hspost() 117 problem function-size /src/feature/control/control_cmd.c:handle_control_add_onion() 293 problem function-size /src/feature/control/control_cmd.c:add_onion_helper_keyarg() 125 -problem function-size /src/feature/control/control.c:connection_control_process_inbuf() 239 +problem function-size /src/feature/control/control_cmd.c:handle_control_command() 104 problem function-size /src/feature/control/control_events.c:control_event_stream_status() 119 problem include-count /src/feature/control/control_getinfo.c 52 -problem function-size /src/feature/stats/rephist.c:rep_hist_load_mtbf_data() 185 -problem function-size /src/feature/stats/rephist.c:rep_hist_format_exit_stats() 148 -problem function-size /src/feature/dircache/consdiffmgr.c:consdiffmgr_cleanup() 115 -problem function-size /src/feature/dircache/consdiffmgr.c:consdiffmgr_rescan_flavor_() 111 -problem function-size /src/feature/dircache/consdiffmgr.c:consensus_diff_worker_threadfn() 132 -problem function-size /src/feature/dircache/dircache.c:handle_get_current_consensus() 166 -problem function-size /src/feature/dircache/dircache.c:directory_handle_command_post() 120 -problem function-size /src/feature/hibernate/hibernate.c:accounting_parse_options() 109 -problem function-size /src/feature/relay/routerkeys.c:load_ed_keys() 294 -problem file-size /src/feature/relay/router.c 3412 -problem include-count /src/feature/relay/router.c 56 -problem function-size /src/feature/relay/router.c:init_keys() 252 -problem function-size /src/feature/relay/router.c:get_my_declared_family() 114 -problem function-size /src/feature/relay/router.c:router_build_fresh_descriptor() 190 -problem function-size /src/feature/relay/router.c:router_build_fresh_unsigned_routerinfo() 140 -problem function-size /src/feature/relay/router.c:router_dump_router_to_string() 375 -problem function-size /src/feature/relay/router.c:extrainfo_dump_to_string() 208 -problem function-size /src/feature/relay/dns.c:dns_resolve_impl() 134 -problem function-size /src/feature/relay/dns.c:configure_nameservers() 161 -problem function-size /src/feature/relay/dns.c:evdns_callback() 109 -problem function-size /src/feature/dirparse/authcert_parse.c:authority_cert_parse_from_string() 182 -problem function-size /src/feature/dirparse/ns_parse.c:routerstatus_parse_entry_from_string() 286 -problem function-size /src/feature/dirparse/ns_parse.c:networkstatus_verify_bw_weights() 389 -problem function-size /src/feature/dirparse/ns_parse.c:networkstatus_parse_vote_from_string() 638 -problem function-size /src/feature/dirparse/parsecommon.c:tokenize_string() 103 -problem function-size /src/feature/dirparse/parsecommon.c:get_next_token() 159 -problem function-size /src/feature/dirparse/routerparse.c:router_parse_entry_from_string() 554 -problem function-size /src/feature/dirparse/routerparse.c:extrainfo_parse_entry_from_string() 210 -problem function-size /src/feature/dirparse/microdesc_parse.c:microdescs_parse_from_string() 154 +problem function-size /src/feature/control/control_getinfo.c:getinfo_helper_misc() 109 +problem function-size /src/feature/control/control_getinfo.c:getinfo_helper_dir() 304 +problem function-size /src/feature/control/control_getinfo.c:getinfo_helper_events() 236 +problem function-size /src/feature/dirauth/bwauth.c:dirserv_read_measured_bandwidths() 127 problem file-size /src/feature/dirauth/dirvote.c 4729 problem include-count /src/feature/dirauth/dirvote.c 53 problem function-size /src/feature/dirauth/dirvote.c:format_networkstatus_vote() 251 @@ -161,26 +165,16 @@ problem function-size /src/feature/dirauth/dirvote.c:networkstatus_add_detached_ problem function-size /src/feature/dirauth/dirvote.c:dirvote_add_vote() 162 problem function-size /src/feature/dirauth/dirvote.c:dirvote_compute_consensuses() 164 problem function-size /src/feature/dirauth/dirvote.c:dirserv_generate_networkstatus_vote_obj() 293 -problem function-size /src/feature/dirauth/guardfraction.c:dirserv_read_guardfraction_file_from_str() 110 -problem function-size /src/feature/dirauth/shared_random.c:should_keep_commit() 110 -problem function-size /src/feature/dirauth/process_descs.c:dirserv_add_descriptor() 125 -problem function-size /src/feature/dirauth/bwauth.c:dirserv_read_measured_bandwidths() 127 problem function-size /src/feature/dirauth/dsigs_parse.c:networkstatus_parse_detached_signatures() 196 +problem function-size /src/feature/dirauth/guardfraction.c:dirserv_read_guardfraction_file_from_str() 110 +problem function-size /src/feature/dirauth/process_descs.c:dirserv_add_descriptor() 125 +problem function-size /src/feature/dirauth/shared_random.c:should_keep_commit() 110 problem function-size /src/feature/dirauth/voteflags.c:dirserv_compute_performance_thresholds() 172 -problem file-size /src/feature/hs/hs_service.c 4171 -problem function-size /src/feature/hs/hs_cell.c:hs_cell_parse_introduce2() 154 -problem function-size /src/feature/hs/hs_common.c:hs_get_responsible_hsdirs() 104 -problem function-size /src/feature/hs/hs_common.c:hs_get_extend_info_from_lspecs() 101 -problem function-size /src/feature/hs/hs_client.c:send_introduce1() 104 -problem function-size /src/feature/hs/hs_client.c:hs_config_client_authorization() 108 -problem function-size /src/feature/hs/hs_config.c:config_generic_service() 149 -problem function-size /src/feature/hs/hs_cell.c:hs_cell_build_establish_intro() 115 -problem file-size /src/feature/hs/hs_descriptor.c 3108 -problem function-size /src/feature/hs/hs_descriptor.c:desc_encode_v3() 108 -problem function-size /src/feature/hs/hs_descriptor.c:decrypt_desc_layer() 110 -problem function-size /src/feature/hs/hs_descriptor.c:decode_introduction_point() 122 -problem function-size /src/feature/hs/hs_descriptor.c:desc_decode_superencrypted_v3() 109 -problem function-size /src/feature/hs/hs_descriptor.c:desc_decode_encrypted_v3() 109 +problem function-size /src/feature/dircache/consdiffmgr.c:consdiffmgr_cleanup() 115 +problem function-size /src/feature/dircache/consdiffmgr.c:consdiffmgr_rescan_flavor_() 111 +problem function-size /src/feature/dircache/consdiffmgr.c:consensus_diff_worker_threadfn() 132 +problem function-size /src/feature/dircache/dircache.c:handle_get_current_consensus() 166 +problem function-size /src/feature/dircache/dircache.c:directory_handle_command_post() 120 problem file-size /src/feature/dirclient/dirclient.c 3215 problem include-count /src/feature/dirclient/dirclient.c 51 problem function-size /src/feature/dirclient/dirclient.c:directory_get_from_dirserver() 131 @@ -189,6 +183,63 @@ problem function-size /src/feature/dirclient/dirclient.c:directory_send_command( problem function-size /src/feature/dirclient/dirclient.c:dir_client_decompress_response_body() 114 problem function-size /src/feature/dirclient/dirclient.c:connection_dir_client_reached_eof() 189 problem function-size /src/feature/dirclient/dirclient.c:handle_response_fetch_consensus() 105 +problem function-size /src/feature/dircommon/consdiff.c:gen_ed_diff() 204 +problem function-size /src/feature/dircommon/consdiff.c:apply_ed_diff() 159 +problem function-size /src/feature/dirparse/authcert_parse.c:authority_cert_parse_from_string() 182 +problem function-size /src/feature/dirparse/microdesc_parse.c:microdescs_parse_from_string() 154 +problem function-size /src/feature/dirparse/ns_parse.c:routerstatus_parse_entry_from_string() 286 +problem function-size /src/feature/dirparse/ns_parse.c:networkstatus_verify_bw_weights() 389 +problem function-size /src/feature/dirparse/ns_parse.c:networkstatus_parse_vote_from_string() 638 +problem function-size /src/feature/dirparse/parsecommon.c:tokenize_string() 103 +problem function-size /src/feature/dirparse/parsecommon.c:get_next_token() 159 +problem function-size /src/feature/dirparse/routerparse.c:router_parse_entry_from_string() 554 +problem function-size /src/feature/dirparse/routerparse.c:extrainfo_parse_entry_from_string() 210 +problem function-size /src/feature/hibernate/hibernate.c:accounting_parse_options() 109 +problem function-size /src/feature/hs/hs_cell.c:hs_cell_build_establish_intro() 115 +problem function-size /src/feature/hs/hs_cell.c:hs_cell_parse_introduce2() 154 +problem function-size /src/feature/hs/hs_client.c:send_introduce1() 104 +problem function-size /src/feature/hs/hs_client.c:hs_config_client_authorization() 108 +problem function-size /src/feature/hs/hs_common.c:hs_get_responsible_hsdirs() 104 +problem function-size /src/feature/hs/hs_common.c:hs_get_extend_info_from_lspecs() 101 +problem function-size /src/feature/hs/hs_config.c:config_generic_service() 149 +problem function-size /src/feature/hs/hs_descriptor.c:desc_encode_v3() 108 +problem function-size /src/feature/hs/hs_descriptor.c:decrypt_desc_layer() 110 +problem function-size /src/feature/hs/hs_descriptor.c:decode_introduction_point() 122 +problem function-size /src/feature/hs/hs_descriptor.c:desc_decode_superencrypted_v3() 109 +problem function-size /src/feature/hs/hs_descriptor.c:desc_decode_encrypted_v3() 109 +problem file-size /src/feature/hs/hs_service.c 4109 +problem function-size /src/feature/keymgt/loadkey.c:ed_key_init_from_file() 333 +problem function-size /src/feature/nodelist/authcert.c:trusted_dirs_load_certs_from_string() 124 +problem function-size /src/feature/nodelist/authcert.c:authority_certs_fetch_missing() 296 +problem function-size /src/feature/nodelist/fmt_routerstatus.c:routerstatus_format_entry() 166 +problem function-size /src/feature/nodelist/microdesc.c:microdesc_cache_rebuild() 134 +problem include-count /src/feature/nodelist/networkstatus.c 61 +problem function-size /src/feature/nodelist/networkstatus.c:networkstatus_check_consensus_signature() 176 +problem function-size /src/feature/nodelist/networkstatus.c:networkstatus_set_current_consensus() 293 +problem function-size /src/feature/nodelist/node_select.c:router_pick_directory_server_impl() 123 +problem function-size /src/feature/nodelist/node_select.c:compute_weighted_bandwidths() 205 +problem function-size /src/feature/nodelist/node_select.c:router_pick_trusteddirserver_impl() 114 +problem function-size /src/feature/nodelist/nodelist.c:compute_frac_paths_available() 193 +problem file-size /src/feature/nodelist/routerlist.c 3234 +problem function-size /src/feature/nodelist/routerlist.c:router_rebuild_store() 148 +problem function-size /src/feature/nodelist/routerlist.c:router_add_to_routerlist() 169 +problem function-size /src/feature/nodelist/routerlist.c:routerlist_remove_old_routers() 121 +problem function-size /src/feature/nodelist/routerlist.c:update_consensus_router_descriptor_downloads() 136 +problem function-size /src/feature/nodelist/routerlist.c:update_extrainfo_downloads() 103 +problem function-size /src/feature/relay/dns.c:dns_resolve_impl() 134 +problem function-size /src/feature/relay/dns.c:configure_nameservers() 161 +problem function-size /src/feature/relay/dns.c:evdns_callback() 109 +problem file-size /src/feature/relay/router.c 3412 +problem include-count /src/feature/relay/router.c 56 +problem function-size /src/feature/relay/router.c:init_keys() 252 +problem function-size /src/feature/relay/router.c:get_my_declared_family() 114 +problem function-size /src/feature/relay/router.c:router_build_fresh_unsigned_routerinfo() 136 +problem function-size /src/feature/relay/router.c:router_dump_router_to_string() 375 +problem function-size /src/feature/relay/router.c:extrainfo_dump_to_string() 207 +problem function-size /src/feature/relay/routerkeys.c:load_ed_keys() 294 +problem function-size /src/feature/rend/rendcache.c:rend_cache_store_v2_desc_as_client() 193 +problem function-size /src/feature/rend/rendclient.c:rend_client_send_introduction() 220 +problem function-size /src/feature/rend/rendcommon.c:rend_encode_v2_descriptors() 225 problem function-size /src/feature/rend/rendmid.c:rend_mid_establish_intro_legacy() 104 problem function-size /src/feature/rend/rendparse.c:rend_parse_v2_service_descriptor() 187 problem function-size /src/feature/rend/rendparse.c:rend_decrypt_introduction_points() 104 @@ -205,59 +256,35 @@ problem function-size /src/feature/rend/rendservice.c:rend_service_rendezvous_ha problem function-size /src/feature/rend/rendservice.c:directory_post_to_hs_dir() 108 problem function-size /src/feature/rend/rendservice.c:upload_service_descriptor() 111 problem function-size /src/feature/rend/rendservice.c:rend_consider_services_intro_points() 169 -problem function-size /src/feature/rend/rendcache.c:rend_cache_store_v2_desc_as_client() 193 -problem function-size /src/feature/rend/rendclient.c:rend_client_send_introduction() 220 -problem function-size /src/feature/rend/rendcommon.c:rend_encode_v2_descriptors() 225 -problem function-size /src/feature/nodelist/authcert.c:trusted_dirs_load_certs_from_string() 124 -problem function-size /src/feature/nodelist/authcert.c:authority_certs_fetch_missing() 296 -problem function-size /src/feature/nodelist/microdesc.c:microdesc_cache_rebuild() 134 -problem function-size /src/feature/nodelist/fmt_routerstatus.c:routerstatus_format_entry() 166 -problem include-count /src/feature/nodelist/networkstatus.c 61 -problem function-size /src/feature/nodelist/networkstatus.c:networkstatus_check_consensus_signature() 176 -problem function-size /src/feature/nodelist/networkstatus.c:networkstatus_set_current_consensus() 293 -problem file-size /src/feature/nodelist/routerlist.c 3234 -problem function-size /src/feature/nodelist/routerlist.c:router_rebuild_store() 148 -problem function-size /src/feature/nodelist/routerlist.c:router_add_to_routerlist() 169 -problem function-size /src/feature/nodelist/routerlist.c:routerlist_remove_old_routers() 121 -problem function-size /src/feature/nodelist/routerlist.c:update_consensus_router_descriptor_downloads() 136 -problem function-size /src/feature/nodelist/routerlist.c:update_extrainfo_downloads() 103 -problem function-size /src/feature/nodelist/nodelist.c:compute_frac_paths_available() 193 -problem function-size /src/feature/nodelist/node_select.c:router_pick_directory_server_impl() 123 -problem function-size /src/feature/nodelist/node_select.c:compute_weighted_bandwidths() 205 -problem function-size /src/feature/nodelist/node_select.c:router_pick_trusteddirserver_impl() 114 -problem function-size /src/feature/client/dnsserv.c:evdns_server_callback() 153 -problem function-size /src/feature/client/transports.c:handle_proxy_line() 108 -problem function-size /src/feature/client/transports.c:parse_method_line_helper() 112 -problem function-size /src/feature/client/transports.c:create_managed_proxy_environment() 109 -problem function-size /src/feature/client/bridges.c:rewrite_node_address_for_bridge() 126 -problem function-size /src/feature/client/addressmap.c:addressmap_rewrite() 112 -problem file-size /src/feature/client/entrynodes.c 3817 -problem function-size /src/feature/client/entrynodes.c:entry_guards_upgrade_waiting_circuits() 153 -problem function-size /src/feature/client/entrynodes.c:entry_guard_parse_from_state() 246 -problem function-size /src/feature/client/circpathbias.c:pathbias_measure_close_rate() 108 +problem function-size /src/feature/stats/rephist.c:rep_hist_load_mtbf_data() 185 +problem function-size /src/feature/stats/rephist.c:rep_hist_format_exit_stats() 148 +problem function-size /src/lib/compress/compress.c:tor_compress_impl() 133 +problem function-size /src/lib/compress/compress_zstd.c:tor_zstd_compress_process() 126 +problem function-size /src/lib/container/smartlist.c:smartlist_bsearch_idx() 109 +problem function-size /src/lib/crypt_ops/crypto_rand.c:crypto_strongest_rand_syscall() 102 +problem function-size /src/lib/encoding/binascii.c:base64_encode() 107 +problem function-size /src/lib/encoding/confline.c:parse_config_line_from_str_verbose() 119 +problem function-size /src/lib/encoding/cstring.c:unescape_string() 108 +problem function-size /src/lib/fs/dir.c:check_private_dir() 231 problem function-size /src/lib/log/log.c:parse_log_severity_config() 101 problem function-size /src/lib/math/prob_distr.c:sample_uniform_interval() 145 -problem function-size /src/lib/process/process_win32.c:process_win32_exec() 138 -problem function-size /src/lib/process/process_win32.c:process_win32_create_pipe() 112 -problem function-size /src/lib/process/setuid.c:switch_id() 156 -problem function-size /src/lib/process/restrict.c:set_max_file_descriptors() 102 -problem function-size /src/lib/process/process_unix.c:process_unix_exec() 220 -problem function-size /src/lib/container/smartlist.c:smartlist_bsearch_idx() 109 -problem function-size /src/lib/sandbox/sandbox.c:prot_strings() 104 -problem function-size /src/lib/compress/compress_zstd.c:tor_zstd_compress_process() 126 -problem function-size /src/lib/compress/compress.c:tor_compress_impl() 133 -problem function-size /src/lib/crypt_ops/crypto_rand.c:crypto_strongest_rand_syscall() 102 -problem function-size /src/lib/string/scanf.c:tor_vsscanf() 112 -problem function-size /src/lib/encoding/confline.c:parse_config_line_from_str_verbose() 119 -problem function-size /src/lib/encoding/binascii.c:base64_encode() 107 -problem function-size /src/lib/encoding/cstring.c:unescape_string() 108 -problem function-size /src/lib/tls/tortls_openssl.c:tor_tls_context_new() 171 -problem function-size /src/lib/tls/x509_nss.c:tor_tls_create_certificate_internal() 126 -problem function-size /src/lib/tls/tortls_nss.c:tor_tls_context_new() 147 -problem function-size /src/lib/osinfo/uname.c:get_uname() 116 problem function-size /src/lib/net/address.c:tor_addr_parse_mask_ports() 198 problem function-size /src/lib/net/address.c:tor_addr_compare_masked() 111 -problem function-size /src/lib/net/resolve.c:tor_addr_lookup() 110 problem function-size /src/lib/net/inaddr.c:tor_inet_pton() 107 +problem function-size /src/lib/net/resolve.c:tor_addr_lookup() 110 problem function-size /src/lib/net/socketpair.c:tor_ersatz_socketpair() 102 -problem function-size /src/lib/fs/dir.c:check_private_dir() 231 +problem function-size /src/lib/osinfo/uname.c:get_uname() 116 +problem function-size /src/lib/process/process_unix.c:process_unix_exec() 220 +problem function-size /src/lib/process/process_win32.c:process_win32_exec() 138 +problem function-size /src/lib/process/process_win32.c:process_win32_create_pipe() 112 +problem function-size /src/lib/process/restrict.c:set_max_file_descriptors() 102 +problem function-size /src/lib/process/setuid.c:switch_id() 156 +problem function-size /src/lib/sandbox/sandbox.c:prot_strings() 104 +problem function-size /src/lib/string/scanf.c:tor_vsscanf() 112 +problem function-size /src/lib/tls/tortls_nss.c:tor_tls_context_new() 147 +problem function-size /src/lib/tls/tortls_openssl.c:tor_tls_context_new() 171 +problem function-size /src/lib/tls/x509_nss.c:tor_tls_create_certificate_internal() 126 +problem function-size /src/tools/tor-gencert.c:parse_commandline() 111 +problem function-size /src/tools/tor-resolve.c:build_socks5_resolve_request() 104 +problem function-size /src/tools/tor-resolve.c:do_resolve() 173 +problem function-size /src/tools/tor-resolve.c:main() 112 From beceb079e15aef0460f1faddb747761a63708526 Mon Sep 17 00:00:00 2001 From: teor Date: Wed, 27 Mar 2019 09:40:03 +1000 Subject: [PATCH 0694/2557] practracker: regen in master, for bug28925 merged to 0.4.0 and later python3 scripts/maint/practracker/practracker.py --regen --- scripts/maint/practracker/exceptions.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/maint/practracker/exceptions.txt b/scripts/maint/practracker/exceptions.txt index 9e372728ce..93d27e7642 100644 --- a/scripts/maint/practracker/exceptions.txt +++ b/scripts/maint/practracker/exceptions.txt @@ -53,7 +53,7 @@ problem function-size /src/app/main/main.c:tor_init() 136 problem function-size /src/app/main/main.c:sandbox_init_filter() 291 problem function-size /src/app/main/main.c:run_tor_main_loop() 105 problem function-size /src/app/main/ntmain.c:nt_service_install() 125 -problem file-size /src/core/mainloop/connection.c 5548 +problem file-size /src/core/mainloop/connection.c 5554 problem include-count /src/core/mainloop/connection.c 61 problem function-size /src/core/mainloop/connection.c:connection_free_minimal() 184 problem function-size /src/core/mainloop/connection.c:connection_listener_new() 328 @@ -111,7 +111,7 @@ problem function-size /src/core/or/connection_edge.c:connection_ap_handshake_sen problem function-size /src/core/or/connection_edge.c:connection_ap_handshake_socks_resolved() 106 problem function-size /src/core/or/connection_edge.c:connection_exit_begin_conn() 184 problem function-size /src/core/or/connection_edge.c:connection_exit_connect() 102 -problem file-size /src/core/or/connection_or.c 3111 +problem file-size /src/core/or/connection_or.c 3121 problem include-count /src/core/or/connection_or.c 51 problem function-size /src/core/or/connection_or.c:connection_or_group_set_badness_() 105 problem function-size /src/core/or/connection_or.c:connection_or_client_learned_peer_id() 144 From 203e9138d1f714e5ad42884f75d50cd31c3bae3b Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 26 Mar 2019 19:56:39 -0400 Subject: [PATCH 0695/2557] Remove message/subsystem numbers from log messages Having the numbers in those messages makes some of the unit test unstable, by causing them to depend on the initialization order of the naming objects. --- src/lib/pubsub/pubsub_check.c | 46 +++++++++++++++++------------------ src/test/test_pubsub_build.c | 18 ++++++++------ 2 files changed, 33 insertions(+), 31 deletions(-) diff --git a/src/lib/pubsub/pubsub_check.c b/src/lib/pubsub/pubsub_check.c index 94a55beebc..d308dc58aa 100644 --- a/src/lib/pubsub/pubsub_check.c +++ b/src/lib/pubsub/pubsub_check.c @@ -190,10 +190,10 @@ get_message_bitarray(const pubsub_adjmap_t *map, SMARTLIST_FOREACH_BEGIN(items, const pubsub_cfg_t *, cfg) { if (bitarray_is_set(*out, cfg->subsys)) { log_warn(LD_MESG|LD_BUG, - "Message %u (%s) is configured to be %s by subsystem " - "%u (%s) more than once.", - msg, get_message_id_name(msg), operation, - cfg->subsys, get_subsys_id_name(cfg->subsys)); + "Message \"%s\" is configured to be %s by subsystem " + "\"%s\" more than once.", + get_message_id_name(msg), operation, + get_subsys_id_name(cfg->subsys)); ok = false; } bitarray_set(*out, cfg->subsys); @@ -234,10 +234,10 @@ lint_message_graph(const pubsub_adjmap_t *map, if (bitarray_is_set(published_by, i) && bitarray_is_set(subscribed_by, i)) { log_warn(LD_MESG|LD_BUG, - "Message %u (%s) is published and subscribed by the same " - "subsystem %u (%s)", - msg, get_message_id_name(msg), - i, get_subsys_id_name(i)); + "Message \"%s\" is published and subscribed by the same " + "subsystem \"%s\".", + get_message_id_name(msg), + get_subsys_id_name(i)); ok = false; } } @@ -287,16 +287,16 @@ lint_message_consistency(message_id_t msg, if (! chan_same) { log_warn(LD_MESG|LD_BUG, - "Message %u (%s) is associated with multiple inconsistent " + "Message \"%s\" is associated with multiple inconsistent " "channels.", - msg, get_message_id_name(msg)); + get_message_id_name(msg)); ok = false; } if (! type_same) { log_warn(LD_MESG|LD_BUG, - "Message %u (%s) is associated with multiple inconsistent " + "Message \"%s\" is associated with multiple inconsistent " "message types.", - msg, get_message_id_name(msg)); + get_message_id_name(msg)); ok = false; } @@ -305,16 +305,16 @@ lint_message_consistency(message_id_t msg, */ if (pub_excl && smartlist_len(pub) > 1) { log_warn(LD_MESG|LD_BUG, - "Message %u (%s) has multiple publishers, but at least one is " + "Message \"%s\" has multiple publishers, but at least one is " "marked as exclusive.", - msg, get_message_id_name(msg)); + get_message_id_name(msg)); ok = false; } if (sub_excl && smartlist_len(sub) > 1) { log_warn(LD_MESG|LD_BUG, - "Message %u (%s) has multiple subscribers, but at least one is " + "Message \"%s\" has multiple subscribers, but at least one is " "marked as exclusive.", - msg, get_message_id_name(msg)); + get_message_id_name(msg)); ok = false; } @@ -344,9 +344,9 @@ lint_message(const pubsub_adjmap_t *map, message_id_t msg) const size_t n_sub = smartlist_len_opt(sub); if (n_pub == 0 && n_sub == 0) { - log_info(LD_MESG, "Nobody is publishing or subscribing to message %u " - "(%s).", - msg, get_message_id_name(msg)); + log_info(LD_MESG, "Nobody is publishing or subscribing to message " + "\"%s\".", + get_message_id_name(msg)); return 0; // No publishers or subscribers: nothing to do. } /* We'll set this to false if there are any problems. */ @@ -355,13 +355,13 @@ lint_message(const pubsub_adjmap_t *map, message_id_t msg) /* First make sure that if there are publishers, there are subscribers. */ if (n_pub == 0) { log_warn(LD_MESG|LD_BUG, - "Message %u (%s) has subscribers, but no publishers.", - msg, get_message_id_name(msg)); + "Message \"%s\" has subscribers, but no publishers.", + get_message_id_name(msg)); ok = false; } else if (n_sub == 0) { log_warn(LD_MESG|LD_BUG, - "Message %u (%s) has publishers, but no subscribers.", - msg, get_message_id_name(msg)); + "Message \"%s\" has publishers, but no subscribers.", + get_message_id_name(msg)); ok = false; } diff --git a/src/test/test_pubsub_build.c b/src/test/test_pubsub_build.c index 9ffe428c0a..ce5bf60080 100644 --- a/src/test/test_pubsub_build.c +++ b/src/test/test_pubsub_build.c @@ -242,9 +242,9 @@ test_pubsub_build_missing_pubsub(void *arg) tt_assert(dispatcher == NULL); expect_log_msg_containing( - "Message 0 (bunch_of_coconuts) has publishers, but no subscribers."); + "Message \"bunch_of_coconuts\" has publishers, but no subscribers."); expect_log_msg_containing( - "Message 1 (yes_we_have_no) has subscribers, but no publishers."); + "Message \"yes_we_have_no\" has subscribers, but no publishers."); done: pubsub_builder_free(b); @@ -311,7 +311,7 @@ test_pubsub_build_channels_conflict(void *arg) b = NULL; tt_assert(dispatcher == NULL); - expect_log_msg_containing("Message 0 (bunch_of_coconuts) is associated " + expect_log_msg_containing("Message \"bunch_of_coconuts\" is associated " "with multiple inconsistent channels."); done: @@ -350,7 +350,7 @@ test_pubsub_build_types_conflict(void *arg) b = NULL; tt_assert(dispatcher == NULL); - expect_log_msg_containing("Message 0 (bunch_of_coconuts) is associated " + expect_log_msg_containing("Message \"bunch_of_coconuts\" is associated " "with multiple inconsistent message types."); done: @@ -383,8 +383,8 @@ test_pubsub_build_pubsub_same(void *arg) b = NULL; tt_assert(dispatcher == NULL); - expect_log_msg_containing("Message 0 (bunch_of_coconuts) is published " - "and subscribed by the same subsystem 0 (sys1)"); + expect_log_msg_containing("Message \"bunch_of_coconuts\" is published " + "and subscribed by the same subsystem \"sys1\"."); done: pubsub_builder_free(b); @@ -523,9 +523,11 @@ test_pubsub_build_pubsub_redundant(void *arg) tt_assert(dispatcher == NULL); expect_log_msg_containing( - "is configured to be published by subsystem 1 (sys2) more than once"); + "Message \"yes_we_have_no\" is configured to be published by " + "subsystem \"sys2\" more than once."); expect_log_msg_containing( - "is configured to be subscribed by subsystem 1 (sys2) more than once"); + "Message \"bunch_of_coconuts\" is configured to be subscribed by " + "subsystem \"sys2\" more than once."); done: pubsub_builder_free(b); From 3767eff9bb712bccc86718647c7dc84998a7f95f Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 26 Mar 2019 20:07:26 -0400 Subject: [PATCH 0696/2557] changes file for pubsub code --- changes/pubsub | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 changes/pubsub diff --git a/changes/pubsub b/changes/pubsub new file mode 100644 index 0000000000..f67b36b988 --- /dev/null +++ b/changes/pubsub @@ -0,0 +1,5 @@ + o Major features (code organization): + - Tor now includes a generic publish-subscribe message-passing subsystem + that we can use to organize intermodule dependencies. We hope to use + this to reduce dependencies between modules that don't need to be + related, and to generally simplify our codebase. Closes ticket 28226. From 7b9732063cf32c319d36246b314ff2cb1fbebb08 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 26 Mar 2019 20:04:09 -0400 Subject: [PATCH 0697/2557] practracker updates from messaging_v3 merge (main.c is a bit better, but shutdown.c is ugly) --- scripts/maint/practracker/exceptions.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/maint/practracker/exceptions.txt b/scripts/maint/practracker/exceptions.txt index 93d27e7642..cd56eba0e0 100644 --- a/scripts/maint/practracker/exceptions.txt +++ b/scripts/maint/practracker/exceptions.txt @@ -47,12 +47,13 @@ problem function-size /src/app/config/config.c:parse_ports() 170 problem function-size /src/app/config/config.c:getinfo_helper_config() 116 problem function-size /src/app/config/confparse.c:config_assign_value() 205 problem function-size /src/app/config/confparse.c:config_get_assigned_option() 129 -problem include-count /src/app/main/main.c 85 +problem include-count /src/app/main/main.c 67 problem function-size /src/app/main/main.c:dumpstats() 102 problem function-size /src/app/main/main.c:tor_init() 136 problem function-size /src/app/main/main.c:sandbox_init_filter() 291 problem function-size /src/app/main/main.c:run_tor_main_loop() 105 problem function-size /src/app/main/ntmain.c:nt_service_install() 125 +problem include-count /src/app/main/shutdown.c 52 problem file-size /src/core/mainloop/connection.c 5554 problem include-count /src/core/mainloop/connection.c 61 problem function-size /src/core/mainloop/connection.c:connection_free_minimal() 184 From a10d4adc25500e160c71d0d3f32a6f3ebbf3c9fd Mon Sep 17 00:00:00 2001 From: teor Date: Wed, 27 Mar 2019 11:07:55 +1000 Subject: [PATCH 0698/2557] Stop assuming that /usr/bin/python3 exists For scripts that work with python2, use /usr/bin/python. Otherwise, use /usr/bin/env python3. Fixes bug 29913; bugfix on 0.2.5.3-alpha. --- changes/ticket29913 | 4 ++++ scripts/maint/checkIncludes.py | 2 +- scripts/maint/practracker/practracker.py | 2 +- scripts/maint/rectify_include_paths.py | 2 +- src/config/mmdb-convert.py | 2 +- src/test/ope_ref.py | 2 +- 6 files changed, 9 insertions(+), 5 deletions(-) create mode 100644 changes/ticket29913 diff --git a/changes/ticket29913 b/changes/ticket29913 new file mode 100644 index 0000000000..a713b0ccef --- /dev/null +++ b/changes/ticket29913 @@ -0,0 +1,4 @@ + o Minor bugfixes (python): + - Stop assuming that /usr/bin/python3 exists. For scripts that work with + python2, use /usr/bin/python. Otherwise, use /usr/bin/env python3. + Fixes bug 29913; bugfix on 0.2.5.3-alpha. diff --git a/scripts/maint/checkIncludes.py b/scripts/maint/checkIncludes.py index 3afd9bbebe..ec9350b9b1 100755 --- a/scripts/maint/checkIncludes.py +++ b/scripts/maint/checkIncludes.py @@ -1,4 +1,4 @@ -#!/usr/bin/python3 +#!/usr/bin/python # Copyright 2018 The Tor Project, Inc. See LICENSE file for licensing info. """This script looks through all the directories for files matching *.c or diff --git a/scripts/maint/practracker/practracker.py b/scripts/maint/practracker/practracker.py index 22cb46c749..febb14639d 100755 --- a/scripts/maint/practracker/practracker.py +++ b/scripts/maint/practracker/practracker.py @@ -1,4 +1,4 @@ -#!/usr/bin/python3 +#!/usr/bin/python """ Best-practices tracker for Tor source code. diff --git a/scripts/maint/rectify_include_paths.py b/scripts/maint/rectify_include_paths.py index 401fadae6d..d1a2328ca6 100755 --- a/scripts/maint/rectify_include_paths.py +++ b/scripts/maint/rectify_include_paths.py @@ -1,4 +1,4 @@ -#!/usr/bin/python3 +#!/usr/bin/python import os import os.path diff --git a/src/config/mmdb-convert.py b/src/config/mmdb-convert.py index 706a8b03cc..b861e9433e 100644 --- a/src/config/mmdb-convert.py +++ b/src/config/mmdb-convert.py @@ -1,4 +1,4 @@ -#!/usr/bin/python3 +#!/usr/bin/python # This software has been dedicated to the public domain under the CC0 # public domain dedication. diff --git a/src/test/ope_ref.py b/src/test/ope_ref.py index f9bd97c546..b2f7012563 100644 --- a/src/test/ope_ref.py +++ b/src/test/ope_ref.py @@ -1,4 +1,4 @@ -#!/usr/bin/python3 +#!/usr/bin/env python3 # Copyright 2018-2019, The Tor Project, Inc. See LICENSE for licensing info. # Reference implementation for our rudimentary OPE code, used to From 283ee0ba0a1540a9e8e8d68625d89315a5356162 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Wed, 27 Mar 2019 19:27:57 +0200 Subject: [PATCH 0699/2557] Fix SC2086 warnings in asciidoc-helper.sh --- changes/bug29926 | 2 ++ doc/asciidoc-helper.sh | 6 +++--- 2 files changed, 5 insertions(+), 3 deletions(-) create mode 100644 changes/bug29926 diff --git a/changes/bug29926 b/changes/bug29926 new file mode 100644 index 0000000000..ab1417c603 --- /dev/null +++ b/changes/bug29926 @@ -0,0 +1,2 @@ + o Code simplification and refactoring (shell scripts): + - Fix shellcheck warnings in asciidoc-helper.sh. Resolves issue 29926. diff --git a/doc/asciidoc-helper.sh b/doc/asciidoc-helper.sh index a3ef53f884..765850a125 100755 --- a/doc/asciidoc-helper.sh +++ b/doc/asciidoc-helper.sh @@ -19,7 +19,7 @@ if [ "$1" = "html" ]; then base=${output%%.html.in} if [ "$2" != none ]; then - TZ=UTC "$2" -d manpage -o $output $input; + TZ=UTC "$2" -d manpage -o "$output" "$input"; else echo "=================================="; echo; @@ -44,8 +44,8 @@ elif [ "$1" = "man" ]; then echo "=================================="; exit 1; fi - if "$2" -f manpage $input; then - mv $base.1 $output; + if "$2" -f manpage "$input"; then + mv "$base.1" "$output"; else cat< Date: Thu, 28 Mar 2019 09:19:23 -0400 Subject: [PATCH 0700/2557] Don't unconditionally deref pub and sub in lint_message_consistency This can't actually result in a null pointer dereference, since pub_excl and sub_excl are only set when the corresponding smartlists are nonempty. But coverity isn't smart enough to figure that out, and we shouldn't really be depending on it. Bug 29938; CID 1444257. Bug not in any released Tor. --- src/lib/pubsub/pubsub_check.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/pubsub/pubsub_check.c b/src/lib/pubsub/pubsub_check.c index d308dc58aa..a3c22d4f25 100644 --- a/src/lib/pubsub/pubsub_check.c +++ b/src/lib/pubsub/pubsub_check.c @@ -303,14 +303,14 @@ lint_message_consistency(message_id_t msg, /* Enforce exclusive-ness for publishers and subscribers that have asked for * it. */ - if (pub_excl && smartlist_len(pub) > 1) { + if (pub_excl && smartlist_len_opt(pub) > 1) { log_warn(LD_MESG|LD_BUG, "Message \"%s\" has multiple publishers, but at least one is " "marked as exclusive.", get_message_id_name(msg)); ok = false; } - if (sub_excl && smartlist_len(sub) > 1) { + if (sub_excl && smartlist_len_opt(sub) > 1) { log_warn(LD_MESG|LD_BUG, "Message \"%s\" has multiple subscribers, but at least one is " "marked as exclusive.", From a7bc47532b1e3d7db89fe9fe4521306fd6170617 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 28 Mar 2019 09:29:00 -0400 Subject: [PATCH 0701/2557] test_routerkeys.c: Always check mkdir() return value After this fix, we have no more unchecked mkdir() calls. Bug 29939; CID 144254. Bugfix on 0.2.7.2-alpha. --- changes/bug29939 | 4 ++++ src/test/test_routerkeys.c | 8 ++++---- 2 files changed, 8 insertions(+), 4 deletions(-) create mode 100644 changes/bug29939 diff --git a/changes/bug29939 b/changes/bug29939 new file mode 100644 index 0000000000..0e9b46c075 --- /dev/null +++ b/changes/bug29939 @@ -0,0 +1,4 @@ + o Minor bugfixes (unit tests): + - In the "routerkeys/*" tests, check the return values of mkdir() for + possible failures. Fixes bug 29939; bugfix on 0.2.7.2-alpha. Found by + Coverity as CID 1444254. diff --git a/src/test/test_routerkeys.c b/src/test/test_routerkeys.c index 727fa5660f..102d9334a1 100644 --- a/src/test/test_routerkeys.c +++ b/src/test/test_routerkeys.c @@ -455,11 +455,11 @@ test_routerkeys_ed_keys_init_all(void *arg) options->TestingLinkKeySlop = 2*3600; #ifdef _WIN32 - mkdir(dir); - mkdir(keydir); + tt_int_op(0, OP_EQ, mkdir(dir)); + tt_int_op(0, OP_EQ, mkdir(keydir)); #else - mkdir(dir, 0700); - mkdir(keydir, 0700); + tt_int_op(0, OP_EQ, mkdir(dir, 0700)); + tt_int_op(0, OP_EQ, mkdir(keydir, 0700)); #endif /* defined(_WIN32) */ options->DataDirectory = dir; From c66df27c906b1140ea33f4e5d5ab9486594ada9d Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 28 Mar 2019 16:35:24 -0400 Subject: [PATCH 0702/2557] Fix checkIncludes warning about "unusual pattern in src/ext/timeouts/" Closes ticket 28806. --- src/ext/timeouts/.may_include | 5 ++--- src/ext/timeouts/test-timeout.c | 2 +- src/ext/timeouts/timeout.c | 6 +++--- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/ext/timeouts/.may_include b/src/ext/timeouts/.may_include index 42f44befd4..92c7116555 100644 --- a/src/ext/timeouts/.may_include +++ b/src/ext/timeouts/.may_include @@ -1,6 +1,5 @@ orconfig.h ext/tor_queue.h -timeout-bitops.c -timeout-debug.h -timeout.h +ext/timeouts/*.h +ext/timeouts/timeout-bitops.c diff --git a/src/ext/timeouts/test-timeout.c b/src/ext/timeouts/test-timeout.c index 8077129376..52d2e31e0c 100644 --- a/src/ext/timeouts/test-timeout.c +++ b/src/ext/timeouts/test-timeout.c @@ -4,7 +4,7 @@ #include #include -#include "timeout.h" +#include "ext/timeouts/timeout.h" #define THE_END_OF_TIME ((timeout_t)-1) diff --git a/src/ext/timeouts/timeout.c b/src/ext/timeouts/timeout.c index 07d06772c5..79fcc168ed 100644 --- a/src/ext/timeouts/timeout.c +++ b/src/ext/timeouts/timeout.c @@ -40,14 +40,14 @@ #include "ext/tor_queue.h" /* TAILQ(3) */ -#include "timeout.h" +#include "ext/timeouts/timeout.h" #ifndef TIMEOUT_DEBUG #define TIMEOUT_DEBUG 0 #endif #if TIMEOUT_DEBUG - 0 -#include "timeout-debug.h" +#include "ext/timeouts/timeout-debug.h" #endif #ifdef TIMEOUT_DISABLE_RELATIVE_ACCESS @@ -141,7 +141,7 @@ #define WHEEL_MASK (WHEEL_LEN - 1) #define TIMEOUT_MAX ((TIMEOUT_C(1) << (WHEEL_BIT * WHEEL_NUM)) - 1) -#include "timeout-bitops.c" +#include "ext/timeouts/timeout-bitops.c" #if WHEEL_BIT == 6 #define ctz(n) ctz64(n) From 6ab1929f00e625a0056eb29c5308ae9f3155a049 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Sat, 9 Feb 2019 16:06:32 +0200 Subject: [PATCH 0703/2557] Add connection_dir_buf_add() helper function --- src/core/mainloop/connection.c | 17 +++++++++++++++++ src/core/mainloop/connection.h | 2 ++ 2 files changed, 19 insertions(+) diff --git a/src/core/mainloop/connection.c b/src/core/mainloop/connection.c index c8b19344bd..7f2e96b4d1 100644 --- a/src/core/mainloop/connection.c +++ b/src/core/mainloop/connection.c @@ -4341,6 +4341,23 @@ connection_write_to_buf_impl_,(const char *string, size_t len, connection_write_to_buf_commit(conn, written); } +/** + * Write a string (of size len to directory connection + * dir_conn. Apply compression if connection is configured to use + * it and finalize it if done is true. + */ +void +connection_dir_buf_add(const char *string, size_t len, + dir_connection_t *dir_conn, int done) +{ + if (dir_conn->compress_state != NULL) { + connection_buf_add_compress(string, len, dir_conn, done); + return; + } + + connection_buf_add(string, len, TO_CONN(dir_conn)); +} + void connection_buf_add_compress(const char *string, size_t len, dir_connection_t *conn, int done) diff --git a/src/core/mainloop/connection.h b/src/core/mainloop/connection.h index 411f13a8b8..c93f1ef8e8 100644 --- a/src/core/mainloop/connection.h +++ b/src/core/mainloop/connection.h @@ -226,6 +226,8 @@ MOCK_DECL(void, connection_write_to_buf_impl_, /* DOCDOC connection_write_to_buf */ static void connection_buf_add(const char *string, size_t len, connection_t *conn); +void connection_dir_buf_add(const char *string, size_t len, + dir_connection_t *dir_conn, int done); static inline void connection_buf_add(const char *string, size_t len, connection_t *conn) { From c2222ba169a059e1a6ce04cb9014fe8870674d87 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Sat, 9 Feb 2019 16:56:54 +0200 Subject: [PATCH 0704/2557] Add changes file --- changes/ticket28816 | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 changes/ticket28816 diff --git a/changes/ticket28816 b/changes/ticket28816 new file mode 100644 index 0000000000..02878ccfdc --- /dev/null +++ b/changes/ticket28816 @@ -0,0 +1,4 @@ + o Code simplification and refactoring: + - Introduce a connection_dir_buf_add() helper function that checks for + compress_state of dir_connection_t and automatically writes a string to + directory connection with or without compression. Resolves issue 28816. From 194b25f0c7043b2485bb90da6fb24c0e96957875 Mon Sep 17 00:00:00 2001 From: teor Date: Tue, 26 Mar 2019 18:12:16 +1000 Subject: [PATCH 0705/2557] dircache: Refactor handle_get_next_bandwidth() to use connection_dir_buf_add() Implements ticket 29897. --- changes/ticket29897 | 3 +++ src/feature/dircache/dircache.c | 5 +---- 2 files changed, 4 insertions(+), 4 deletions(-) create mode 100644 changes/ticket29897 diff --git a/changes/ticket29897 b/changes/ticket29897 new file mode 100644 index 0000000000..232a79fbce --- /dev/null +++ b/changes/ticket29897 @@ -0,0 +1,3 @@ + o Code simplification and refactoring: + - Refactor handle_get_next_bandwidth() to use connection_dir_buf_add(). + Implements ticket 29897. diff --git a/src/feature/dircache/dircache.c b/src/feature/dircache/dircache.c index caa085dd63..62eb4c86c2 100644 --- a/src/feature/dircache/dircache.c +++ b/src/feature/dircache/dircache.c @@ -1486,13 +1486,10 @@ handle_get_next_bandwidth(dir_connection_t *conn, conn->compress_state = tor_compress_new(1, compress_method, choose_compression_level(len/2)); log_debug(LD_DIR, "Compressing bandwidth file."); - connection_buf_add_compress(bandwidth, len, conn, 0); - /* Flush the compression state. */ - connection_buf_add_compress("", 0, conn, 1); } else { log_debug(LD_DIR, "Not compressing bandwidth file."); - connection_buf_add(bandwidth, len, TO_CONN(conn)); } + connection_dir_buf_add((const char*)bandwidth, len, conn, 1); tor_free(bandwidth); return 0; } From f66a17444e4a83e548985784956f321e9e1e1ab2 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Tue, 5 Mar 2019 17:02:03 +0200 Subject: [PATCH 0706/2557] Silence compiler warnings --- src/lib/log/util_bug.c | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/lib/log/util_bug.c b/src/lib/log/util_bug.c index bbc744f0cc..1ce48c0ebb 100644 --- a/src/lib/log/util_bug.c +++ b/src/lib/log/util_bug.c @@ -69,6 +69,7 @@ tor_set_failed_assertion_callback(void (*fn)(void)) /** Helper for tor_assert: report the assertion failure. */ void +CHECK_PRINTF(5, 6) tor_assertion_failed_(const char *fname, unsigned int line, const char *func, const char *expr, const char *fmt, ...) @@ -77,11 +78,18 @@ tor_assertion_failed_(const char *fname, unsigned int line, char *extra = NULL; va_list ap; +#ifdef __clang__ +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wformat-nonliteral" +#endif if (fmt) { va_start(ap,fmt); tor_vasprintf(&extra, fmt, ap); va_end(ap); } +#ifdef __clang__ +#pragma clang diagnostic pop +#endif log_err(LD_BUG, "%s:%u: %s: Assertion %s failed; aborting.", fname, line, func, expr); @@ -94,6 +102,7 @@ tor_assertion_failed_(const char *fname, unsigned int line, /** Helper for tor_assert_nonfatal: report the assertion failure. */ void +CHECK_PRINTF(6, 7) tor_bug_occurred_(const char *fname, unsigned int line, const char *func, const char *expr, int once, const char *fmt, ...) @@ -120,11 +129,18 @@ tor_bug_occurred_(const char *fname, unsigned int line, va_list ap; char *extra = NULL; +#ifdef __clang__ +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wformat-nonliteral" +#endif if (fmt) { va_start(ap,fmt); tor_vasprintf(&extra, fmt, ap); va_end(ap); } +#ifdef __clang__ +#pragma clang diagnostic pop +#endif log_warn(LD_BUG, "%s:%u: %s: Non-fatal assertion %s failed.%s", fname, line, func, expr, once_str); From a959d7cb98ed0e5b6e2fabf36301accd28f0641c Mon Sep 17 00:00:00 2001 From: rl1987 Date: Tue, 5 Mar 2019 17:58:31 +0200 Subject: [PATCH 0707/2557] Use tor_assertf{_nonfatal} in code --- src/core/or/circuituse.c | 4 +++- src/test/test_bt_cl.c | 2 +- src/test/testing_rsakeys.c | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/core/or/circuituse.c b/src/core/or/circuituse.c index 2fdf6f7e8c..f77decf530 100644 --- a/src/core/or/circuituse.c +++ b/src/core/or/circuituse.c @@ -3117,7 +3117,9 @@ circuit_sent_valid_data(origin_circuit_t *circ, uint16_t relay_body_len) { if (!circ) return; - tor_assert_nonfatal(relay_body_len <= RELAY_PAYLOAD_SIZE); + tor_assertf_nonfatal(relay_body_len <= RELAY_PAYLOAD_SIZE, + "Wrong relay_body_len: %d (should be at most %d)", + relay_body_len, RELAY_PAYLOAD_SIZE); circ->n_delivered_written_circ_bw = tor_add_u32_nowrap(circ->n_delivered_written_circ_bw, relay_body_len); diff --git a/src/test/test_bt_cl.c b/src/test/test_bt_cl.c index 08b08ba423..b29c2c6cbc 100644 --- a/src/test/test_bt_cl.c +++ b/src/test/test_bt_cl.c @@ -46,7 +46,7 @@ crash(int x) *(volatile int *)0 = 0; #endif /* defined(__clang_analyzer__) || defined(__COVERITY__) */ } else if (crashtype == 1) { - tor_assert(1 == 0); + tor_assertf(1 == 0, "%d != %d", 1, 0); } else if (crashtype == -1) { ; } diff --git a/src/test/testing_rsakeys.c b/src/test/testing_rsakeys.c index 0f22d4e01b..8ba6bf9fe4 100644 --- a/src/test/testing_rsakeys.c +++ b/src/test/testing_rsakeys.c @@ -448,7 +448,8 @@ static int next_key_idx_2048; static crypto_pk_t * pk_generate_internal(int bits) { - tor_assert(bits == 2048 || bits == 1024); + tor_assertf(bits == 2048 || bits == 1024, + "Wrong key size: %d", bits); #ifdef USE_PREGENERATED_RSA_KEYS int *idxp; From bf953fe602a973259723dacb4389d3c4bee7e0a4 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Tue, 5 Mar 2019 18:02:24 +0200 Subject: [PATCH 0708/2557] Add changes file --- changes/ticket29662 | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 changes/ticket29662 diff --git a/changes/ticket29662 b/changes/ticket29662 new file mode 100644 index 0000000000..872df9ad82 --- /dev/null +++ b/changes/ticket29662 @@ -0,0 +1,5 @@ + o Minor features (debugging): + - Introduce tor_assertf() and tor_assertf_nonfatal() to enable logging of + additional information during assert failure. Now we can use format + strings to include pieces of information that are relevant for trouble + shooting. Resolves ticket 29662. From 9c132a5f9e87a2cd0f59bd8d21226ba2256f7bce Mon Sep 17 00:00:00 2001 From: rl1987 Date: Fri, 22 Mar 2019 13:44:02 +0200 Subject: [PATCH 0709/2557] Refrain from using static buffer for assert failure message; call tor_asprintf() instead --- src/lib/log/util_bug.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/lib/log/util_bug.c b/src/lib/log/util_bug.c index 1ce48c0ebb..e5d5a2958c 100644 --- a/src/lib/log/util_bug.c +++ b/src/lib/log/util_bug.c @@ -74,7 +74,7 @@ tor_assertion_failed_(const char *fname, unsigned int line, const char *func, const char *expr, const char *fmt, ...) { - char buf[256]; + char *buf = NULL; char *extra = NULL; va_list ap; @@ -93,11 +93,11 @@ tor_assertion_failed_(const char *fname, unsigned int line, log_err(LD_BUG, "%s:%u: %s: Assertion %s failed; aborting.", fname, line, func, expr); - tor_snprintf(buf, sizeof(buf), - "Assertion %s failed in %s at %s:%u: %s", + tor_asprintf(&buf, "Assertion %s failed in %s at %s:%u: %s", expr, func, fname, line, extra ? extra : ""); tor_free(extra); log_backtrace(LOG_ERR, LD_BUG, buf); + tor_free(buf); } /** Helper for tor_assert_nonfatal: report the assertion failure. */ @@ -107,7 +107,7 @@ tor_bug_occurred_(const char *fname, unsigned int line, const char *func, const char *expr, int once, const char *fmt, ...) { - char buf[256]; + char *buf = NULL; const char *once_str = once ? " (Future instances of this warning will be silenced.)": ""; if (! expr) { @@ -144,12 +144,12 @@ tor_bug_occurred_(const char *fname, unsigned int line, log_warn(LD_BUG, "%s:%u: %s: Non-fatal assertion %s failed.%s", fname, line, func, expr, once_str); - tor_snprintf(buf, sizeof(buf), - "Non-fatal assertion %s failed in %s at %s:%u%s%s", + tor_asprintf(&buf, "Non-fatal assertion %s failed in %s at %s:%u%s%s", expr, func, fname, line, fmt ? " : " : "", extra); tor_free(extra); } log_backtrace(LOG_WARN, LD_BUG, buf); + tor_free(buf); #ifdef TOR_UNIT_TESTS if (failed_assertion_cb) { From 537ad0bca3479c10fc7d1093aff534be11800dcb Mon Sep 17 00:00:00 2001 From: rl1987 Date: Fri, 29 Mar 2019 18:38:27 +0200 Subject: [PATCH 0710/2557] Check for NULL in tor_assertf_nonfatal() --- src/lib/log/util_bug.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/lib/log/util_bug.c b/src/lib/log/util_bug.c index e5d5a2958c..b9dc296fc6 100644 --- a/src/lib/log/util_bug.c +++ b/src/lib/log/util_bug.c @@ -145,7 +145,8 @@ tor_bug_occurred_(const char *fname, unsigned int line, log_warn(LD_BUG, "%s:%u: %s: Non-fatal assertion %s failed.%s", fname, line, func, expr, once_str); tor_asprintf(&buf, "Non-fatal assertion %s failed in %s at %s:%u%s%s", - expr, func, fname, line, fmt ? " : " : "", extra); + expr, func, fname, line, fmt ? " : " : "", + extra ? extra : ""); tor_free(extra); } log_backtrace(LOG_WARN, LD_BUG, buf); From f0e39df5ce7b884ae6ff1c8579200886e298f44c Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 29 Mar 2019 14:30:33 -0400 Subject: [PATCH 0711/2557] allow circuituse.c to get even longer. --- scripts/maint/practracker/exceptions.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/maint/practracker/exceptions.txt b/scripts/maint/practracker/exceptions.txt index cd56eba0e0..ad5d3e9725 100644 --- a/scripts/maint/practracker/exceptions.txt +++ b/scripts/maint/practracker/exceptions.txt @@ -92,7 +92,7 @@ problem function-size /src/core/or/circuitlist.c:circuits_handle_oom() 117 problem function-size /src/core/or/circuitmux.c:circuitmux_set_policy() 110 problem function-size /src/core/or/circuitmux.c:circuitmux_attach_circuit() 114 problem function-size /src/core/or/circuitstats.c:circuit_build_times_parse_state() 124 -problem file-size /src/core/or/circuituse.c 3150 +problem file-size /src/core/or/circuituse.c 3152 problem function-size /src/core/or/circuituse.c:circuit_is_acceptable() 129 problem function-size /src/core/or/circuituse.c:circuit_expire_building() 394 problem function-size /src/core/or/circuituse.c:circuit_log_ancient_one_hop_circuits() 126 From 809a3a748d5f0cbaa0357ec63388ed291fb7b0bd Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Sat, 30 Mar 2019 21:07:15 -0400 Subject: [PATCH 0712/2557] bug_occurred: a place where we assumed that "buf" was still a buffer In 9c132a5f9e87a2cd0f we replaced "buf" with a pointer and replaced one instance of snprintf with asprintf -- but there was still one snprintf left over, being crashy. Fixes bug 29967; bug not in any released Tor. This is CID 1444262. --- src/lib/log/util_bug.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/log/util_bug.c b/src/lib/log/util_bug.c index b9dc296fc6..65ab7bc9c6 100644 --- a/src/lib/log/util_bug.c +++ b/src/lib/log/util_bug.c @@ -117,7 +117,7 @@ tor_bug_occurred_(const char *fname, unsigned int line, } log_warn(LD_BUG, "%s:%u: %s: This line should not have been reached.%s", fname, line, func, once_str); - tor_snprintf(buf, sizeof(buf), + tor_asprintf(&buf, "Line unexpectedly reached at %s at %s:%u", func, fname, line); } else { From eaf071d7da6b09cfbdc60db701ae053b340ff37d Mon Sep 17 00:00:00 2001 From: rl1987 Date: Tue, 2 Apr 2019 12:31:57 +0300 Subject: [PATCH 0713/2557] Stop requiring bash in test-network.sh. Make it POSIX compliant --- src/test/test-network.sh | 38 ++++++++++++-------------------------- 1 file changed, 12 insertions(+), 26 deletions(-) diff --git a/src/test/test-network.sh b/src/test/test-network.sh index 4d56e83806..3a765d51f0 100755 --- a/src/test/test-network.sh +++ b/src/test/test-network.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/bin/sh # This script calls the equivalent script in chutney/tools @@ -18,32 +18,18 @@ ECHO="${ECHO:-echo}" # Output is prefixed with the name of the script myname=$(basename "$0") -# Save the arguments before we destroy them -# This might not preserve arguments with spaces in them -ORIGINAL_ARGS=( "$@" ) - # We need to find CHUTNEY_PATH, so that we can call the version of this script # in chutney/tools with the same arguments. We also need to respect --quiet. -until [ -z "$1" ] -do - case "$1" in - --chutney-path) - CHUTNEY_PATH="$2" - shift - ;; - --tor-path) - TOR_DIR="$2" - shift - ;; - --quiet) - ECHO=true - ;; - *) - # maybe chutney's test-network.sh can handle it - ;; - esac - shift -done +CHUTNEY_PATH=$(echo "$@" | awk -F '--chutney-path ' '{sub(" .*","",$2); print $2}') +TOR_DIR=$(echo "$@" | awk -F '--tor-dir ' '{sub(" .*","",$2); print $2}') + +if echo "$@" | grep -e "--quiet" > /dev/null; then + ECHO=true +fi + +echo "CHUTNEY_PATH = $CHUTNEY_PATH" +echo "TOR_DIR = $TOR_DIR" +echo "ECHO = $ECHO" # optional: $TOR_DIR is the tor build directory # it's used to find the location of tor binaries @@ -99,7 +85,7 @@ if [ -d "$CHUTNEY_PATH" ] && [ -x "$TEST_NETWORK" ]; then # this may fail if some arguments have spaces in them # if so, set CHUTNEY_PATH before calling test-network.sh, and spaces # will be handled correctly - exec "$TEST_NETWORK" "${ORIGINAL_ARGS[@]}" # $ORIGINAL_ARGS + exec "$TEST_NETWORK" "$@" else $ECHO "$myname: Could not find tools/test-network.sh in CHUTNEY_PATH." $ECHO "$myname: Please update your chutney using 'git pull'." From a549e4f7a3ebd088218cc16fdbf6964c53dba6a6 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Tue, 2 Apr 2019 12:59:37 +0300 Subject: [PATCH 0714/2557] Remove no-longer needed logging statements --- src/test/test-network.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/test/test-network.sh b/src/test/test-network.sh index 3a765d51f0..5ef995f1a4 100755 --- a/src/test/test-network.sh +++ b/src/test/test-network.sh @@ -27,10 +27,6 @@ if echo "$@" | grep -e "--quiet" > /dev/null; then ECHO=true fi -echo "CHUTNEY_PATH = $CHUTNEY_PATH" -echo "TOR_DIR = $TOR_DIR" -echo "ECHO = $ECHO" - # optional: $TOR_DIR is the tor build directory # it's used to find the location of tor binaries # if it's not set: From 0ebe290198ca852c27422fb3a82670f113f78727 Mon Sep 17 00:00:00 2001 From: Elichai Turkel Date: Mon, 1 Apr 2019 21:50:02 +0300 Subject: [PATCH 0715/2557] Removed the use of expect from CodingStandardsRust --- doc/HACKING/CodingStandardsRust.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/HACKING/CodingStandardsRust.md b/doc/HACKING/CodingStandardsRust.md index fc562816db..b570e10dc7 100644 --- a/doc/HACKING/CodingStandardsRust.md +++ b/doc/HACKING/CodingStandardsRust.md @@ -256,7 +256,7 @@ Here are some additional bits of advice and rules: or 2) should fail (i.e. in a unittest). You SHOULD NOT use `unwrap()` anywhere in which it is possible to handle the - potential error with either `expect()` or the eel operator, `?`. + potential error with the eel operator, `?` or another non panicking way. For example, consider a function which parses a string into an integer: fn parse_port_number(config_string: &str) -> u16 { @@ -264,12 +264,12 @@ Here are some additional bits of advice and rules: } There are numerous ways this can fail, and the `unwrap()` will cause the - whole program to byte the dust! Instead, either you SHOULD use `expect()` + whole program to byte the dust! Instead, either you SHOULD use `ok()` (or another equivalent function which will return an `Option` or a `Result`) and change the return type to be compatible: fn parse_port_number(config_string: &str) -> Option { - u16::from_str_radix(config_string, 10).expect("Couldn't parse port into a u16") + u16::from_str_radix(config_string, 10).ok() } or you SHOULD use `or()` (or another similar method): From 367dd9cf0226c087d49225a20415e9a199d2d71f Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 3 Apr 2019 10:16:18 -0400 Subject: [PATCH 0716/2557] 30001: Fix a race condition in test_dir_handle_get.c Previously we used time(NULL) to set the Expires: header in our HTTP responses. This made the actual contents of that header untestable, since the unit tests have no good way to override time(), or to see what time() was at the exact moment of the call to time() in dircache.c. This gave us a race in dir_handle_get/status_vote_next_bandwidth, where the time() call in dircache.c got one value, and the call in the tests got another value. I'm applying our regular solution here: using approx_time() so that the value stays the same between the code and the test. Since approx_time() is updated on every event callback, we shouldn't be losing any accuracy here. Fixes bug 30001. Bug introduced in fb4a40c32c4a7e5; not in any released Tor. --- changes/bug30001 | 4 ++++ src/feature/dircache/dircache.c | 2 +- src/test/test_dir_handle_get.c | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) create mode 100644 changes/bug30001 diff --git a/changes/bug30001 b/changes/bug30001 new file mode 100644 index 0000000000..e3304701e7 --- /dev/null +++ b/changes/bug30001 @@ -0,0 +1,4 @@ + o Minor features (testing): + - Use the approx_time() function when setting the "Expires" header + in directory replies, to make them more testable. Needed for + ticket 30001. diff --git a/src/feature/dircache/dircache.c b/src/feature/dircache/dircache.c index caa085dd63..1123d034e0 100644 --- a/src/feature/dircache/dircache.c +++ b/src/feature/dircache/dircache.c @@ -124,7 +124,7 @@ write_http_response_header_impl(dir_connection_t *conn, ssize_t length, long cache_lifetime) { char date[RFC1123_TIME_LEN+1]; - time_t now = time(NULL); + time_t now = approx_time(); buf_t *buf = buf_new_with_capacity(1024); tor_assert(conn); diff --git a/src/test/test_dir_handle_get.c b/src/test/test_dir_handle_get.c index c3a17e7309..e57bd02584 100644 --- a/src/test/test_dir_handle_get.c +++ b/src/test/test_dir_handle_get.c @@ -2526,7 +2526,7 @@ test_dir_handle_get_status_vote_next_bandwidth(void* data) /* Check cache lifetime */ char expbuf[RFC1123_TIME_LEN+1]; - time_t now = time(NULL); + time_t now = approx_time(); /* BANDWIDTH_CACHE_LIFETIME is defined in dircache.c. */ format_rfc1123_time(expbuf, (time_t)(now + 30*60)); char *expires = NULL; From 9e0f0a565632327263c1c3466978f8bc5e759194 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Wed, 3 Apr 2019 17:52:31 +0300 Subject: [PATCH 0717/2557] Fix SC2086 warnings in test_key_expiration.sh --- src/test/test_key_expiration.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/test_key_expiration.sh b/src/test/test_key_expiration.sh index cf6608634d..5e89bbbd6a 100755 --- a/src/test/test_key_expiration.sh +++ b/src/test/test_key_expiration.sh @@ -6,7 +6,7 @@ umask 077 set -e -if [ $# -eq 0 ] || [ ! -f ${1} ] || [ ! -x ${1} ]; then +if [ $# -eq 0 ] || [ ! -f "${1}" ] || [ ! -x "${1}" ]; then if [ "$TESTING_TOR_BINARY" = "" ] ; then echo "Usage: ${0} PATH_TO_TOR [case-number]" exit 1 @@ -48,7 +48,7 @@ die() { echo "$1" >&2 ; exit 5; } check_dir() { [ -d "$1" ] || die "$1 did not exist"; } check_file() { [ -e "$1" ] || die "$1 did not exist"; } check_no_file() { [ -e "$1" ] && die "$1 was not supposed to exist" || true; } -check_files_eq() { cmp "$1" "$2" || die "$1 and $2 did not match: `dump $1` vs `dump $2`"; } +check_files_eq() { cmp "$1" "$2" || die "$1 and $2 did not match: `dump "$1"` vs `dump "$2"`"; } check_keys_eq() { check_files_eq "${SRC}/keys/${1}" "${ME}/keys/${1}"; } DATA_DIR=`mktemp -d -t tor_key_expiration_tests.XXXXXX` From 700310df613e379dc1930917f0f8a25f0c894a52 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Wed, 3 Apr 2019 17:56:52 +0300 Subject: [PATCH 0718/2557] Fix SC2006 warnings --- src/test/test_key_expiration.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/test/test_key_expiration.sh b/src/test/test_key_expiration.sh index 5e89bbbd6a..6e49919941 100755 --- a/src/test/test_key_expiration.sh +++ b/src/test/test_key_expiration.sh @@ -13,7 +13,7 @@ if [ $# -eq 0 ] || [ ! -f "${1}" ] || [ ! -x "${1}" ]; then fi fi -UNAME_OS=`uname -s | cut -d_ -f1` +UNAME_OS=$(uname -s | cut -d_ -f1) if test "$UNAME_OS" = 'CYGWIN' || \ test "$UNAME_OS" = 'MSYS' || \ test "$UNAME_OS" = 'MINGW'; then @@ -48,10 +48,10 @@ die() { echo "$1" >&2 ; exit 5; } check_dir() { [ -d "$1" ] || die "$1 did not exist"; } check_file() { [ -e "$1" ] || die "$1 did not exist"; } check_no_file() { [ -e "$1" ] && die "$1 was not supposed to exist" || true; } -check_files_eq() { cmp "$1" "$2" || die "$1 and $2 did not match: `dump "$1"` vs `dump "$2"`"; } +check_files_eq() { cmp "$1" "$2" || die "$1 and $2 did not match: $(dump "$1") vs $(dump "$2")"; } check_keys_eq() { check_files_eq "${SRC}/keys/${1}" "${ME}/keys/${1}"; } -DATA_DIR=`mktemp -d -t tor_key_expiration_tests.XXXXXX` +DATA_DIR=$(mktemp -d -t tor_key_expiration_tests.XXXXXX) if [ -z "$DATA_DIR" ]; then echo "Failure: mktemp invocation returned empty string" >&2 exit 3 @@ -63,7 +63,7 @@ fi trap "rm -rf '$DATA_DIR'" 0 # Use an absolute path for this or Tor will complain -DATA_DIR=`cd "${DATA_DIR}" && pwd` +DATA_DIR=$(cd "${DATA_DIR}" && pwd) touch "${DATA_DIR}/empty_torrc" From 9e04a8722081cd761ee3a2e173b53c7859a8c916 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Wed, 3 Apr 2019 17:58:05 +0300 Subject: [PATCH 0719/2557] Fix SC2064 warning --- src/test/test_key_expiration.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/test_key_expiration.sh b/src/test/test_key_expiration.sh index 6e49919941..8614a4fa48 100755 --- a/src/test/test_key_expiration.sh +++ b/src/test/test_key_expiration.sh @@ -60,7 +60,7 @@ if [ ! -d "$DATA_DIR" ]; then echo "Failure: mktemp invocation result doesn't point to directory" >&2 exit 3 fi -trap "rm -rf '$DATA_DIR'" 0 +trap 'rm -rf "$DATA_DIR"' 0 # Use an absolute path for this or Tor will complain DATA_DIR=$(cd "${DATA_DIR}" && pwd) From 4172b638b8221e2d56c20e2a408948977c68358a Mon Sep 17 00:00:00 2001 From: rl1987 Date: Wed, 3 Apr 2019 18:03:34 +0300 Subject: [PATCH 0720/2557] Fix SC2015 warning --- src/test/test_key_expiration.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/test_key_expiration.sh b/src/test/test_key_expiration.sh index 8614a4fa48..1ed81c81c7 100755 --- a/src/test/test_key_expiration.sh +++ b/src/test/test_key_expiration.sh @@ -47,7 +47,7 @@ dump() { xxd -p "$1" | tr -d '\n '; } die() { echo "$1" >&2 ; exit 5; } check_dir() { [ -d "$1" ] || die "$1 did not exist"; } check_file() { [ -e "$1" ] || die "$1 did not exist"; } -check_no_file() { [ -e "$1" ] && die "$1 was not supposed to exist" || true; } +check_no_file() { if [ -e "$1" ]; then die "$1 was not supposed to exist"; fi } check_files_eq() { cmp "$1" "$2" || die "$1 and $2 did not match: $(dump "$1") vs $(dump "$2")"; } check_keys_eq() { check_files_eq "${SRC}/keys/${1}" "${ME}/keys/${1}"; } From b2eced6c0747159ab7b3942a45c1a147a5ecfc96 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Wed, 3 Apr 2019 18:05:23 +0300 Subject: [PATCH 0721/2557] Add changes file --- changes/bug30002 | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 changes/bug30002 diff --git a/changes/bug30002 b/changes/bug30002 new file mode 100644 index 0000000000..da61c9e4b2 --- /dev/null +++ b/changes/bug30002 @@ -0,0 +1,2 @@ + o Code simplification and refactoring (shell scripts): + - Fix shellcheck warnings in test_key_expiration.sh. Resolves issue 30002. From 99b87d7ca48c5ed80aec5f24902843f72975bfea Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 3 Apr 2019 13:53:06 -0400 Subject: [PATCH 0722/2557] Even more diagnostic messages for bug 28223. Try to figure out _where exactly_ we are first encountering NULs in microdescriptors, and what we are doing when that happens. --- changes/diagnostic_28223_redux | 4 ++++ src/feature/dirparse/microdesc_parse.c | 17 ++++++++++++++++- src/feature/nodelist/microdesc.c | 21 +++++++++++++++++++++ 3 files changed, 41 insertions(+), 1 deletion(-) create mode 100644 changes/diagnostic_28223_redux diff --git a/changes/diagnostic_28223_redux b/changes/diagnostic_28223_redux new file mode 100644 index 0000000000..0d7499832e --- /dev/null +++ b/changes/diagnostic_28223_redux @@ -0,0 +1,4 @@ + o Minor features (diagnostic): + - Add more diagnostic log messages in an attempt to solve + the issue of NUL bytes appearing in a microdescriptor cache. + Related to ticket 28223. diff --git a/src/feature/dirparse/microdesc_parse.c b/src/feature/dirparse/microdesc_parse.c index 5a75af3994..1e26901ce8 100644 --- a/src/feature/dirparse/microdesc_parse.c +++ b/src/feature/dirparse/microdesc_parse.c @@ -159,7 +159,22 @@ microdescs_parse_from_string(const char *s, const char *eos, if (tokenize_string(area, s, start_of_next_microdesc, tokens, microdesc_token_table, flags)) { - log_warn(LD_DIR, "Unparseable microdescriptor"); + const char *location; + switch (where) { + case SAVED_NOWHERE: + location = "download or generated string"; + break; + case SAVED_IN_CACHE: + location = "cache"; + break; + case SAVED_IN_JOURNAL: + location = "journal"; + break; + default: + location = "unknown location"; + break; + } + log_warn(LD_DIR, "Unparseable microdescriptor found in %s", location); goto next; } diff --git a/src/feature/nodelist/microdesc.c b/src/feature/nodelist/microdesc.c index dafaabb5e5..af4f608d2f 100644 --- a/src/feature/nodelist/microdesc.c +++ b/src/feature/nodelist/microdesc.c @@ -221,6 +221,13 @@ dump_microdescriptor(int fd, microdesc_t *md, size_t *annotation_len_out) } md->off = tor_fd_getpos(fd); + const char *nulpos = memchr(md->body, 0, md->bodylen); + if (BUG(nulpos)) { + log_warn(LD_BUG, "About to dump a NUL into a microdescriptor file. " + "offset %"PRId64", bodylen %zu, nul position %zu", + (int64_t)md->off, md->bodylen, + (size_t)(nulpos - md->body)); + } written = write_all_to_fd(fd, md->body, md->bodylen); if (written != (ssize_t)md->bodylen) { written = written < 0 ? 0 : written; @@ -480,6 +487,17 @@ microdesc_cache_clear(microdesc_cache_t *cache) cache->bytes_dropped = 0; } +static void +warn_if_nul_found(const char *inp, size_t len, const char *description) +{ + const char *nul_found = memchr(inp, 0, len); + if (BUG(nul_found)) { + log_warn(LD_BUG, "Found unexpected NUL while reading %s, at " + "position %zu/%zu.", + description, (nul_found - inp), len); + } +} + /** Reload the contents of cache from disk. If it is empty, load it * for the first time. Return 0 on success, -1 on failure. */ int @@ -497,6 +515,7 @@ microdesc_cache_reload(microdesc_cache_t *cache) mm = cache->cache_content = tor_mmap_file(cache->cache_fname); if (mm) { + warn_if_nul_found(mm->data, mm->size, "microdesc cache"); added = microdescs_add_to_cache(cache, mm->data, mm->data+mm->size, SAVED_IN_CACHE, 0, -1, NULL); if (added) { @@ -509,6 +528,8 @@ microdesc_cache_reload(microdesc_cache_t *cache) RFTS_IGNORE_MISSING, &st); if (journal_content) { cache->journal_len = (size_t) st.st_size; + warn_if_nul_found(journal_content, cache->journal_len, + "microdesc journal"); added = microdescs_add_to_cache(cache, journal_content, journal_content+st.st_size, SAVED_IN_JOURNAL, 0, -1, NULL); From 821d29e42040789c59d271ba2f1d87a54d6d1435 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 3 Apr 2019 13:53:36 -0400 Subject: [PATCH 0723/2557] fdio.c: add more includes. This is just in case there is some rogue platform that uses a nonstandard value for SEEK_*, and does not define that macro in unistd.h. I think that's unlikely, but it's conceivable. --- src/lib/fdio/fdio.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/lib/fdio/fdio.c b/src/lib/fdio/fdio.c index 6c87af791d..078af6a9ba 100644 --- a/src/lib/fdio/fdio.c +++ b/src/lib/fdio/fdio.c @@ -17,12 +17,16 @@ #ifdef _WIN32 #include #endif +#ifdef HAVE_SYS_TYPES_H +#include +#endif #include "lib/fdio/fdio.h" #include "lib/cc/torint.h" #include "lib/err/torerr.h" #include +#include /** @{ */ /** Some old versions of Unix didn't define constants for these values, From 5613968d57cb07cc5a66e97c58bd39c22b86a50a Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 3 Apr 2019 14:30:56 -0400 Subject: [PATCH 0724/2557] Improve logging for 28614. When we fixed 28614, our answer was "if we failed to load the consensus on windows and it had a CRLF, retry it." But we logged the failure at "warn", and we only logged the retry at "info". Now we log the retry at "notice", with more useful information. Fixes bug 30004. --- changes/bug28614_better_logging | 6 ++++++ src/feature/nodelist/networkstatus.c | 7 +++++-- 2 files changed, 11 insertions(+), 2 deletions(-) create mode 100644 changes/bug28614_better_logging diff --git a/changes/bug28614_better_logging b/changes/bug28614_better_logging new file mode 100644 index 0000000000..26d19c3c11 --- /dev/null +++ b/changes/bug28614_better_logging @@ -0,0 +1,6 @@ + o Minor bugfixes (logging): + - On Windows, when errors cause us to reload a consensus from disk, tell + the user that we are retrying at log level "notice". Previously we only + logged this information at "info", which was confusing because the + errors themselves were logged at "warning". Improves previous fix for + 28614. Fixes bug 30004; bugfix on 0.4.0.2-alpha. diff --git a/src/feature/nodelist/networkstatus.c b/src/feature/nodelist/networkstatus.c index 023115978c..a988f700f3 100644 --- a/src/feature/nodelist/networkstatus.c +++ b/src/feature/nodelist/networkstatus.c @@ -1761,8 +1761,11 @@ reload_consensus_from_file(const char *fname, flavor, flags, source_dir); #ifdef _WIN32 if (rv < 0 && tor_memstr(map->data, map->size, "\r\n")) { - log_info(LD_GENERAL, "Found CRLF in consensus file %s; falling back to " - "read_file_to_string.", escaped(fname)); + log_notice(LD_GENERAL, "Looks like the above failures are probably " + "because of a CRLF in consensus file %s; falling back to " + "read_file_to_string. Nothing to worry about: this file " + "was probably saved by an earlier version of Tor.", + escaped(fname)); char *content = read_file_to_str(fname, RFTS_IGNORE_MISSING, NULL); rv = networkstatus_set_current_consensus(content, strlen(content), flavor, flags, source_dir); From d4d77b277e72c74a47bd724531426d7f561607e4 Mon Sep 17 00:00:00 2001 From: Neel Chauhan Date: Wed, 3 Apr 2019 11:36:52 -0400 Subject: [PATCH 0725/2557] Stop setting bridges running in networkstatus_getinfo_by_purpose() --- changes/bug24490 | 5 +++++ scripts/maint/practracker/exceptions.txt | 6 +++--- src/core/mainloop/mainloop.c | 25 ++++++++++++++++++++++++ src/feature/nodelist/networkstatus.c | 4 ---- 4 files changed, 33 insertions(+), 7 deletions(-) create mode 100644 changes/bug24490 diff --git a/changes/bug24490 b/changes/bug24490 new file mode 100644 index 0000000000..9ae09dbd17 --- /dev/null +++ b/changes/bug24490 @@ -0,0 +1,5 @@ + o Minor bugfixes (bridge authority): + - We set bridges as running in a callback which runs every 5 minutes. + Previously, we set bridges as running in a GETINFO controller as + these shouldn't modify vital data structures. Fixes bug 24490; + bugfix on 0.2.0.13-alpha. Patch by Neel Chauhan diff --git a/scripts/maint/practracker/exceptions.txt b/scripts/maint/practracker/exceptions.txt index ad5d3e9725..a9fcf9095a 100644 --- a/scripts/maint/practracker/exceptions.txt +++ b/scripts/maint/practracker/exceptions.txt @@ -67,11 +67,11 @@ problem function-size /src/core/mainloop/connection.c:connection_handle_read_imp problem function-size /src/core/mainloop/connection.c:connection_buf_read_from_socket() 177 problem function-size /src/core/mainloop/connection.c:connection_handle_write_impl() 241 problem function-size /src/core/mainloop/connection.c:assert_connection_ok() 143 -problem file-size /src/core/mainloop/mainloop.c 3051 -problem include-count /src/core/mainloop/mainloop.c 66 +problem file-size /src/core/mainloop/mainloop.c 3076 +problem include-count /src/core/mainloop/mainloop.c 68 problem function-size /src/core/mainloop/mainloop.c:conn_close_if_marked() 108 problem function-size /src/core/mainloop/mainloop.c:run_connection_housekeeping() 123 -problem function-size /src/core/mainloop/mainloop.c:CALLBACK() 116 +problem function-size /src/core/mainloop/mainloop.c:CALLBACK() 118 problem file-size /src/core/or/channel.c 3476 problem function-size /src/core/or/channeltls.c:channel_tls_handle_var_cell() 160 problem function-size /src/core/or/channeltls.c:channel_tls_process_versions_cell() 170 diff --git a/src/core/mainloop/mainloop.c b/src/core/mainloop/mainloop.c index c9f2b0d896..800304a736 100644 --- a/src/core/mainloop/mainloop.c +++ b/src/core/mainloop/mainloop.c @@ -76,6 +76,7 @@ #include "feature/control/control_events.h" #include "feature/dirauth/authmode.h" #include "feature/dirauth/reachability.h" +#include "feature/dirauth/voteflags.h" #include "feature/dircache/consdiffmgr.h" #include "feature/dircache/dirserv.h" #include "feature/dircommon/directory.h" @@ -87,6 +88,7 @@ #include "feature/nodelist/networkstatus.h" #include "feature/nodelist/nodelist.h" #include "feature/nodelist/routerlist.h" +#include "feature/nodelist/routerlist_st.h" #include "feature/relay/dns.h" #include "feature/relay/routerkeys.h" #include "feature/relay/routermode.h" @@ -1375,6 +1377,7 @@ CALLBACK(rotate_onion_key); CALLBACK(rotate_x509_certificate); CALLBACK(save_stability); CALLBACK(save_state); +CALLBACK(set_bridge_running); CALLBACK(write_bridge_ns); CALLBACK(write_stats_file); CALLBACK(control_per_second_events); @@ -1453,6 +1456,7 @@ STATIC periodic_event_item_t periodic_events[] = { /* Bridge Authority only. */ CALLBACK(write_bridge_ns, BRIDGEAUTH, 0), + CALLBACK(set_bridge_running, BRIDGEAUTH, 0), /* Directory server only. */ CALLBACK(clean_consdiffmgr, DIRSERVER, 0), @@ -2583,6 +2587,27 @@ write_bridge_ns_callback(time_t now, const or_options_t *options) return PERIODIC_EVENT_NO_UPDATE; } +/** + * Periodic callback: if we're the bridge authority, set the running flag on + * bridges if they're reachable + */ +static int +set_bridge_running_callback(time_t now, const or_options_t *options) +{ + if (options->BridgeAuthoritativeDir) { + routerlist_t *rl = router_get_routerlist(); + + SMARTLIST_FOREACH_BEGIN(rl->routers, routerinfo_t *, ri) { + if (ri->purpose == ROUTER_PURPOSE_BRIDGE) + dirserv_set_router_is_running(ri, now); + } SMARTLIST_FOREACH_END(ri); + +#define SET_BRIDGES_RUNNING_INTERVAL (3*60) + return SET_BRIDGES_RUNNING_INTERVAL; + } + return PERIODIC_EVENT_NO_UPDATE; +} + static int heartbeat_callback_first_time = 1; /** diff --git a/src/feature/nodelist/networkstatus.c b/src/feature/nodelist/networkstatus.c index ea9f12367f..bc12fa4075 100644 --- a/src/feature/nodelist/networkstatus.c +++ b/src/feature/nodelist/networkstatus.c @@ -2378,7 +2378,6 @@ networkstatus_getinfo_by_purpose(const char *purpose_string, time_t now) smartlist_t *statuses; const uint8_t purpose = router_purpose_from_string(purpose_string); routerstatus_t rs; - const int bridge_auth = authdir_mode_bridge(get_options()); if (purpose == ROUTER_PURPOSE_UNKNOWN) { log_info(LD_DIR, "Unrecognized purpose '%s' when listing router statuses.", @@ -2395,9 +2394,6 @@ networkstatus_getinfo_by_purpose(const char *purpose_string, time_t now) continue; if (ri->purpose != purpose) continue; - /* TODO: modifying the running flag in a getinfo is a bad idea */ - if (bridge_auth && ri->purpose == ROUTER_PURPOSE_BRIDGE) - dirserv_set_router_is_running(ri, now); /* then generate and write out status lines for each of them */ set_routerstatus_from_routerinfo(&rs, node, ri, now, 0); smartlist_add(statuses, networkstatus_getinfo_helper_single(&rs)); From d194f6bedf48410132b03fdab06b679512c14db1 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Sun, 17 Mar 2019 13:48:00 -0400 Subject: [PATCH 0726/2557] Implement an DormantCanceledByStartup option Closes ticket 29357, and comes with appropriate notions of caution. --- changes/ticket29357 | 7 +++++++ doc/tor.1.txt | 15 +++++++++++++++ src/app/config/config.c | 1 + src/app/config/or_options_st.h | 5 +++++ src/core/mainloop/netstatus.c | 4 ++++ src/test/test_mainloop.c | 8 ++++++++ 6 files changed, 40 insertions(+) create mode 100644 changes/ticket29357 diff --git a/changes/ticket29357 b/changes/ticket29357 new file mode 100644 index 0000000000..3aab930cd4 --- /dev/null +++ b/changes/ticket29357 @@ -0,0 +1,7 @@ + o Minor features (dormant mode): + - Add a DormantCanceledByStartup option to tell Tor that it should + treat a startup event as cancelling any previous dormant state. + Integrators should use this option with caution: it should + only be used if Tor is being started because of something that the + user did, and not if Tor is being automatically started in the + background. Closes ticket 29357. diff --git a/doc/tor.1.txt b/doc/tor.1.txt index 52f5bfa0c3..ea9942a28d 100644 --- a/doc/tor.1.txt +++ b/doc/tor.1.txt @@ -1850,6 +1850,21 @@ The following options are useful only for clients (that is, if After the first time Tor starts, it begins in dormant mode if it was dormant before, and not otherwise. (Default: 0) +[[DormantCanceledByStartup]] **DormantCanceledByStartup** **0**|**1**:: + By default, Tor starts in active mode if it was active the last time + it was shut down, and in dormant mode if it was dormant. But if + this option is true, Tor treats every startup event as user + activity, and Tor will never start in Dormant mode, even if it has + been unused for a long time on previous runs. (Default: 0) + + + Note: Packagers and application developers should change the value of + this option only with great caution: it has the potential to + create spurious traffic on the network. This option should only + be used if Tor is started by an affirmative user activity (like + clicking on an applcation or running a command), and not if Tor + is launched for some other reason (for example, by a startup + process, or by an application that launches itself on every login.) + SERVER OPTIONS -------------- diff --git a/src/app/config/config.c b/src/app/config/config.c index 8e0bfbe797..dc213ce2fc 100644 --- a/src/app/config/config.c +++ b/src/app/config/config.c @@ -396,6 +396,7 @@ static config_var_t option_vars_[] = { V(DormantClientTimeout, INTERVAL, "24 hours"), V(DormantTimeoutDisabledByIdleStreams, BOOL, "1"), V(DormantOnFirstStartup, BOOL, "0"), + V(DormantCanceledByStartup, BOOL, "0"), /* DoS circuit creation options. */ V(DoSCircuitCreationEnabled, AUTOBOOL, "auto"), V(DoSCircuitCreationMinConnections, UINT, "0"), diff --git a/src/app/config/or_options_st.h b/src/app/config/or_options_st.h index 06e11d3c75..bd707fd193 100644 --- a/src/app/config/or_options_st.h +++ b/src/app/config/or_options_st.h @@ -1092,6 +1092,11 @@ struct or_options_t { /** Boolean: true if Tor should be dormant the first time it starts with * a datadirectory; false otherwise. */ int DormantOnFirstStartup; + /** + * Boolean: true if Tor should treat every startup event as cancelling + * a possible previous dormant state. + **/ + int DormantCanceledByStartup; }; #endif diff --git a/src/core/mainloop/netstatus.c b/src/core/mainloop/netstatus.c index fc5a465ff7..4924888598 100644 --- a/src/core/mainloop/netstatus.c +++ b/src/core/mainloop/netstatus.c @@ -144,6 +144,10 @@ netstatus_load_from_state(const or_state_t *state, time_t now) last_activity = now - 60 * state->MinutesSinceUserActivity; participating_on_network = true; } + if (get_options()->DormantCanceledByStartup) { + last_activity = now; + participating_on_network = true; + } reset_user_activity(last_activity); } diff --git a/src/test/test_mainloop.c b/src/test/test_mainloop.c index 2c3449305a..ed6b8a9b66 100644 --- a/src/test/test_mainloop.c +++ b/src/test/test_mainloop.c @@ -317,6 +317,14 @@ test_mainloop_dormant_load_state(void *arg) tt_assert(is_participating_on_network()); tt_i64_op(get_last_user_activity_time(), OP_EQ, start - 123*60); + // If we would start dormant, but DormantCanceledByStartup is set, then + // we start up non-dormant. + state->Dormant = 1; + get_options_mutable()->DormantCanceledByStartup = 1; + netstatus_load_from_state(state, start); + tt_assert(is_participating_on_network()); + tt_i64_op(get_last_user_activity_time(), OP_EQ, start); + done: or_state_free(state); } From ab6ad3c040de68b1f06b8f910407bff570b24b43 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 6 Mar 2019 09:59:10 -0500 Subject: [PATCH 0727/2557] Drop thread-local fast_rng on fork. This will cause the child process to construct a new one in a nice safe way. Closes ticket 29668; bug not in any released Tor. --- src/lib/crypt_ops/crypto_init.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/lib/crypt_ops/crypto_init.c b/src/lib/crypt_ops/crypto_init.c index cf491f32d1..5c2780b2ca 100644 --- a/src/lib/crypt_ops/crypto_init.c +++ b/src/lib/crypt_ops/crypto_init.c @@ -152,6 +152,12 @@ crypto_prefork(void) #ifdef ENABLE_NSS crypto_nss_prefork(); #endif + /* It is not safe to share a fast_rng object across a fork boundary unless + * we actually have zero-on-fork support in map_anon.c. If we have + * drop-on-fork support, we will crash; if we have neither, we will yield + * a copy of the parent process's rng, which is scary and insecure. + */ + destroy_thread_fast_rng(); } /** Run operations that the crypto library requires to be happy again From 12205c3cbee4e71ded2b5710a57342b510e9d6df Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 6 Mar 2019 10:35:02 -0500 Subject: [PATCH 0728/2557] Make map_anon expose the result of a noinherit attempt Previously we did this for tests only, but it's valuable for getting proper fork behavior in rand_fast. --- src/lib/crypt_ops/crypto_rand_fast.c | 4 +-- src/lib/malloc/map_anon.c | 48 +++++++++++----------------- src/lib/malloc/map_anon.h | 17 +++++++--- src/test/test_util.c | 21 ++++++------ 4 files changed, 45 insertions(+), 45 deletions(-) diff --git a/src/lib/crypt_ops/crypto_rand_fast.c b/src/lib/crypt_ops/crypto_rand_fast.c index 760e1025ed..f1acc9faf2 100644 --- a/src/lib/crypt_ops/crypto_rand_fast.c +++ b/src/lib/crypt_ops/crypto_rand_fast.c @@ -134,8 +134,8 @@ crypto_fast_rng_new_from_seed(const uint8_t *seed) * having it get dumped, swapped, or shared after fork. */ crypto_fast_rng_t *result = tor_mmap_anonymous(sizeof(*result), - ANONMAP_PRIVATE | ANONMAP_NOINHERIT); - + ANONMAP_PRIVATE | ANONMAP_NOINHERIT, + NULL); memcpy(result->buf.seed, seed, SEED_LEN); /* Causes an immediate refill once the user asks for data. */ result->bytes_left = 0; diff --git a/src/lib/malloc/map_anon.c b/src/lib/malloc/map_anon.c index 5dac5256a6..bb0a3d5741 100644 --- a/src/lib/malloc/map_anon.c +++ b/src/lib/malloc/map_anon.c @@ -107,54 +107,34 @@ nodump_mem(void *mem, size_t sz) #endif } -#ifdef TOR_UNIT_TESTS -static unsigned last_anon_map_noinherit = ~0; -/* Testing helper: return the outcome of the last call to noinherit_mem(): - * 0 if it did no good; 1 if it caused the memory not to be inherited, and - * 2 if it caused the memory to be cleared on fork */ -unsigned -get_last_anon_map_noinherit(void) -{ - return last_anon_map_noinherit; -} -static void -set_last_anon_map_noinherit(unsigned f) -{ - last_anon_map_noinherit = f; -} -#else -static void -set_last_anon_map_noinherit(unsigned f) -{ - (void)f; -} -#endif - /** * Helper: try to prevent the sz bytes at mem from being * accessible in child processes -- ideally by having them set to 0 after a * fork, and if that doesn't work, by having them unmapped after a fork. * Return 0 on success or if the facility is not available on this OS; return * -1 on failure. + * + * If we successfully make the memory uninheritable, adjust the value of + * *inherit_result_out. */ static int -noinherit_mem(void *mem, size_t sz) +noinherit_mem(void *mem, size_t sz, unsigned *inherit_result_out) { - set_last_anon_map_noinherit(0); #ifdef FLAG_ZERO int r = MINHERIT(mem, sz, FLAG_ZERO); if (r == 0) { - set_last_anon_map_noinherit(2); + *inherit_result_out = INHERIT_ZERO; return 0; } #endif #ifdef FLAG_NOINHERIT int r2 = MINHERIT(mem, sz, FLAG_NOINHERIT); if (r2 == 0) { - set_last_anon_map_noinherit(1); + *inherit_result_out = INHERIT_DROP; } return r2; #else + (void)inherit_result_out; (void)mem; (void)sz; return 0; @@ -174,14 +154,24 @@ noinherit_mem(void *mem, size_t sz) * Memory returned from this function must be released with * tor_munmap_anonymous(). * + * If inherit_result_out is non-NULL, set it to one of INHERIT_KEEP, + * INHERIT_DROP, and INHERIT_ZERO, depending on the properties of the returned + * memory. + * * [Note: OS people use the word "anonymous" here to mean that the memory * isn't associated with any file. This has *nothing* to do with the kind of * anonymity that Tor is trying to provide.] */ void * -tor_mmap_anonymous(size_t sz, unsigned flags) +tor_mmap_anonymous(size_t sz, unsigned flags, unsigned *inherit_result_out) { void *ptr; + unsigned itmp=0; + if (inherit_result_out == NULL) { + inherit_result_out = &itmp; + } + *inherit_result_out = INHERIT_KEEP; + #if defined(_WIN32) HANDLE mapping = CreateFileMapping(INVALID_HANDLE_VALUE, NULL, /*attributes*/ @@ -214,7 +204,7 @@ tor_mmap_anonymous(size_t sz, unsigned flags) } if (flags & ANONMAP_NOINHERIT) { - int noinherit_result = noinherit_mem(ptr, sz); + int noinherit_result = noinherit_mem(ptr, sz, inherit_result_out); raw_assert(noinherit_result == 0); } diff --git a/src/lib/malloc/map_anon.h b/src/lib/malloc/map_anon.h index 395145bd71..edd7500821 100644 --- a/src/lib/malloc/map_anon.h +++ b/src/lib/malloc/map_anon.h @@ -31,11 +31,18 @@ */ #define ANONMAP_NOINHERIT (1u<<1) -void *tor_mmap_anonymous(size_t sz, unsigned flags); +/** Possible value for inherit_result_out: the memory will be kept + * by any child process. */ +#define INHERIT_KEEP 0 +/** Possible value for inherit_result_out: the memory will be dropped in + * the child process. Attempting to access it will likely cause a segfault. */ +#define INHERIT_DROP 1 +/** Possible value for inherit_result_out: the memory will be cleared in + * the child process. */ +#define INHERIT_ZERO 2 + +void *tor_mmap_anonymous(size_t sz, unsigned flags, + unsigned *inherit_result_out); void tor_munmap_anonymous(void *mapping, size_t sz); -#ifdef TOR_UNIT_TESTS -unsigned get_last_anon_map_noinherit(void); -#endif - #endif /* !defined(TOR_MAP_ANON_H) */ diff --git a/src/test/test_util.c b/src/test/test_util.c index 4990aa709a..039fc435ce 100644 --- a/src/test/test_util.c +++ b/src/test/test_util.c @@ -6130,10 +6130,12 @@ test_util_map_anon(void *arg) (void)arg; char *ptr = NULL; size_t sz = 16384; + unsigned inherit=0; /* Basic checks. */ - ptr = tor_mmap_anonymous(sz, 0); + ptr = tor_mmap_anonymous(sz, 0, &inherit); tt_ptr_op(ptr, OP_NE, 0); + tt_int_op(inherit, OP_EQ, INHERIT_KEEP); ptr[sz-1] = 3; tt_int_op(ptr[0], OP_EQ, 0); tt_int_op(ptr[sz-2], OP_EQ, 0); @@ -6141,8 +6143,9 @@ test_util_map_anon(void *arg) /* Try again, with a private (non-swappable) mapping. */ tor_munmap_anonymous(ptr, sz); - ptr = tor_mmap_anonymous(sz, ANONMAP_PRIVATE); + ptr = tor_mmap_anonymous(sz, ANONMAP_PRIVATE, &inherit); tt_ptr_op(ptr, OP_NE, 0); + tt_int_op(inherit, OP_EQ, INHERIT_KEEP); ptr[sz-1] = 10; tt_int_op(ptr[0], OP_EQ, 0); tt_int_op(ptr[sz/2], OP_EQ, 0); @@ -6150,7 +6153,7 @@ test_util_map_anon(void *arg) /* Now let's test a drop-on-fork mapping. */ tor_munmap_anonymous(ptr, sz); - ptr = tor_mmap_anonymous(sz, ANONMAP_NOINHERIT); + ptr = tor_mmap_anonymous(sz, ANONMAP_NOINHERIT, &inherit); tt_ptr_op(ptr, OP_NE, 0); ptr[sz-1] = 10; tt_int_op(ptr[0], OP_EQ, 0); @@ -6179,10 +6182,10 @@ test_util_map_anon_nofork(void *arg) char *ptr = NULL; size_t sz = 16384; int pipefd[2] = {-1, -1}; + unsigned inherit=0; tor_munmap_anonymous(ptr, sz); - ptr = tor_mmap_anonymous(sz, ANONMAP_NOINHERIT); - int outcome = get_last_anon_map_noinherit(); + ptr = tor_mmap_anonymous(sz, ANONMAP_NOINHERIT, &inherit); tt_ptr_op(ptr, OP_NE, 0); memset(ptr, 0xd0, sz); @@ -6204,16 +6207,16 @@ test_util_map_anon_nofork(void *arg) char buf[1]; ssize_t r = read(pipefd[0], buf, 1); - if (outcome == 2) { + if (inherit == INHERIT_ZERO) { // We should be seeing clear-on-fork behavior. tt_int_op((int)r, OP_EQ, 1); // child should send us a byte. tt_int_op(buf[0], OP_EQ, 0); // that byte should be zero. - } else if (outcome == 1) { + } else if (inherit == INHERIT_DROP) { // We should be seeing noinherit behavior. tt_int_op(r, OP_LE, 0); // child said nothing; it should have crashed. } else { // noinherit isn't implemented. - tt_int_op(outcome, OP_EQ, 0); + tt_int_op(inherit, OP_EQ, INHERIT_KEEP); tt_int_op((int)r, OP_EQ, 1); // child should send us a byte. tt_int_op(buf[0], OP_EQ, 0xd0); // that byte should what we set it to. } @@ -6221,7 +6224,7 @@ test_util_map_anon_nofork(void *arg) int ws; waitpid(child, &ws, 0); - if (outcome == 0) { + if (inherit == INHERIT_KEEP) { /* Call this test "skipped", not "passed", since noinherit wasn't * implemented. */ tt_skip(); From 361e955cf3f852caebb63f618fbae883237bf28b Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 6 Mar 2019 11:03:26 -0500 Subject: [PATCH 0729/2557] map_anon: define a macro if it is possible for noinherit to fail. --- src/lib/malloc/map_anon.h | 21 +++++++++++++++++++++ src/test/test_util.c | 6 ++++++ 2 files changed, 27 insertions(+) diff --git a/src/lib/malloc/map_anon.h b/src/lib/malloc/map_anon.h index edd7500821..89fb9da0f0 100644 --- a/src/lib/malloc/map_anon.h +++ b/src/lib/malloc/map_anon.h @@ -41,6 +41,27 @@ * the child process. */ #define INHERIT_ZERO 2 +/* Here we define the NOINHERIT_CAN_FAIL macro if and only if + * it's possible that ANONMAP_NOINHERIT might yield inheritable memory. + */ +#ifdef _WIN32 +/* Windows can't fork, so NOINHERIT is never needed. */ +#elif defined(HAVE_MINHERIT) +/* minherit() will always have a working MAP_INHERIT_NONE or MAP_INHERIT_ZERO. + * NOINHERIT should always work. + */ +#elif defined(HAVE_MADVISE) +/* madvise() sometimes has neither MADV_DONTFORK and MADV_WIPEONFORK. + * We need to be ready for the possibility it failed. + * + * (Linux added DONTFORK in 2.6.16 and WIPEONFORK in 4.14. If we someday + * require 2.6.16 or later, we can assume that DONTFORK will work.) + */ +#define NOINHERIT_CAN_FAIL +#else +#define NOINHERIT_CAN_FAIL +#endif + void *tor_mmap_anonymous(size_t sz, unsigned flags, unsigned *inherit_result_out); void tor_munmap_anonymous(void *mapping, size_t sz); diff --git a/src/test/test_util.c b/src/test/test_util.c index 039fc435ce..f6085fdb90 100644 --- a/src/test/test_util.c +++ b/src/test/test_util.c @@ -6224,11 +6224,17 @@ test_util_map_anon_nofork(void *arg) int ws; waitpid(child, &ws, 0); +#ifndef NOINHERIT_CAN_FAIL + /* Only if NOINHERIT_CAN_FAIL should it be possible for us to get + * INHERIT_KEEP behavior in this case. */ + tt_assert(inherit, OP_NE, INHERIT_KEEP); +#else if (inherit == INHERIT_KEEP) { /* Call this test "skipped", not "passed", since noinherit wasn't * implemented. */ tt_skip(); } +#endif done: tor_munmap_anonymous(ptr, sz); From 785c3f84de958e90c45e82d6126a8c66958be4e1 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 6 Mar 2019 11:03:42 -0500 Subject: [PATCH 0730/2557] fast_rng: if noinherit has failed, then check getpid() for bad forks getpid() can be really expensive sometimes, and it can fail to detect some kind of fork+prng mistakes, so we need to avoid it if it's safe to do so. This patch might slow down fast_prng a lot on any old operating system that lacks a way to prevent ram from being inherited, AND requires a syscall for any getpid() calls. But it should make sure that we either crash or continue safely on incorrect fork+prng usage elsewhere in the future. --- src/lib/crypt_ops/crypto_rand_fast.c | 63 +++++++++++++++++++++++++++- 1 file changed, 61 insertions(+), 2 deletions(-) diff --git a/src/lib/crypt_ops/crypto_rand_fast.c b/src/lib/crypt_ops/crypto_rand_fast.c index f1acc9faf2..384c172c32 100644 --- a/src/lib/crypt_ops/crypto_rand_fast.c +++ b/src/lib/crypt_ops/crypto_rand_fast.c @@ -46,8 +46,25 @@ #include "lib/log/util_bug.h" +#ifdef HAVE_SYS_TYPES_H +#include +#endif +#ifdef HAVE_UNISTD_H +#include +#endif + #include +#ifdef NOINHERIT_CAN_FAIL +#define CHECK_PID +#endif + +#ifdef CHECK_PID +#define PID_FIELD_LEN sizeof(pid_t) +#else +#define PID_FIELD_LEN 0 +#endif + /* Alias for CRYPTO_FAST_RNG_SEED_LEN to make our code shorter. */ #define SEED_LEN (CRYPTO_FAST_RNG_SEED_LEN) @@ -59,7 +76,7 @@ /* The number of random bytes that we can yield to the user after each * time we fill a crypto_fast_rng_t's buffer. */ -#define BUFLEN (MAPLEN - 2*sizeof(uint16_t) - SEED_LEN) +#define BUFLEN (MAPLEN - 2*sizeof(uint16_t) - SEED_LEN - PID_FIELD_LEN) /* The number of buffer refills after which we should fetch more * entropy from crypto_strongest_rand(). @@ -82,6 +99,11 @@ struct crypto_fast_rng_t { uint16_t n_till_reseed; /** How many bytes are remaining in cbuf.bytes? */ uint16_t bytes_left; +#ifdef CHECK_PID + /** Which process owns this fast_rng? If this value is zero, we do not + * need to test the owner. */ + pid_t owner; +#endif struct cbuf { /** The seed (key and IV) that we will use the next time that we refill * cbuf. */ @@ -130,16 +152,32 @@ crypto_fast_rng_new(void) crypto_fast_rng_t * crypto_fast_rng_new_from_seed(const uint8_t *seed) { + unsigned inherit = INHERIT_KEEP; /* We try to allocate this object as securely as we can, to avoid * having it get dumped, swapped, or shared after fork. */ crypto_fast_rng_t *result = tor_mmap_anonymous(sizeof(*result), ANONMAP_PRIVATE | ANONMAP_NOINHERIT, - NULL); + &inherit); memcpy(result->buf.seed, seed, SEED_LEN); /* Causes an immediate refill once the user asks for data. */ result->bytes_left = 0; result->n_till_reseed = RESEED_AFTER; +#ifdef CHECK_PID + if (inherit == INHERIT_KEEP) { + /* This value will neither be dropped nor zeroed after fork, so we need to + * check our pid to make sure we are not sharing it across a fork. This + * can be expensive if the pid value isn't cached, sadly. + */ + result->owner = getpid(); + } +#elif defined(_WIN32) + /* Windows can't fork(), so there's no need to noinherit. */ +#else + /* We decided above that noinherit would always do _something_. Assert here + * that we were correct. */ + tor_assert(inherit != INHERIT_KEEP); +#endif return result; } @@ -211,6 +249,27 @@ static void crypto_fast_rng_getbytes_impl(crypto_fast_rng_t *rng, uint8_t *out, const size_t n) { +#ifdef CHECK_PID + if (rng->owner) { + /* Note that we only need to do this check when we have owner set: that + * is, when our attempt to block inheriting failed, and the result was + * INHERIT_KEEP. + * + * If the result was INHERIT_DROP, then any attempt to access the rng + * memory after forking will crash. + * + * If the result was INHERIT_ZERO, then forking will set the bytes_left + * and n_till_reseed fields to zero. This function will call + * crypto_fast_rng_refill(), which will in turn reseed the PRNG. + * + * So we only need to do this test in the case when mmap_anonymous() + * returned INHERIT_KEEP. We avoid doing it needlessly, since getpid() is + * often a system call, and that can be slow. + */ + tor_assert(rng->owner == getpid()); + } +#endif + size_t bytes_to_yield = n; while (bytes_to_yield) { From 027c536598f336014d7ef406610207ec4559d490 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 6 Mar 2019 12:08:25 -0500 Subject: [PATCH 0731/2557] rename inherit values to avoid conflict with system defines --- src/lib/crypt_ops/crypto_rand_fast.c | 12 ++++++------ src/lib/malloc/map_anon.c | 12 ++++++------ src/lib/malloc/map_anon.h | 6 +++--- src/test/test_util.c | 14 +++++++------- 4 files changed, 22 insertions(+), 22 deletions(-) diff --git a/src/lib/crypt_ops/crypto_rand_fast.c b/src/lib/crypt_ops/crypto_rand_fast.c index 384c172c32..01817c618f 100644 --- a/src/lib/crypt_ops/crypto_rand_fast.c +++ b/src/lib/crypt_ops/crypto_rand_fast.c @@ -152,7 +152,7 @@ crypto_fast_rng_new(void) crypto_fast_rng_t * crypto_fast_rng_new_from_seed(const uint8_t *seed) { - unsigned inherit = INHERIT_KEEP; + unsigned inherit = INHERIT_RES_KEEP; /* We try to allocate this object as securely as we can, to avoid * having it get dumped, swapped, or shared after fork. */ @@ -164,7 +164,7 @@ crypto_fast_rng_new_from_seed(const uint8_t *seed) result->bytes_left = 0; result->n_till_reseed = RESEED_AFTER; #ifdef CHECK_PID - if (inherit == INHERIT_KEEP) { + if (inherit == INHERIT_RES_KEEP) { /* This value will neither be dropped nor zeroed after fork, so we need to * check our pid to make sure we are not sharing it across a fork. This * can be expensive if the pid value isn't cached, sadly. @@ -176,7 +176,7 @@ crypto_fast_rng_new_from_seed(const uint8_t *seed) #else /* We decided above that noinherit would always do _something_. Assert here * that we were correct. */ - tor_assert(inherit != INHERIT_KEEP); + tor_assert(inherit != INHERIT_RES_KEEP); #endif return result; } @@ -253,12 +253,12 @@ crypto_fast_rng_getbytes_impl(crypto_fast_rng_t *rng, uint8_t *out, if (rng->owner) { /* Note that we only need to do this check when we have owner set: that * is, when our attempt to block inheriting failed, and the result was - * INHERIT_KEEP. + * INHERIT_RES_KEEP. * - * If the result was INHERIT_DROP, then any attempt to access the rng + * If the result was INHERIT_RES_DROP, then any attempt to access the rng * memory after forking will crash. * - * If the result was INHERIT_ZERO, then forking will set the bytes_left + * If the result was INHERIT_RES_ZERO, then forking will set the bytes_left * and n_till_reseed fields to zero. This function will call * crypto_fast_rng_refill(), which will in turn reseed the PRNG. * diff --git a/src/lib/malloc/map_anon.c b/src/lib/malloc/map_anon.c index bb0a3d5741..3e1c22648d 100644 --- a/src/lib/malloc/map_anon.c +++ b/src/lib/malloc/map_anon.c @@ -123,14 +123,14 @@ noinherit_mem(void *mem, size_t sz, unsigned *inherit_result_out) #ifdef FLAG_ZERO int r = MINHERIT(mem, sz, FLAG_ZERO); if (r == 0) { - *inherit_result_out = INHERIT_ZERO; + *inherit_result_out = INHERIT_RES_ZERO; return 0; } #endif #ifdef FLAG_NOINHERIT int r2 = MINHERIT(mem, sz, FLAG_NOINHERIT); if (r2 == 0) { - *inherit_result_out = INHERIT_DROP; + *inherit_result_out = INHERIT_RES_DROP; } return r2; #else @@ -154,9 +154,9 @@ noinherit_mem(void *mem, size_t sz, unsigned *inherit_result_out) * Memory returned from this function must be released with * tor_munmap_anonymous(). * - * If inherit_result_out is non-NULL, set it to one of INHERIT_KEEP, - * INHERIT_DROP, and INHERIT_ZERO, depending on the properties of the returned - * memory. + * If inherit_result_out is non-NULL, set it to one of + * INHERIT_RES_KEEP, INHERIT_RES_DROP, or INHERIT_RES_ZERO, depending on the + * properties of the returned memory. * * [Note: OS people use the word "anonymous" here to mean that the memory * isn't associated with any file. This has *nothing* to do with the kind of @@ -170,7 +170,7 @@ tor_mmap_anonymous(size_t sz, unsigned flags, unsigned *inherit_result_out) if (inherit_result_out == NULL) { inherit_result_out = &itmp; } - *inherit_result_out = INHERIT_KEEP; + *inherit_result_out = INHERIT_RES_KEEP; #if defined(_WIN32) HANDLE mapping = CreateFileMapping(INVALID_HANDLE_VALUE, diff --git a/src/lib/malloc/map_anon.h b/src/lib/malloc/map_anon.h index 89fb9da0f0..f101654eb8 100644 --- a/src/lib/malloc/map_anon.h +++ b/src/lib/malloc/map_anon.h @@ -33,13 +33,13 @@ /** Possible value for inherit_result_out: the memory will be kept * by any child process. */ -#define INHERIT_KEEP 0 +#define INHERIT_RES_KEEP 0 /** Possible value for inherit_result_out: the memory will be dropped in * the child process. Attempting to access it will likely cause a segfault. */ -#define INHERIT_DROP 1 +#define INHERIT_RES_DROP 1 /** Possible value for inherit_result_out: the memory will be cleared in * the child process. */ -#define INHERIT_ZERO 2 +#define INHERIT_RES_ZERO 2 /* Here we define the NOINHERIT_CAN_FAIL macro if and only if * it's possible that ANONMAP_NOINHERIT might yield inheritable memory. diff --git a/src/test/test_util.c b/src/test/test_util.c index f6085fdb90..4f81e54d2b 100644 --- a/src/test/test_util.c +++ b/src/test/test_util.c @@ -6135,7 +6135,7 @@ test_util_map_anon(void *arg) /* Basic checks. */ ptr = tor_mmap_anonymous(sz, 0, &inherit); tt_ptr_op(ptr, OP_NE, 0); - tt_int_op(inherit, OP_EQ, INHERIT_KEEP); + tt_int_op(inherit, OP_EQ, INHERIT_RES_KEEP); ptr[sz-1] = 3; tt_int_op(ptr[0], OP_EQ, 0); tt_int_op(ptr[sz-2], OP_EQ, 0); @@ -6145,7 +6145,7 @@ test_util_map_anon(void *arg) tor_munmap_anonymous(ptr, sz); ptr = tor_mmap_anonymous(sz, ANONMAP_PRIVATE, &inherit); tt_ptr_op(ptr, OP_NE, 0); - tt_int_op(inherit, OP_EQ, INHERIT_KEEP); + tt_int_op(inherit, OP_EQ, INHERIT_RES_KEEP); ptr[sz-1] = 10; tt_int_op(ptr[0], OP_EQ, 0); tt_int_op(ptr[sz/2], OP_EQ, 0); @@ -6207,16 +6207,16 @@ test_util_map_anon_nofork(void *arg) char buf[1]; ssize_t r = read(pipefd[0], buf, 1); - if (inherit == INHERIT_ZERO) { + if (inherit == INHERIT_RES_ZERO) { // We should be seeing clear-on-fork behavior. tt_int_op((int)r, OP_EQ, 1); // child should send us a byte. tt_int_op(buf[0], OP_EQ, 0); // that byte should be zero. - } else if (inherit == INHERIT_DROP) { + } else if (inherit == INHERIT_RES_DROP) { // We should be seeing noinherit behavior. tt_int_op(r, OP_LE, 0); // child said nothing; it should have crashed. } else { // noinherit isn't implemented. - tt_int_op(inherit, OP_EQ, INHERIT_KEEP); + tt_int_op(inherit, OP_EQ, INHERIT_RES_KEEP); tt_int_op((int)r, OP_EQ, 1); // child should send us a byte. tt_int_op(buf[0], OP_EQ, 0xd0); // that byte should what we set it to. } @@ -6227,9 +6227,9 @@ test_util_map_anon_nofork(void *arg) #ifndef NOINHERIT_CAN_FAIL /* Only if NOINHERIT_CAN_FAIL should it be possible for us to get * INHERIT_KEEP behavior in this case. */ - tt_assert(inherit, OP_NE, INHERIT_KEEP); + tt_assert(inherit, OP_NE, INHERIT_RES_KEEP); #else - if (inherit == INHERIT_KEEP) { + if (inherit == INHERIT_RES_KEEP) { /* Call this test "skipped", not "passed", since noinherit wasn't * implemented. */ tt_skip(); From 8c06f02c94067d7ad1bf39982e7238b7fafae0e0 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 6 Mar 2019 12:09:46 -0500 Subject: [PATCH 0732/2557] Syntax fix in test. --- src/test/test_util.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/test_util.c b/src/test/test_util.c index 4f81e54d2b..c683c9475b 100644 --- a/src/test/test_util.c +++ b/src/test/test_util.c @@ -6227,7 +6227,7 @@ test_util_map_anon_nofork(void *arg) #ifndef NOINHERIT_CAN_FAIL /* Only if NOINHERIT_CAN_FAIL should it be possible for us to get * INHERIT_KEEP behavior in this case. */ - tt_assert(inherit, OP_NE, INHERIT_RES_KEEP); + tt_int_op(inherit, OP_NE, INHERIT_RES_KEEP); #else if (inherit == INHERIT_RES_KEEP) { /* Call this test "skipped", not "passed", since noinherit wasn't From 76912bf140ec61856c0bb0d25354283d024229f5 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 3 Apr 2019 10:57:06 -0400 Subject: [PATCH 0733/2557] Use an enum for inherit_result_out. --- src/lib/malloc/map_anon.c | 7 ++++--- src/lib/malloc/map_anon.h | 22 ++++++++++++---------- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/src/lib/malloc/map_anon.c b/src/lib/malloc/map_anon.c index 3e1c22648d..f4fda00bff 100644 --- a/src/lib/malloc/map_anon.c +++ b/src/lib/malloc/map_anon.c @@ -118,7 +118,7 @@ nodump_mem(void *mem, size_t sz) * *inherit_result_out. */ static int -noinherit_mem(void *mem, size_t sz, unsigned *inherit_result_out) +noinherit_mem(void *mem, size_t sz, inherit_res_t *inherit_result_out) { #ifdef FLAG_ZERO int r = MINHERIT(mem, sz, FLAG_ZERO); @@ -163,10 +163,11 @@ noinherit_mem(void *mem, size_t sz, unsigned *inherit_result_out) * anonymity that Tor is trying to provide.] */ void * -tor_mmap_anonymous(size_t sz, unsigned flags, unsigned *inherit_result_out) +tor_mmap_anonymous(size_t sz, unsigned flags, + inherit_res_t *inherit_result_out) { void *ptr; - unsigned itmp=0; + inherit_res_t itmp=0; if (inherit_result_out == NULL) { inherit_result_out = &itmp; } diff --git a/src/lib/malloc/map_anon.h b/src/lib/malloc/map_anon.h index f101654eb8..6c02cd6c16 100644 --- a/src/lib/malloc/map_anon.h +++ b/src/lib/malloc/map_anon.h @@ -31,15 +31,17 @@ */ #define ANONMAP_NOINHERIT (1u<<1) -/** Possible value for inherit_result_out: the memory will be kept - * by any child process. */ -#define INHERIT_RES_KEEP 0 -/** Possible value for inherit_result_out: the memory will be dropped in - * the child process. Attempting to access it will likely cause a segfault. */ -#define INHERIT_RES_DROP 1 -/** Possible value for inherit_result_out: the memory will be cleared in - * the child process. */ -#define INHERIT_RES_ZERO 2 +typedef enum { + /** Possible value for inherit_result_out: the memory will be kept + * by any child process. */ + INHERIT_RES_KEEP=0, + /** Possible value for inherit_result_out: the memory will be dropped in the + * child process. Attempting to access it will likely cause a segfault. */ + INHERIT_RES_DROP, + /** Possible value for inherit_result_out: the memory will be cleared in + * the child process. */ + INHERIT_RES_ZERO +} inherit_res_t; /* Here we define the NOINHERIT_CAN_FAIL macro if and only if * it's possible that ANONMAP_NOINHERIT might yield inheritable memory. @@ -63,7 +65,7 @@ #endif void *tor_mmap_anonymous(size_t sz, unsigned flags, - unsigned *inherit_result_out); + inherit_res_t *inherit_result_out); void tor_munmap_anonymous(void *mapping, size_t sz); #endif /* !defined(TOR_MAP_ANON_H) */ From 8e961b2174f6714fdff295f032abe993f864b81b Mon Sep 17 00:00:00 2001 From: teor Date: Sat, 30 Mar 2019 12:09:47 +1000 Subject: [PATCH 0734/2557] bwauth: Actually include the bandwidth-file-digest in authority votes Fixes bug 29959; bugfix on 0.4.0.2-alpha. --- changes/bug29959-040 | 3 +++ src/feature/dirauth/dirvote.c | 21 ++++++++++----------- 2 files changed, 13 insertions(+), 11 deletions(-) create mode 100644 changes/bug29959-040 diff --git a/changes/bug29959-040 b/changes/bug29959-040 new file mode 100644 index 0000000000..3740e0169a --- /dev/null +++ b/changes/bug29959-040 @@ -0,0 +1,3 @@ + o Minor bugfixes (directory authorities): + - Actually include the bandwidth-file-digest line in directory authority + votes. Fixes bug 29959; bugfix on 0.4.0.2-alpha. diff --git a/src/feature/dirauth/dirvote.c b/src/feature/dirauth/dirvote.c index ba7b2f1de6..29f5d04509 100644 --- a/src/feature/dirauth/dirvote.c +++ b/src/feature/dirauth/dirvote.c @@ -320,18 +320,17 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key, if (!tor_digest256_is_zero((const char *)v3_ns->bw_file_digest256)) { /* Encode the digest. */ char b64_digest_bw_file[BASE64_DIGEST256_LEN+1] = {0}; - if (digest256_to_base64(b64_digest_bw_file, - (const char *)v3_ns->bw_file_digest256)>0) { - /* "bandwidth-file-digest" 1*(SP algorithm "=" digest) NL */ - char *digest_algo_b64_digest_bw_file = NULL; - tor_asprintf(&digest_algo_b64_digest_bw_file, "%s=%s", - crypto_digest_algorithm_get_name(DIGEST_ALG_BW_FILE), - b64_digest_bw_file); - /* No need for tor_strdup(""), format_line_if_present does it. */ - bw_file_digest = format_line_if_present( + digest256_to_base64(b64_digest_bw_file, + (const char *)v3_ns->bw_file_digest256); + /* "bandwidth-file-digest" 1*(SP algorithm "=" digest) NL */ + char *digest_algo_b64_digest_bw_file = NULL; + tor_asprintf(&digest_algo_b64_digest_bw_file, "%s=%s", + crypto_digest_algorithm_get_name(DIGEST_ALG_BW_FILE), + b64_digest_bw_file); + /* No need for tor_strdup(""), format_line_if_present does it. */ + bw_file_digest = format_line_if_present( "bandwidth-file-digest", digest_algo_b64_digest_bw_file); - tor_free(digest_algo_b64_digest_bw_file); - } + tor_free(digest_algo_b64_digest_bw_file); } smartlist_add_asprintf(chunks, From 1f48c6cd83d66c904e7339e1ff60f5e7c5f6bbf4 Mon Sep 17 00:00:00 2001 From: Mike Perry Date: Wed, 27 Feb 2019 00:27:22 +0000 Subject: [PATCH 0735/2557] Bug 29500: Attempt to fix the tokens test. Cancel the padding timer by changing order of sent vs recv (sent cancels). --- src/test/test_circuitpadding.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/test/test_circuitpadding.c b/src/test/test_circuitpadding.c index eee1edc50c..077529a0dc 100644 --- a/src/test/test_circuitpadding.c +++ b/src/test/test_circuitpadding.c @@ -1004,9 +1004,8 @@ test_circuitpadding_tokens(void *arg) mi = client_side->padding_info[0]; // Pretend a non-padding cell was sent - // XXX: This messes us up.. Padding gets scheduled.. - circpad_cell_event_nonpadding_sent((circuit_t*)client_side); circpad_cell_event_nonpadding_received((circuit_t*)client_side); + circpad_cell_event_nonpadding_sent((circuit_t*)client_side); /* We have to save the infinity bin because one inf delay * could have been chosen when we transition to burst */ circpad_hist_token_t inf_bin = mi->histogram[4]; @@ -1105,7 +1104,8 @@ test_circuitpadding_tokens(void *arg) /* 2.c. Bin 0 */ { - tt_int_op(mi->histogram[0], OP_EQ, 1); + tt_int_op(mi->histogram[0], OP_EQ, 0); + mi->histogram[0] = 1; circpad_machine_remove_higher_token(mi, state->start_usec/2); tt_int_op(mi->histogram[0], OP_EQ, 0); From b027b06dbbbfcac6be91a9a89bda0b0949bd8b7d Mon Sep 17 00:00:00 2001 From: Mike Perry Date: Mon, 4 Mar 2019 21:54:19 +0000 Subject: [PATCH 0736/2557] Bug 29500: Start monotime at 1000 nsec. Hopefully this will stop monotime_absolute_usec() from returning 0 on some platforms in the tests. --- src/test/test_circuitpadding.c | 36 +++++++++++++++++----------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/src/test/test_circuitpadding.c b/src/test/test_circuitpadding.c index 077529a0dc..cd150b0646 100644 --- a/src/test/test_circuitpadding.c +++ b/src/test/test_circuitpadding.c @@ -300,9 +300,9 @@ test_circuitpadding_rtt(void *arg) monotime_init(); monotime_enable_test_mocking(); - monotime_set_mock_time_nsec(1*TOR_NSEC_PER_USEC); - monotime_coarse_set_mock_time_nsec(1*TOR_NSEC_PER_USEC); - curr_mocked_time = 1*TOR_NSEC_PER_USEC; + monotime_set_mock_time_nsec(1000*TOR_NSEC_PER_USEC); + monotime_coarse_set_mock_time_nsec(1000*TOR_NSEC_PER_USEC); + curr_mocked_time = 1000*TOR_NSEC_PER_USEC; timers_initialize(); circpad_machines_init(); @@ -990,9 +990,9 @@ test_circuitpadding_tokens(void *arg) monotime_init(); monotime_enable_test_mocking(); - monotime_set_mock_time_nsec(1*TOR_NSEC_PER_USEC); - monotime_coarse_set_mock_time_nsec(1*TOR_NSEC_PER_USEC); - curr_mocked_time = 1*TOR_NSEC_PER_USEC; + monotime_set_mock_time_nsec(1000*TOR_NSEC_PER_USEC); + monotime_coarse_set_mock_time_nsec(1000*TOR_NSEC_PER_USEC); + curr_mocked_time = 1000*TOR_NSEC_PER_USEC; timers_initialize(); @@ -1254,9 +1254,9 @@ test_circuitpadding_wronghop(void *arg) monotime_init(); monotime_enable_test_mocking(); - monotime_set_mock_time_nsec(1*TOR_NSEC_PER_USEC); - monotime_coarse_set_mock_time_nsec(1*TOR_NSEC_PER_USEC); - curr_mocked_time = 1*TOR_NSEC_PER_USEC; + monotime_set_mock_time_nsec(1000*TOR_NSEC_PER_USEC); + monotime_coarse_set_mock_time_nsec(1000*TOR_NSEC_PER_USEC); + curr_mocked_time = 1000*TOR_NSEC_PER_USEC; timers_initialize(); circpad_machines_init(); @@ -1441,9 +1441,9 @@ test_circuitpadding_negotiation(void *arg) monotime_init(); monotime_enable_test_mocking(); - monotime_set_mock_time_nsec(1*TOR_NSEC_PER_USEC); - monotime_coarse_set_mock_time_nsec(1*TOR_NSEC_PER_USEC); - curr_mocked_time = 1*TOR_NSEC_PER_USEC; + monotime_set_mock_time_nsec(1000*TOR_NSEC_PER_USEC); + monotime_coarse_set_mock_time_nsec(1000*TOR_NSEC_PER_USEC); + curr_mocked_time = 1000*TOR_NSEC_PER_USEC; timers_initialize(); circpad_machines_init(); @@ -1716,9 +1716,9 @@ test_circuitpadding_conditions(void *arg) monotime_init(); monotime_enable_test_mocking(); - monotime_set_mock_time_nsec(1*TOR_NSEC_PER_USEC); - monotime_coarse_set_mock_time_nsec(1*TOR_NSEC_PER_USEC); - curr_mocked_time = 1*TOR_NSEC_PER_USEC; + monotime_set_mock_time_nsec(1000*TOR_NSEC_PER_USEC); + monotime_coarse_set_mock_time_nsec(1000*TOR_NSEC_PER_USEC); + curr_mocked_time = 1000*TOR_NSEC_PER_USEC; timers_initialize(); helper_create_conditional_machines(); @@ -2261,9 +2261,9 @@ test_circuitpadding_global_rate_limiting(void *arg) monotime_init(); monotime_enable_test_mocking(); - monotime_set_mock_time_nsec(1*TOR_NSEC_PER_USEC); - monotime_coarse_set_mock_time_nsec(1*TOR_NSEC_PER_USEC); - curr_mocked_time = 1*TOR_NSEC_PER_USEC; + monotime_set_mock_time_nsec(1000*TOR_NSEC_PER_USEC); + monotime_coarse_set_mock_time_nsec(1000*TOR_NSEC_PER_USEC); + curr_mocked_time = 1000*TOR_NSEC_PER_USEC; timers_initialize(); client_side = (circuit_t *)origin_circuit_new(); From 28db7646ba64673188be95d3204abb94ce890b86 Mon Sep 17 00:00:00 2001 From: Mike Perry Date: Mon, 4 Mar 2019 21:54:56 +0000 Subject: [PATCH 0737/2557] Changes file for bug 29500. --- changes/bug29500 | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 changes/bug29500 diff --git a/changes/bug29500 b/changes/bug29500 new file mode 100644 index 0000000000..16550935b2 --- /dev/null +++ b/changes/bug29500 @@ -0,0 +1,3 @@ + o Minor bugfixes (circuitpadding testing): + - Minor tweaks to avoid very rare test failures related to timers and + monotime. Fixes bug 29500; bugfix on 0.4.0.1-alpha From b733044f7a41d65b6f3937056e9499613218ab0b Mon Sep 17 00:00:00 2001 From: Mike Perry Date: Fri, 5 Apr 2019 00:21:07 +0000 Subject: [PATCH 0738/2557] Bug #29500: Fix monotime mocking in circpad unittests. Our monotime mocking forces us to call monotime_init() *before* we set the mocked time value. monotime_init() thus stores the first ratchet value at whatever the platform is at, and then we set fake mocked time to some later value. If monotime_init() gets a value from the host that is greater than what we choose to mock time at for our unittests, all subsequent monotime_abosolute() calls return zero, which breaks all unittests that depend on time moving forward by updating mocked monotime values. So, we need to adjust our mocked time to take the weird monotime_init() time into account, when we set fake time. --- src/test/test_circuitpadding.c | 62 ++++++++++++++++++++++------------ 1 file changed, 41 insertions(+), 21 deletions(-) diff --git a/src/test/test_circuitpadding.c b/src/test/test_circuitpadding.c index cd150b0646..6272c0c2ac 100644 --- a/src/test/test_circuitpadding.c +++ b/src/test/test_circuitpadding.c @@ -31,6 +31,12 @@ #include "core/or/or_circuit_st.h" #include "core/or/origin_circuit_st.h" +/* Start our monotime mocking at 1 second past whatever monotime_init() + * thought the actual wall clock time was, for platforms with bad resolution + * and weird timevalues during monotime_init() before mocking. */ +#define MONOTIME_MOCK_START (monotime_absolute_nsec()+\ + TOR_NSEC_PER_USEC*TOR_USEC_PER_SEC) + extern smartlist_t *connection_array; circid_t get_unique_circ_id_by_chan(channel_t *chan); @@ -287,6 +293,7 @@ test_circuitpadding_rtt(void *arg) * 3. Test client side circuit and non-application of RTT.. */ circpad_delay_t rtt_estimate; + int64_t actual_mocked_monotime_start; (void)arg; MOCK(circuitmux_attach_circuit, circuitmux_attach_circuit_mock); @@ -300,9 +307,10 @@ test_circuitpadding_rtt(void *arg) monotime_init(); monotime_enable_test_mocking(); - monotime_set_mock_time_nsec(1000*TOR_NSEC_PER_USEC); - monotime_coarse_set_mock_time_nsec(1000*TOR_NSEC_PER_USEC); - curr_mocked_time = 1000*TOR_NSEC_PER_USEC; + actual_mocked_monotime_start = MONOTIME_MOCK_START; + monotime_set_mock_time_nsec(actual_mocked_monotime_start); + monotime_coarse_set_mock_time_nsec(actual_mocked_monotime_start); + curr_mocked_time = actual_mocked_monotime_start; timers_initialize(); circpad_machines_init(); @@ -963,6 +971,7 @@ test_circuitpadding_tokens(void *arg) { const circpad_state_t *state; circpad_machine_state_t *mi; + int64_t actual_mocked_monotime_start; (void)arg; /** Test plan: @@ -990,9 +999,10 @@ test_circuitpadding_tokens(void *arg) monotime_init(); monotime_enable_test_mocking(); - monotime_set_mock_time_nsec(1000*TOR_NSEC_PER_USEC); - monotime_coarse_set_mock_time_nsec(1000*TOR_NSEC_PER_USEC); - curr_mocked_time = 1000*TOR_NSEC_PER_USEC; + actual_mocked_monotime_start = MONOTIME_MOCK_START; + monotime_set_mock_time_nsec(actual_mocked_monotime_start); + monotime_coarse_set_mock_time_nsec(actual_mocked_monotime_start); + curr_mocked_time = actual_mocked_monotime_start; timers_initialize(); @@ -1235,6 +1245,7 @@ test_circuitpadding_wronghop(void *arg) cell_t cell; signed_error_t ret; origin_circuit_t *orig_client; + int64_t actual_mocked_monotime_start; MOCK(circuitmux_attach_circuit, circuitmux_attach_circuit_mock); @@ -1254,9 +1265,10 @@ test_circuitpadding_wronghop(void *arg) monotime_init(); monotime_enable_test_mocking(); - monotime_set_mock_time_nsec(1000*TOR_NSEC_PER_USEC); - monotime_coarse_set_mock_time_nsec(1000*TOR_NSEC_PER_USEC); - curr_mocked_time = 1000*TOR_NSEC_PER_USEC; + actual_mocked_monotime_start = MONOTIME_MOCK_START; + monotime_set_mock_time_nsec(actual_mocked_monotime_start); + monotime_coarse_set_mock_time_nsec(actual_mocked_monotime_start); + curr_mocked_time = actual_mocked_monotime_start; timers_initialize(); circpad_machines_init(); @@ -1427,6 +1439,7 @@ test_circuitpadding_negotiation(void *arg) * a. Make sure padding negotiation is not sent * 3. Test failure to negotiate a machine due to desync. */ + int64_t actual_mocked_monotime_start; (void)arg; MOCK(circuitmux_attach_circuit, circuitmux_attach_circuit_mock); @@ -1441,9 +1454,10 @@ test_circuitpadding_negotiation(void *arg) monotime_init(); monotime_enable_test_mocking(); - monotime_set_mock_time_nsec(1000*TOR_NSEC_PER_USEC); - monotime_coarse_set_mock_time_nsec(1000*TOR_NSEC_PER_USEC); - curr_mocked_time = 1000*TOR_NSEC_PER_USEC; + actual_mocked_monotime_start = MONOTIME_MOCK_START; + monotime_set_mock_time_nsec(actual_mocked_monotime_start); + monotime_coarse_set_mock_time_nsec(actual_mocked_monotime_start); + curr_mocked_time = actual_mocked_monotime_start; timers_initialize(); circpad_machines_init(); @@ -1703,6 +1717,7 @@ test_circuitpadding_conditions(void *arg) * 2. Test marking a circuit before padding callback fires * 3. Test freeing a circuit before padding callback fires */ + int64_t actual_mocked_monotime_start; (void)arg; MOCK(circuitmux_attach_circuit, circuitmux_attach_circuit_mock); @@ -1716,9 +1731,10 @@ test_circuitpadding_conditions(void *arg) monotime_init(); monotime_enable_test_mocking(); - monotime_set_mock_time_nsec(1000*TOR_NSEC_PER_USEC); - monotime_coarse_set_mock_time_nsec(1000*TOR_NSEC_PER_USEC); - curr_mocked_time = 1000*TOR_NSEC_PER_USEC; + actual_mocked_monotime_start = MONOTIME_MOCK_START; + monotime_set_mock_time_nsec(actual_mocked_monotime_start); + monotime_coarse_set_mock_time_nsec(actual_mocked_monotime_start); + curr_mocked_time = actual_mocked_monotime_start; timers_initialize(); helper_create_conditional_machines(); @@ -1813,6 +1829,7 @@ test_circuitpadding_conditions(void *arg) void test_circuitpadding_circuitsetup_machine(void *arg) { + int64_t actual_mocked_monotime_start; /** * Test case plan: * @@ -1838,9 +1855,10 @@ test_circuitpadding_circuitsetup_machine(void *arg) monotime_init(); monotime_enable_test_mocking(); - monotime_set_mock_time_nsec(1*TOR_NSEC_PER_USEC); - monotime_coarse_set_mock_time_nsec(1*TOR_NSEC_PER_USEC); - curr_mocked_time = 1*TOR_NSEC_PER_USEC; + actual_mocked_monotime_start = MONOTIME_MOCK_START; + monotime_set_mock_time_nsec(actual_mocked_monotime_start); + monotime_coarse_set_mock_time_nsec(actual_mocked_monotime_start); + curr_mocked_time = actual_mocked_monotime_start; timers_initialize(); circpad_machines_init(); @@ -2250,6 +2268,7 @@ test_circuitpadding_global_rate_limiting(void *arg) bool retval; circpad_machine_state_t *mi; int i; + int64_t actual_mocked_monotime_start; /* Ignore machine transitions for the purposes of this function, we only * really care about padding counts */ @@ -2261,9 +2280,10 @@ test_circuitpadding_global_rate_limiting(void *arg) monotime_init(); monotime_enable_test_mocking(); - monotime_set_mock_time_nsec(1000*TOR_NSEC_PER_USEC); - monotime_coarse_set_mock_time_nsec(1000*TOR_NSEC_PER_USEC); - curr_mocked_time = 1000*TOR_NSEC_PER_USEC; + actual_mocked_monotime_start = MONOTIME_MOCK_START; + monotime_set_mock_time_nsec(actual_mocked_monotime_start); + monotime_coarse_set_mock_time_nsec(actual_mocked_monotime_start); + curr_mocked_time = actual_mocked_monotime_start; timers_initialize(); client_side = (circuit_t *)origin_circuit_new(); From 387d9448de4c5044800706b4ed21b48b4426cb7e Mon Sep 17 00:00:00 2001 From: teor Date: Tue, 2 Apr 2019 19:56:38 +1000 Subject: [PATCH 0739/2557] test/circuitpadding: Delete circuitpadding_circuitsetup_machine() This test was disabled in 0.4.0 and later, but the fix in #29298 was only merged to 0.4.1. So this test will never be re-enabled in 0.4.0. Part of 29500. --- src/test/test_circuitpadding.c | 253 --------------------------------- 1 file changed, 253 deletions(-) diff --git a/src/test/test_circuitpadding.c b/src/test/test_circuitpadding.c index 6272c0c2ac..ed048ab968 100644 --- a/src/test/test_circuitpadding.c +++ b/src/test/test_circuitpadding.c @@ -1824,257 +1824,6 @@ test_circuitpadding_conditions(void *arg) return; } -/** Disabled unstable test until #29298 is implemented (see #29122) */ -#if 0 -void -test_circuitpadding_circuitsetup_machine(void *arg) -{ - int64_t actual_mocked_monotime_start; - /** - * Test case plan: - * - * 1. Simulate a normal circuit setup pattern - * a. Application traffic - * - * FIXME: This should focus more on exercising the machine - * features rather than actual traffic patterns. For example, - * test cancellation and bins empty/refill - */ - (void)arg; - - MOCK(circuitmux_attach_circuit, circuitmux_attach_circuit_mock); - - dummy_channel.cmux = circuitmux_alloc(); - client_side = TO_CIRCUIT(origin_circuit_new()); - relay_side = TO_CIRCUIT(new_fake_orcirc(&dummy_channel, &dummy_channel)); - - relay_side->purpose = CIRCUIT_PURPOSE_OR; - client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL; - - nodes_init(); - - monotime_init(); - monotime_enable_test_mocking(); - actual_mocked_monotime_start = MONOTIME_MOCK_START; - monotime_set_mock_time_nsec(actual_mocked_monotime_start); - monotime_coarse_set_mock_time_nsec(actual_mocked_monotime_start); - curr_mocked_time = actual_mocked_monotime_start; - - timers_initialize(); - circpad_machines_init(); - - MOCK(circuit_package_relay_cell, - circuit_package_relay_cell_mock); - MOCK(node_get_by_id, - node_get_by_id_mock); - - /* Test case #1: Build a 3 hop circuit, then wait and let pad */ - simulate_single_hop_extend(client_side, relay_side, 1); - simulate_single_hop_extend(client_side, relay_side, 1); - simulate_single_hop_extend(client_side, relay_side, 1); - - tt_int_op(n_client_cells, OP_EQ, 1); - tt_int_op(n_relay_cells, OP_EQ, 1); - tt_int_op(client_side->padding_info[0]->current_state, OP_EQ, - CIRCPAD_STATE_BURST); - tt_int_op(relay_side->padding_info[0]->current_state, OP_EQ, - CIRCPAD_STATE_BURST); - - tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec, - OP_NE, 0); - tt_int_op(relay_side->padding_info[0]->is_padding_timer_scheduled, - OP_EQ, 0); - timers_advance_and_run(2000); - tt_int_op(n_client_cells, OP_EQ, 2); - tt_int_op(n_relay_cells, OP_EQ, 1); - - tt_int_op(relay_side->padding_info[0]->current_state, OP_EQ, - CIRCPAD_STATE_GAP); - - tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec, - OP_EQ, 0); - tt_u64_op(relay_side->padding_info[0]->padding_scheduled_at_usec, - OP_NE, 0); - timers_advance_and_run(5000); - tt_int_op(n_client_cells, OP_EQ, 2); - tt_int_op(n_relay_cells, OP_EQ, 2); - - tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec, - OP_NE, 0); - tt_u64_op(relay_side->padding_info[0]->padding_scheduled_at_usec, - OP_EQ, 0); - timers_advance_and_run(2000); - tt_int_op(n_client_cells, OP_EQ, 3); - tt_int_op(n_relay_cells, OP_EQ, 2); - - tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec, - OP_EQ, 0); - tt_u64_op(relay_side->padding_info[0]->padding_scheduled_at_usec, - OP_NE, 0); - timers_advance_and_run(5000); - tt_int_op(n_client_cells, OP_EQ, 3); - tt_int_op(n_relay_cells, OP_EQ, 3); - - tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec, - OP_NE, 0); - tt_u64_op(relay_side->padding_info[0]->padding_scheduled_at_usec, - OP_EQ, 0); - timers_advance_and_run(2000); - tt_int_op(n_client_cells, OP_EQ, 4); - tt_int_op(n_relay_cells, OP_EQ, 3); - - tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec, - OP_EQ, 0); - tt_u64_op(relay_side->padding_info[0]->padding_scheduled_at_usec, - OP_NE, 0); - timers_advance_and_run(5000); - tt_int_op(n_client_cells, OP_EQ, 4); - tt_int_op(n_relay_cells, OP_EQ, 4); - - tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec, - OP_NE, 0); - tt_u64_op(relay_side->padding_info[0]->padding_scheduled_at_usec, - OP_EQ, 0); - timers_advance_and_run(2000); - tt_int_op(n_client_cells, OP_EQ, 5); - tt_int_op(n_relay_cells, OP_EQ, 4); - - tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec, - OP_EQ, 0); - tt_u64_op(relay_side->padding_info[0]->padding_scheduled_at_usec, - OP_NE, 0); - timers_advance_and_run(5000); - tt_int_op(n_client_cells, OP_EQ, 5); - tt_int_op(n_relay_cells, OP_EQ, 5); - - tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec, - OP_NE, 0); - tt_u64_op(relay_side->padding_info[0]->padding_scheduled_at_usec, - OP_EQ, 0); - timers_advance_and_run(2000); - tt_int_op(n_client_cells, OP_EQ, 6); - tt_int_op(n_relay_cells, OP_EQ, 5); - - tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec, - OP_EQ, 0); - tt_u64_op(relay_side->padding_info[0]->padding_scheduled_at_usec, - OP_NE, 0); - timers_advance_and_run(5000); - tt_int_op(n_client_cells, OP_EQ, 6); - tt_int_op(n_relay_cells, OP_EQ, 6); - - tt_int_op(client_side->padding_info[0]->current_state, - OP_EQ, CIRCPAD_STATE_END); - tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec, - OP_EQ, 0); - tt_int_op(relay_side->padding_info[0]->current_state, - OP_EQ, CIRCPAD_STATE_GAP); - tt_u64_op(relay_side->padding_info[0]->padding_scheduled_at_usec, - OP_EQ, 0); - - /* Verify we can't schedule padding in END state */ - circpad_decision_t ret = - circpad_machine_schedule_padding(client_side->padding_info[0]); - tt_int_op(ret, OP_EQ, CIRCPAD_STATE_UNCHANGED); - - /* Simulate application traffic */ - circpad_cell_event_nonpadding_sent(client_side); - circpad_deliver_unrecognized_cell_events(relay_side, CELL_DIRECTION_OUT); - circpad_deliver_unrecognized_cell_events(relay_side, CELL_DIRECTION_IN); - circpad_deliver_recognized_relay_cell_events(client_side, RELAY_COMMAND_DATA, - TO_ORIGIN_CIRCUIT(client_side)->cpath->next); - - tt_ptr_op(client_side->padding_info[0], OP_EQ, NULL); - tt_ptr_op(client_side->padding_machine[0], OP_EQ, NULL); - - tt_ptr_op(relay_side->padding_info[0], OP_EQ, NULL); - tt_ptr_op(relay_side->padding_machine[0], OP_EQ, NULL); - tt_int_op(n_client_cells, OP_EQ, 6); - tt_int_op(n_relay_cells, OP_EQ, 7); - - // Test timer cancellation - simulate_single_hop_extend(client_side, relay_side, 1); - simulate_single_hop_extend(client_side, relay_side, 1); - timers_advance_and_run(5000); - circpad_cell_event_padding_received(client_side); - - tt_int_op(client_side->padding_info[0]->current_state, OP_EQ, - CIRCPAD_STATE_BURST); - tt_int_op(relay_side->padding_info[0]->current_state, OP_EQ, - CIRCPAD_STATE_GAP); - - tt_int_op(n_client_cells, OP_EQ, 8); - tt_int_op(n_relay_cells, OP_EQ, 8); - tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec, - OP_NE, 0); - tt_u64_op(relay_side->padding_info[0]->padding_scheduled_at_usec, - OP_NE, 0); - - /* Test timer cancel due to state rules */ - circpad_cell_event_nonpadding_sent(client_side); - tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec, - OP_EQ, 0); - circpad_cell_event_padding_received(client_side); - tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec, - OP_NE, 0); - - /* Simulate application traffic to cancel timer */ - circpad_cell_event_nonpadding_sent(client_side); - circpad_deliver_unrecognized_cell_events(relay_side, CELL_DIRECTION_OUT); - circpad_deliver_unrecognized_cell_events(relay_side, CELL_DIRECTION_IN); - circpad_deliver_recognized_relay_cell_events(client_side, RELAY_COMMAND_DATA, - TO_ORIGIN_CIRCUIT(client_side)->cpath->next); - - tt_ptr_op(client_side->padding_info[0], OP_EQ, NULL); - tt_ptr_op(client_side->padding_machine[0], OP_EQ, NULL); - - tt_ptr_op(relay_side->padding_info[0], OP_EQ, NULL); - tt_ptr_op(relay_side->padding_machine[0], OP_EQ, NULL); - - /* No cells sent, except negotiate end from relay */ - tt_int_op(n_client_cells, OP_EQ, 8); - tt_int_op(n_relay_cells, OP_EQ, 9); - - /* Test mark for close and free */ - simulate_single_hop_extend(client_side, relay_side, 1); - simulate_single_hop_extend(client_side, relay_side, 1); - timers_advance_and_run(5000); - circpad_cell_event_padding_received(client_side); - - tt_int_op(n_client_cells, OP_EQ, 10); - tt_int_op(n_relay_cells, OP_EQ, 10); - - tt_int_op(client_side->padding_info[0]->current_state, OP_EQ, - CIRCPAD_STATE_BURST); - tt_int_op(relay_side->padding_info[0]->current_state, OP_EQ, - CIRCPAD_STATE_GAP); - - tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec, - OP_NE, 0); - tt_u64_op(relay_side->padding_info[0]->padding_scheduled_at_usec, - OP_NE, 0); - circuit_mark_for_close(client_side, END_CIRC_REASON_FLAG_REMOTE); - free_fake_orcirc(relay_side); - timers_advance_and_run(5000); - - /* No cells sent */ - tt_int_op(n_client_cells, OP_EQ, 10); - tt_int_op(n_relay_cells, OP_EQ, 10); - - done: - free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side)); - - circuitmux_detach_all_circuits(dummy_channel.cmux, NULL); - circuitmux_free(dummy_channel.cmux); - timers_shutdown(); - monotime_disable_test_mocking(); - UNMOCK(circuit_package_relay_cell); - UNMOCK(circuitmux_attach_circuit); - - return; -} -#endif - /** Helper function: Initializes a padding machine where every state uses the * uniform probability distribution. */ static void @@ -2365,8 +2114,6 @@ struct testcase_t circuitpadding_tests[] = { TEST_CIRCUITPADDING(circuitpadding_tokens, TT_FORK), TEST_CIRCUITPADDING(circuitpadding_negotiation, TT_FORK), TEST_CIRCUITPADDING(circuitpadding_wronghop, TT_FORK), - /** Disabled unstable test until #29298 is implemented (see #29122) */ - // TEST_CIRCUITPADDING(circuitpadding_circuitsetup_machine, TT_FORK), TEST_CIRCUITPADDING(circuitpadding_conditions, TT_FORK), TEST_CIRCUITPADDING(circuitpadding_rtt, TT_FORK), TEST_CIRCUITPADDING(circuitpadding_sample_distribution, TT_FORK), From da678213e00cfe54ccc80286bed2b36603bba115 Mon Sep 17 00:00:00 2001 From: teor Date: Wed, 3 Apr 2019 13:40:06 +1000 Subject: [PATCH 0740/2557] circuitpadding: comment fixes --- src/core/or/circuitpadding.c | 3 ++- src/test/test_circuitpadding.c | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/core/or/circuitpadding.c b/src/core/or/circuitpadding.c index ba6bfe1f53..0e3dc502ce 100644 --- a/src/core/or/circuitpadding.c +++ b/src/core/or/circuitpadding.c @@ -1466,7 +1466,8 @@ circpad_estimate_circ_rtt_on_send(circuit_t *circ, /* If the old RTT estimate is lower than this one, use this one, because * the circuit is getting longer. If this estimate is somehow - * faster than the previous, then maybe that was network jitter. + * faster than the previous, then maybe that was network jitter, or a + * bad monotonic clock source (so our ratchet returned a zero delta). * In that case, average them. */ if (mi->rtt_estimate_usec < (circpad_delay_t)rtt_time) { mi->rtt_estimate_usec = (circpad_delay_t)rtt_time; diff --git a/src/test/test_circuitpadding.c b/src/test/test_circuitpadding.c index ed048ab968..09a4c9a0ca 100644 --- a/src/test/test_circuitpadding.c +++ b/src/test/test_circuitpadding.c @@ -576,7 +576,7 @@ test_circuitpadding_token_removal_higher(void *arg) } } - /* Check that all lowe bins are not touched */ + /* Check that all lower bins are not touched */ for (i=0; i < 4 ; i++) { tt_int_op(mi->histogram[i], OP_EQ, 2); } From 593b33608d6b0696e8cf29780e4881f93cbd4055 Mon Sep 17 00:00:00 2001 From: teor Date: Fri, 5 Apr 2019 12:22:18 +1000 Subject: [PATCH 0741/2557] Revert "test/circuitpadding: Delete circuitpadding_circuitsetup_machine()" This reverts commit 387d9448de4c5044800706b4ed21b48b4426cb7e. --- src/test/test_circuitpadding.c | 253 +++++++++++++++++++++++++++++++++ 1 file changed, 253 insertions(+) diff --git a/src/test/test_circuitpadding.c b/src/test/test_circuitpadding.c index 09a4c9a0ca..22fa817cd6 100644 --- a/src/test/test_circuitpadding.c +++ b/src/test/test_circuitpadding.c @@ -1824,6 +1824,257 @@ test_circuitpadding_conditions(void *arg) return; } +/** Disabled unstable test until #29298 is implemented (see #29122) */ +#if 0 +void +test_circuitpadding_circuitsetup_machine(void *arg) +{ + int64_t actual_mocked_monotime_start; + /** + * Test case plan: + * + * 1. Simulate a normal circuit setup pattern + * a. Application traffic + * + * FIXME: This should focus more on exercising the machine + * features rather than actual traffic patterns. For example, + * test cancellation and bins empty/refill + */ + (void)arg; + + MOCK(circuitmux_attach_circuit, circuitmux_attach_circuit_mock); + + dummy_channel.cmux = circuitmux_alloc(); + client_side = TO_CIRCUIT(origin_circuit_new()); + relay_side = TO_CIRCUIT(new_fake_orcirc(&dummy_channel, &dummy_channel)); + + relay_side->purpose = CIRCUIT_PURPOSE_OR; + client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL; + + nodes_init(); + + monotime_init(); + monotime_enable_test_mocking(); + actual_mocked_monotime_start = MONOTIME_MOCK_START; + monotime_set_mock_time_nsec(actual_mocked_monotime_start); + monotime_coarse_set_mock_time_nsec(actual_mocked_monotime_start); + curr_mocked_time = actual_mocked_monotime_start; + + timers_initialize(); + circpad_machines_init(); + + MOCK(circuit_package_relay_cell, + circuit_package_relay_cell_mock); + MOCK(node_get_by_id, + node_get_by_id_mock); + + /* Test case #1: Build a 3 hop circuit, then wait and let pad */ + simulate_single_hop_extend(client_side, relay_side, 1); + simulate_single_hop_extend(client_side, relay_side, 1); + simulate_single_hop_extend(client_side, relay_side, 1); + + tt_int_op(n_client_cells, OP_EQ, 1); + tt_int_op(n_relay_cells, OP_EQ, 1); + tt_int_op(client_side->padding_info[0]->current_state, OP_EQ, + CIRCPAD_STATE_BURST); + tt_int_op(relay_side->padding_info[0]->current_state, OP_EQ, + CIRCPAD_STATE_BURST); + + tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec, + OP_NE, 0); + tt_int_op(relay_side->padding_info[0]->is_padding_timer_scheduled, + OP_EQ, 0); + timers_advance_and_run(2000); + tt_int_op(n_client_cells, OP_EQ, 2); + tt_int_op(n_relay_cells, OP_EQ, 1); + + tt_int_op(relay_side->padding_info[0]->current_state, OP_EQ, + CIRCPAD_STATE_GAP); + + tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec, + OP_EQ, 0); + tt_u64_op(relay_side->padding_info[0]->padding_scheduled_at_usec, + OP_NE, 0); + timers_advance_and_run(5000); + tt_int_op(n_client_cells, OP_EQ, 2); + tt_int_op(n_relay_cells, OP_EQ, 2); + + tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec, + OP_NE, 0); + tt_u64_op(relay_side->padding_info[0]->padding_scheduled_at_usec, + OP_EQ, 0); + timers_advance_and_run(2000); + tt_int_op(n_client_cells, OP_EQ, 3); + tt_int_op(n_relay_cells, OP_EQ, 2); + + tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec, + OP_EQ, 0); + tt_u64_op(relay_side->padding_info[0]->padding_scheduled_at_usec, + OP_NE, 0); + timers_advance_and_run(5000); + tt_int_op(n_client_cells, OP_EQ, 3); + tt_int_op(n_relay_cells, OP_EQ, 3); + + tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec, + OP_NE, 0); + tt_u64_op(relay_side->padding_info[0]->padding_scheduled_at_usec, + OP_EQ, 0); + timers_advance_and_run(2000); + tt_int_op(n_client_cells, OP_EQ, 4); + tt_int_op(n_relay_cells, OP_EQ, 3); + + tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec, + OP_EQ, 0); + tt_u64_op(relay_side->padding_info[0]->padding_scheduled_at_usec, + OP_NE, 0); + timers_advance_and_run(5000); + tt_int_op(n_client_cells, OP_EQ, 4); + tt_int_op(n_relay_cells, OP_EQ, 4); + + tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec, + OP_NE, 0); + tt_u64_op(relay_side->padding_info[0]->padding_scheduled_at_usec, + OP_EQ, 0); + timers_advance_and_run(2000); + tt_int_op(n_client_cells, OP_EQ, 5); + tt_int_op(n_relay_cells, OP_EQ, 4); + + tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec, + OP_EQ, 0); + tt_u64_op(relay_side->padding_info[0]->padding_scheduled_at_usec, + OP_NE, 0); + timers_advance_and_run(5000); + tt_int_op(n_client_cells, OP_EQ, 5); + tt_int_op(n_relay_cells, OP_EQ, 5); + + tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec, + OP_NE, 0); + tt_u64_op(relay_side->padding_info[0]->padding_scheduled_at_usec, + OP_EQ, 0); + timers_advance_and_run(2000); + tt_int_op(n_client_cells, OP_EQ, 6); + tt_int_op(n_relay_cells, OP_EQ, 5); + + tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec, + OP_EQ, 0); + tt_u64_op(relay_side->padding_info[0]->padding_scheduled_at_usec, + OP_NE, 0); + timers_advance_and_run(5000); + tt_int_op(n_client_cells, OP_EQ, 6); + tt_int_op(n_relay_cells, OP_EQ, 6); + + tt_int_op(client_side->padding_info[0]->current_state, + OP_EQ, CIRCPAD_STATE_END); + tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec, + OP_EQ, 0); + tt_int_op(relay_side->padding_info[0]->current_state, + OP_EQ, CIRCPAD_STATE_GAP); + tt_u64_op(relay_side->padding_info[0]->padding_scheduled_at_usec, + OP_EQ, 0); + + /* Verify we can't schedule padding in END state */ + circpad_decision_t ret = + circpad_machine_schedule_padding(client_side->padding_info[0]); + tt_int_op(ret, OP_EQ, CIRCPAD_STATE_UNCHANGED); + + /* Simulate application traffic */ + circpad_cell_event_nonpadding_sent(client_side); + circpad_deliver_unrecognized_cell_events(relay_side, CELL_DIRECTION_OUT); + circpad_deliver_unrecognized_cell_events(relay_side, CELL_DIRECTION_IN); + circpad_deliver_recognized_relay_cell_events(client_side, RELAY_COMMAND_DATA, + TO_ORIGIN_CIRCUIT(client_side)->cpath->next); + + tt_ptr_op(client_side->padding_info[0], OP_EQ, NULL); + tt_ptr_op(client_side->padding_machine[0], OP_EQ, NULL); + + tt_ptr_op(relay_side->padding_info[0], OP_EQ, NULL); + tt_ptr_op(relay_side->padding_machine[0], OP_EQ, NULL); + tt_int_op(n_client_cells, OP_EQ, 6); + tt_int_op(n_relay_cells, OP_EQ, 7); + + // Test timer cancellation + simulate_single_hop_extend(client_side, relay_side, 1); + simulate_single_hop_extend(client_side, relay_side, 1); + timers_advance_and_run(5000); + circpad_cell_event_padding_received(client_side); + + tt_int_op(client_side->padding_info[0]->current_state, OP_EQ, + CIRCPAD_STATE_BURST); + tt_int_op(relay_side->padding_info[0]->current_state, OP_EQ, + CIRCPAD_STATE_GAP); + + tt_int_op(n_client_cells, OP_EQ, 8); + tt_int_op(n_relay_cells, OP_EQ, 8); + tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec, + OP_NE, 0); + tt_u64_op(relay_side->padding_info[0]->padding_scheduled_at_usec, + OP_NE, 0); + + /* Test timer cancel due to state rules */ + circpad_cell_event_nonpadding_sent(client_side); + tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec, + OP_EQ, 0); + circpad_cell_event_padding_received(client_side); + tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec, + OP_NE, 0); + + /* Simulate application traffic to cancel timer */ + circpad_cell_event_nonpadding_sent(client_side); + circpad_deliver_unrecognized_cell_events(relay_side, CELL_DIRECTION_OUT); + circpad_deliver_unrecognized_cell_events(relay_side, CELL_DIRECTION_IN); + circpad_deliver_recognized_relay_cell_events(client_side, RELAY_COMMAND_DATA, + TO_ORIGIN_CIRCUIT(client_side)->cpath->next); + + tt_ptr_op(client_side->padding_info[0], OP_EQ, NULL); + tt_ptr_op(client_side->padding_machine[0], OP_EQ, NULL); + + tt_ptr_op(relay_side->padding_info[0], OP_EQ, NULL); + tt_ptr_op(relay_side->padding_machine[0], OP_EQ, NULL); + + /* No cells sent, except negotiate end from relay */ + tt_int_op(n_client_cells, OP_EQ, 8); + tt_int_op(n_relay_cells, OP_EQ, 9); + + /* Test mark for close and free */ + simulate_single_hop_extend(client_side, relay_side, 1); + simulate_single_hop_extend(client_side, relay_side, 1); + timers_advance_and_run(5000); + circpad_cell_event_padding_received(client_side); + + tt_int_op(n_client_cells, OP_EQ, 10); + tt_int_op(n_relay_cells, OP_EQ, 10); + + tt_int_op(client_side->padding_info[0]->current_state, OP_EQ, + CIRCPAD_STATE_BURST); + tt_int_op(relay_side->padding_info[0]->current_state, OP_EQ, + CIRCPAD_STATE_GAP); + + tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec, + OP_NE, 0); + tt_u64_op(relay_side->padding_info[0]->padding_scheduled_at_usec, + OP_NE, 0); + circuit_mark_for_close(client_side, END_CIRC_REASON_FLAG_REMOTE); + free_fake_orcirc(relay_side); + timers_advance_and_run(5000); + + /* No cells sent */ + tt_int_op(n_client_cells, OP_EQ, 10); + tt_int_op(n_relay_cells, OP_EQ, 10); + + done: + free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side)); + + circuitmux_detach_all_circuits(dummy_channel.cmux, NULL); + circuitmux_free(dummy_channel.cmux); + timers_shutdown(); + monotime_disable_test_mocking(); + UNMOCK(circuit_package_relay_cell); + UNMOCK(circuitmux_attach_circuit); + + return; +} +#endif + /** Helper function: Initializes a padding machine where every state uses the * uniform probability distribution. */ static void @@ -2114,6 +2365,8 @@ struct testcase_t circuitpadding_tests[] = { TEST_CIRCUITPADDING(circuitpadding_tokens, TT_FORK), TEST_CIRCUITPADDING(circuitpadding_negotiation, TT_FORK), TEST_CIRCUITPADDING(circuitpadding_wronghop, TT_FORK), + /** Disabled unstable test until #29298 is implemented (see #29122) */ + // TEST_CIRCUITPADDING(circuitpadding_circuitsetup_machine, TT_FORK), TEST_CIRCUITPADDING(circuitpadding_conditions, TT_FORK), TEST_CIRCUITPADDING(circuitpadding_rtt, TT_FORK), TEST_CIRCUITPADDING(circuitpadding_sample_distribution, TT_FORK), From 0d136a12bbc498991bcb364098021d0532d85ab0 Mon Sep 17 00:00:00 2001 From: teor Date: Fri, 5 Apr 2019 13:58:23 +1000 Subject: [PATCH 0742/2557] crypto_format: Remove outdated comments (These functions look pretty unified to me.) Part of 29660. --- src/lib/crypt_ops/crypto_format.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/lib/crypt_ops/crypto_format.c b/src/lib/crypt_ops/crypto_format.c index 84f73e5272..f37ec1046f 100644 --- a/src/lib/crypt_ops/crypto_format.c +++ b/src/lib/crypt_ops/crypto_format.c @@ -253,7 +253,6 @@ ed25519_signature_from_base64(ed25519_signature_t *sig, /** Base64 encode DIGEST_LINE bytes from digest, remove the trailing = * characters, and store the nul-terminated result in the first * BASE64_DIGEST_LEN+1 bytes of d64. */ -/* XXXX unify with crypto_format.c code */ int digest_to_base64(char *d64, const char *digest) { @@ -267,7 +266,6 @@ digest_to_base64(char *d64, const char *digest) /** Given a base64 encoded, nul-terminated digest in d64 (without * trailing newline or = characters), decode it and store the result in the * first DIGEST_LEN bytes at digest. */ -/* XXXX unify with crypto_format.c code */ int digest_from_base64(char *digest, const char *d64) { @@ -280,7 +278,6 @@ digest_from_base64(char *digest, const char *d64) /** Base64 encode DIGEST256_LINE bytes from digest, remove the * trailing = characters, and store the nul-terminated result in the first * BASE64_DIGEST256_LEN+1 bytes of d64. */ - /* XXXX unify with crypto_format.c code */ int digest256_to_base64(char *d64, const char *digest) { @@ -294,7 +291,6 @@ digest256_to_base64(char *d64, const char *digest) /** Given a base64 encoded, nul-terminated digest in d64 (without * trailing newline or = characters), decode it and store the result in the * first DIGEST256_LEN bytes at digest. */ -/* XXXX unify with crypto_format.c code */ int digest256_from_base64(char *digest, const char *d64) { From abaed046a6f0f6f6fea5eaf3631b10a514e962e9 Mon Sep 17 00:00:00 2001 From: teor Date: Fri, 5 Apr 2019 14:58:20 +1000 Subject: [PATCH 0743/2557] crypto_format: Remove unused return value from digest_to_base64() Part of 29660. --- src/lib/crypt_ops/crypto_format.c | 16 ++++++++++------ src/lib/crypt_ops/crypto_format.h | 2 +- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/src/lib/crypt_ops/crypto_format.c b/src/lib/crypt_ops/crypto_format.c index f37ec1046f..1467b3d0a6 100644 --- a/src/lib/crypt_ops/crypto_format.c +++ b/src/lib/crypt_ops/crypto_format.c @@ -250,17 +250,21 @@ ed25519_signature_from_base64(ed25519_signature_t *sig, return 0; } -/** Base64 encode DIGEST_LINE bytes from digest, remove the trailing = +/** Base64 encode DIGEST_LEN bytes from digest, remove the trailing = * characters, and store the nul-terminated result in the first - * BASE64_DIGEST_LEN+1 bytes of d64. */ -int + * BASE64_DIGEST_LEN+1 bytes of d64. + * Can not fail. */ +void digest_to_base64(char *d64, const char *digest) { char buf[256]; - base64_encode(buf, sizeof(buf), digest, DIGEST_LEN, 0); - buf[BASE64_DIGEST_LEN] = '\0'; + int n = base64_encode_nopad(buf, sizeof(buf), + (const uint8_t *)digest, DIGEST_LEN); + /* These asserts should always succeed, unless there is a bug in + * base64_encode_nopad(). */ + tor_assert(n == BASE64_DIGEST_LEN); + tor_assert(buf[BASE64_DIGEST_LEN] == '\0'); memcpy(d64, buf, BASE64_DIGEST_LEN+1); - return 0; } /** Given a base64 encoded, nul-terminated digest in d64 (without diff --git a/src/lib/crypt_ops/crypto_format.h b/src/lib/crypt_ops/crypto_format.h index fe852e6a61..53238cb5ab 100644 --- a/src/lib/crypt_ops/crypto_format.h +++ b/src/lib/crypt_ops/crypto_format.h @@ -42,7 +42,7 @@ int ed25519_signature_from_base64(struct ed25519_signature_t *sig, int ed25519_signature_to_base64(char *output, const struct ed25519_signature_t *sig); -int digest_to_base64(char *d64, const char *digest); +void digest_to_base64(char *d64, const char *digest); int digest_from_base64(char *digest, const char *d64); int digest256_to_base64(char *d64, const char *digest); int digest256_from_base64(char *digest, const char *d64); From 7d513a5d5541d17c4e9622a9af76303042fd380b Mon Sep 17 00:00:00 2001 From: teor Date: Fri, 5 Apr 2019 15:02:43 +1000 Subject: [PATCH 0744/2557] crypto_format: Remove the return values from digest256_to_base64() ... and ed25519_public_to_base64(). Also remove all checks for the return values, which were redundant anyway, because the functions never failed. Part of 29960. --- src/feature/dirauth/dirvote.c | 3 +-- src/feature/hs/hs_client.c | 16 +++------------- src/feature/hs/hs_control.c | 23 +++++------------------ src/feature/relay/router.c | 7 ++----- src/lib/crypt_ops/crypto_format.c | 27 +++++++++++++++++---------- src/lib/crypt_ops/crypto_format.h | 6 +++--- src/test/test_crypto.c | 2 +- src/test/test_dir.c | 4 +--- src/test/test_hs_cache.c | 4 +--- src/test/test_hs_control.c | 3 +-- 10 files changed, 35 insertions(+), 60 deletions(-) diff --git a/src/feature/dirauth/dirvote.c b/src/feature/dirauth/dirvote.c index 29f5d04509..1f861d2417 100644 --- a/src/feature/dirauth/dirvote.c +++ b/src/feature/dirauth/dirvote.c @@ -3914,8 +3914,7 @@ dirvote_format_microdesc_vote_line(char *out_buf, size_t out_buf_len, ","); tor_assert(microdesc_consensus_methods); - if (digest256_to_base64(d64, md->digest)<0) - goto out; + digest256_to_base64(d64, md->digest); if (tor_snprintf(out_buf, out_buf_len, "m %s sha256=%s\n", microdesc_consensus_methods, d64)<0) diff --git a/src/feature/hs/hs_client.c b/src/feature/hs/hs_client.c index c34271efca..38b9646c17 100644 --- a/src/feature/hs/hs_client.c +++ b/src/feature/hs/hs_client.c @@ -165,9 +165,7 @@ purge_hid_serv_request(const ed25519_public_key_t *identity_pk) * some point and we don't care about those anymore. */ hs_build_blinded_pubkey(identity_pk, NULL, 0, hs_get_time_period_num(0), &blinded_pk); - if (BUG(ed25519_public_to_base64(base64_blinded_pk, &blinded_pk) < 0)) { - return; - } + ed25519_public_to_base64(base64_blinded_pk, &blinded_pk); /* Purge last hidden service request from cache for this blinded key. */ hs_purge_hid_serv_from_last_hid_serv_requests(base64_blinded_pk); } @@ -354,7 +352,6 @@ directory_launch_v3_desc_fetch(const ed25519_public_key_t *onion_identity_pk, ed25519_public_key_t blinded_pubkey; char base64_blinded_pubkey[ED25519_BASE64_LEN + 1]; hs_ident_dir_conn_t hs_conn_dir_ident; - int retval; tor_assert(hsdir); tor_assert(onion_identity_pk); @@ -363,10 +360,7 @@ directory_launch_v3_desc_fetch(const ed25519_public_key_t *onion_identity_pk, hs_build_blinded_pubkey(onion_identity_pk, NULL, 0, current_time_period, &blinded_pubkey); /* ...and base64 it. */ - retval = ed25519_public_to_base64(base64_blinded_pubkey, &blinded_pubkey); - if (BUG(retval < 0)) { - return HS_CLIENT_FETCH_ERROR; - } + ed25519_public_to_base64(base64_blinded_pubkey, &blinded_pubkey); /* Copy onion pk to a dir_ident so that we attach it to the dir conn */ hs_ident_dir_conn_init(onion_identity_pk, &blinded_pubkey, @@ -405,7 +399,6 @@ directory_launch_v3_desc_fetch(const ed25519_public_key_t *onion_identity_pk, STATIC routerstatus_t * pick_hsdir_v3(const ed25519_public_key_t *onion_identity_pk) { - int retval; char base64_blinded_pubkey[ED25519_BASE64_LEN + 1]; uint64_t current_time_period = hs_get_time_period_num(0); smartlist_t *responsible_hsdirs = NULL; @@ -418,10 +411,7 @@ pick_hsdir_v3(const ed25519_public_key_t *onion_identity_pk) hs_build_blinded_pubkey(onion_identity_pk, NULL, 0, current_time_period, &blinded_pubkey); /* ...and base64 it. */ - retval = ed25519_public_to_base64(base64_blinded_pubkey, &blinded_pubkey); - if (BUG(retval < 0)) { - return NULL; - } + ed25519_public_to_base64(base64_blinded_pubkey, &blinded_pubkey); /* Get responsible hsdirs of service for this time period */ responsible_hsdirs = smartlist_new(); diff --git a/src/feature/hs/hs_control.c b/src/feature/hs/hs_control.c index 20a1061609..abb421345c 100644 --- a/src/feature/hs/hs_control.c +++ b/src/feature/hs/hs_control.c @@ -74,10 +74,7 @@ hs_control_desc_event_failed(const hs_ident_dir_conn_t *ident, tor_assert(reason); /* Build onion address and encoded blinded key. */ - IF_BUG_ONCE(ed25519_public_to_base64(base64_blinded_pk, - &ident->blinded_pk) < 0) { - return; - } + ed25519_public_to_base64(base64_blinded_pk, &ident->blinded_pk); hs_build_address(&ident->identity_pk, HS_VERSION_THREE, onion_address); control_event_hsv3_descriptor_failed(onion_address, base64_blinded_pk, @@ -99,10 +96,7 @@ hs_control_desc_event_received(const hs_ident_dir_conn_t *ident, tor_assert(hsdir_id_digest); /* Build onion address and encoded blinded key. */ - IF_BUG_ONCE(ed25519_public_to_base64(base64_blinded_pk, - &ident->blinded_pk) < 0) { - return; - } + ed25519_public_to_base64(base64_blinded_pk, &ident->blinded_pk); hs_build_address(&ident->identity_pk, HS_VERSION_THREE, onion_address); control_event_hsv3_descriptor_received(onion_address, base64_blinded_pk, @@ -123,9 +117,7 @@ hs_control_desc_event_created(const char *onion_address, tor_assert(blinded_pk); /* Build base64 encoded blinded key. */ - IF_BUG_ONCE(ed25519_public_to_base64(base64_blinded_pk, blinded_pk) < 0) { - return; - } + ed25519_public_to_base64(base64_blinded_pk, blinded_pk); /* Version 3 doesn't use the replica number in its descriptor ID computation * so we pass negative value so the control port subsystem can ignore it. */ @@ -151,9 +143,7 @@ hs_control_desc_event_upload(const char *onion_address, tor_assert(hsdir_index); /* Build base64 encoded blinded key. */ - IF_BUG_ONCE(ed25519_public_to_base64(base64_blinded_pk, blinded_pk) < 0) { - return; - } + ed25519_public_to_base64(base64_blinded_pk, blinded_pk); control_event_hs_descriptor_upload(onion_address, hsdir_id_digest, base64_blinded_pk, @@ -196,10 +186,7 @@ hs_control_desc_event_content(const hs_ident_dir_conn_t *ident, tor_assert(hsdir_id_digest); /* Build onion address and encoded blinded key. */ - IF_BUG_ONCE(ed25519_public_to_base64(base64_blinded_pk, - &ident->blinded_pk) < 0) { - return; - } + ed25519_public_to_base64(base64_blinded_pk, &ident->blinded_pk); hs_build_address(&ident->identity_pk, HS_VERSION_THREE, onion_address); control_event_hs_descriptor_content(onion_address, base64_blinded_pk, diff --git a/src/feature/relay/router.c b/src/feature/relay/router.c index e5cf72ad18..837465cfe9 100644 --- a/src/feature/relay/router.c +++ b/src/feature/relay/router.c @@ -2728,11 +2728,8 @@ router_dump_router_to_string(routerinfo_t *router, log_err(LD_BUG,"Couldn't base64-encode signing key certificate!"); goto err; } - if (ed25519_public_to_base64(ed_fp_base64, - &router->cache_info.signing_key_cert->signing_key)<0) { - log_err(LD_BUG,"Couldn't base64-encode identity key\n"); - goto err; - } + ed25519_public_to_base64(ed_fp_base64, + &router->cache_info.signing_key_cert->signing_key); tor_asprintf(&ed_cert_line, "identity-ed25519\n" "-----BEGIN ED25519 CERT-----\n" "%s" diff --git a/src/lib/crypt_ops/crypto_format.c b/src/lib/crypt_ops/crypto_format.c index 1467b3d0a6..217ac8959c 100644 --- a/src/lib/crypt_ops/crypto_format.c +++ b/src/lib/crypt_ops/crypto_format.c @@ -181,8 +181,7 @@ ed25519_fmt(const ed25519_public_key_t *pkey) if (ed25519_public_key_is_zero(pkey)) { strlcpy(formatted, "", sizeof(formatted)); } else { - int r = ed25519_public_to_base64(formatted, pkey); - tor_assert(!r); + ed25519_public_to_base64(formatted, pkey); } } else { strlcpy(formatted, "", sizeof(formatted)); @@ -202,13 +201,17 @@ ed25519_public_from_base64(ed25519_public_key_t *pkey, /** Encode the public key pkey into the buffer at output, * which must have space for ED25519_BASE64_LEN bytes of encoded key, - * plus one byte for a terminating NUL. Return 0 on success, -1 on failure. + * plus one byte for a terminating NUL. + * Can not fail. + * + * Careful! ED25519_BASE64_LEN is one byte shorter than + * CURVE25519_BASE64_PADDED_LEN. */ -int +void ed25519_public_to_base64(char *output, const ed25519_public_key_t *pkey) { - return digest256_to_base64(output, (const char *)pkey->pubkey); + digest256_to_base64(output, (const char *)pkey->pubkey); } /** Encode the signature sig into the buffer at output, @@ -281,15 +284,19 @@ digest_from_base64(char *digest, const char *d64) /** Base64 encode DIGEST256_LINE bytes from digest, remove the * trailing = characters, and store the nul-terminated result in the first - * BASE64_DIGEST256_LEN+1 bytes of d64. */ -int + * BASE64_DIGEST256_LEN+1 bytes of d64. + * Can not fail. */ +void digest256_to_base64(char *d64, const char *digest) { char buf[256]; - base64_encode(buf, sizeof(buf), digest, DIGEST256_LEN, 0); - buf[BASE64_DIGEST256_LEN] = '\0'; + int n = base64_encode_nopad(buf, sizeof(buf), + (const uint8_t *)digest, DIGEST256_LEN); + /* These asserts should always succeed, unless there is a bug in + * base64_encode_nopad(). */ + tor_assert(n == BASE64_DIGEST256_LEN); + tor_assert(buf[BASE64_DIGEST256_LEN] == '\0'); memcpy(d64, buf, BASE64_DIGEST256_LEN+1); - return 0; } /** Given a base64 encoded, nul-terminated digest in d64 (without diff --git a/src/lib/crypt_ops/crypto_format.h b/src/lib/crypt_ops/crypto_format.h index 53238cb5ab..41c2b06ec8 100644 --- a/src/lib/crypt_ops/crypto_format.h +++ b/src/lib/crypt_ops/crypto_format.h @@ -33,8 +33,8 @@ ssize_t crypto_read_tagged_contents_from_file(const char *fname, int ed25519_public_from_base64(struct ed25519_public_key_t *pkey, const char *input); -int ed25519_public_to_base64(char *output, - const struct ed25519_public_key_t *pkey); +void ed25519_public_to_base64(char *output, + const struct ed25519_public_key_t *pkey); const char *ed25519_fmt(const struct ed25519_public_key_t *pkey); int ed25519_signature_from_base64(struct ed25519_signature_t *sig, @@ -44,7 +44,7 @@ int ed25519_signature_to_base64(char *output, void digest_to_base64(char *d64, const char *digest); int digest_from_base64(char *digest, const char *d64); -int digest256_to_base64(char *d64, const char *digest); +void digest256_to_base64(char *d64, const char *digest); int digest256_from_base64(char *digest, const char *d64); #endif /* !defined(TOR_CRYPTO_FORMAT_H) */ diff --git a/src/test/test_crypto.c b/src/test/test_crypto.c index a5c17b3e6a..fb8046fd22 100644 --- a/src/test/test_crypto.c +++ b/src/test/test_crypto.c @@ -2455,7 +2455,7 @@ test_crypto_ed25519_encode(void *arg) /* Test roundtrip. */ tt_int_op(0, OP_EQ, ed25519_keypair_generate(&kp, 0)); - tt_int_op(0, OP_EQ, ed25519_public_to_base64(buf, &kp.pubkey)); + ed25519_public_to_base64(buf, &kp.pubkey); tt_int_op(ED25519_BASE64_LEN, OP_EQ, strlen(buf)); tt_int_op(0, OP_EQ, ed25519_public_from_base64(&pk, buf)); tt_mem_op(kp.pubkey.pubkey, OP_EQ, pk.pubkey, ED25519_PUBKEY_LEN); diff --git a/src/test/test_dir.c b/src/test/test_dir.c index 6518977b6f..17d6db1e4d 100644 --- a/src/test/test_dir.c +++ b/src/test/test_dir.c @@ -995,9 +995,7 @@ test_dir_formats_rsa_ed25519(void *arg) smartlist_add_strdup(chunks, "master-key-ed25519 "); { char k[ED25519_BASE64_LEN+1]; - tt_int_op(ed25519_public_to_base64(k, - &r2->cache_info.signing_key_cert->signing_key), - OP_GE, 0); + ed25519_public_to_base64(k, &r2->cache_info.signing_key_cert->signing_key); smartlist_add_strdup(chunks, k); smartlist_add_strdup(chunks, "\n"); } diff --git a/src/test/test_hs_cache.c b/src/test/test_hs_cache.c index 9182829116..48e8d3b8c4 100644 --- a/src/test/test_hs_cache.c +++ b/src/test/test_hs_cache.c @@ -238,9 +238,7 @@ helper_fetch_desc_from_hsdir(const ed25519_public_key_t *blinded_key) { char hsdir_cache_key[ED25519_BASE64_LEN+1]; - retval = ed25519_public_to_base64(hsdir_cache_key, - blinded_key); - tt_int_op(retval, OP_EQ, 0); + ed25519_public_to_base64(hsdir_cache_key, blinded_key); tor_asprintf(&hsdir_query_str, GET("/tor/hs/3/%s"), hsdir_cache_key); } diff --git a/src/test/test_hs_control.c b/src/test/test_hs_control.c index 481ef1eb39..7cedc987bb 100644 --- a/src/test/test_hs_control.c +++ b/src/test/test_hs_control.c @@ -107,8 +107,7 @@ test_hs_desc_event(void *arg) memset(&blinded_pk, 'B', sizeof(blinded_pk)); memset(&hsdir_rs, 0, sizeof(hsdir_rs)); memcpy(hsdir_rs.identity_digest, HSDIR_EXIST_ID, DIGEST_LEN); - ret = ed25519_public_to_base64(base64_blinded_pk, &blinded_pk); - tt_int_op(ret, OP_EQ, 0); + ed25519_public_to_base64(base64_blinded_pk, &blinded_pk); memcpy(&ident.identity_pk, &identity_kp.pubkey, sizeof(ed25519_public_key_t)); memcpy(&ident.blinded_pk, &blinded_pk, sizeof(blinded_pk)); From e3124fef54f90828f7b06c41fd4e39ef7778f2e3 Mon Sep 17 00:00:00 2001 From: teor Date: Fri, 5 Apr 2019 15:06:30 +1000 Subject: [PATCH 0745/2557] crypto_format: Remove the return value from curve25519_public_to_base64() And fix the documentation on the function: it does produce trailing "="s as padding. Also remove all checks for the return value, which were redundant anyway, because the function never failed. Part of 29660. --- src/feature/hs/hs_descriptor.c | 21 ++++++--------------- src/lib/crypt_ops/crypto_curve25519.h | 4 ++-- src/lib/crypt_ops/crypto_format.c | 23 +++++++++++++++-------- src/test/test_crypto.c | 2 +- 4 files changed, 24 insertions(+), 26 deletions(-) diff --git a/src/feature/hs/hs_descriptor.c b/src/feature/hs/hs_descriptor.c index 8f7bdf86ef..2793597028 100644 --- a/src/feature/hs/hs_descriptor.c +++ b/src/feature/hs/hs_descriptor.c @@ -403,9 +403,7 @@ encode_enc_key(const hs_desc_intro_point_t *ip) tor_assert(ip); /* Base64 encode the encryption key for the "enc-key" field. */ - if (curve25519_public_to_base64(key_b64, &ip->enc_key) < 0) { - goto done; - } + curve25519_public_to_base64(key_b64, &ip->enc_key); if (tor_cert_encode_ed22519(ip->enc_key_cert, &encoded_cert) < 0) { goto done; } @@ -421,7 +419,7 @@ encode_enc_key(const hs_desc_intro_point_t *ip) } /* Encode an introduction point onion key. Return a newly allocated string - * with it. On failure, return NULL. */ + * with it. Can not fail. */ static char * encode_onion_key(const hs_desc_intro_point_t *ip) { @@ -431,12 +429,9 @@ encode_onion_key(const hs_desc_intro_point_t *ip) tor_assert(ip); /* Base64 encode the encryption key for the "onion-key" field. */ - if (curve25519_public_to_base64(key_b64, &ip->onion_key) < 0) { - goto done; - } + curve25519_public_to_base64(key_b64, &ip->onion_key); tor_asprintf(&encoded, "%s ntor %s", str_ip_onion_key, key_b64); - done: return encoded; } @@ -797,8 +792,8 @@ get_inner_encrypted_layer_plaintext(const hs_descriptor_t *desc) /* Create the middle layer of the descriptor, which includes the client auth * data and the encrypted inner layer (provided as a base64 string at * layer2_b64_ciphertext). Return a newly-allocated string with the - * layer plaintext, or NULL if an error occurred. It's the responsibility of - * the caller to free the returned string. */ + * layer plaintext. It's the responsibility of the caller to free the returned + * string. Can not fail. */ static char * get_outer_encrypted_layer_plaintext(const hs_descriptor_t *desc, const char *layer2_b64_ciphertext) @@ -817,10 +812,7 @@ get_outer_encrypted_layer_plaintext(const hs_descriptor_t *desc, tor_assert(!tor_mem_is_zero((char *) ephemeral_pubkey->public_key, CURVE25519_PUBKEY_LEN)); - if (curve25519_public_to_base64(ephemeral_key_base64, - ephemeral_pubkey) < 0) { - goto done; - } + curve25519_public_to_base64(ephemeral_key_base64, ephemeral_pubkey); smartlist_add_asprintf(lines, "%s %s\n", str_desc_auth_key, ephemeral_key_base64); @@ -845,7 +837,6 @@ get_outer_encrypted_layer_plaintext(const hs_descriptor_t *desc, layer1_str = smartlist_join_strings(lines, "", 0, NULL); - done: /* We need to memwipe all lines because it contains the ephemeral key */ SMARTLIST_FOREACH(lines, char *, a, memwipe(a, 0, strlen(a))); SMARTLIST_FOREACH(lines, char *, a, tor_free(a)); diff --git a/src/lib/crypt_ops/crypto_curve25519.h b/src/lib/crypt_ops/crypto_curve25519.h index 061a7a3505..cd23169cd5 100644 --- a/src/lib/crypt_ops/crypto_curve25519.h +++ b/src/lib/crypt_ops/crypto_curve25519.h @@ -76,8 +76,8 @@ STATIC int curve25519_basepoint_impl(uint8_t *output, const uint8_t *secret); int curve25519_public_from_base64(curve25519_public_key_t *pkey, const char *input); -int curve25519_public_to_base64(char *output, - const curve25519_public_key_t *pkey); +void curve25519_public_to_base64(char *output, + const curve25519_public_key_t *pkey); void curve25519_set_impl_params(int use_ed); void curve25519_init(void); diff --git a/src/lib/crypt_ops/crypto_format.c b/src/lib/crypt_ops/crypto_format.c index 217ac8959c..800f4ad5bc 100644 --- a/src/lib/crypt_ops/crypto_format.c +++ b/src/lib/crypt_ops/crypto_format.c @@ -131,20 +131,27 @@ crypto_read_tagged_contents_from_file(const char *fname, return r; } -/** Encode pkey as a base64-encoded string, without trailing "=" +/** Encode pkey as a base64-encoded string, including trailing "=" * characters, in the buffer output, which must have at least - * CURVE25519_BASE64_PADDED_LEN+1 bytes available. Return 0 on success, -1 on - * failure. */ -int + * CURVE25519_BASE64_PADDED_LEN+1 bytes available. + * Can not fail. + * + * Careful! CURVE25519_BASE64_PADDED_LEN is one byte longer than + * ED25519_BASE64_LEN. + */ +void curve25519_public_to_base64(char *output, const curve25519_public_key_t *pkey) { char buf[128]; - base64_encode(buf, sizeof(buf), - (const char*)pkey->public_key, CURVE25519_PUBKEY_LEN, 0); - buf[CURVE25519_BASE64_PADDED_LEN] = '\0'; + int n = base64_encode(buf, sizeof(buf), + (const char*)pkey->public_key, + CURVE25519_PUBKEY_LEN, 0); + /* These asserts should always succeed, unless there is a bug in + * base64_encode(). */ + tor_assert(n == CURVE25519_BASE64_PADDED_LEN); + tor_assert(buf[CURVE25519_BASE64_PADDED_LEN] == '\0'); memcpy(output, buf, CURVE25519_BASE64_PADDED_LEN+1); - return 0; } /** Try to decode a base64-encoded curve25519 public key from input diff --git a/src/test/test_crypto.c b/src/test/test_crypto.c index fb8046fd22..5f53ba688e 100644 --- a/src/test/test_crypto.c +++ b/src/test/test_crypto.c @@ -2075,7 +2075,7 @@ test_crypto_curve25519_encode(void *arg) curve25519_secret_key_generate(&seckey, 0); curve25519_public_key_generate(&key1, &seckey); - tt_int_op(0, OP_EQ, curve25519_public_to_base64(buf, &key1)); + curve25519_public_to_base64(buf, &key1); tt_int_op(CURVE25519_BASE64_PADDED_LEN, OP_EQ, strlen(buf)); tt_int_op(0, OP_EQ, curve25519_public_from_base64(&key2, buf)); From ce5e38642d1f5e48a7e5c98422e0fa23145f0363 Mon Sep 17 00:00:00 2001 From: teor Date: Fri, 5 Apr 2019 15:08:54 +1000 Subject: [PATCH 0746/2557] crypto_format: Remove the return value from ed25519_signature_to_base64() Also remove all checks for the return value, which were redundant anyway, because the function never failed. Part of 29660. --- src/feature/hs/hs_descriptor.c | 6 +----- src/feature/relay/router.c | 6 ++---- src/lib/crypt_ops/crypto_format.c | 9 ++++++--- src/lib/crypt_ops/crypto_format.h | 4 ++-- src/test/test_crypto.c | 2 +- src/test/test_hs_descriptor.c | 3 +-- 6 files changed, 13 insertions(+), 17 deletions(-) diff --git a/src/feature/hs/hs_descriptor.c b/src/feature/hs/hs_descriptor.c index 2793597028..b526da6661 100644 --- a/src/feature/hs/hs_descriptor.c +++ b/src/feature/hs/hs_descriptor.c @@ -1082,11 +1082,7 @@ desc_encode_v3(const hs_descriptor_t *desc, tor_free(encoded_str); goto err; } - if (ed25519_signature_to_base64(ed_sig_b64, &sig) < 0) { - log_warn(LD_BUG, "Can't base64 encode descriptor signature!"); - tor_free(encoded_str); - goto err; - } + ed25519_signature_to_base64(ed_sig_b64, &sig); /* Create the signature line. */ smartlist_add_asprintf(lines, "%s %s", str_signature, ed_sig_b64); } diff --git a/src/feature/relay/router.c b/src/feature/relay/router.c index 837465cfe9..ac4b3b7a02 100644 --- a/src/feature/relay/router.c +++ b/src/feature/relay/router.c @@ -2974,8 +2974,7 @@ router_dump_router_to_string(routerinfo_t *router, if (ed25519_sign(&sig, (const uint8_t*)digest, DIGEST256_LEN, signing_keypair) < 0) goto err; - if (ed25519_signature_to_base64(buf, &sig) < 0) - goto err; + ed25519_signature_to_base64(buf, &sig); smartlist_add_asprintf(chunks, "%s\n", buf); } @@ -3249,8 +3248,7 @@ extrainfo_dump_to_string(char **s_out, extrainfo_t *extrainfo, if (ed25519_sign(&ed_sig, (const uint8_t*)sha256_digest, DIGEST256_LEN, signing_keypair) < 0) goto err; - if (ed25519_signature_to_base64(buf, &ed_sig) < 0) - goto err; + ed25519_signature_to_base64(buf, &ed_sig); smartlist_add_asprintf(chunks, "%s\n", buf); } diff --git a/src/lib/crypt_ops/crypto_format.c b/src/lib/crypt_ops/crypto_format.c index 800f4ad5bc..269e6d9da9 100644 --- a/src/lib/crypt_ops/crypto_format.c +++ b/src/lib/crypt_ops/crypto_format.c @@ -223,17 +223,20 @@ ed25519_public_to_base64(char *output, /** Encode the signature sig into the buffer at output, * which must have space for ED25519_SIG_BASE64_LEN bytes of encoded signature, - * plus one byte for a terminating NUL. Return 0 on success, -1 on failure. + * plus one byte for a terminating NUL. + * Can not fail. */ -int +void ed25519_signature_to_base64(char *output, const ed25519_signature_t *sig) { char buf[256]; int n = base64_encode_nopad(buf, sizeof(buf), sig->sig, ED25519_SIG_LEN); + /* These asserts should always succeed, unless there is a bug in + * base64_encode_nopad(). */ tor_assert(n == ED25519_SIG_BASE64_LEN); + tor_assert(buf[ED25519_SIG_BASE64_LEN] == '\0'); memcpy(output, buf, ED25519_SIG_BASE64_LEN+1); - return 0; } /** Try to decode the string input into an ed25519 signature. On diff --git a/src/lib/crypt_ops/crypto_format.h b/src/lib/crypt_ops/crypto_format.h index 41c2b06ec8..b4b3aa189c 100644 --- a/src/lib/crypt_ops/crypto_format.h +++ b/src/lib/crypt_ops/crypto_format.h @@ -39,8 +39,8 @@ const char *ed25519_fmt(const struct ed25519_public_key_t *pkey); int ed25519_signature_from_base64(struct ed25519_signature_t *sig, const char *input); -int ed25519_signature_to_base64(char *output, - const struct ed25519_signature_t *sig); +void ed25519_signature_to_base64(char *output, + const struct ed25519_signature_t *sig); void digest_to_base64(char *d64, const char *digest); int digest_from_base64(char *digest, const char *d64); diff --git a/src/test/test_crypto.c b/src/test/test_crypto.c index 5f53ba688e..08dfb6bcdd 100644 --- a/src/test/test_crypto.c +++ b/src/test/test_crypto.c @@ -2461,7 +2461,7 @@ test_crypto_ed25519_encode(void *arg) tt_mem_op(kp.pubkey.pubkey, OP_EQ, pk.pubkey, ED25519_PUBKEY_LEN); tt_int_op(0, OP_EQ, ed25519_sign(&sig1, (const uint8_t*)"ABC", 3, &kp)); - tt_int_op(0, OP_EQ, ed25519_signature_to_base64(buf, &sig1)); + ed25519_signature_to_base64(buf, &sig1); tt_int_op(0, OP_EQ, ed25519_signature_from_base64(&sig2, buf)); tt_mem_op(sig1.sig, OP_EQ, sig2.sig, ED25519_SIG_LEN); diff --git a/src/test/test_hs_descriptor.c b/src/test/test_hs_descriptor.c index 09c6c3e700..86965d7d66 100644 --- a/src/test/test_hs_descriptor.c +++ b/src/test/test_hs_descriptor.c @@ -739,8 +739,7 @@ test_desc_signature(void *arg) ret = ed25519_sign_prefixed(&sig, (const uint8_t *) data, strlen(data), "Tor onion service descriptor sig v3", &kp); tt_int_op(ret, OP_EQ, 0); - ret = ed25519_signature_to_base64(sig_b64, &sig); - tt_int_op(ret, OP_EQ, 0); + ed25519_signature_to_base64(sig_b64, &sig); /* Build the descriptor that should be valid. */ tor_asprintf(&desc, "%ssignature %s\n", data, sig_b64); ret = desc_sig_is_valid(sig_b64, &kp.pubkey, desc, strlen(desc)); From 5e2cba8eb4b80990fa63626054d070ad77578028 Mon Sep 17 00:00:00 2001 From: teor Date: Fri, 5 Apr 2019 15:10:00 +1000 Subject: [PATCH 0747/2557] crypto_format: Stop adding padding in ed25519_signature_from_base64() base64_decode() does not require padding. Part of 29660. --- src/lib/crypt_ops/crypto_format.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/lib/crypt_ops/crypto_format.c b/src/lib/crypt_ops/crypto_format.c index 269e6d9da9..1827168c75 100644 --- a/src/lib/crypt_ops/crypto_format.c +++ b/src/lib/crypt_ops/crypto_format.c @@ -246,14 +246,11 @@ int ed25519_signature_from_base64(ed25519_signature_t *sig, const char *input) { - if (strlen(input) != ED25519_SIG_BASE64_LEN) return -1; - char buf[ED25519_SIG_BASE64_LEN+3]; + char buf[ED25519_SIG_BASE64_LEN+1]; memcpy(buf, input, ED25519_SIG_BASE64_LEN); - buf[ED25519_SIG_BASE64_LEN+0] = '='; - buf[ED25519_SIG_BASE64_LEN+1] = '='; - buf[ED25519_SIG_BASE64_LEN+2] = 0; + buf[ED25519_SIG_BASE64_LEN] = 0; char decoded[128]; int n = base64_decode(decoded, sizeof(decoded), buf, strlen(buf)); if (n < 0 || n != ED25519_SIG_LEN) From c616f457762117cbcbb982a0224d9f8eb2cd9419 Mon Sep 17 00:00:00 2001 From: teor Date: Fri, 5 Apr 2019 15:10:36 +1000 Subject: [PATCH 0748/2557] binascii: Fix the base64_encode_nopad() buffer length requirement Comment-only change. Part of 29660. --- src/lib/encoding/binascii.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/lib/encoding/binascii.c b/src/lib/encoding/binascii.c index 187df34243..fc64e014e7 100644 --- a/src/lib/encoding/binascii.c +++ b/src/lib/encoding/binascii.c @@ -321,8 +321,10 @@ base64_encode(char *dest, size_t destlen, const char *src, size_t srclen, return (int) enclen; } -/** As base64_encode, but do not add any internal spaces or external padding - * to the output stream. */ +/** As base64_encode, but do not add any internal spaces, and remove external + * padding from the output stream. + * dest must be at least base64_encode_size(srclen, 0), including space for + * the removed external padding. */ int base64_encode_nopad(char *dest, size_t destlen, const uint8_t *src, size_t srclen) From ac269d5c30d09194b6ab80232f72bfbd6fa69ed2 Mon Sep 17 00:00:00 2001 From: teor Date: Fri, 5 Apr 2019 15:16:49 +1000 Subject: [PATCH 0749/2557] changes: file for 29660 --- changes/ticket29660 | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 changes/ticket29660 diff --git a/changes/ticket29660 b/changes/ticket29660 new file mode 100644 index 0000000000..84b8059106 --- /dev/null +++ b/changes/ticket29660 @@ -0,0 +1,5 @@ + o Code simplification and refactoring: + - Remove redundant return values in crypto_format, and the associated + return value checks elsewhere in the code. Make the implementations in + crypto_format consistent, and remove redundant code. + Resolves ticket 29660. From f021ca2d52da262e14706498190de5fe7859e00b Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 5 Apr 2019 09:59:45 -0400 Subject: [PATCH 0750/2557] practracker: allow config.c to be a touch larger. --- scripts/maint/practracker/exceptions.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/maint/practracker/exceptions.txt b/scripts/maint/practracker/exceptions.txt index ad5d3e9725..8d6464e3f1 100644 --- a/scripts/maint/practracker/exceptions.txt +++ b/scripts/maint/practracker/exceptions.txt @@ -29,7 +29,7 @@ # # Remember: It is better to fix the problem than to add a new exception! -problem file-size /src/app/config/config.c 8490 +problem file-size /src/app/config/config.c 8491 problem include-count /src/app/config/config.c 86 problem function-size /src/app/config/config.c:options_act_reversible() 296 problem function-size /src/app/config/config.c:options_act() 588 @@ -157,9 +157,9 @@ problem function-size /src/feature/control/control_getinfo.c:getinfo_helper_misc problem function-size /src/feature/control/control_getinfo.c:getinfo_helper_dir() 304 problem function-size /src/feature/control/control_getinfo.c:getinfo_helper_events() 236 problem function-size /src/feature/dirauth/bwauth.c:dirserv_read_measured_bandwidths() 127 -problem file-size /src/feature/dirauth/dirvote.c 4729 +problem file-size /src/feature/dirauth/dirvote.c 4728 problem include-count /src/feature/dirauth/dirvote.c 53 -problem function-size /src/feature/dirauth/dirvote.c:format_networkstatus_vote() 251 +problem function-size /src/feature/dirauth/dirvote.c:format_networkstatus_vote() 250 problem function-size /src/feature/dirauth/dirvote.c:networkstatus_compute_bw_weights_v10() 235 problem function-size /src/feature/dirauth/dirvote.c:networkstatus_compute_consensus() 962 problem function-size /src/feature/dirauth/dirvote.c:networkstatus_add_detached_signatures() 123 @@ -276,7 +276,7 @@ problem function-size /src/lib/net/resolve.c:tor_addr_lookup() 110 problem function-size /src/lib/net/socketpair.c:tor_ersatz_socketpair() 102 problem function-size /src/lib/osinfo/uname.c:get_uname() 116 problem function-size /src/lib/process/process_unix.c:process_unix_exec() 220 -problem function-size /src/lib/process/process_win32.c:process_win32_exec() 138 +problem function-size /src/lib/process/process_win32.c:process_win32_exec() 133 problem function-size /src/lib/process/process_win32.c:process_win32_create_pipe() 112 problem function-size /src/lib/process/restrict.c:set_max_file_descriptors() 102 problem function-size /src/lib/process/setuid.c:switch_id() 156 From 705c1c28c710b70406712027833ab739729ee583 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Fri, 5 Apr 2019 18:51:24 +0300 Subject: [PATCH 0751/2557] Call pre-commit hook from pre-push script --- scripts/git/pre-push.git-hook | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/scripts/git/pre-push.git-hook b/scripts/git/pre-push.git-hook index e7a72efa08..f329829346 100755 --- a/scripts/git/pre-push.git-hook +++ b/scripts/git/pre-push.git-hook @@ -27,6 +27,13 @@ ref_is_upstream_branch() { fi } +workdir=$(git rev-parse --show-toplevel) +if [ -x "$workdir/.git/hooks/pre-commit" ]; then + if ! "$workdir"/.git/hooks/pre-commit; then + exit 1 + fi +fi + # shellcheck disable=SC2034 while read -r local_ref local_sha remote_ref remote_sha do From e8e6931638a14a2c996e55d7c8342654fb546b34 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Fri, 5 Apr 2019 19:03:41 +0300 Subject: [PATCH 0752/2557] Add changes file --- changes/ticket30033 | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 changes/ticket30033 diff --git a/changes/ticket30033 b/changes/ticket30033 new file mode 100644 index 0000000000..3f66d049c8 --- /dev/null +++ b/changes/ticket30033 @@ -0,0 +1,4 @@ + o Minor features (developer tooling): + - Call pre-commit git hook from pre-push hook to make sure we're + running documentation and code style checks before pushing to remote + git repository. Implements feature 30033. From 7741b21d0e3afbfc6d60a852fce6992724c4ae71 Mon Sep 17 00:00:00 2001 From: teor Date: Sat, 6 Apr 2019 12:19:47 +1000 Subject: [PATCH 0753/2557] practracker: accept 6 extra lines in tortls_nss.c:tor_tls_context_new() These lines were added to fix bug 29241. --- scripts/maint/practracker/exceptions.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/maint/practracker/exceptions.txt b/scripts/maint/practracker/exceptions.txt index 8d6464e3f1..678d97849a 100644 --- a/scripts/maint/practracker/exceptions.txt +++ b/scripts/maint/practracker/exceptions.txt @@ -282,7 +282,7 @@ problem function-size /src/lib/process/restrict.c:set_max_file_descriptors() 102 problem function-size /src/lib/process/setuid.c:switch_id() 156 problem function-size /src/lib/sandbox/sandbox.c:prot_strings() 104 problem function-size /src/lib/string/scanf.c:tor_vsscanf() 112 -problem function-size /src/lib/tls/tortls_nss.c:tor_tls_context_new() 147 +problem function-size /src/lib/tls/tortls_nss.c:tor_tls_context_new() 153 problem function-size /src/lib/tls/tortls_openssl.c:tor_tls_context_new() 171 problem function-size /src/lib/tls/x509_nss.c:tor_tls_create_certificate_internal() 126 problem function-size /src/tools/tor-gencert.c:parse_commandline() 111 From 0e0a0b9802e7e7bf192df9c93f257913e926033a Mon Sep 17 00:00:00 2001 From: rl1987 Date: Mon, 8 Apr 2019 11:16:45 +0300 Subject: [PATCH 0754/2557] Fix SC2006 in minimize.sh --- changes/ticket30079 | 3 +++ src/test/fuzz/minimize.sh | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 changes/ticket30079 diff --git a/changes/ticket30079 b/changes/ticket30079 new file mode 100644 index 0000000000..56b88e7f53 --- /dev/null +++ b/changes/ticket30079 @@ -0,0 +1,3 @@ + o Code simplification and refactoring (shell scripts): + - Fix shellcheck warning SC2006 in src/test/fuzz/minimize.sh. Resolves + issue 30079. diff --git a/src/test/fuzz/minimize.sh b/src/test/fuzz/minimize.sh index 87d3dda13c..ce43812bb8 100755 --- a/src/test/fuzz/minimize.sh +++ b/src/test/fuzz/minimize.sh @@ -7,7 +7,7 @@ if [ ! -d "$1" ] ; then exit 1 fi -which=`basename "$1"` +which=$(basename "$1") mkdir "$1.out" afl-cmin -i "$1" -o "$1.out" -m none "./src/test/fuzz/fuzz-${which}" From 61e6b217c5c83bc49e888f594f931c00c3e9b971 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Mon, 25 Mar 2019 15:40:46 +0200 Subject: [PATCH 0755/2557] manpage: Clarify that Tor does stream isolation between *Port listeners by default cherry-pick of tor-github/pr/841 to maint-0.4.0. --- changes/doc29121 | 3 +++ doc/tor.1.txt | 14 +++++++++++--- 2 files changed, 14 insertions(+), 3 deletions(-) create mode 100644 changes/doc29121 diff --git a/changes/doc29121 b/changes/doc29121 new file mode 100644 index 0000000000..dd31cc9c70 --- /dev/null +++ b/changes/doc29121 @@ -0,0 +1,3 @@ + o Documentation: + - Clarify that Tor performs stream isolation between *Port listeners by + default. Resolves issue 29121. diff --git a/doc/tor.1.txt b/doc/tor.1.txt index ea9942a28d..c2df7687fe 100644 --- a/doc/tor.1.txt +++ b/doc/tor.1.txt @@ -1230,6 +1230,8 @@ The following options are useful only for clients (that is, if information to anybody watching your network, and allow anybody to use your computer as an open proxy. + + + If multiple entries of this option are present in your configuration + file, Tor will perform stream isolation between listeners by default. The _isolation flags_ arguments give Tor rules for which streams received on this SocksPort are allowed to share circuits with one another. Recognized isolation flags are: @@ -1472,14 +1474,18 @@ The following options are useful only for clients (that is, if protocol instead of SOCKS. Set this to 0 if you don't want to allow "HTTP CONNECT" connections. Set the port to "auto" to have Tor pick a port for you. This directive can be - specified multiple times to bind to multiple addresses/ports. See + specified multiple times to bind to multiple addresses/ports. If multiple + entries of this option are present in your configuration file, Tor will + perform stream isolation between listeners by default. See SOCKSPort for an explanation of isolation flags. (Default: 0) [[TransPort]] **TransPort** \['address':]__port__|**auto** [_isolation flags_]:: Open this port to listen for transparent proxy connections. Set this to 0 if you don't want to allow transparent proxy connections. Set the port to "auto" to have Tor pick a port for you. This directive can be - specified multiple times to bind to multiple addresses/ports. See + specified multiple times to bind to multiple addresses/ports. If multiple + entries of this option are present in your configuration file, Tor will + perform stream isolation between listeners by default. See SOCKSPort for an explanation of isolation flags. + + TransPort requires OS support for transparent proxies, such as BSDs' pf or @@ -1516,7 +1522,9 @@ The following options are useful only for clients (that is, if included in old versions of FreeBSD, etc) using the NATD protocol. Use 0 if you don't want to allow NATD connections. Set the port to "auto" to have Tor pick a port for you. This directive can be - specified multiple times to bind to multiple addresses/ports. See + specified multiple times to bind to multiple addresses/ports. If multiple + entries of this option are present in your configuration file, Tor will + perform stream isolation between listeners by default. See SocksPort for an explanation of isolation flags. + + This option is only for people who cannot use TransPort. (Default: 0) From ce9b1015741b663e4f1dbd2923886aedd00c7094 Mon Sep 17 00:00:00 2001 From: teor Date: Wed, 10 Apr 2019 15:57:54 +1000 Subject: [PATCH 0756/2557] bwauth: update measured bandwidth file comments We forgot to update function header comments and code comments when we made changes in 0.3.5.1-alpha and later. Closes 30112. --- src/feature/dirauth/bwauth.c | 33 ++++++++++++++++++++++++++++----- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/src/feature/dirauth/bwauth.c b/src/feature/dirauth/bwauth.c index 1cfd8119df..e60c8b86bd 100644 --- a/src/feature/dirauth/bwauth.c +++ b/src/feature/dirauth/bwauth.c @@ -199,9 +199,32 @@ dirserv_get_credible_bandwidth_kb(const routerinfo_t *ri) } /** - * Read the measured bandwidth list file, apply it to the list of - * vote_routerstatus_t and store all the headers in bw_file_headers. + * Read the measured bandwidth list from_file: + * - store all the headers in bw_file_headers, + * - apply bandwidth lines to the list of vote_routerstatus_t in + * routerstatuses, + * - cache bandwidth lines for dirserv_get_bandwidth_for_router(), + * - expire old entries in the measured bandwidth cache, and + * - store the DIGEST_SHA256 of the contents of the file in digest_out. + * * Returns -1 on error, 0 otherwise. + * + * If the file can't be read, or is empty: + * - bw_file_headers is empty, + * - routerstatuses is not modified, + * - the measured bandwidth cache is not modified, and + * - digest_out is the zero-byte digest. + * + * Otherwise, if there is an error later in the file: + * - bw_file_headers contains all the headers up to the error, + * - routerstatuses is updated with all the relay lines up to the error, + * - the measured bandwidth cache is updated with all the relay lines up to + * the error, + * - if the timestamp is valid and recent, old entries in the measured + * bandwidth cache are expired, and + * - digest_out is the digest up to the first read error (if any). + * The digest is taken over all the readable file contents, even if the + * file is outdated or unparseable. */ int dirserv_read_measured_bandwidths(const char *from_file, @@ -223,15 +246,12 @@ dirserv_read_measured_bandwidths(const char *from_file, size_t n = 0; crypto_digest_t *digest = crypto_digest256_new(DIGEST_SHA256); - /* Initialise line, so that we can't possibly run off the end. */ - if (fp == NULL) { log_warn(LD_CONFIG, "Can't open bandwidth file at configured location: %s", from_file); goto err; } - /* If fgets fails, line is either unmodified, or indeterminate. */ if (tor_getline(&line,&n,fp) <= 0) { log_warn(LD_DIRSERV, "Empty bandwidth file"); goto err; @@ -345,6 +365,9 @@ dirserv_read_measured_bandwidths(const char *from_file, * the header block yet. If we encounter an incomplete bw line, return -1 but * don't warn since there could be additional header lines coming. If we * encounter a proper bw line, return 0 (and we got past the headers). + * + * If the line contains "vote=0", stop parsing it, and return -1, so that the + * line is ignored during voting. */ STATIC int measured_bw_line_parse(measured_bw_line_t *out, const char *orig_line, From 8cfab3b7c3ed626b8f17018a627634de9b4502af Mon Sep 17 00:00:00 2001 From: teor Date: Wed, 10 Apr 2019 16:12:56 +1000 Subject: [PATCH 0757/2557] doc: Improve the documentation for MapAddress .exit Fixes bug 30109; bugfix on 0.1.0.1-rc. --- changes/bug30109 | 3 +++ doc/tor.1.txt | 17 ++++++++++++++--- 2 files changed, 17 insertions(+), 3 deletions(-) create mode 100644 changes/bug30109 diff --git a/changes/bug30109 b/changes/bug30109 new file mode 100644 index 0000000000..b25aa803bb --- /dev/null +++ b/changes/bug30109 @@ -0,0 +1,3 @@ + o Minor bugfixes (documentation): + - Improve the documentation for MapAddress .exit. + Fixes bug 30109; bugfix on 0.1.0.1-rc. diff --git a/doc/tor.1.txt b/doc/tor.1.txt index c2df7687fe..a80da23a03 100644 --- a/doc/tor.1.txt +++ b/doc/tor.1.txt @@ -1056,8 +1056,8 @@ The following options are useful only for clients (that is, if [[StrictNodes]] **StrictNodes** **0**|**1**:: If StrictNodes is set to 1, Tor will treat solely the ExcludeNodes option as a requirement to follow for all the circuits you generate, even if - doing so will break functionality for you (StrictNodes applies to neither - ExcludeExitNodes nor to ExitNodes, nor to MiddleNodes). If StrictNodes + doing so will break functionality for you (StrictNodes does not apply to + ExcludeExitNodes, ExitNodes, MiddleNodes, or MapAddress). If StrictNodes is set to 0, Tor will still try to avoid nodes in the ExcludeNodes list, but it will err on the side of avoiding unexpected errors. Specifically, StrictNodes 0 tells Tor that it is okay to use an excluded @@ -1153,7 +1153,9 @@ The following options are useful only for clients (that is, if "MapAddress \*.example.com \*.example.com.torserver.exit". (Note the leading "*." in each part of the directive.) You can also redirect all subdomains of a domain to a single address. For example, "MapAddress - *.example.com www.example.com". + + *.example.com www.example.com". If the specified exit is not available, + or the exit can not connect to the site, Tor will fail any connections + to the mapped address.+ + NOTES: @@ -1181,6 +1183,15 @@ The following options are useful only for clients (that is, if 4. Using a wildcard to match only part of a string (as in *ample.com) is also invalid. + 5. Tor maps hostnames and IP addresses separately. If you MapAddress + a DNS name, but use an IP address to connect, then Tor will ignore the + DNS name mapping. + + 6. MapAddress does not apply to redirects in the application protocol. + For example, HTTP redirects and alt-svc headers will ignore mappings + for the original address. You can use a wildcard mapping to handle + redirects within the same site. + [[NewCircuitPeriod]] **NewCircuitPeriod** __NUM__:: Every NUM seconds consider whether to build a new circuit. (Default: 30 seconds) From 60c46c6cd01092ccc17d5e8bd15778fd93d30d77 Mon Sep 17 00:00:00 2001 From: teor Date: Wed, 10 Apr 2019 18:29:11 +1000 Subject: [PATCH 0758/2557] practracker: accept 4 extra lines due to 30041 --- scripts/maint/practracker/exceptions.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/maint/practracker/exceptions.txt b/scripts/maint/practracker/exceptions.txt index 678d97849a..7d03bf27d6 100644 --- a/scripts/maint/practracker/exceptions.txt +++ b/scripts/maint/practracker/exceptions.txt @@ -54,7 +54,7 @@ problem function-size /src/app/main/main.c:sandbox_init_filter() 291 problem function-size /src/app/main/main.c:run_tor_main_loop() 105 problem function-size /src/app/main/ntmain.c:nt_service_install() 125 problem include-count /src/app/main/shutdown.c 52 -problem file-size /src/core/mainloop/connection.c 5554 +problem file-size /src/core/mainloop/connection.c 5558 problem include-count /src/core/mainloop/connection.c 61 problem function-size /src/core/mainloop/connection.c:connection_free_minimal() 184 problem function-size /src/core/mainloop/connection.c:connection_listener_new() 328 @@ -64,7 +64,7 @@ problem function-size /src/core/mainloop/connection.c:connection_proxy_connect() problem function-size /src/core/mainloop/connection.c:connection_read_proxy_handshake() 153 problem function-size /src/core/mainloop/connection.c:retry_listener_ports() 116 problem function-size /src/core/mainloop/connection.c:connection_handle_read_impl() 111 -problem function-size /src/core/mainloop/connection.c:connection_buf_read_from_socket() 177 +problem function-size /src/core/mainloop/connection.c:connection_buf_read_from_socket() 181 problem function-size /src/core/mainloop/connection.c:connection_handle_write_impl() 241 problem function-size /src/core/mainloop/connection.c:assert_connection_ok() 143 problem file-size /src/core/mainloop/mainloop.c 3051 From 5722c6d12d75ad1ddb28c32a11aae7aed7ccf5db Mon Sep 17 00:00:00 2001 From: teor Date: Wed, 10 Apr 2019 19:26:47 +1000 Subject: [PATCH 0759/2557] scripts: In git-pull-all.sh, also fetch the latest tor-github pull requests Implements ticket 30114. --- changes/ticket30114 | 3 +++ scripts/git/git-pull-all.sh | 18 +++++++++++++++++- 2 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 changes/ticket30114 diff --git a/changes/ticket30114 b/changes/ticket30114 new file mode 100644 index 0000000000..a80f7f4dcf --- /dev/null +++ b/changes/ticket30114 @@ -0,0 +1,3 @@ + o Minor features (git scripts): + - In git-pull-all.sh, also fetch the latest tor-github pull requests. + Implements ticket 30114. diff --git a/scripts/git/git-pull-all.sh b/scripts/git/git-pull-all.sh index 0a4898a111..5d1d58e4bf 100755 --- a/scripts/git/git-pull-all.sh +++ b/scripts/git/git-pull-all.sh @@ -174,6 +174,19 @@ function fetch_origin fi } +# Fetch tor-github pull requests. No arguments. +function fetch_tor_github +{ + local cmd="git fetch tor-github" + printf " %s Fetching tor-github..." "$MARKER" + if [ $DRY_RUN -eq 0 ]; then + msg=$( eval "$cmd" 2>&1 ) + validate_ret $? "$msg" + else + printf "\\n %s\\n" "${IWTH}$cmd${CNRM}" + fi +} + ############### # Entry point # ############### @@ -188,8 +201,11 @@ while getopts "n" opt; do esac done -# First, fetch the origin. +# First, fetch tor-github. goto_repo "$ORIGIN_PATH" +fetch_tor_github + +# Then, fetch the origin. fetch_origin # Go over all configured worktree. From 98e08b452f81316f108113d7ecff15a5e2855d63 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 10 Apr 2019 11:49:23 -0400 Subject: [PATCH 0760/2557] Fix pre-commit hook to correctly allow empty changes files. Fixes bug 30120; bugfix not in any released Tor. --- scripts/git/pre-commit.git-hook | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/git/pre-commit.git-hook b/scripts/git/pre-commit.git-hook index 65fa99f4c4..4a962851b0 100755 --- a/scripts/git/pre-commit.git-hook +++ b/scripts/git/pre-commit.git-hook @@ -12,7 +12,7 @@ cd "$workdir" || exit 1 set -e -if [ ! -z "ls ./changes/*" ]; then +if [ -n "$(ls ./changes/)" ]; then python scripts/maint/lintChanges.py ./changes/* fi @@ -26,7 +26,7 @@ if [ -d src/lib ]; then src/test/*.[ch] \ src/test/*/*.[ch] \ src/tools/*.[ch] -elif [ -d src/common]; then +elif [ -d src/common ]; then # This was the layout before 0.3.5 perl scripts/maint/checkSpace.pl -C \ src/common/*/*.[ch] \ From b2fc57426c29f28ade658c642cd692c022d88b42 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 10 Apr 2019 11:51:49 -0400 Subject: [PATCH 0761/2557] Bump version to 0.4.0.4-rc --- configure.ac | 4 ++-- contrib/win32build/tor-mingw.nsi.in | 2 +- src/win32/orconfig.h | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/configure.ac b/configure.ac index 3304b5cabb..dc8fa70863 100644 --- a/configure.ac +++ b/configure.ac @@ -4,7 +4,7 @@ dnl Copyright (c) 2007-2019, The Tor Project, Inc. dnl See LICENSE for licensing information AC_PREREQ([2.63]) -AC_INIT([tor],[0.4.0.3-alpha-dev]) +AC_INIT([tor],[0.4.0.4-rc]) AC_CONFIG_SRCDIR([src/app/main/tor_main.c]) AC_CONFIG_MACRO_DIR([m4]) @@ -14,7 +14,7 @@ AC_CONFIG_MACRO_DIR([m4]) # version number changes. Tor uses it to make sure that it # only shuts down for missing "required protocols" when those protocols # are listed as required by a consensus after this date. -AC_DEFINE(APPROX_RELEASE_DATE, ["2019-03-23"], # for 0.4.0.3-alpha-dev +AC_DEFINE(APPROX_RELEASE_DATE, ["2019-04-10"], # for 0.4.0.4-rc [Approximate date when this software was released. (Updated when the version changes.)]) # "foreign" means we don't follow GNU package layout standards diff --git a/contrib/win32build/tor-mingw.nsi.in b/contrib/win32build/tor-mingw.nsi.in index 0d0d4f493e..4100b46191 100644 --- a/contrib/win32build/tor-mingw.nsi.in +++ b/contrib/win32build/tor-mingw.nsi.in @@ -8,7 +8,7 @@ !include "LogicLib.nsh" !include "FileFunc.nsh" !insertmacro GetParameters -!define VERSION "0.4.0.3-alpha-dev" +!define VERSION "0.4.0.4-rc" !define INSTALLER "tor-${VERSION}-win32.exe" !define WEBSITE "https://www.torproject.org/" !define LICENSE "LICENSE" diff --git a/src/win32/orconfig.h b/src/win32/orconfig.h index d3441610b6..09d38cb1eb 100644 --- a/src/win32/orconfig.h +++ b/src/win32/orconfig.h @@ -218,7 +218,7 @@ #define USING_TWOS_COMPLEMENT /* Version number of package */ -#define VERSION "0.4.0.3-alpha-dev" +#define VERSION "0.4.0.4-rc" From 011307dd5fa608739456b98d259b013286320b91 Mon Sep 17 00:00:00 2001 From: Neel Chauhan Date: Thu, 11 Apr 2019 15:20:31 -0400 Subject: [PATCH 0762/2557] Make repeated/rate limited HSFETCH queries fail with QUERY_RATE_LIMITED --- changes/bug28269 | 7 +++++++ src/feature/hs/hs_client.c | 2 +- src/feature/hs/hs_common.c | 20 ++++++++++++++++---- src/feature/hs/hs_common.h | 2 +- src/feature/rend/rendclient.c | 9 ++++++--- src/test/test_hs.c | 10 ++++++++++ 6 files changed, 41 insertions(+), 9 deletions(-) create mode 100644 changes/bug28269 diff --git a/changes/bug28269 b/changes/bug28269 new file mode 100644 index 0000000000..bdfe9e1aae --- /dev/null +++ b/changes/bug28269 @@ -0,0 +1,7 @@ + o Minor bugfixes (onion services): + - If we are launching repeated HSFETCH queries and are rate-limited, + we introduce a new controller response QUERY_RATE_LIMITED instead + of QUERY_NO_HSDIR, while keeping the latter for when onion service + directories are missing a descriptor. Previously, we returned + QUERY_NO_HSDIR for both cases. Fixes bug 28269; bugfix on + 0.3.1.1-alpha. Patch by Neel Chauhan diff --git a/src/feature/hs/hs_client.c b/src/feature/hs/hs_client.c index c34271efca..b4b9f0a948 100644 --- a/src/feature/hs/hs_client.c +++ b/src/feature/hs/hs_client.c @@ -434,7 +434,7 @@ pick_hsdir_v3(const ed25519_public_key_t *onion_identity_pk) /* Pick an HSDir from the responsible ones. The ownership of * responsible_hsdirs is given to this function so no need to free it. */ - hsdir_rs = hs_pick_hsdir(responsible_hsdirs, base64_blinded_pubkey); + hsdir_rs = hs_pick_hsdir(responsible_hsdirs, base64_blinded_pubkey, NULL); return hsdir_rs; } diff --git a/src/feature/hs/hs_common.c b/src/feature/hs/hs_common.c index b2227432d2..cffec2b878 100644 --- a/src/feature/hs/hs_common.c +++ b/src/feature/hs/hs_common.c @@ -1589,20 +1589,25 @@ hs_purge_last_hid_serv_requests(void) /** Given the list of responsible HSDirs in responsible_dirs, pick the * one that we should use to fetch a descriptor right now. Take into account * previous failed attempts at fetching this descriptor from HSDirs using the - * string identifier req_key_str. + * string identifier req_key_str. We return whether we are rate limited + * into *is_rate_limited if it is not NULL. * * Steals ownership of responsible_dirs. * * Return the routerstatus of the chosen HSDir if successful, otherwise return * NULL if no HSDirs are worth trying right now. */ routerstatus_t * -hs_pick_hsdir(smartlist_t *responsible_dirs, const char *req_key_str) +hs_pick_hsdir(smartlist_t *responsible_dirs, const char *req_key_str, + int *is_rate_limited) { smartlist_t *usable_responsible_dirs = smartlist_new(); const or_options_t *options = get_options(); routerstatus_t *hs_dir; time_t now = time(NULL); int excluded_some; + int rate_limited; + int rate_limited_count = 0; + int responsible_dirs_count = smartlist_len(responsible_dirs); tor_assert(req_key_str); @@ -1622,6 +1627,7 @@ hs_pick_hsdir(smartlist_t *responsible_dirs, const char *req_key_str) if (last + hs_hsdir_requery_period(options) >= now || !node || !node_has_preferred_descriptor(node, 0)) { SMARTLIST_DEL_CURRENT(responsible_dirs, dir); + rate_limited_count++; continue; } if (!routerset_contains_node(options->ExcludeNodes, node)) { @@ -1629,6 +1635,7 @@ hs_pick_hsdir(smartlist_t *responsible_dirs, const char *req_key_str) } } SMARTLIST_FOREACH_END(dir); + rate_limited = rate_limited_count == responsible_dirs_count; excluded_some = smartlist_len(usable_responsible_dirs) < smartlist_len(responsible_dirs); @@ -1640,9 +1647,10 @@ hs_pick_hsdir(smartlist_t *responsible_dirs, const char *req_key_str) smartlist_free(responsible_dirs); smartlist_free(usable_responsible_dirs); if (!hs_dir) { + const char *warn_str = (rate_limited) ? "we are rate limited." : + "we requested them all recently without success"; log_info(LD_REND, "Could not pick one of the responsible hidden " - "service directories, because we requested them all " - "recently without success."); + "service directories, because %s.", warn_str); if (options->StrictNodes && excluded_some) { log_warn(LD_REND, "Could not pick a hidden service directory for the " "requested hidden service: they are all either down or " @@ -1654,6 +1662,10 @@ hs_pick_hsdir(smartlist_t *responsible_dirs, const char *req_key_str) hs_lookup_last_hid_serv_request(hs_dir, req_key_str, now, 1); } + if (is_rate_limited != NULL) { + *is_rate_limited = rate_limited; + } + return hs_dir; } diff --git a/src/feature/hs/hs_common.h b/src/feature/hs/hs_common.h index abf39fa431..f96fc8beb7 100644 --- a/src/feature/hs/hs_common.h +++ b/src/feature/hs/hs_common.h @@ -241,7 +241,7 @@ void hs_get_responsible_hsdirs(const struct ed25519_public_key_t *blinded_pk, int use_second_hsdir_index, int for_fetching, smartlist_t *responsible_dirs); routerstatus_t *hs_pick_hsdir(smartlist_t *responsible_dirs, - const char *req_key_str); + const char *req_key_str, int *is_rate_limited); time_t hs_hsdir_requery_period(const or_options_t *options); time_t hs_lookup_last_hid_serv_request(routerstatus_t *hs_dir, diff --git a/src/feature/rend/rendclient.c b/src/feature/rend/rendclient.c index 5a8b234544..9863fc1c11 100644 --- a/src/feature/rend/rendclient.c +++ b/src/feature/rend/rendclient.c @@ -469,16 +469,19 @@ directory_get_from_hs_dir(const char *desc_id, /* Automatically pick an hs dir if none given. */ if (!rs_hsdir) { + int rate_limited; + /* Determine responsible dirs. Even if we can't get all we want, work with * the ones we have. If it's empty, we'll notice in hs_pick_hsdir(). */ smartlist_t *responsible_dirs = smartlist_new(); hid_serv_get_responsible_directories(responsible_dirs, desc_id); - hs_dir = hs_pick_hsdir(responsible_dirs, desc_id_base32); + hs_dir = hs_pick_hsdir(responsible_dirs, desc_id_base32, &rate_limited); if (!hs_dir) { /* No suitable hs dir can be found, stop right now. */ - control_event_hsv2_descriptor_failed(rend_query, NULL, - "QUERY_NO_HSDIR"); + const char *query_response = (rate_limited) ? "QUERY_RATE_LIMITED" : + "QUERY_NO_HSDIR"; + control_event_hsv2_descriptor_failed(rend_query, NULL, query_response); control_event_hs_descriptor_content(rend_data_get_address(rend_query), desc_id_base32, NULL, NULL); return 0; diff --git a/src/test/test_hs.c b/src/test/test_hs.c index aeb3387471..5d3327c777 100644 --- a/src/test/test_hs.c +++ b/src/test/test_hs.c @@ -323,6 +323,16 @@ test_hs_desc_event(void *arg) tt_str_op(received_msg,OP_EQ, expected_msg); tor_free(received_msg); + /* test HSDir rate limited */ + rend_query.auth_type = REND_NO_AUTH; + control_event_hsv2_descriptor_failed(&rend_query.base_, NULL, + "QUERY_RATE_LIMITED"); + expected_msg = "650 HS_DESC FAILED "STR_HS_ADDR" NO_AUTH " \ + "UNKNOWN REASON=QUERY_RATE_LIMITED\r\n"; + tt_assert(received_msg); + tt_str_op(received_msg,OP_EQ, expected_msg); + tor_free(received_msg); + /* Test invalid content with no HSDir fingerprint. */ char *exp_msg; control_event_hs_descriptor_content(rend_query.onion_address, From 40471d73e5270d615a92be2ee1e7c70232dcbbcc Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 11 Apr 2019 17:05:38 -0400 Subject: [PATCH 0763/2557] bump to 0.4.0.4-rc-dev --- configure.ac | 4 ++-- contrib/win32build/tor-mingw.nsi.in | 2 +- src/win32/orconfig.h | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/configure.ac b/configure.ac index dc8fa70863..062ff5168a 100644 --- a/configure.ac +++ b/configure.ac @@ -4,7 +4,7 @@ dnl Copyright (c) 2007-2019, The Tor Project, Inc. dnl See LICENSE for licensing information AC_PREREQ([2.63]) -AC_INIT([tor],[0.4.0.4-rc]) +AC_INIT([tor],[0.4.0.4-rc-dev]) AC_CONFIG_SRCDIR([src/app/main/tor_main.c]) AC_CONFIG_MACRO_DIR([m4]) @@ -14,7 +14,7 @@ AC_CONFIG_MACRO_DIR([m4]) # version number changes. Tor uses it to make sure that it # only shuts down for missing "required protocols" when those protocols # are listed as required by a consensus after this date. -AC_DEFINE(APPROX_RELEASE_DATE, ["2019-04-10"], # for 0.4.0.4-rc +AC_DEFINE(APPROX_RELEASE_DATE, ["2019-04-11"], # for 0.4.0.4-rc-dev [Approximate date when this software was released. (Updated when the version changes.)]) # "foreign" means we don't follow GNU package layout standards diff --git a/contrib/win32build/tor-mingw.nsi.in b/contrib/win32build/tor-mingw.nsi.in index 4100b46191..38bbca4c66 100644 --- a/contrib/win32build/tor-mingw.nsi.in +++ b/contrib/win32build/tor-mingw.nsi.in @@ -8,7 +8,7 @@ !include "LogicLib.nsh" !include "FileFunc.nsh" !insertmacro GetParameters -!define VERSION "0.4.0.4-rc" +!define VERSION "0.4.0.4-rc-dev" !define INSTALLER "tor-${VERSION}-win32.exe" !define WEBSITE "https://www.torproject.org/" !define LICENSE "LICENSE" diff --git a/src/win32/orconfig.h b/src/win32/orconfig.h index 09d38cb1eb..3a7410b8d8 100644 --- a/src/win32/orconfig.h +++ b/src/win32/orconfig.h @@ -218,7 +218,7 @@ #define USING_TWOS_COMPLEMENT /* Version number of package */ -#define VERSION "0.4.0.4-rc" +#define VERSION "0.4.0.4-rc-dev" From adeecce53b68eb7183bf8d5b3051303b5f2f1d45 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 11 Apr 2019 17:06:19 -0400 Subject: [PATCH 0764/2557] forward-port the 0.4.0.4-rc changelog --- ChangeLog | 138 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 138 insertions(+) diff --git a/ChangeLog b/ChangeLog index 9b512c6aed..accde75b97 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,141 @@ +Changes in version 0.4.0.4-rc - 2019-04-11 + Tor 0.4.0.4-rc is the first release candidate in its series; it fixes + several bugs from earlier versions, including some that had affected + stability, and one that prevented relays from working with NSS. + + o Major bugfixes (NSS, relay): + - When running with NSS, disable TLS 1.2 ciphersuites that use + SHA384 for their PRF. Due to an NSS bug, the TLS key exporters for + these ciphersuites don't work -- which caused relays to fail to + handshake with one another when these ciphersuites were enabled. + Fixes bug 29241; bugfix on 0.3.5.1-alpha. + + o Minor features (bandwidth authority): + - Make bandwidth authorities ignore relays that are reported in the + bandwidth file with the flag "vote=0". This change allows us to + report unmeasured relays for diagnostic reasons without including + their bandwidth in the bandwidth authorities' vote. Closes + ticket 29806. + - When a directory authority is using a bandwidth file to obtain the + bandwidth values that will be included in the next vote, serve + this bandwidth file at /tor/status-vote/next/bandwidth. Closes + ticket 21377. + + o Minor features (circuit padding): + - Stop warning about undefined behavior in the probability + distribution tests. Float division by zero may technically be + undefined behavior in C, but it's well defined in IEEE 754. + Partial backport of 29298. Closes ticket 29527; bugfix + on 0.4.0.1-alpha. + + o Minor features (continuous integration): + - On Travis Rust builds, cleanup Rust registry and refrain from + caching the "target/" directory to speed up builds. Resolves + issue 29962. + + o Minor features (dormant mode): + - Add a DormantCanceledByStartup option to tell Tor that it should + treat a startup event as cancelling any previous dormant state. + Integrators should use this option with caution: it should only be + used if Tor is being started because of something that the user + did, and not if Tor is being automatically started in the + background. Closes ticket 29357. + + o Minor features (geoip): + - Update geoip and geoip6 to the April 2 2019 Maxmind GeoLite2 + Country database. Closes ticket 29992. + + o Minor features (NSS, diagnostic): + - Try to log an error from NSS (if there is any) and a more useful + description of our situation if we are using NSS and a call to + SSL_ExportKeyingMaterial() fails. Diagnostic for ticket 29241. + + o Minor bugfixes (security): + - Fix a potential double free bug when reading huge bandwidth files. + The issue is not exploitable in the current Tor network because + the vulnerable code is only reached when directory authorities + read bandwidth files, but bandwidth files come from a trusted + source (usually the authorities themselves). Furthermore, the + issue is only exploitable in rare (non-POSIX) 32-bit architectures, + which are not used by any of the current authorities. Fixes bug + 30040; bugfix on 0.3.5.1-alpha. Bug found and fixed by + Tobias Stoeckmann. + - Verify in more places that we are not about to create a buffer + with more than INT_MAX bytes, to avoid possible OOB access in the + event of bugs. Fixes bug 30041; bugfix on 0.2.0.16. Found and + fixed by Tobias Stoeckmann. + + o Minor bugfix (continuous integration): + - Reset coverage state on disk after Travis CI has finished. This + should prevent future coverage merge errors from causing the test + suite for the "process" subsystem to fail. The process subsystem + was introduced in 0.4.0.1-alpha. Fixes bug 29036; bugfix + on 0.2.9.15. + - Terminate test-stem if it takes more than 9.5 minutes to run. + (Travis terminates the job after 10 minutes of no output.) + Diagnostic for 29437. Fixes bug 30011; bugfix on 0.3.5.4-alpha. + + o Minor bugfixes (bootstrap reporting): + - During bootstrap reporting, correctly distinguish pluggable + transports from plain proxies. Fixes bug 28925; bugfix + on 0.4.0.1-alpha. + + o Minor bugfixes (C correctness): + - Fix an unlikely memory leak in consensus_diff_apply(). Fixes bug + 29824; bugfix on 0.3.1.1-alpha. This is Coverity warning + CID 1444119. + + o Minor bugfixes (circuitpadding testing): + - Minor tweaks to avoid rare test failures related to timers and + monotonic time. Fixes bug 29500; bugfix on 0.4.0.1-alpha. + + o Minor bugfixes (directory authorities): + - Actually include the bandwidth-file-digest line in directory + authority votes. Fixes bug 29959; bugfix on 0.4.0.2-alpha. + + o Minor bugfixes (logging): + - On Windows, when errors cause us to reload a consensus from disk, + tell the user that we are retrying at log level "notice". + Previously we only logged this information at "info", which was + confusing because the errors themselves were logged at "warning". + Improves previous fix for 28614. Fixes bug 30004; bugfix + on 0.4.0.2-alpha. + + o Minor bugfixes (pluggable transports): + - Restore old behavior when it comes to discovering the path of a + given Pluggable Transport executable file. A change in + 0.4.0.1-alpha had broken this behavior on paths containing a + space. Fixes bug 29874; bugfix on 0.4.0.1-alpha. + + o Minor bugfixes (testing): + - Backport the 0.3.4 src/test/test-network.sh to 0.2.9. We need a + recent test-network.sh to use new chutney features in CI. Fixes + bug 29703; bugfix on 0.2.9.1-alpha. + - Fix a test failure on Windows caused by an unexpected "BUG" + warning in our tests for tor_gmtime_r(-1). Fixes bug 29922; bugfix + on 0.2.9.3-alpha. + + o Minor bugfixes (TLS protocol): + - When classifying a client's selection of TLS ciphers, if the + client ciphers are not yet available, do not cache the result. + Previously, we had cached the unavailability of the cipher list + and never looked again, which in turn led us to assume that the + client only supported the ancient V1 link protocol. This, in turn, + was causing Stem integration tests to stall in some cases. Fixes + bug 30021; bugfix on 0.2.4.8-alpha. + + o Code simplification and refactoring: + - Introduce a connection_dir_buf_add() helper function that detects + whether compression is in use, and adds a string accordingly. + Resolves issue 28816. + - Refactor handle_get_next_bandwidth() to use + connection_dir_buf_add(). Implements ticket 29897. + + o Documentation: + - Clarify that Tor performs stream isolation among *Port listeners + by default. Resolves issue 29121. + + Changes in version 0.4.0.3-alpha - 2019-03-22 Tor 0.4.0.3-alpha is the third in its series; it fixes several small bugs from earlier versions. From 7c98105d5617a0423c2d45941e7c0906de649128 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 11 Apr 2019 17:35:19 -0400 Subject: [PATCH 0765/2557] On failure to create extend info for an introduction point, don't leak. This is CID 1438152. No backport needed: this path is already inside a BUG() guard. --- src/feature/rend/rendservice.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/feature/rend/rendservice.c b/src/feature/rend/rendservice.c index 57475a64b0..996e7b9a28 100644 --- a/src/feature/rend/rendservice.c +++ b/src/feature/rend/rendservice.c @@ -4227,6 +4227,7 @@ rend_consider_services_intro_points(time_t now) * directly ourselves. */ intro->extend_info = extend_info_from_node(node, 0); if (BUG(intro->extend_info == NULL)) { + tor_free(intro); break; } intro->intro_key = crypto_pk_new(); From 4e3d144fb0940d8ee5a89427d471ea3656e8e122 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 11 Apr 2019 17:40:05 -0400 Subject: [PATCH 0766/2557] Don't leak on logic error in string_is_valid_nonrfc_hostname() This is CID 1437438. No backport needed: this is unreachable, and guarded with a BUG() check. --- src/lib/net/address.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/lib/net/address.c b/src/lib/net/address.c index 214d8aa3eb..7dec4c8e25 100644 --- a/src/lib/net/address.c +++ b/src/lib/net/address.c @@ -2027,8 +2027,12 @@ string_is_valid_nonrfc_hostname(const char *string) smartlist_split_string(components,string,".",0,0); - if (BUG(smartlist_len(components) == 0)) - return 0; // LCOV_EXCL_LINE should be impossible given the earlier checks. + if (BUG(smartlist_len(components) == 0)) { + // LCOV_EXCL_START should be impossible given the earlier checks. + smartlist_free(components); + return 0; + // LCOV_EXCL_STOP + } /* Allow a single terminating '.' used rarely to indicate domains * are FQDNs rather than relative. */ From 781d69f3a7e1c9d70120cda970b203dea4180b99 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 11 Apr 2019 17:51:11 -0400 Subject: [PATCH 0767/2557] Make it clear to coverity we aren't leaking in protover_all_supported() The logic here should be "use versions or free it". The "free it" part was previously in a kind of obfuscated place, so coverity wasn't sure it was invoked as appropriate. CID 1437436. --- src/core/or/protover.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/core/or/protover.c b/src/core/or/protover.c index 53709ad002..1edf78ec87 100644 --- a/src/core/or/protover.c +++ b/src/core/or/protover.c @@ -820,6 +820,8 @@ protover_all_supported(const char *s, char **missing_out) * ones and, if so, add them to unsupported->ranges. */ if (versions->low != 0 && versions->high != 0) { smartlist_add(unsupported->ranges, versions); + } else { + tor_free(versions); } /* Finally, if we had something unsupported, add it to the list of * missing_some things and mark that there was something missing. */ @@ -828,7 +830,6 @@ protover_all_supported(const char *s, char **missing_out) all_supported = 0; } else { proto_entry_free(unsupported); - tor_free(versions); } } SMARTLIST_FOREACH_END(range); From e39b53ef7de4ac6705869fd153da7b0f87c48ee6 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 11 Apr 2019 17:59:21 -0400 Subject: [PATCH 0768/2557] changes file and practracker updates for 30147. --- changes/coverity_falsepos | 4 ++++ scripts/maint/practracker/exceptions.txt | 6 +++--- 2 files changed, 7 insertions(+), 3 deletions(-) create mode 100644 changes/coverity_falsepos diff --git a/changes/coverity_falsepos b/changes/coverity_falsepos new file mode 100644 index 0000000000..9fbb01a0c1 --- /dev/null +++ b/changes/coverity_falsepos @@ -0,0 +1,4 @@ + o Code simplification and refactoring: + - Refactor several places in our code that coverity incorrectly believed + that we might have memory leaks, so that we can analyze our software + more easily. Closes ticket 30147. diff --git a/scripts/maint/practracker/exceptions.txt b/scripts/maint/practracker/exceptions.txt index 7d03bf27d6..b84396272d 100644 --- a/scripts/maint/practracker/exceptions.txt +++ b/scripts/maint/practracker/exceptions.txt @@ -119,7 +119,7 @@ problem function-size /src/core/or/connection_or.c:connection_or_client_learned_ problem function-size /src/core/or/connection_or.c:connection_or_compute_authenticate_cell_body() 235 problem file-size /src/core/or/policies.c 3163 problem function-size /src/core/or/policies.c:policy_summarize() 107 -problem function-size /src/core/or/protover.c:protover_all_supported() 116 +problem function-size /src/core/or/protover.c:protover_all_supported() 117 problem file-size /src/core/or/relay.c 3173 problem function-size /src/core/or/relay.c:circuit_receive_relay_cell() 123 problem function-size /src/core/or/relay.c:relay_send_command_from_edge_() 101 @@ -245,7 +245,7 @@ problem function-size /src/feature/rend/rendmid.c:rend_mid_establish_intro_legac problem function-size /src/feature/rend/rendparse.c:rend_parse_v2_service_descriptor() 187 problem function-size /src/feature/rend/rendparse.c:rend_decrypt_introduction_points() 104 problem function-size /src/feature/rend/rendparse.c:rend_parse_introduction_points() 131 -problem file-size /src/feature/rend/rendservice.c 4509 +problem file-size /src/feature/rend/rendservice.c 4510 problem function-size /src/feature/rend/rendservice.c:rend_service_prune_list_impl_() 107 problem function-size /src/feature/rend/rendservice.c:rend_config_service() 164 problem function-size /src/feature/rend/rendservice.c:rend_service_load_auth_keys() 178 @@ -256,7 +256,7 @@ problem function-size /src/feature/rend/rendservice.c:rend_service_intro_has_ope problem function-size /src/feature/rend/rendservice.c:rend_service_rendezvous_has_opened() 117 problem function-size /src/feature/rend/rendservice.c:directory_post_to_hs_dir() 108 problem function-size /src/feature/rend/rendservice.c:upload_service_descriptor() 111 -problem function-size /src/feature/rend/rendservice.c:rend_consider_services_intro_points() 169 +problem function-size /src/feature/rend/rendservice.c:rend_consider_services_intro_points() 170 problem function-size /src/feature/stats/rephist.c:rep_hist_load_mtbf_data() 185 problem function-size /src/feature/stats/rephist.c:rep_hist_format_exit_stats() 148 problem function-size /src/lib/compress/compress.c:tor_compress_impl() 133 From 55690d05bd8856798d2e6f6b56261629847c068b Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 11 Apr 2019 18:38:59 -0400 Subject: [PATCH 0769/2557] Add an assertion to pathbias_get_scale_ratio() This should please coverity, and fix CID 1415723. It didn't understand that networkstatus_get_param() always returns a value between its minimum and maximum values. --- src/feature/client/circpathbias.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/feature/client/circpathbias.c b/src/feature/client/circpathbias.c index 1743ab5a81..e6af649ba7 100644 --- a/src/feature/client/circpathbias.c +++ b/src/feature/client/circpathbias.c @@ -176,6 +176,7 @@ pathbias_get_scale_threshold(const or_options_t *options) static double pathbias_get_scale_ratio(const or_options_t *options) { + (void) options; /* * The scale factor is the denominator for our scaling * of circuit counts for our path bias window. @@ -185,7 +186,8 @@ pathbias_get_scale_ratio(const or_options_t *options) */ int denominator = networkstatus_get_param(NULL, "pb_scalefactor", 2, 2, INT32_MAX); - (void) options; + tor_assert(denominator > 0); + /** * The mult factor is the numerator for our scaling * of circuit counts for our path bias window. It From 96e310911fa14b8a8fb1861d08fbda7abe61eb10 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 11 Apr 2019 18:38:59 -0400 Subject: [PATCH 0770/2557] Add an assertion to compute_weighted_bandwidths() This should please coverity, and fix CID 1415722. It didn't understand that networkstatus_get_param() always returns a value between its minimum and maximum values. --- src/feature/nodelist/node_select.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/feature/nodelist/node_select.c b/src/feature/nodelist/node_select.c index e31abb247f..93ddb066d4 100644 --- a/src/feature/nodelist/node_select.c +++ b/src/feature/nodelist/node_select.c @@ -585,6 +585,7 @@ compute_weighted_bandwidths(const smartlist_t *sl, } weight_scale = networkstatus_get_weight_scale_param(NULL); + tor_assert(weight_scale >= 1); if (rule == WEIGHT_FOR_GUARD) { Wg = networkstatus_get_bw_weight(NULL, "Wgg", -1); From 66b07e7ec13feb57d696b920493590d232e62763 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 11 Apr 2019 18:38:59 -0400 Subject: [PATCH 0771/2557] Add an assertion to num_ntors_per_tap(). This should please coverity, and fix CID 1415721. It didn't understand that networkstatus_get_param() always returns a value between its minimum and maximum values. --- src/feature/relay/onion_queue.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/feature/relay/onion_queue.c b/src/feature/relay/onion_queue.c index 696905cf5e..c37745cf33 100644 --- a/src/feature/relay/onion_queue.c +++ b/src/feature/relay/onion_queue.c @@ -212,10 +212,12 @@ num_ntors_per_tap(void) #define MIN_NUM_NTORS_PER_TAP 1 #define MAX_NUM_NTORS_PER_TAP 100000 - return networkstatus_get_param(NULL, "NumNTorsPerTAP", - DEFAULT_NUM_NTORS_PER_TAP, - MIN_NUM_NTORS_PER_TAP, - MAX_NUM_NTORS_PER_TAP); + int result = networkstatus_get_param(NULL, "NumNTorsPerTAP", + DEFAULT_NUM_NTORS_PER_TAP, + MIN_NUM_NTORS_PER_TAP, + MAX_NUM_NTORS_PER_TAP); + tor_assert(result > 0); + return result; } /** Choose which onion queue we'll pull from next. If one is empty choose From 48a574604bef9a4fe77d9bb43dfbfc50328e13e0 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 11 Apr 2019 18:56:02 -0400 Subject: [PATCH 0772/2557] Remove an extraneous _ from __COVERITY__ We had a typo in this check, so that coverity wasn't taking the right path. Bug not in any released Tor. --- src/lib/math/prob_distr.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/math/prob_distr.h b/src/lib/math/prob_distr.h index 2eb935e4a8..8fccf8d015 100644 --- a/src/lib/math/prob_distr.h +++ b/src/lib/math/prob_distr.h @@ -53,7 +53,7 @@ struct dist { * We define this conditionally to suppress false positives from * Coverity, which gets confused by the sizeof business. */ -#ifdef __COVERITY___ +#ifdef __COVERITY__ #define TYPE_CHECK_OBJ(OPS, OBJ, TYPE) 0 #else #define TYPE_CHECK_OBJ(OPS, OBJ, TYPE) \ From 73323460023807259d3cd1dd52d33749ffe53886 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 11 Apr 2019 18:58:44 -0400 Subject: [PATCH 0773/2557] Changes file and practracker updates for 30149. --- changes/ticket30149 | 3 +++ scripts/maint/practracker/exceptions.txt | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 changes/ticket30149 diff --git a/changes/ticket30149 b/changes/ticket30149 new file mode 100644 index 0000000000..a21687ac2f --- /dev/null +++ b/changes/ticket30149 @@ -0,0 +1,3 @@ + o Code simplification and refactoring: + - Add several assertions in an attempt to fix some Coverity warnings. + Closes ticket 30149. diff --git a/scripts/maint/practracker/exceptions.txt b/scripts/maint/practracker/exceptions.txt index 7d03bf27d6..ecc58012c6 100644 --- a/scripts/maint/practracker/exceptions.txt +++ b/scripts/maint/practracker/exceptions.txt @@ -218,7 +218,7 @@ problem include-count /src/feature/nodelist/networkstatus.c 61 problem function-size /src/feature/nodelist/networkstatus.c:networkstatus_check_consensus_signature() 176 problem function-size /src/feature/nodelist/networkstatus.c:networkstatus_set_current_consensus() 293 problem function-size /src/feature/nodelist/node_select.c:router_pick_directory_server_impl() 123 -problem function-size /src/feature/nodelist/node_select.c:compute_weighted_bandwidths() 205 +problem function-size /src/feature/nodelist/node_select.c:compute_weighted_bandwidths() 206 problem function-size /src/feature/nodelist/node_select.c:router_pick_trusteddirserver_impl() 114 problem function-size /src/feature/nodelist/nodelist.c:compute_frac_paths_available() 193 problem file-size /src/feature/nodelist/routerlist.c 3234 From cdafcc49bc273e472d40ea8c01219bbc165c92cb Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 11 Apr 2019 19:09:23 -0400 Subject: [PATCH 0774/2557] Fix a memory leak in tor-resolve.c Closes bug 30151/coverity CID 1441830. Bugfix on 0.4.0.1-alpha when we started doing trunnel parsing in tor-resolve.c. --- changes/bug30151 | 5 +++++ src/tools/tor-resolve.c | 1 + 2 files changed, 6 insertions(+) create mode 100644 changes/bug30151 diff --git a/changes/bug30151 b/changes/bug30151 new file mode 100644 index 0000000000..8ac9a320a0 --- /dev/null +++ b/changes/bug30151 @@ -0,0 +1,5 @@ + o Minor bugfixes (tor-resolve): + - Fix a memory leak in tor-resolve that could happen if Tor gave it a + malformed SOCKS response. (Memory leaks in tor-resolve don't actually + matter, but it's good to fix them anyway.) Fixes bug 30151; bugfix on + 0.4.0.1-alpha. diff --git a/src/tools/tor-resolve.c b/src/tools/tor-resolve.c index 98b3a4a74c..5d97696c18 100644 --- a/src/tools/tor-resolve.c +++ b/src/tools/tor-resolve.c @@ -424,6 +424,7 @@ do_resolve(const char *hostname, if (parsed < 2) { log_err(LD_NET, "Failed to parse SOCKS5 method selection " "message"); + socks5_server_method_free(m); goto err; } From aa9940ed21e890ddbb0a3541bf15e7cca4c3b489 Mon Sep 17 00:00:00 2001 From: Neel Chauhan Date: Thu, 11 Apr 2019 20:24:08 -0400 Subject: [PATCH 0775/2557] Make SET_BRIDGES_RUNNING_INTERVAL 5 minutes --- src/core/mainloop/mainloop.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/mainloop/mainloop.c b/src/core/mainloop/mainloop.c index 800304a736..7609c630c8 100644 --- a/src/core/mainloop/mainloop.c +++ b/src/core/mainloop/mainloop.c @@ -2602,7 +2602,7 @@ set_bridge_running_callback(time_t now, const or_options_t *options) dirserv_set_router_is_running(ri, now); } SMARTLIST_FOREACH_END(ri); -#define SET_BRIDGES_RUNNING_INTERVAL (3*60) +#define SET_BRIDGES_RUNNING_INTERVAL (5*60) return SET_BRIDGES_RUNNING_INTERVAL; } return PERIODIC_EVENT_NO_UPDATE; From 30279a7c575d2b27055e511a2f8cf84336387bb8 Mon Sep 17 00:00:00 2001 From: Neel Chauhan Date: Thu, 11 Apr 2019 20:28:11 -0400 Subject: [PATCH 0776/2557] Use authdir_mode_bridge() in set_bridge_running_callback() --- src/core/mainloop/mainloop.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/mainloop/mainloop.c b/src/core/mainloop/mainloop.c index 7609c630c8..fd711fd3bc 100644 --- a/src/core/mainloop/mainloop.c +++ b/src/core/mainloop/mainloop.c @@ -2594,7 +2594,7 @@ write_bridge_ns_callback(time_t now, const or_options_t *options) static int set_bridge_running_callback(time_t now, const or_options_t *options) { - if (options->BridgeAuthoritativeDir) { + if (authdir_mode_bridge(options)) { routerlist_t *rl = router_get_routerlist(); SMARTLIST_FOREACH_BEGIN(rl->routers, routerinfo_t *, ri) { From e16f5184dad6d0052df37496327e2652d9c79d00 Mon Sep 17 00:00:00 2001 From: Neel Chauhan Date: Thu, 11 Apr 2019 20:32:38 -0400 Subject: [PATCH 0777/2557] Fix grammar in bug24490 changes file --- changes/bug24490 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changes/bug24490 b/changes/bug24490 index 9ae09dbd17..1167e9f8d3 100644 --- a/changes/bug24490 +++ b/changes/bug24490 @@ -1,5 +1,5 @@ o Minor bugfixes (bridge authority): - We set bridges as running in a callback which runs every 5 minutes. - Previously, we set bridges as running in a GETINFO controller as + Previously, we set bridges as running in a GETINFO controller, but these shouldn't modify vital data structures. Fixes bug 24490; bugfix on 0.2.0.13-alpha. Patch by Neel Chauhan From 4172dcaa62b02593910736110d9d2c94052dbdcb Mon Sep 17 00:00:00 2001 From: Neel Chauhan Date: Thu, 11 Apr 2019 20:44:30 -0400 Subject: [PATCH 0778/2557] Move code for setting bridges as running to voteflags.c --- scripts/maint/practracker/exceptions.txt | 4 ++-- src/core/mainloop/mainloop.c | 7 +------ src/feature/dirauth/voteflags.c | 18 ++++++++++++++++++ src/feature/dirauth/voteflags.h | 2 ++ 4 files changed, 23 insertions(+), 8 deletions(-) diff --git a/scripts/maint/practracker/exceptions.txt b/scripts/maint/practracker/exceptions.txt index a9fcf9095a..7b06683eb7 100644 --- a/scripts/maint/practracker/exceptions.txt +++ b/scripts/maint/practracker/exceptions.txt @@ -67,7 +67,7 @@ problem function-size /src/core/mainloop/connection.c:connection_handle_read_imp problem function-size /src/core/mainloop/connection.c:connection_buf_read_from_socket() 177 problem function-size /src/core/mainloop/connection.c:connection_handle_write_impl() 241 problem function-size /src/core/mainloop/connection.c:assert_connection_ok() 143 -problem file-size /src/core/mainloop/mainloop.c 3076 +problem file-size /src/core/mainloop/mainloop.c 3071 problem include-count /src/core/mainloop/mainloop.c 68 problem function-size /src/core/mainloop/mainloop.c:conn_close_if_marked() 108 problem function-size /src/core/mainloop/mainloop.c:run_connection_housekeeping() 123 @@ -276,7 +276,7 @@ problem function-size /src/lib/net/resolve.c:tor_addr_lookup() 110 problem function-size /src/lib/net/socketpair.c:tor_ersatz_socketpair() 102 problem function-size /src/lib/osinfo/uname.c:get_uname() 116 problem function-size /src/lib/process/process_unix.c:process_unix_exec() 220 -problem function-size /src/lib/process/process_win32.c:process_win32_exec() 138 +problem function-size /src/lib/process/process_win32.c:process_win32_exec() 133 problem function-size /src/lib/process/process_win32.c:process_win32_create_pipe() 112 problem function-size /src/lib/process/restrict.c:set_max_file_descriptors() 102 problem function-size /src/lib/process/setuid.c:switch_id() 156 diff --git a/src/core/mainloop/mainloop.c b/src/core/mainloop/mainloop.c index fd711fd3bc..e845ff416e 100644 --- a/src/core/mainloop/mainloop.c +++ b/src/core/mainloop/mainloop.c @@ -2595,12 +2595,7 @@ static int set_bridge_running_callback(time_t now, const or_options_t *options) { if (authdir_mode_bridge(options)) { - routerlist_t *rl = router_get_routerlist(); - - SMARTLIST_FOREACH_BEGIN(rl->routers, routerinfo_t *, ri) { - if (ri->purpose == ROUTER_PURPOSE_BRIDGE) - dirserv_set_router_is_running(ri, now); - } SMARTLIST_FOREACH_END(ri); + dirserv_set_bridges_running(now); #define SET_BRIDGES_RUNNING_INTERVAL (5*60) return SET_BRIDGES_RUNNING_INTERVAL; diff --git a/src/feature/dirauth/voteflags.c b/src/feature/dirauth/voteflags.c index 0a53c588d6..4040f162fa 100644 --- a/src/feature/dirauth/voteflags.c +++ b/src/feature/dirauth/voteflags.c @@ -29,6 +29,7 @@ #include "feature/nodelist/node_st.h" #include "feature/nodelist/routerinfo_st.h" +#include "feature/nodelist/routerlist_st.h" #include "feature/nodelist/vote_routerstatus_st.h" #include "lib/container/order.h" @@ -658,3 +659,20 @@ dirserv_set_routerstatus_testing(routerstatus_t *rs) rs->is_hs_dir = 0; } } + +/** Use dirserv_set_router_is_running() to set bridges as running if they're + * reachable. + * + * This function is called from set_bridge_running_callback() when running as + * a bridge authority. + */ +void +dirserv_set_bridges_running(time_t now) +{ + routerlist_t *rl = router_get_routerlist(); + + SMARTLIST_FOREACH_BEGIN(rl->routers, routerinfo_t *, ri) { + if (ri->purpose == ROUTER_PURPOSE_BRIDGE) + dirserv_set_router_is_running(ri, now); + } SMARTLIST_FOREACH_END(ri); +} diff --git a/src/feature/dirauth/voteflags.h b/src/feature/dirauth/voteflags.h index cca6f53746..18b29a5183 100644 --- a/src/feature/dirauth/voteflags.h +++ b/src/feature/dirauth/voteflags.h @@ -25,6 +25,8 @@ void set_routerstatus_from_routerinfo(routerstatus_t *rs, void dirserv_compute_performance_thresholds(digestmap_t *omit_as_sybil); +void dirserv_set_bridges_running(time_t now); + #ifdef VOTEFLAGS_PRIVATE /** Any descriptor older than this age causes the authorities to set the * StaleDesc flag. */ From c07d854772bda558ef8cf4fd71f2673c7ed00083 Mon Sep 17 00:00:00 2001 From: Neel Chauhan Date: Thu, 11 Apr 2019 21:28:35 -0400 Subject: [PATCH 0779/2557] Remove callback for setting bridges as running --- changes/bug24490 | 2 +- scripts/maint/practracker/exceptions.txt | 8 ++++---- src/core/mainloop/mainloop.c | 20 -------------------- src/feature/nodelist/networkstatus.c | 5 ++++- 4 files changed, 9 insertions(+), 26 deletions(-) diff --git a/changes/bug24490 b/changes/bug24490 index 1167e9f8d3..cf9281c878 100644 --- a/changes/bug24490 +++ b/changes/bug24490 @@ -1,5 +1,5 @@ o Minor bugfixes (bridge authority): - - We set bridges as running in a callback which runs every 5 minutes. + - We set bridges as running when we dump the bridge status to a file. Previously, we set bridges as running in a GETINFO controller, but these shouldn't modify vital data structures. Fixes bug 24490; bugfix on 0.2.0.13-alpha. Patch by Neel Chauhan diff --git a/scripts/maint/practracker/exceptions.txt b/scripts/maint/practracker/exceptions.txt index 7b06683eb7..ad5d3e9725 100644 --- a/scripts/maint/practracker/exceptions.txt +++ b/scripts/maint/practracker/exceptions.txt @@ -67,11 +67,11 @@ problem function-size /src/core/mainloop/connection.c:connection_handle_read_imp problem function-size /src/core/mainloop/connection.c:connection_buf_read_from_socket() 177 problem function-size /src/core/mainloop/connection.c:connection_handle_write_impl() 241 problem function-size /src/core/mainloop/connection.c:assert_connection_ok() 143 -problem file-size /src/core/mainloop/mainloop.c 3071 -problem include-count /src/core/mainloop/mainloop.c 68 +problem file-size /src/core/mainloop/mainloop.c 3051 +problem include-count /src/core/mainloop/mainloop.c 66 problem function-size /src/core/mainloop/mainloop.c:conn_close_if_marked() 108 problem function-size /src/core/mainloop/mainloop.c:run_connection_housekeeping() 123 -problem function-size /src/core/mainloop/mainloop.c:CALLBACK() 118 +problem function-size /src/core/mainloop/mainloop.c:CALLBACK() 116 problem file-size /src/core/or/channel.c 3476 problem function-size /src/core/or/channeltls.c:channel_tls_handle_var_cell() 160 problem function-size /src/core/or/channeltls.c:channel_tls_process_versions_cell() 170 @@ -276,7 +276,7 @@ problem function-size /src/lib/net/resolve.c:tor_addr_lookup() 110 problem function-size /src/lib/net/socketpair.c:tor_ersatz_socketpair() 102 problem function-size /src/lib/osinfo/uname.c:get_uname() 116 problem function-size /src/lib/process/process_unix.c:process_unix_exec() 220 -problem function-size /src/lib/process/process_win32.c:process_win32_exec() 133 +problem function-size /src/lib/process/process_win32.c:process_win32_exec() 138 problem function-size /src/lib/process/process_win32.c:process_win32_create_pipe() 112 problem function-size /src/lib/process/restrict.c:set_max_file_descriptors() 102 problem function-size /src/lib/process/setuid.c:switch_id() 156 diff --git a/src/core/mainloop/mainloop.c b/src/core/mainloop/mainloop.c index e845ff416e..c9f2b0d896 100644 --- a/src/core/mainloop/mainloop.c +++ b/src/core/mainloop/mainloop.c @@ -76,7 +76,6 @@ #include "feature/control/control_events.h" #include "feature/dirauth/authmode.h" #include "feature/dirauth/reachability.h" -#include "feature/dirauth/voteflags.h" #include "feature/dircache/consdiffmgr.h" #include "feature/dircache/dirserv.h" #include "feature/dircommon/directory.h" @@ -88,7 +87,6 @@ #include "feature/nodelist/networkstatus.h" #include "feature/nodelist/nodelist.h" #include "feature/nodelist/routerlist.h" -#include "feature/nodelist/routerlist_st.h" #include "feature/relay/dns.h" #include "feature/relay/routerkeys.h" #include "feature/relay/routermode.h" @@ -1377,7 +1375,6 @@ CALLBACK(rotate_onion_key); CALLBACK(rotate_x509_certificate); CALLBACK(save_stability); CALLBACK(save_state); -CALLBACK(set_bridge_running); CALLBACK(write_bridge_ns); CALLBACK(write_stats_file); CALLBACK(control_per_second_events); @@ -1456,7 +1453,6 @@ STATIC periodic_event_item_t periodic_events[] = { /* Bridge Authority only. */ CALLBACK(write_bridge_ns, BRIDGEAUTH, 0), - CALLBACK(set_bridge_running, BRIDGEAUTH, 0), /* Directory server only. */ CALLBACK(clean_consdiffmgr, DIRSERVER, 0), @@ -2587,22 +2583,6 @@ write_bridge_ns_callback(time_t now, const or_options_t *options) return PERIODIC_EVENT_NO_UPDATE; } -/** - * Periodic callback: if we're the bridge authority, set the running flag on - * bridges if they're reachable - */ -static int -set_bridge_running_callback(time_t now, const or_options_t *options) -{ - if (authdir_mode_bridge(options)) { - dirserv_set_bridges_running(now); - -#define SET_BRIDGES_RUNNING_INTERVAL (5*60) - return SET_BRIDGES_RUNNING_INTERVAL; - } - return PERIODIC_EVENT_NO_UPDATE; -} - static int heartbeat_callback_first_time = 1; /** diff --git a/src/feature/nodelist/networkstatus.c b/src/feature/nodelist/networkstatus.c index bc12fa4075..20881112aa 100644 --- a/src/feature/nodelist/networkstatus.c +++ b/src/feature/nodelist/networkstatus.c @@ -2409,7 +2409,7 @@ networkstatus_getinfo_by_purpose(const char *purpose_string, time_t now) void networkstatus_dump_bridge_status_to_file(time_t now) { - char *status = networkstatus_getinfo_by_purpose("bridge", now); + char *status; char *fname = NULL; char *thresholds = NULL; char *published_thresholds_and_status = NULL; @@ -2418,6 +2418,9 @@ networkstatus_dump_bridge_status_to_file(time_t now) char fingerprint[FINGERPRINT_LEN+1]; char *fingerprint_line = NULL; + dirserv_set_bridges_running(now); + status = networkstatus_getinfo_by_purpose("bridge", now); + if (me && crypto_pk_get_fingerprint(me->identity_pkey, fingerprint, 0) >= 0) { tor_asprintf(&fingerprint_line, "fingerprint %s\n", fingerprint); From 14d700804568ef5624856976b723eafab9e41972 Mon Sep 17 00:00:00 2001 From: Neel Chauhan Date: Thu, 11 Apr 2019 21:30:48 -0400 Subject: [PATCH 0780/2557] Stop setting routers as running in list_server_status_v1() --- src/feature/control/fmt_serverstatus.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/feature/control/fmt_serverstatus.c b/src/feature/control/fmt_serverstatus.c index a1ddd2119a..fc1029ae84 100644 --- a/src/feature/control/fmt_serverstatus.c +++ b/src/feature/control/fmt_serverstatus.c @@ -78,10 +78,7 @@ list_server_status_v1(smartlist_t *routers, char **router_status_out, SMARTLIST_FOREACH_BEGIN(routers, routerinfo_t *, ri) { const node_t *node = node_get_by_id(ri->cache_info.identity_digest); tor_assert(node); - if (authdir) { - /* Update router status in routerinfo_t. */ - dirserv_set_router_is_running(ri, now); - } + if (for_controller) { char name_buf[MAX_VERBOSE_NICKNAME_LEN+2]; char *cp = name_buf; From 994b8ba424e0e225b3a3b3923044ff900cedfc12 Mon Sep 17 00:00:00 2001 From: Neel Chauhan Date: Thu, 11 Apr 2019 21:36:38 -0400 Subject: [PATCH 0781/2557] Update networkstatus_getinfo_by_purpose() comment --- src/feature/nodelist/networkstatus.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/feature/nodelist/networkstatus.c b/src/feature/nodelist/networkstatus.c index 20881112aa..db510c975b 100644 --- a/src/feature/nodelist/networkstatus.c +++ b/src/feature/nodelist/networkstatus.c @@ -2405,7 +2405,8 @@ networkstatus_getinfo_by_purpose(const char *purpose_string, time_t now) return answer; } -/** Write out router status entries for all our bridge descriptors. */ +/** Write out router status entries for all our bridge descriptors. Here, we + * also mark routers as running. */ void networkstatus_dump_bridge_status_to_file(time_t now) { From 398c736230719e6b8eb539c7b08a5acfc0463db1 Mon Sep 17 00:00:00 2001 From: Neel Chauhan Date: Thu, 11 Apr 2019 22:11:27 -0400 Subject: [PATCH 0782/2557] Remove unused variable in fmt_serverstatus.c --- src/feature/control/fmt_serverstatus.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/feature/control/fmt_serverstatus.c b/src/feature/control/fmt_serverstatus.c index fc1029ae84..1fc5d4a157 100644 --- a/src/feature/control/fmt_serverstatus.c +++ b/src/feature/control/fmt_serverstatus.c @@ -70,7 +70,6 @@ list_server_status_v1(smartlist_t *routers, char **router_status_out, /* We include v2 dir auths here too, because they need to answer * controllers. Eventually we'll deprecate this whole function; * see also networkstatus_getinfo_by_purpose(). */ - int authdir = authdir_mode_publishes_statuses(options); tor_assert(router_status_out); rs_entries = smartlist_new(); From ebbc2c3d8f8268c09b6df8270907a1123f926caa Mon Sep 17 00:00:00 2001 From: teor Date: Fri, 12 Apr 2019 13:00:02 +1000 Subject: [PATCH 0783/2557] crypt_ops: Stop using a separate buffer in ed25519_signature_from_base64() Part of 29960. --- src/lib/crypt_ops/crypto_format.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/lib/crypt_ops/crypto_format.c b/src/lib/crypt_ops/crypto_format.c index 1827168c75..e11b391194 100644 --- a/src/lib/crypt_ops/crypto_format.c +++ b/src/lib/crypt_ops/crypto_format.c @@ -248,11 +248,9 @@ ed25519_signature_from_base64(ed25519_signature_t *sig, { if (strlen(input) != ED25519_SIG_BASE64_LEN) return -1; - char buf[ED25519_SIG_BASE64_LEN+1]; - memcpy(buf, input, ED25519_SIG_BASE64_LEN); - buf[ED25519_SIG_BASE64_LEN] = 0; char decoded[128]; - int n = base64_decode(decoded, sizeof(decoded), buf, strlen(buf)); + int n = base64_decode(decoded, sizeof(decoded), input, + ED25519_SIG_BASE64_LEN); if (n < 0 || n != ED25519_SIG_LEN) return -1; memcpy(sig->sig, decoded, ED25519_SIG_LEN); From d1f5957c4ea82d1622233c0dabd1e761df5200d4 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 1 Apr 2019 17:27:08 -0400 Subject: [PATCH 0784/2557] Improve handling of controller commands Use a table-based lookup to find the right command handler. This will serve as the basement for several future improvements, as we improve the API for parsing commands. --- src/feature/control/control_cmd.c | 298 ++++++++++++++++++++---------- 1 file changed, 198 insertions(+), 100 deletions(-) diff --git a/src/feature/control/control_cmd.c b/src/feature/control/control_cmd.c index 95cf0d561e..4fcfa73105 100644 --- a/src/feature/control/control_cmd.c +++ b/src/feature/control/control_cmd.c @@ -82,7 +82,7 @@ handle_control_resetconf(control_connection_t *conn, uint32_t len, char *body) * reply with a CONFVALUE or an ERROR message */ static int handle_control_getconf(control_connection_t *conn, uint32_t body_len, - const char *body) + char *body) { smartlist_t *questions = smartlist_new(); smartlist_t *answers = smartlist_new(); @@ -2206,110 +2206,208 @@ handle_control_del_onion(control_connection_t *conn, return 0; } +/** + * Called when we get an obsolete command: tell the controller that it is + * obsolete. + */ +static int +handle_control_obsolete(control_connection_t *conn, + uint32_t arg_len, + const char *args) +{ + (void)arg_len; + (void)args; + char *command = tor_strdup(conn->incoming_cmd); + tor_strupper(command); + connection_printf_to_buf(conn, "511 %s is obsolete.\r\n", command); + tor_free(command); + return 0; +} + +/** + * Selects an API to a controller command. See handler_fn_t for the + * possible types. + **/ +typedef enum handler_type_t { + hnd_legacy, + hnd_legacy_mut +} handler_type_t; + +/** + * Union: a function pointer to a handler function for a controller command. + * + * This needs to be a union (rather than just a single pointer) since not + * all controller commands have the same type. + **/ +typedef union handler_fn_t { + /** + * A "legacy" handler takes a command's arguments as a nul-terminated + * string, and their length. It may not change the contents of the + * arguments. If the command is a multiline one, then the arguments may + * extend across multiple lines. + */ + int (*legacy)(control_connection_t *conn, + uint32_t arg_len, + const char *args); + /** + * A "legacy_mut" handler is the same as a "legacy" one, except that it may + * change the contents of the command's arguments -- for example, by + * inserting NULs. It may not deallocate them. + */ + int (*legacy_mut)(control_connection_t *conn, + uint32_t arg_len, + char *args); +} handler_fn_t; + +/** + * Definition for a controller command. + */ +typedef struct control_cmd_def_t { + /** + * The name of the command. If the command is multiline, the name must + * begin with "+". This is not case-sensitive. */ + const char *name; + /** + * Which API to use when calling the handler function. + */ + handler_type_t handler_type; + /** + * A function to execute the command. + */ + handler_fn_t handler; + /** + * Zero or more CMD_FL_* flags, or'd together. + */ + unsigned flags; +} control_cmd_def_t; + +/** + * Indicates that the command's arguments are sensitive, and should be + * memwiped after use. + */ +#define CMD_FL_WIPE (1u<<0) + +/** + * Macro: declare a command with a one-line argument and a given set of + * flags. + **/ +#define ONE_LINE(name, htype, flags) \ + { #name, \ + hnd_ ##htype, \ + { .htype = handle_control_ ##name }, \ + flags \ + } +/** + * Macro: declare a command with a multi-line argument and a given set of + * flags. + **/ +#define MULTLINE(name, htype, flags) \ + { "+"#name, \ + hnd_ ##htype, \ + { .htype = handle_control_ ##name }, \ + flags \ + } +/** + * Macro: declare an obsolete command. (Obsolete commands give a different + * error than non-existent ones.) + **/ +#define OBSOLETE(name) \ + { #name, \ + hnd_legacy, \ + { .legacy = handle_control_obsolete }, \ + 0 \ + } + +/** + * An array defining all the recognized controller commands. + **/ +static const control_cmd_def_t CONTROL_COMMANDS[] = +{ + ONE_LINE(setconf, legacy_mut, 0), + ONE_LINE(resetconf, legacy_mut, 0), + ONE_LINE(getconf, legacy_mut, 0), + MULTLINE(loadconf, legacy, 0), + ONE_LINE(setevents, legacy, 0), + ONE_LINE(authenticate, legacy, 0), + ONE_LINE(saveconf, legacy, 0), + ONE_LINE(signal, legacy, 0), + ONE_LINE(takeownership, legacy, 0), + ONE_LINE(dropownership, legacy, 0), + ONE_LINE(mapaddress, legacy, 0), + ONE_LINE(getinfo, legacy, 0), + ONE_LINE(extendcircuit, legacy, 0), + ONE_LINE(setcircuitpurpose, legacy, 0), + OBSOLETE(setrouterpurpose), + ONE_LINE(attachstream, legacy, 0), + MULTLINE(postdescriptor, legacy, 0), + ONE_LINE(redirectstream, legacy, 0), + ONE_LINE(closestream, legacy, 0), + ONE_LINE(closecircuit, legacy, 0), + ONE_LINE(usefeature, legacy, 0), + ONE_LINE(resolve, legacy, 0), + ONE_LINE(protocolinfo, legacy, 0), + ONE_LINE(authchallenge, legacy, 0), + ONE_LINE(dropguards, legacy, 0), + ONE_LINE(hsfetch, legacy, 0), + MULTLINE(hspost, legacy, 0), + ONE_LINE(add_onion, legacy, CMD_FL_WIPE), + ONE_LINE(del_onion, legacy, CMD_FL_WIPE), +}; + +/** + * The number of entries in CONTROL_COMMANDS. + **/ +static const size_t N_CONTROL_COMMANDS = ARRAY_LENGTH(CONTROL_COMMANDS); + +/** + * Run a single control command, as defined by a control_cmd_def_t, + * with a given set of arguments. + */ +static int +handle_single_control_command(const control_cmd_def_t *def, + control_connection_t *conn, + uint32_t cmd_data_len, + char *args) +{ + int rv = 0; + switch (def->handler_type) { + case hnd_legacy: + if (def->handler.legacy(conn, cmd_data_len, args)) + rv = -1; + break; + case hnd_legacy_mut: + if (def->handler.legacy_mut(conn, cmd_data_len, args)) + rv = -1; + break; + default: + tor_assert_unreached(); + } + + if (def->flags & CMD_FL_WIPE) + memwipe(args, 0, cmd_data_len); + + return rv; +} + +/** + * Run a given controller command, as selected by the incoming_cmd field of + * conn. + */ int handle_control_command(control_connection_t *conn, - uint32_t cmd_data_len, - char *args) + uint32_t cmd_data_len, + char *args) { - /* XXXX Why is this not implemented as a table like the GETINFO - * items are? Even handling the plus signs at the beginnings of - * commands wouldn't be very hard with proper macros. */ - - if (!strcasecmp(conn->incoming_cmd, "SETCONF")) { - if (handle_control_setconf(conn, cmd_data_len, args)) - return -1; - } else if (!strcasecmp(conn->incoming_cmd, "RESETCONF")) { - if (handle_control_resetconf(conn, cmd_data_len, args)) - return -1; - } else if (!strcasecmp(conn->incoming_cmd, "GETCONF")) { - if (handle_control_getconf(conn, cmd_data_len, args)) - return -1; - } else if (!strcasecmp(conn->incoming_cmd, "+LOADCONF")) { - if (handle_control_loadconf(conn, cmd_data_len, args)) - return -1; - } else if (!strcasecmp(conn->incoming_cmd, "SETEVENTS")) { - if (handle_control_setevents(conn, cmd_data_len, args)) - return -1; - } else if (!strcasecmp(conn->incoming_cmd, "AUTHENTICATE")) { - if (handle_control_authenticate(conn, cmd_data_len, args)) - return -1; - } else if (!strcasecmp(conn->incoming_cmd, "SAVECONF")) { - if (handle_control_saveconf(conn, cmd_data_len, args)) - return -1; - } else if (!strcasecmp(conn->incoming_cmd, "SIGNAL")) { - if (handle_control_signal(conn, cmd_data_len, args)) - return -1; - } else if (!strcasecmp(conn->incoming_cmd, "TAKEOWNERSHIP")) { - if (handle_control_takeownership(conn, cmd_data_len, args)) - return -1; - } else if (!strcasecmp(conn->incoming_cmd, "DROPOWNERSHIP")) { - if (handle_control_dropownership(conn, cmd_data_len, args)) - return -1; - } else if (!strcasecmp(conn->incoming_cmd, "MAPADDRESS")) { - if (handle_control_mapaddress(conn, cmd_data_len, args)) - return -1; - } else if (!strcasecmp(conn->incoming_cmd, "GETINFO")) { - if (handle_control_getinfo(conn, cmd_data_len, args)) - return -1; - } else if (!strcasecmp(conn->incoming_cmd, "EXTENDCIRCUIT")) { - if (handle_control_extendcircuit(conn, cmd_data_len, args)) - return -1; - } else if (!strcasecmp(conn->incoming_cmd, "SETCIRCUITPURPOSE")) { - if (handle_control_setcircuitpurpose(conn, cmd_data_len, args)) - return -1; - } else if (!strcasecmp(conn->incoming_cmd, "SETROUTERPURPOSE")) { - connection_write_str_to_buf("511 SETROUTERPURPOSE is obsolete.\r\n", conn); - } else if (!strcasecmp(conn->incoming_cmd, "ATTACHSTREAM")) { - if (handle_control_attachstream(conn, cmd_data_len, args)) - return -1; - } else if (!strcasecmp(conn->incoming_cmd, "+POSTDESCRIPTOR")) { - if (handle_control_postdescriptor(conn, cmd_data_len, args)) - return -1; - } else if (!strcasecmp(conn->incoming_cmd, "REDIRECTSTREAM")) { - if (handle_control_redirectstream(conn, cmd_data_len, args)) - return -1; - } else if (!strcasecmp(conn->incoming_cmd, "CLOSESTREAM")) { - if (handle_control_closestream(conn, cmd_data_len, args)) - return -1; - } else if (!strcasecmp(conn->incoming_cmd, "CLOSECIRCUIT")) { - if (handle_control_closecircuit(conn, cmd_data_len, args)) - return -1; - } else if (!strcasecmp(conn->incoming_cmd, "USEFEATURE")) { - if (handle_control_usefeature(conn, cmd_data_len, args)) - return -1; - } else if (!strcasecmp(conn->incoming_cmd, "RESOLVE")) { - if (handle_control_resolve(conn, cmd_data_len, args)) - return -1; - } else if (!strcasecmp(conn->incoming_cmd, "PROTOCOLINFO")) { - if (handle_control_protocolinfo(conn, cmd_data_len, args)) - return -1; - } else if (!strcasecmp(conn->incoming_cmd, "AUTHCHALLENGE")) { - if (handle_control_authchallenge(conn, cmd_data_len, args)) - return -1; - } else if (!strcasecmp(conn->incoming_cmd, "DROPGUARDS")) { - if (handle_control_dropguards(conn, cmd_data_len, args)) - return -1; - } else if (!strcasecmp(conn->incoming_cmd, "HSFETCH")) { - if (handle_control_hsfetch(conn, cmd_data_len, args)) - return -1; - } else if (!strcasecmp(conn->incoming_cmd, "+HSPOST")) { - if (handle_control_hspost(conn, cmd_data_len, args)) - return -1; - } else if (!strcasecmp(conn->incoming_cmd, "ADD_ONION")) { - int ret = handle_control_add_onion(conn, cmd_data_len, args); - memwipe(args, 0, cmd_data_len); /* Scrub the private key. */ - if (ret) - return -1; - } else if (!strcasecmp(conn->incoming_cmd, "DEL_ONION")) { - int ret = handle_control_del_onion(conn, cmd_data_len, args); - memwipe(args, 0, cmd_data_len); /* Scrub the service id/pk. */ - if (ret) - return -1; - } else { - connection_printf_to_buf(conn, "510 Unrecognized command \"%s\"\r\n", - conn->incoming_cmd); + for (unsigned i = 0; i < N_CONTROL_COMMANDS; ++i) { + const control_cmd_def_t *def = &CONTROL_COMMANDS[i]; + if (!strcasecmp(conn->incoming_cmd, def->name)) { + return handle_single_control_command(def, conn, cmd_data_len, args); + } } + connection_printf_to_buf(conn, "510 Unrecognized command \"%s\"\r\n", + conn->incoming_cmd); + return 0; } From f3bd0240a6089910f3075a779b32e7aed07338e0 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 1 Apr 2019 17:29:46 -0400 Subject: [PATCH 0785/2557] Add assertions for correct input to handle_control_command. --- src/feature/control/control_cmd.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/feature/control/control_cmd.c b/src/feature/control/control_cmd.c index 4fcfa73105..56202a7ef4 100644 --- a/src/feature/control/control_cmd.c +++ b/src/feature/control/control_cmd.c @@ -2398,6 +2398,10 @@ handle_control_command(control_connection_t *conn, uint32_t cmd_data_len, char *args) { + tor_assert(conn); + tor_assert(args); + tor_assert(args[cmd_data_len] == '\0'); + for (unsigned i = 0; i < N_CONTROL_COMMANDS; ++i) { const control_cmd_def_t *def = &CONTROL_COMMANDS[i]; if (!strcasecmp(conn->incoming_cmd, def->name)) { From e9ca904dbfc99ebef567d7bb2c6d87819d0d832c Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 1 Apr 2019 17:30:46 -0400 Subject: [PATCH 0786/2557] Define two more commands as wipe-after-parse. --- src/feature/control/control_cmd.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/feature/control/control_cmd.c b/src/feature/control/control_cmd.c index 56202a7ef4..930276b104 100644 --- a/src/feature/control/control_cmd.c +++ b/src/feature/control/control_cmd.c @@ -2328,7 +2328,7 @@ static const control_cmd_def_t CONTROL_COMMANDS[] = ONE_LINE(getconf, legacy_mut, 0), MULTLINE(loadconf, legacy, 0), ONE_LINE(setevents, legacy, 0), - ONE_LINE(authenticate, legacy, 0), + ONE_LINE(authenticate, legacy, CMD_FL_WIPE), ONE_LINE(saveconf, legacy, 0), ONE_LINE(signal, legacy, 0), ONE_LINE(takeownership, legacy, 0), @@ -2346,7 +2346,7 @@ static const control_cmd_def_t CONTROL_COMMANDS[] = ONE_LINE(usefeature, legacy, 0), ONE_LINE(resolve, legacy, 0), ONE_LINE(protocolinfo, legacy, 0), - ONE_LINE(authchallenge, legacy, 0), + ONE_LINE(authchallenge, legacy, CMD_FL_WIPE), ONE_LINE(dropguards, legacy, 0), ONE_LINE(hsfetch, legacy, 0), MULTLINE(hspost, legacy, 0), From 1ee991ed4b65767f1a82e23191271fa9183d847f Mon Sep 17 00:00:00 2001 From: rl1987 Date: Fri, 12 Apr 2019 22:38:06 +0300 Subject: [PATCH 0787/2557] Add shebang line to fuzz_multi.sh (fixes SC2148) --- src/test/fuzz/fuzz_multi.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/test/fuzz/fuzz_multi.sh b/src/test/fuzz/fuzz_multi.sh index b4a17ed8cb..b7f59512db 100755 --- a/src/test/fuzz/fuzz_multi.sh +++ b/src/test/fuzz/fuzz_multi.sh @@ -1,3 +1,5 @@ +#!/bin/sh + MEMLIMIT_BYTES=21990500990976 N_CPUS=1 From 4fa4fe09453c2363d6a939482459a1fbe27ea6b4 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Fri, 12 Apr 2019 22:41:39 +0300 Subject: [PATCH 0788/2557] Fix remaining shellcheck warnings in fuzz_multi.sh --- src/test/fuzz/fuzz_multi.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/fuzz/fuzz_multi.sh b/src/test/fuzz/fuzz_multi.sh index b7f59512db..406ab498d9 100755 --- a/src/test/fuzz/fuzz_multi.sh +++ b/src/test/fuzz/fuzz_multi.sh @@ -8,9 +8,9 @@ if [ $# -ge 1 ]; then shift fi -FILTER=echo +FILTER="echo" -for i in `seq -w "$N_CPUS"`; do +for i in $(seq -w "$N_CPUS"); do if [ "$i" -eq 1 ]; then if [ "$N_CPUS" -eq 1 ]; then INSTANCE="" From db52180abe2bc9e82c7ce44d278af7e408974b12 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Fri, 12 Apr 2019 22:43:49 +0300 Subject: [PATCH 0789/2557] Add changes file --- changes/ticket30077 | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 changes/ticket30077 diff --git a/changes/ticket30077 b/changes/ticket30077 new file mode 100644 index 0000000000..9be014730e --- /dev/null +++ b/changes/ticket30077 @@ -0,0 +1,2 @@ + o Code simplification and refactoring (shell scripts): + - Fix shellcheck warnings in fuzz_multi.sh. Resolves issue 30077. From 88dc7bc171ea8bb4696aac5b142d5b39b632dc79 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Sun, 14 Apr 2019 11:31:50 -0400 Subject: [PATCH 0790/2557] Add an assertion to test_hs_cache.c to appease coverity. Coverity doesn't like to see a path where we test a pointer for NULL if we have already ready dereferenced the pointer on that path. While in this case, the check is not needed, it's best not to remove checks from the unit tests IMO. Instead, I'm adding an earlier check, so that coverity, when analyzing this function, will think that we have always checked the pointer before dereferencing it. Closes ticket 30180; CID 1444641. --- src/test/test_hs_cache.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test/test_hs_cache.c b/src/test/test_hs_cache.c index 48e8d3b8c4..2187c2be39 100644 --- a/src/test/test_hs_cache.c +++ b/src/test/test_hs_cache.c @@ -244,6 +244,7 @@ helper_fetch_desc_from_hsdir(const ed25519_public_key_t *blinded_key) /* Simulate an HTTP GET request to the HSDir */ conn = dir_connection_new(AF_INET); + tt_assert(conn); tor_addr_from_ipv4h(&conn->base_.addr, 0x7f000001); TO_CONN(conn)->linked = 1;/* Pretend the conn is encrypted :) */ retval = directory_handle_command_get(conn, hsdir_query_str, From 5cad9fb4776979285596c4b26230683de6bb84c6 Mon Sep 17 00:00:00 2001 From: Neel Chauhan Date: Thu, 11 Apr 2019 20:21:51 -0400 Subject: [PATCH 0791/2557] Become an exit relay if IPv6Exit is 1 --- src/core/or/policies.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/core/or/policies.c b/src/core/or/policies.c index a6d66d36de..ab320dd2ca 100644 --- a/src/core/or/policies.c +++ b/src/core/or/policies.c @@ -1182,9 +1182,9 @@ validate_addr_policies(const or_options_t *options, char **msg) static int warned_about_nonexit = 0; - if (public_server_mode(options) && - !warned_about_nonexit && options->ExitPolicy == NULL && - options->ExitRelay == -1 && options->ReducedExitPolicy == 0) { + if (public_server_mode(options) && !warned_about_nonexit && + options->ExitPolicy == NULL && options->ExitRelay == -1 && + options->ReducedExitPolicy == 0 && options->IPv6Exit == 0) { warned_about_nonexit = 1; log_notice(LD_CONFIG, "By default, Tor does not run as an exit relay. " "If you want to be an exit relay, " @@ -2142,8 +2142,9 @@ policies_parse_exit_policy_from_options(const or_options_t *or_options, /* Short-circuit for non-exit relays, or for relays where we didn't specify * ExitPolicy or ReducedExitPolicy and ExitRelay is auto. */ - if (or_options->ExitRelay == 0 || (or_options->ExitPolicy == NULL && - or_options->ExitRelay == -1 && or_options->ReducedExitPolicy == 0)) { + if (or_options->ExitRelay == 0 || + (or_options->ExitPolicy == NULL && or_options->ExitRelay == -1 && + or_options->ReducedExitPolicy == 0 && or_options->IPv6Exit == 0)) { append_exit_policy_string(result, "reject *4:*"); append_exit_policy_string(result, "reject *6:*"); return 0; From 03464a916571a5f682b42513c47568e317f38b18 Mon Sep 17 00:00:00 2001 From: Neel Chauhan Date: Thu, 11 Apr 2019 20:22:27 -0400 Subject: [PATCH 0792/2557] Update torrc.sample.in to IPv6Exit 1 being an exit by default --- src/config/torrc.sample.in | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/config/torrc.sample.in b/src/config/torrc.sample.in index c2ae707e93..95167d03fd 100644 --- a/src/config/torrc.sample.in +++ b/src/config/torrc.sample.in @@ -174,13 +174,11 @@ ## Uncomment this if you want your relay to be an exit, with the default ## exit policy (or whatever exit policy you set below). -## (If ReducedExitPolicy or ExitPolicy are set, relays are exits. +## (If ReducedExitPolicy, ExitPolicy, or IPv6Exit are set, relays are exits. ## If neither exit policy option is set, relays are non-exits.) #ExitRelay 1 ## Uncomment this if you want your relay to allow IPv6 exit traffic. -## You must also set ExitRelay, ReducedExitPolicy, or ExitPolicy to make your -## relay into an exit. ## (Relays do not allow any exit traffic by default.) #IPv6Exit 1 From e7288111101b2a0fe74ec87a314d6d33c1a2920a Mon Sep 17 00:00:00 2001 From: Neel Chauhan Date: Thu, 11 Apr 2019 20:22:46 -0400 Subject: [PATCH 0793/2557] Add changes file for Bug #29613 --- changes/bug29613 | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 changes/bug29613 diff --git a/changes/bug29613 b/changes/bug29613 new file mode 100644 index 0000000000..e966973255 --- /dev/null +++ b/changes/bug29613 @@ -0,0 +1,5 @@ + o Minor bugfixes (relay): + - If we are are a relay and have IPv6Exit to 1 while ExitRelay is + auto, we act as if ExitRelay is 1. Previously, we ignored IPv6Exit + if ExitRelay was 0 or auto. Fixes bug 29613; bugfix on 0.3.5.1-alpha. + Patch by Neel Chauhan. From d4f980d29ac4f9c850fff967018f88115bf3cc2d Mon Sep 17 00:00:00 2001 From: Neel Chauhan Date: Sun, 14 Apr 2019 14:27:05 -0400 Subject: [PATCH 0794/2557] Update exceptions.txt for Bug #29613 --- scripts/maint/practracker/exceptions.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/maint/practracker/exceptions.txt b/scripts/maint/practracker/exceptions.txt index 7d03bf27d6..8601c58033 100644 --- a/scripts/maint/practracker/exceptions.txt +++ b/scripts/maint/practracker/exceptions.txt @@ -117,7 +117,7 @@ problem include-count /src/core/or/connection_or.c 51 problem function-size /src/core/or/connection_or.c:connection_or_group_set_badness_() 105 problem function-size /src/core/or/connection_or.c:connection_or_client_learned_peer_id() 144 problem function-size /src/core/or/connection_or.c:connection_or_compute_authenticate_cell_body() 235 -problem file-size /src/core/or/policies.c 3163 +problem file-size /src/core/or/policies.c 3164 problem function-size /src/core/or/policies.c:policy_summarize() 107 problem function-size /src/core/or/protover.c:protover_all_supported() 116 problem file-size /src/core/or/relay.c 3173 From cc87acf29b40a135745f74412caaececf8ea0329 Mon Sep 17 00:00:00 2001 From: Neel Chauhan Date: Sun, 14 Apr 2019 14:51:42 -0400 Subject: [PATCH 0795/2557] Remove unused get_options() --- src/feature/control/fmt_serverstatus.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/feature/control/fmt_serverstatus.c b/src/feature/control/fmt_serverstatus.c index 1fc5d4a157..d224a1d234 100644 --- a/src/feature/control/fmt_serverstatus.c +++ b/src/feature/control/fmt_serverstatus.c @@ -66,7 +66,6 @@ list_server_status_v1(smartlist_t *routers, char **router_status_out, smartlist_t *rs_entries; time_t now = time(NULL); time_t cutoff = now - ROUTER_MAX_AGE_TO_PUBLISH; - const or_options_t *options = get_options(); /* We include v2 dir auths here too, because they need to answer * controllers. Eventually we'll deprecate this whole function; * see also networkstatus_getinfo_by_purpose(). */ From 55b4f02ba6cbbd4cae942a9e84e48e237d8e38aa Mon Sep 17 00:00:00 2001 From: rl1987 Date: Mon, 15 Apr 2019 12:13:35 +0300 Subject: [PATCH 0796/2557] Fix shellcheck warnings in fixup_filenames.sh --- changes/ticket30078 | 3 +++ src/test/fuzz/fixup_filenames.sh | 6 +++--- 2 files changed, 6 insertions(+), 3 deletions(-) create mode 100644 changes/ticket30078 diff --git a/changes/ticket30078 b/changes/ticket30078 new file mode 100644 index 0000000000..5ab5abdbfd --- /dev/null +++ b/changes/ticket30078 @@ -0,0 +1,3 @@ + o Code simplification and refactoring (shell scripts): + - Fix shellcheck warnings in src/test/fuzz/fixup_filenames.sh. Resolves + issue 30078. diff --git a/src/test/fuzz/fixup_filenames.sh b/src/test/fuzz/fixup_filenames.sh index 68efc1abc5..f730d532a5 100755 --- a/src/test/fuzz/fixup_filenames.sh +++ b/src/test/fuzz/fixup_filenames.sh @@ -8,9 +8,9 @@ if [ ! -d "$1" ] ; then fi for fn in "$1"/* ; do - prev=`basename "$fn"` - post=`sha256sum "$fn" | sed -e 's/ .*//;'` - if [ "$prev" == "$post" ] ; then + prev=$(basename "$fn") + post=$(sha256sum "$fn" | sed -e 's/ .*//;') + if [ "$prev" = "$post" ] ; then echo "OK $prev" else echo "mv $prev $post" From 3105081c2f075409f182d497eb101e3a690bb2f7 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 15 Apr 2019 14:37:46 -0400 Subject: [PATCH 0797/2557] Fix assertf() issues when ALL_BUGS_ARE_FATAL is defined. Fix from Gisle Vanem; fixes bug 30179. Bug not in any released version of Tor. --- src/lib/log/util_bug.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/lib/log/util_bug.h b/src/lib/log/util_bug.h index 2e220b7286..fb223b35f4 100644 --- a/src/lib/log/util_bug.h +++ b/src/lib/log/util_bug.h @@ -143,12 +143,13 @@ #ifdef ALL_BUGS_ARE_FATAL #define tor_assert_nonfatal_unreached() tor_assert(0) #define tor_assert_nonfatal(cond) tor_assert((cond)) -#define tor_assertf_nonfatal(cond, fmt, ...) tor_assertf(cond, fmt, ...) +#define tor_assertf_nonfatal(cond, fmt, ...) \ + tor_assertf(cond, fmt, ##__VA_ARGS__) #define tor_assert_nonfatal_unreached_once() tor_assert(0) #define tor_assert_nonfatal_once(cond) tor_assert((cond)) #define BUG(cond) \ (ASSERT_PREDICT_UNLIKELY_(cond) ? \ - (tor_assertion_failed_(SHORT_FILE__,__LINE__,__func__,"!("#cond")"), \ + (tor_assertion_failed_(SHORT_FILE__,__LINE__,__func__,"!("#cond")",NULL), \ tor_abort_(), 1) \ : 0) #elif defined(TOR_UNIT_TESTS) && defined(DISABLE_ASSERTS_IN_UNIT_TESTS) From 5a0c8579964aa1d27dd9be35afda5e7226c31fb4 Mon Sep 17 00:00:00 2001 From: Tobias Stoeckmann Date: Sat, 13 Apr 2019 16:54:05 +0200 Subject: [PATCH 0798/2557] Add test to verify that unused pointers are NULL. The smartlist code takes great care to set all unused pointers inside the smartlist memory to NULL. Check if this is also the case after modifying the smartlist multiple times. Signed-off-by: Tobias Stoeckmann --- src/test/test_containers.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test/test_containers.c b/src/test/test_containers.c index 7892a08853..ad272eeaf5 100644 --- a/src/test/test_containers.c +++ b/src/test/test_containers.c @@ -1006,6 +1006,7 @@ test_container_smartlist_remove(void *arg) tt_ptr_op(smartlist_get(sl, 1), OP_EQ, &array[2]); tt_ptr_op(smartlist_get(sl, 2), OP_EQ, &array[1]); tt_ptr_op(smartlist_get(sl, 3), OP_EQ, &array[2]); + tt_ptr_op(sl->list[4], OP_EQ, NULL); done: smartlist_free(sl); From 670d0f9f5bb7d73ad236a035ed7bd69e96cadd41 Mon Sep 17 00:00:00 2001 From: Tobias Stoeckmann Date: Sat, 13 Apr 2019 16:55:36 +0200 Subject: [PATCH 0799/2557] Clear memory in smartlist_remove_keeporder. The smartlist functions take great care to reset unused pointers inside the smartlist memory to NULL. The function smartlist_remove_keeporder does not clear memory in such way when elements have been removed. Therefore call memset after the for-loop that removes elements. If no element is removed, it is effectively a no-op. Signed-off-by: Tobias Stoeckmann --- changes/ticket30176 | 4 ++++ src/lib/smartlist_core/smartlist_core.c | 2 ++ 2 files changed, 6 insertions(+) create mode 100644 changes/ticket30176 diff --git a/changes/ticket30176 b/changes/ticket30176 new file mode 100644 index 0000000000..da23760ce5 --- /dev/null +++ b/changes/ticket30176 @@ -0,0 +1,4 @@ + o Minor features (defense in depth): + - In smartlist_remove_keeporder(), set any pointers that become + unused to NULL, in case a bug causes them to be used later. Closes + ticket 30176. Patch from Tobias Stoeckmann. diff --git a/src/lib/smartlist_core/smartlist_core.c b/src/lib/smartlist_core/smartlist_core.c index 5947e76271..6b0a305a93 100644 --- a/src/lib/smartlist_core/smartlist_core.c +++ b/src/lib/smartlist_core/smartlist_core.c @@ -177,6 +177,8 @@ smartlist_remove_keeporder(smartlist_t *sl, const void *element) sl->list[i++] = sl->list[j]; } } + memset(sl->list + sl->num_used, 0, + sizeof(void *) * (num_used_orig - sl->num_used)); } /** If sl is nonempty, remove and return the final element. Otherwise, From 82a3161c41e2c8d8d800a8c645a93b81bd8b2278 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 15 Apr 2019 14:52:08 -0400 Subject: [PATCH 0800/2557] Document check for 30176, since it's a bit subtle. --- src/test/test_containers.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/test/test_containers.c b/src/test/test_containers.c index ad272eeaf5..67ba457975 100644 --- a/src/test/test_containers.c +++ b/src/test/test_containers.c @@ -1006,6 +1006,9 @@ test_container_smartlist_remove(void *arg) tt_ptr_op(smartlist_get(sl, 1), OP_EQ, &array[2]); tt_ptr_op(smartlist_get(sl, 2), OP_EQ, &array[1]); tt_ptr_op(smartlist_get(sl, 3), OP_EQ, &array[2]); + /* Ordinary code should never look at this pointer; we're doing it here + * to make sure that we really cleared the pointer we removed. + */ tt_ptr_op(sl->list[4], OP_EQ, NULL); done: From 0c42ddf28c330280f66feb7cc92b63f3998468e1 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 15 Apr 2019 15:21:18 -0400 Subject: [PATCH 0801/2557] fixup! Even more diagnostic messages for bug 28223. Use TOR_PRIuSZ in place of %zu. --- src/feature/nodelist/microdesc.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/feature/nodelist/microdesc.c b/src/feature/nodelist/microdesc.c index af4f608d2f..eaa7f0cafd 100644 --- a/src/feature/nodelist/microdesc.c +++ b/src/feature/nodelist/microdesc.c @@ -224,7 +224,8 @@ dump_microdescriptor(int fd, microdesc_t *md, size_t *annotation_len_out) const char *nulpos = memchr(md->body, 0, md->bodylen); if (BUG(nulpos)) { log_warn(LD_BUG, "About to dump a NUL into a microdescriptor file. " - "offset %"PRId64", bodylen %zu, nul position %zu", + "offset %"PRId64", bodylen %"TOR_PRIuSZ", " + "nul position %"TOR_PRIuSZ".", (int64_t)md->off, md->bodylen, (size_t)(nulpos - md->body)); } @@ -493,7 +494,7 @@ warn_if_nul_found(const char *inp, size_t len, const char *description) const char *nul_found = memchr(inp, 0, len); if (BUG(nul_found)) { log_warn(LD_BUG, "Found unexpected NUL while reading %s, at " - "position %zu/%zu.", + "position %"TOR_PRIuSZ"/%"TOR_PRIuSZ".", description, (nul_found - inp), len); } } From 950d890f77d8f060702b100e6075d5083d536577 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 15 Apr 2019 15:33:09 -0400 Subject: [PATCH 0802/2557] In warn_if_nul_found, log surrounding context. We need to encode here instead of doing escaped(), since fwict escaped() does not currently handle NUL bytes. Also, use warn_if_nul_found in more cases to avoid duplication. --- src/feature/nodelist/microdesc.c | 36 +++++++++++++++++++------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/src/feature/nodelist/microdesc.c b/src/feature/nodelist/microdesc.c index eaa7f0cafd..f4fa8eeae2 100644 --- a/src/feature/nodelist/microdesc.c +++ b/src/feature/nodelist/microdesc.c @@ -69,6 +69,8 @@ struct microdesc_cache_t { }; static microdesc_cache_t *get_microdesc_cache_noload(void); +static void warn_if_nul_found(const char *inp, size_t len, int64_t offset, + const char *activity); /** Helper: computes a hash of md to place it in a hash table. */ static inline unsigned int @@ -221,14 +223,8 @@ dump_microdescriptor(int fd, microdesc_t *md, size_t *annotation_len_out) } md->off = tor_fd_getpos(fd); - const char *nulpos = memchr(md->body, 0, md->bodylen); - if (BUG(nulpos)) { - log_warn(LD_BUG, "About to dump a NUL into a microdescriptor file. " - "offset %"PRId64", bodylen %"TOR_PRIuSZ", " - "nul position %"TOR_PRIuSZ".", - (int64_t)md->off, md->bodylen, - (size_t)(nulpos - md->body)); - } + warn_if_nul_found(md->body, md->bodylen, (int64_t) md->off, + "dumping a microdescriptor"); written = write_all_to_fd(fd, md->body, md->bodylen); if (written != (ssize_t)md->bodylen) { written = written < 0 ? 0 : written; @@ -489,13 +485,23 @@ microdesc_cache_clear(microdesc_cache_t *cache) } static void -warn_if_nul_found(const char *inp, size_t len, const char *description) +warn_if_nul_found(const char *inp, size_t len, int64_t offset, + const char *activity) { const char *nul_found = memchr(inp, 0, len); if (BUG(nul_found)) { - log_warn(LD_BUG, "Found unexpected NUL while reading %s, at " - "position %"TOR_PRIuSZ"/%"TOR_PRIuSZ".", - description, (nul_found - inp), len); + log_warn(LD_BUG, "Found unexpected NUL while %s, offset %"PRId64 + "at position %"TOR_PRIuSZ"/%"TOR_PRIuSZ".", + activity, offset, (nul_found - inp), len); + const char *start_excerpt_at, *eos = inp + len; + if ((nul_found - inp) >= 16) + start_excerpt_at = nul_found - 16; + else + start_excerpt_at = inp; + size_t excerpt_len = MIN(32, eos - start_excerpt_at); + char tmp[65]; + base16_encode(tmp, sizeof(tmp), start_excerpt_at, excerpt_len); + log_warn(LD_BUG, " surrounding string: %s", tmp); } } @@ -516,7 +522,7 @@ microdesc_cache_reload(microdesc_cache_t *cache) mm = cache->cache_content = tor_mmap_file(cache->cache_fname); if (mm) { - warn_if_nul_found(mm->data, mm->size, "microdesc cache"); + warn_if_nul_found(mm->data, mm->size, 0, "scanning microdesc cache"); added = microdescs_add_to_cache(cache, mm->data, mm->data+mm->size, SAVED_IN_CACHE, 0, -1, NULL); if (added) { @@ -529,8 +535,8 @@ microdesc_cache_reload(microdesc_cache_t *cache) RFTS_IGNORE_MISSING, &st); if (journal_content) { cache->journal_len = (size_t) st.st_size; - warn_if_nul_found(journal_content, cache->journal_len, - "microdesc journal"); + warn_if_nul_found(journal_content, cache->journal_len, 0, + "reading microdesc journal"); added = microdescs_add_to_cache(cache, journal_content, journal_content+st.st_size, SAVED_IN_JOURNAL, 0, -1, NULL); From e7c22e6e4882332cc0e3377f0d3494f3bf47ad96 Mon Sep 17 00:00:00 2001 From: Neel Chauhan Date: Tue, 16 Apr 2019 08:19:44 -0400 Subject: [PATCH 0803/2557] Add policy_using_default_exit_options() to determine if we're using the default exit options --- src/core/or/policies.c | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/core/or/policies.c b/src/core/or/policies.c index ab320dd2ca..41d221b19d 100644 --- a/src/core/or/policies.c +++ b/src/core/or/policies.c @@ -1164,6 +1164,15 @@ authdir_policy_badexit_address(uint32_t addr, uint16_t port) #define REJECT(arg) \ STMT_BEGIN *msg = tor_strdup(arg); goto err; STMT_END +/** Check or_options to determine whether or not we are using the + * default options for exit policy. Return true if so, false otherwise. */ +static int +policy_using_default_exit_options(const or_options_t *or_options) +{ + return (or_options->ExitPolicy == NULL && or_options->ExitRelay == -1 && + or_options->ReducedExitPolicy == 0 && or_options->IPv6Exit == 0); +} + /** Config helper: If there's any problem with the policy configuration * options in options, return -1 and set msg to a newly * allocated description of the error. Else return 0. */ @@ -1183,8 +1192,7 @@ validate_addr_policies(const or_options_t *options, char **msg) static int warned_about_nonexit = 0; if (public_server_mode(options) && !warned_about_nonexit && - options->ExitPolicy == NULL && options->ExitRelay == -1 && - options->ReducedExitPolicy == 0 && options->IPv6Exit == 0) { + policy_using_default_exit_options(options)) { warned_about_nonexit = 1; log_notice(LD_CONFIG, "By default, Tor does not run as an exit relay. " "If you want to be an exit relay, " @@ -2143,8 +2151,7 @@ policies_parse_exit_policy_from_options(const or_options_t *or_options, /* Short-circuit for non-exit relays, or for relays where we didn't specify * ExitPolicy or ReducedExitPolicy and ExitRelay is auto. */ if (or_options->ExitRelay == 0 || - (or_options->ExitPolicy == NULL && or_options->ExitRelay == -1 && - or_options->ReducedExitPolicy == 0 && or_options->IPv6Exit == 0)) { + policy_using_default_exit_options(or_options)) { append_exit_policy_string(result, "reject *4:*"); append_exit_policy_string(result, "reject *6:*"); return 0; From 06c76e79aab3ff9cbed04bd3f670099e57d851b6 Mon Sep 17 00:00:00 2001 From: Neel Chauhan Date: Tue, 16 Apr 2019 08:20:48 -0400 Subject: [PATCH 0804/2557] Clarify torrc comment for IPv6Exit --- src/config/torrc.sample.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config/torrc.sample.in b/src/config/torrc.sample.in index 95167d03fd..9d514e6bda 100644 --- a/src/config/torrc.sample.in +++ b/src/config/torrc.sample.in @@ -175,7 +175,7 @@ ## Uncomment this if you want your relay to be an exit, with the default ## exit policy (or whatever exit policy you set below). ## (If ReducedExitPolicy, ExitPolicy, or IPv6Exit are set, relays are exits. -## If neither exit policy option is set, relays are non-exits.) +## If none of these options are set, relays are non-exits.) #ExitRelay 1 ## Uncomment this if you want your relay to allow IPv6 exit traffic. From ada673291402d1e015e79cadca67c0bdd31b4019 Mon Sep 17 00:00:00 2001 From: Neel Chauhan Date: Tue, 16 Apr 2019 08:22:17 -0400 Subject: [PATCH 0805/2557] Clarify comment about IPv6Exit in policies_parse_exit_policy_from_options() --- src/core/or/policies.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/or/policies.c b/src/core/or/policies.c index 41d221b19d..f59894ea8f 100644 --- a/src/core/or/policies.c +++ b/src/core/or/policies.c @@ -2149,7 +2149,7 @@ policies_parse_exit_policy_from_options(const or_options_t *or_options, int rv = 0; /* Short-circuit for non-exit relays, or for relays where we didn't specify - * ExitPolicy or ReducedExitPolicy and ExitRelay is auto. */ + * ExitPolicy or ReducedExitPolicy or IPv6Exit and ExitRelay is auto. */ if (or_options->ExitRelay == 0 || policy_using_default_exit_options(or_options)) { append_exit_policy_string(result, "reject *4:*"); From f643020e64a3730f262f810ed68ed7910da2a9f4 Mon Sep 17 00:00:00 2001 From: Neel Chauhan Date: Tue, 16 Apr 2019 09:16:52 -0400 Subject: [PATCH 0806/2557] Update practracker exceptions.txt for policies.c --- scripts/maint/practracker/exceptions.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/maint/practracker/exceptions.txt b/scripts/maint/practracker/exceptions.txt index 8601c58033..c99cdc0160 100644 --- a/scripts/maint/practracker/exceptions.txt +++ b/scripts/maint/practracker/exceptions.txt @@ -117,7 +117,7 @@ problem include-count /src/core/or/connection_or.c 51 problem function-size /src/core/or/connection_or.c:connection_or_group_set_badness_() 105 problem function-size /src/core/or/connection_or.c:connection_or_client_learned_peer_id() 144 problem function-size /src/core/or/connection_or.c:connection_or_compute_authenticate_cell_body() 235 -problem file-size /src/core/or/policies.c 3164 +problem file-size /src/core/or/policies.c 3171 problem function-size /src/core/or/policies.c:policy_summarize() 107 problem function-size /src/core/or/protover.c:protover_all_supported() 116 problem file-size /src/core/or/relay.c 3173 From 37d7daa3cd22a7dce833e09392bcdbadf6e047f1 Mon Sep 17 00:00:00 2001 From: teor Date: Wed, 17 Apr 2019 12:34:33 +1000 Subject: [PATCH 0807/2557] changes: update the changes file for 30001 --- changes/bug30001 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/changes/bug30001 b/changes/bug30001 index e3304701e7..52e58872ef 100644 --- a/changes/bug30001 +++ b/changes/bug30001 @@ -2,3 +2,6 @@ - Use the approx_time() function when setting the "Expires" header in directory replies, to make them more testable. Needed for ticket 30001. + o Minor bug fixes (testing): + - Check the time in the "Expires" header with approx_time(). + Fixes bug 30001; bugfix on 0.4.0.4-rc. From 21a4438c58d2c3923499b0a8c3f2240eeca5e2a8 Mon Sep 17 00:00:00 2001 From: Mike Perry Date: Wed, 17 Apr 2019 02:44:42 +0000 Subject: [PATCH 0808/2557] Bug 30173: Add consensus param to disable padding. Disable padding via limit check and machine condition. Limits cause us to stop sending padding. Machine conditions cause the machines to be shut down, and not restarted. --- src/core/or/circuitpadding.c | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/src/core/or/circuitpadding.c b/src/core/or/circuitpadding.c index d30edf9db8..f6385f32e0 100644 --- a/src/core/or/circuitpadding.c +++ b/src/core/or/circuitpadding.c @@ -81,6 +81,7 @@ static void circpad_setup_machine_on_circ(circuit_t *on_circ, static double circpad_distribution_sample(circpad_distribution_t dist); /** Cached consensus params */ +static uint8_t circpad_padding_disabled; static uint8_t circpad_global_max_padding_percent; static uint16_t circpad_global_allowed_cells; static uint16_t circpad_max_circ_queued_cells; @@ -1081,6 +1082,10 @@ circpad_send_padding_callback(tor_timer_t *timer, void *args, void circpad_new_consensus_params(const networkstatus_t *ns) { + circpad_padding_disabled = + networkstatus_get_param(ns, "circpad_padding_disabled", + 0, 0, 1); + circpad_global_allowed_cells = networkstatus_get_param(ns, "circpad_global_allowed_cells", 0, 0, UINT16_MAX-1); @@ -1112,10 +1117,19 @@ circpad_machine_reached_padding_limit(circpad_machine_runtime_t *mi) { const circpad_machine_spec_t *machine = CIRCPAD_GET_MACHINE(mi); + /* If padding has been disabled in the consensus, don't send any more + * padding. Technically the machine should be shut down when the next + * machine condition check happens, but machine checks only happen on + * certain circuit events, and if padding is disabled due to some + * network overload or DoS condition, we really want to stop ASAP. */ + if (circpad_padding_disabled) { + return 1; + } + /* If machine_padding_pct is non-zero, and we've sent more * than the allowed count of padding cells, then check our * percent limits for this machine. */ - if (machine->max_padding_percent && + if (machine->max_padding_percent && mi->padding_sent >= machine->allowed_padding_count) { uint32_t total_cells = mi->padding_sent + mi->nonpadding_sent; @@ -1621,6 +1635,11 @@ static inline bool circpad_machine_conditions_met(origin_circuit_t *circ, const circpad_machine_spec_t *machine) { + /* If padding is disabled, no machines should match/apply. This has + * the effect of shutting down all machines, and not adding any more. */ + if (circpad_padding_disabled) + return 0; + if (!(circpad_circ_purpose_to_mask(TO_CIRCUIT(circ)->purpose) & machine->conditions.purpose_mask)) return 0; From 17a164a8275970aed53076ba58296ffc424c3b75 Mon Sep 17 00:00:00 2001 From: Mike Perry Date: Wed, 17 Apr 2019 02:51:48 +0000 Subject: [PATCH 0809/2557] Bug 30173: Rate limit padding rate limit log message. Gotta limit to the limit when the limit is reached. --- src/core/or/circuitpadding.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/core/or/circuitpadding.c b/src/core/or/circuitpadding.c index f6385f32e0..bf74ecc3ff 100644 --- a/src/core/or/circuitpadding.c +++ b/src/core/or/circuitpadding.c @@ -1196,7 +1196,8 @@ circpad_machine_schedule_padding,(circpad_machine_runtime_t *mi)) "Padding machine has reached padding limit on circuit %u", TO_ORIGIN_CIRCUIT(mi->on_circ)->global_identifier); } else { - log_fn(LOG_INFO, LD_CIRC, + static ratelim_t padding_lim = RATELIM_INIT(600); + log_fn_ratelim(&padding_lim,LOG_INFO,LD_CIRC, "Padding machine has reached padding limit on circuit %"PRIu64 ", %d", mi->on_circ->n_chan ? mi->on_circ->n_chan->global_identifier : 0, From 728d20ed08122d2f34b001217faa009040fb2fee Mon Sep 17 00:00:00 2001 From: teor Date: Wed, 17 Apr 2019 17:58:40 +1000 Subject: [PATCH 0810/2557] connection_edge: Return a web page when HTTPTunnelPort is misconfigured Return an informative web page when the HTTPTunnelPort is used as an HTTP proxy. Closes ticket 27821, patch by "eighthave". --- changes/ticket27821 | 3 +++ src/core/or/connection_edge.c | 27 ++++++++++++++++++++++++++- 2 files changed, 29 insertions(+), 1 deletion(-) create mode 100644 changes/ticket27821 diff --git a/changes/ticket27821 b/changes/ticket27821 new file mode 100644 index 0000000000..158f308fbf --- /dev/null +++ b/changes/ticket27821 @@ -0,0 +1,3 @@ + o Minor features (HTTP tunnel): + - Return an informative web page when the HTTPTunnelPort is used as an + HTTP proxy. Closes ticket 27821, patch by "eighthave". diff --git a/src/core/or/connection_edge.c b/src/core/or/connection_edge.c index 071a8c91ed..4f7cbafe07 100644 --- a/src/core/or/connection_edge.c +++ b/src/core/or/connection_edge.c @@ -2810,6 +2810,31 @@ connection_ap_process_natd(entry_connection_t *conn) return connection_ap_rewrite_and_attach_if_allowed(conn, NULL, NULL); } +static const char HTTP_CONNECT_IS_NOT_AN_HTTP_PROXY_MSG[] = + "HTTP/1.0 405 Method Not Allowed\r\n"; + "Content-Type: text/html; charset=iso-8859-1\r\n\r\n" + "\n" + "\n" + "This is an HTTP CONNECT tunnel, not an full HTTP Proxy\n" + "\n" + "\n" + "

This is an HTTP CONNECT tunnel, not an HTTP proxy.

\n" + "

\n" + "It appears you have configured your web browser to use this Tor port as\n" + "an HTTP proxy.\n" + "

\n" + "This is not correct: This port is configured as a CONNECT tunnel, not\n" + "an HTTP proxy. Please configure your client accordingly. You can also\n" + "use HTTPS, then the client should automatically use HTTP CONNECT." + "

\n" + "

\n" + "See " + "https://www.torproject.org/documentation.html for more " + "information.\n" + "

\n" + "\n" + "\n"; + /** Called on an HTTP CONNECT entry connection when some bytes have arrived, * but we have not yet received a full HTTP CONNECT request. Try to parse an * HTTP CONNECT request from the connection's inbuf. On success, set up the @@ -2850,7 +2875,7 @@ connection_ap_process_http_connect(entry_connection_t *conn) tor_assert(command); tor_assert(addrport); if (strcasecmp(command, "connect")) { - errmsg = "HTTP/1.0 405 Method Not Allowed\r\n\r\n"; + errmsg = HTTP_CONNECT_IS_NOT_AN_HTTP_PROXY_MSG; goto err; } From c483ccf1c9e8fc1c4026dbf8c3e685ec55e8d8bb Mon Sep 17 00:00:00 2001 From: teor Date: Wed, 17 Apr 2019 18:43:20 +1000 Subject: [PATCH 0811/2557] connection_edge: remove an extra ; --- src/core/or/connection_edge.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/or/connection_edge.c b/src/core/or/connection_edge.c index 4f7cbafe07..33ba723971 100644 --- a/src/core/or/connection_edge.c +++ b/src/core/or/connection_edge.c @@ -2811,7 +2811,7 @@ connection_ap_process_natd(entry_connection_t *conn) } static const char HTTP_CONNECT_IS_NOT_AN_HTTP_PROXY_MSG[] = - "HTTP/1.0 405 Method Not Allowed\r\n"; + "HTTP/1.0 405 Method Not Allowed\r\n" "Content-Type: text/html; charset=iso-8859-1\r\n\r\n" "\n" "\n" From f12b990bbf6876fd019b9d9d5e8ef42c1bde2550 Mon Sep 17 00:00:00 2001 From: teor Date: Wed, 17 Apr 2019 18:44:26 +1000 Subject: [PATCH 0812/2557] practracker: accept the extra 25 line string from 27821 --- scripts/maint/practracker/exceptions.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/maint/practracker/exceptions.txt b/scripts/maint/practracker/exceptions.txt index 7d03bf27d6..d77e788654 100644 --- a/scripts/maint/practracker/exceptions.txt +++ b/scripts/maint/practracker/exceptions.txt @@ -102,7 +102,7 @@ problem function-size /src/core/or/circuituse.c:circuit_get_open_circ_or_launch( problem function-size /src/core/or/circuituse.c:connection_ap_handshake_attach_circuit() 244 problem function-size /src/core/or/command.c:command_process_create_cell() 156 problem function-size /src/core/or/command.c:command_process_relay_cell() 132 -problem file-size /src/core/or/connection_edge.c 4550 +problem file-size /src/core/or/connection_edge.c 4575 problem include-count /src/core/or/connection_edge.c 64 problem function-size /src/core/or/connection_edge.c:connection_ap_expire_beginning() 117 problem function-size /src/core/or/connection_edge.c:connection_ap_handshake_rewrite() 192 From 0d88b808e93fd20a3690061d781fe896ad9e2084 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Wed, 17 Apr 2019 13:15:02 +0300 Subject: [PATCH 0813/2557] Remove contrib/dist/tor.sh.in --- changes/ticket30075 | 3 + configure.ac | 1 - contrib/dist/tor.sh.in | 123 ----------------------------------------- contrib/include.am | 1 - 4 files changed, 3 insertions(+), 125 deletions(-) create mode 100644 changes/ticket30075 delete mode 100644 contrib/dist/tor.sh.in diff --git a/changes/ticket30075 b/changes/ticket30075 new file mode 100644 index 0000000000..288abd7674 --- /dev/null +++ b/changes/ticket30075 @@ -0,0 +1,3 @@ + o Removed features: + - Remove the obsolete script at contrib/dist/tor.sh.in. Resolves issue + 30075. diff --git a/configure.ac b/configure.ac index 0b80669f03..3ea578bbba 100644 --- a/configure.ac +++ b/configure.ac @@ -2459,7 +2459,6 @@ AC_CONFIG_FILES([ config.rust contrib/dist/suse/tor.sh contrib/operator-tools/tor.logrotate - contrib/dist/tor.sh contrib/dist/torctl contrib/dist/tor.service src/config/torrc.sample diff --git a/contrib/dist/tor.sh.in b/contrib/dist/tor.sh.in deleted file mode 100644 index 92f890681f..0000000000 --- a/contrib/dist/tor.sh.in +++ /dev/null @@ -1,123 +0,0 @@ -#!/bin/sh -# -# tor The Onion Router -# -# Startup/shutdown script for tor. This is a wrapper around torctl; -# torctl does the actual work in a relatively system-independent, or at least -# distribution-independent, way, and this script deals with fitting the -# whole thing into the conventions of the particular system at hand. -# This particular script is written for Red Hat/Fedora Linux, and may -# also work on Mandrake, but not SuSE. -# -# These next couple of lines "declare" tor for the "chkconfig" program, -# originally from SGI, used on Red Hat/Fedora and probably elsewhere. -# -# chkconfig: 2345 90 10 -# description: Onion Router - A low-latency anonymous proxy -# - -PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin -DAEMON=/usr/sbin/tor -NAME=tor -DESC="tor daemon" -TORPIDDIR=/var/run/tor -TORPID=$TORPIDDIR/tor.pid -WAITFORDAEMON=60 -ARGS="" - -# Library functions -if [ -f /etc/rc.d/init.d/functions ]; then - . /etc/rc.d/init.d/functions -elif [ -f /etc/init.d/functions ]; then - . /etc/init.d/functions -fi - -TORCTL=@BINDIR@/torctl - -# torctl will use these environment variables -TORUSER=@TORUSER@ -export TORUSER - -if [ -x /bin/su ] ; then - SUPROG=/bin/su -elif [ -x /sbin/su ] ; then - SUPROG=/sbin/su -elif [ -x /usr/bin/su ] ; then - SUPROG=/usr/bin/su -elif [ -x /usr/sbin/su ] ; then - SUPROG=/usr/sbin/su -else - SUPROG=/bin/su -fi - -# Raise ulimit based on number of file descriptors available (thanks, Debian) - -if [ -r /proc/sys/fs/file-max ]; then - system_max=`cat /proc/sys/fs/file-max` - if [ "$system_max" -gt "80000" ] ; then - MAX_FILEDESCRIPTORS=32768 - elif [ "$system_max" -gt "40000" ] ; then - MAX_FILEDESCRIPTORS=16384 - elif [ "$system_max" -gt "10000" ] ; then - MAX_FILEDESCRIPTORS=8192 - else - MAX_FILEDESCRIPTORS=1024 - cat << EOF - -Warning: Your system has very few filedescriptors available in total. - -Maybe you should try raising that by adding 'fs.file-max=100000' to your -/etc/sysctl.conf file. Feel free to pick any number that you deem appropriate. -Then run 'sysctl -p'. See /proc/sys/fs/file-max for the current value, and -file-nr in the same directory for how many of those are used at the moment. - -EOF - fi -else - MAX_FILEDESCRIPTORS=8192 -fi - -NICE="" - -case "$1" in - - start) - if [ -n "$MAX_FILEDESCRIPTORS" ]; then - echo -n "Raising maximum number of filedescriptors (ulimit -n) to $MAX_FILEDESCRIPTORS" - if ulimit -n "$MAX_FILEDESCRIPTORS" ; then - echo "." - else - echo ": FAILED." - fi - fi - - action $"Starting tor:" $TORCTL start - RETVAL=$? - ;; - - stop) - action $"Stopping tor:" $TORCTL stop - RETVAL=$? - ;; - - restart) - action $"Restarting tor:" $TORCTL restart - RETVAL=$? - ;; - - reload) - action $"Reloading tor:" $TORCTL reload - RETVAL=$? - ;; - - status) - $TORCTL status - RETVAL=$? - ;; - - *) - echo "Usage: $0 (start|stop|restart|reload|status)" - RETVAL=1 -esac - -exit $RETVAL diff --git a/contrib/include.am b/contrib/include.am index 742bc58163..9f4775632c 100644 --- a/contrib/include.am +++ b/contrib/include.am @@ -4,7 +4,6 @@ EXTRA_DIST+= \ contrib/client-tools/torify \ contrib/dist/rc.subr \ contrib/dist/suse/tor.sh.in \ - contrib/dist/tor.sh \ contrib/dist/torctl \ contrib/dist/tor.service.in \ contrib/operator-tools/tor-exit-notice.html \ From cfada969a9d450ba12548ab4ff112a2fe3642b95 Mon Sep 17 00:00:00 2001 From: Neel Chauhan Date: Wed, 17 Apr 2019 07:26:55 -0400 Subject: [PATCH 0814/2557] Update tor.1 man page for IPv6Exit --- doc/tor.1.txt | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/doc/tor.1.txt b/doc/tor.1.txt index f992172405..23b7e7b607 100644 --- a/doc/tor.1.txt +++ b/doc/tor.1.txt @@ -1935,13 +1935,13 @@ is non-zero): exit according to the ExitPolicy option, the ReducedExitPolicy option, or the default ExitPolicy (if no other exit policy option is specified). + + - If ExitRelay is set to 0, no traffic is allowed to - exit, and the ExitPolicy and ReducedExitPolicy options are ignored. + + If ExitRelay is set to 0, no traffic is allowed to exit, and the + ExitPolicy, ReducedExitPolicy, and IPv6Exit options are ignored. + + - If ExitRelay is set to "auto", then Tor checks the ExitPolicy and - ReducedExitPolicy options. If either is set, Tor behaves as if ExitRelay - were set to 1. If neither exit policy option is set, Tor behaves as if - ExitRelay were set to 0. (Default: auto) + If ExitRelay is set to "auto", then Tor checks the ExitPolicy, + ReducedExitPolicy, and IPv6Exit options. If either is set, Tor behaves + as if ExitRelay were set to 1. If neither exit policy option is set, Tor + behaves as if ExitRelay were set to 0. (Default: auto) [[ExitPolicy]] **ExitPolicy** __policy__,__policy__,__...__:: Set an exit policy for this server. Each policy is of the form @@ -2136,8 +2136,9 @@ is non-zero): (Default: 0) [[IPv6Exit]] **IPv6Exit** **0**|**1**:: - If set, and we are an exit node, allow clients to use us for IPv6 - traffic. (Default: 0) + If set, and we are an exit node, allow clients to use us for IPv6 traffic. + When this option is set and ExitRelay is auto, we act as if ExitRelay + is 1. (Default: 0) [[MaxOnionQueueDelay]] **MaxOnionQueueDelay** __NUM__ [**msec**|**second**]:: If we have more onionskins queued for processing than we can process in From caa55a6d37426073dc264d2adec7201ec65aec19 Mon Sep 17 00:00:00 2001 From: Neel Chauhan Date: Wed, 17 Apr 2019 21:41:31 -0400 Subject: [PATCH 0815/2557] Grammar fixes for IPv6Exit in tor.1.txt --- doc/tor.1.txt | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/doc/tor.1.txt b/doc/tor.1.txt index 23b7e7b607..cbbc3515bb 100644 --- a/doc/tor.1.txt +++ b/doc/tor.1.txt @@ -1939,9 +1939,10 @@ is non-zero): ExitPolicy, ReducedExitPolicy, and IPv6Exit options are ignored. + + If ExitRelay is set to "auto", then Tor checks the ExitPolicy, - ReducedExitPolicy, and IPv6Exit options. If either is set, Tor behaves - as if ExitRelay were set to 1. If neither exit policy option is set, Tor - behaves as if ExitRelay were set to 0. (Default: auto) + ReducedExitPolicy, and IPv6Exit options. If at least one of these options + is set, Tor behaves as if ExitRelay were set to 1. If none of these exit + policy options are set, Tor behaves as if ExitRelay were set to 0. + (Default: auto) [[ExitPolicy]] **ExitPolicy** __policy__,__policy__,__...__:: Set an exit policy for this server. Each policy is of the form From bffba9d26f62d934c686576424a79eab953bd490 Mon Sep 17 00:00:00 2001 From: teor Date: Fri, 19 Apr 2019 10:34:16 +1000 Subject: [PATCH 0816/2557] practracker: accept more lines in microdescs_parse_from_string() Part of 28223. --- scripts/maint/practracker/exceptions.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/maint/practracker/exceptions.txt b/scripts/maint/practracker/exceptions.txt index ecc58012c6..fefd6c3a74 100644 --- a/scripts/maint/practracker/exceptions.txt +++ b/scripts/maint/practracker/exceptions.txt @@ -187,7 +187,7 @@ problem function-size /src/feature/dirclient/dirclient.c:handle_response_fetch_c problem function-size /src/feature/dircommon/consdiff.c:gen_ed_diff() 204 problem function-size /src/feature/dircommon/consdiff.c:apply_ed_diff() 159 problem function-size /src/feature/dirparse/authcert_parse.c:authority_cert_parse_from_string() 182 -problem function-size /src/feature/dirparse/microdesc_parse.c:microdescs_parse_from_string() 154 +problem function-size /src/feature/dirparse/microdesc_parse.c:microdescs_parse_from_string() 169 problem function-size /src/feature/dirparse/ns_parse.c:routerstatus_parse_entry_from_string() 286 problem function-size /src/feature/dirparse/ns_parse.c:networkstatus_verify_bw_weights() 389 problem function-size /src/feature/dirparse/ns_parse.c:networkstatus_parse_vote_from_string() 638 From 943559b18066e126652450df4402dca74ba1baa0 Mon Sep 17 00:00:00 2001 From: Neel Chauhan Date: Fri, 19 Apr 2019 08:33:00 -0400 Subject: [PATCH 0817/2557] Make rate_limited and is_rate_limited a bool --- src/feature/hs/hs_common.c | 4 ++-- src/feature/hs/hs_common.h | 2 +- src/feature/rend/rendclient.c | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/feature/hs/hs_common.c b/src/feature/hs/hs_common.c index cffec2b878..a14ff6a645 100644 --- a/src/feature/hs/hs_common.c +++ b/src/feature/hs/hs_common.c @@ -1598,14 +1598,14 @@ hs_purge_last_hid_serv_requests(void) * NULL if no HSDirs are worth trying right now. */ routerstatus_t * hs_pick_hsdir(smartlist_t *responsible_dirs, const char *req_key_str, - int *is_rate_limited) + bool *is_rate_limited) { smartlist_t *usable_responsible_dirs = smartlist_new(); const or_options_t *options = get_options(); routerstatus_t *hs_dir; time_t now = time(NULL); int excluded_some; - int rate_limited; + bool rate_limited; int rate_limited_count = 0; int responsible_dirs_count = smartlist_len(responsible_dirs); diff --git a/src/feature/hs/hs_common.h b/src/feature/hs/hs_common.h index f96fc8beb7..fc0c39f94f 100644 --- a/src/feature/hs/hs_common.h +++ b/src/feature/hs/hs_common.h @@ -241,7 +241,7 @@ void hs_get_responsible_hsdirs(const struct ed25519_public_key_t *blinded_pk, int use_second_hsdir_index, int for_fetching, smartlist_t *responsible_dirs); routerstatus_t *hs_pick_hsdir(smartlist_t *responsible_dirs, - const char *req_key_str, int *is_rate_limited); + const char *req_key_str, bool *is_rate_limited); time_t hs_hsdir_requery_period(const or_options_t *options); time_t hs_lookup_last_hid_serv_request(routerstatus_t *hs_dir, diff --git a/src/feature/rend/rendclient.c b/src/feature/rend/rendclient.c index 9863fc1c11..090722a4c5 100644 --- a/src/feature/rend/rendclient.c +++ b/src/feature/rend/rendclient.c @@ -469,7 +469,7 @@ directory_get_from_hs_dir(const char *desc_id, /* Automatically pick an hs dir if none given. */ if (!rs_hsdir) { - int rate_limited; + bool rate_limited; /* Determine responsible dirs. Even if we can't get all we want, work with * the ones we have. If it's empty, we'll notice in hs_pick_hsdir(). */ From 144bc5026e90421bb814f35f2b61f3ffa0d078f8 Mon Sep 17 00:00:00 2001 From: Neel Chauhan Date: Fri, 19 Apr 2019 09:17:29 -0400 Subject: [PATCH 0818/2557] Initialize rate_limited to false in directory_get_from_hs_dir() --- src/feature/rend/rendclient.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/feature/rend/rendclient.c b/src/feature/rend/rendclient.c index 090722a4c5..f84d221b1a 100644 --- a/src/feature/rend/rendclient.c +++ b/src/feature/rend/rendclient.c @@ -469,7 +469,7 @@ directory_get_from_hs_dir(const char *desc_id, /* Automatically pick an hs dir if none given. */ if (!rs_hsdir) { - bool rate_limited; + bool rate_limited = false; /* Determine responsible dirs. Even if we can't get all we want, work with * the ones we have. If it's empty, we'll notice in hs_pick_hsdir(). */ From efde686aa537b31c7cc626b863e3e0b750975526 Mon Sep 17 00:00:00 2001 From: Neel Chauhan Date: Fri, 19 Apr 2019 09:21:20 -0400 Subject: [PATCH 0819/2557] Only set rate_limited in hs_pick_hsdir() if rate_limited_count or responsible_dirs_count is greater than 0 --- src/feature/hs/hs_common.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/feature/hs/hs_common.c b/src/feature/hs/hs_common.c index a14ff6a645..f9caf24f1c 100644 --- a/src/feature/hs/hs_common.c +++ b/src/feature/hs/hs_common.c @@ -1635,7 +1635,10 @@ hs_pick_hsdir(smartlist_t *responsible_dirs, const char *req_key_str, } } SMARTLIST_FOREACH_END(dir); - rate_limited = rate_limited_count == responsible_dirs_count; + if (rate_limited_count > 0 || responsible_dirs_count > 0) { + rate_limited = rate_limited_count == responsible_dirs_count; + } + excluded_some = smartlist_len(usable_responsible_dirs) < smartlist_len(responsible_dirs); From 2ab19a48c24ce985ff8328c3c7778f6cd5bf90b3 Mon Sep 17 00:00:00 2001 From: Neel Chauhan Date: Fri, 19 Apr 2019 09:50:54 -0400 Subject: [PATCH 0820/2557] Initialize rate_limited in hs_pick_hsdir() to false --- src/feature/hs/hs_common.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/feature/hs/hs_common.c b/src/feature/hs/hs_common.c index f9caf24f1c..11efe078ae 100644 --- a/src/feature/hs/hs_common.c +++ b/src/feature/hs/hs_common.c @@ -1605,7 +1605,7 @@ hs_pick_hsdir(smartlist_t *responsible_dirs, const char *req_key_str, routerstatus_t *hs_dir; time_t now = time(NULL); int excluded_some; - bool rate_limited; + bool rate_limited = false; int rate_limited_count = 0; int responsible_dirs_count = smartlist_len(responsible_dirs); From ac5753d3ca943982290d24899a2c4d406098424a Mon Sep 17 00:00:00 2001 From: teor Date: Tue, 23 Apr 2019 10:03:18 +1000 Subject: [PATCH 0821/2557] Coding Standards: Document how to find git commits Document how to find git commits and tags for bug fixes in CodingStandards.md. And update some changes file documentation. Closes ticket 30261. --- changes/ticket30261 | 4 ++++ doc/HACKING/CodingStandards.md | 41 +++++++++++++++++++++++++++++----- 2 files changed, 39 insertions(+), 6 deletions(-) create mode 100644 changes/ticket30261 diff --git a/changes/ticket30261 b/changes/ticket30261 new file mode 100644 index 0000000000..e4a2643c88 --- /dev/null +++ b/changes/ticket30261 @@ -0,0 +1,4 @@ + o Documentation: + - Document how to find git commits and tags for bug fixes in + CodingStandards.md. And update some changes file documentation. + Closes ticket 30261. diff --git a/doc/HACKING/CodingStandards.md b/doc/HACKING/CodingStandards.md index 4f229348e4..74db2a39a3 100644 --- a/doc/HACKING/CodingStandards.md +++ b/doc/HACKING/CodingStandards.md @@ -110,12 +110,41 @@ it's a bugfix, mention what bug it fixes and when the bug was introduced. To find out which Git tag the change was introduced in, you can use `git describe --contains `. -If at all possible, try to create this file in the same commit where you are -making the change. Please give it a distinctive name that no other branch will -use for the lifetime of your change. To verify the format of the changes file, -you can use `make check-changes`. This is run automatically as part of -`make check` -- if it fails, we must fix it before we release. These -checks are implemented in `scripts/maint/lintChanges.py`. +If you don't know the commit, you can search the git diffs (-S) for the first +instance of the feature (--reverse). + +For example, for #30224, we wanted to know when the bridge-distribution-request +feature was introduced into Tor: + $ git log -S bridge-distribution-request --reverse + commit ebab521525 + Author: Roger Dingledine + Date: Sun Nov 13 02:39:16 2016 -0500 + + Add new BridgeDistribution config option + + $ git describe --contains ebab521525 + tor-0.3.2.3-alpha~15^2~4 + +If you need to know all the Tor versions that contain a commit, use: + $ git tag --contains 9f2efd02a1 | sort -V + tor-0.2.5.16 + tor-0.2.8.17 + tor-0.2.9.14 + tor-0.2.9.15 + ... + tor-0.3.0.13 + tor-0.3.1.9 + tor-0.3.1.10 + ... + +If at all possible, try to create the changes file in the same commit where +you are making the change. Please give it a distinctive name that no other +branch will use for the lifetime of your change. We usually use "ticketNNNNN" +or "bugNNNNN", where NNNNN is the ticket number. To verify the format of the +changes file, you can use `make check-changes`. This is run automatically as +part of `make check` -- if it fails, we must fix it as soon as possible, so +that our CI passes. These checks are implemented in +`scripts/maint/lintChanges.py`. Changes file style guide: * Changes files begin with " o Header (subheading):". The header From 1788343affdec7ac73dbdf1c50b486f593a2d428 Mon Sep 17 00:00:00 2001 From: teor Date: Tue, 23 Apr 2019 12:31:14 +1000 Subject: [PATCH 0822/2557] Stop looking for scripts in the build directory during "make shellcheck" Fixes bug 30263; bugfix on 0.4.0.1-alpha. --- Makefile.am | 2 +- changes/bug30263 | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 changes/bug30263 diff --git a/Makefile.am b/Makefile.am index d65c08f6bf..8bf2aaf910 100644 --- a/Makefile.am +++ b/Makefile.am @@ -220,7 +220,7 @@ shellcheck: if command -v shellcheck; then \ find $(top_srcdir)/scripts/ -name "*.sh" -exec shellcheck {} +; \ if [ -d "$(top_srcdir)/scripts/test" ]; then \ - shellcheck $(top_srcdir)/scripts/test/cov-diff $(top_builddir)/scripts/test/coverage; \ + shellcheck $(top_srcdir)/scripts/test/cov-diff $(top_srcdir)/scripts/test/coverage; \ fi; \ fi diff --git a/changes/bug30263 b/changes/bug30263 new file mode 100644 index 0000000000..ba81c1b8a1 --- /dev/null +++ b/changes/bug30263 @@ -0,0 +1,3 @@ + o Minor bugfixes (shellcheck): + - Stop looking for scripts in the build directory during + "make shellcheck". Fixes bug 30263; bugfix on 0.4.0.1-alpha. From a85cd5b41bbd71cd16a9a641e067faf4cafb313e Mon Sep 17 00:00:00 2001 From: rl1987 Date: Fri, 19 Apr 2019 09:46:36 +0300 Subject: [PATCH 0823/2557] Call practracker from pre-push and pre-commit hooks --- scripts/git/pre-commit.git-hook | 4 ++++ scripts/git/pre-push.git-hook | 6 ++++++ 2 files changed, 10 insertions(+) diff --git a/scripts/git/pre-commit.git-hook b/scripts/git/pre-commit.git-hook index 65fa99f4c4..76d13adeb9 100755 --- a/scripts/git/pre-commit.git-hook +++ b/scripts/git/pre-commit.git-hook @@ -39,3 +39,7 @@ fi if test -e scripts/maint/checkIncludes.py; then python scripts/maint/checkIncludes.py fi + +if [ -e scripts/maint/practracker/practracker.py ]; then + python3 ./scripts/maint/practracker/practracker.py "$workdir" +fi diff --git a/scripts/git/pre-push.git-hook b/scripts/git/pre-push.git-hook index f329829346..740180d6f6 100755 --- a/scripts/git/pre-push.git-hook +++ b/scripts/git/pre-push.git-hook @@ -34,6 +34,12 @@ if [ -x "$workdir/.git/hooks/pre-commit" ]; then fi fi +if [ -e scripts/maint/practracker/practracker.py ]; then + if ! python3 ./scripts/maint/practracker/practracker.py "$workdir"; then + exit 1 + fi +fi + # shellcheck disable=SC2034 while read -r local_ref local_sha remote_ref remote_sha do From 0c78811ceabaece8bec92cc6a9a42a7b4631cfa7 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Fri, 19 Apr 2019 09:51:04 +0300 Subject: [PATCH 0824/2557] Add changes file --- changes/ticket30051 | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 changes/ticket30051 diff --git a/changes/ticket30051 b/changes/ticket30051 new file mode 100644 index 0000000000..87b6d7611f --- /dev/null +++ b/changes/ticket30051 @@ -0,0 +1,5 @@ + o Minor features (developer tooling): + - Call practracker from pre-push and pre-commit git hooks to let a + developer know if they made any code style violations in their last + commit. This should help preventing code style violations appearing + upstream. Closes ticket 30051. From 8bea0c2fa346b8f67e5c33a5c455d022049e34c6 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 23 Apr 2019 14:14:17 -0400 Subject: [PATCH 0825/2557] Rename outvar to follow _out convention. --- src/feature/hs/hs_common.c | 8 ++++---- src/feature/hs/hs_common.h | 3 ++- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/feature/hs/hs_common.c b/src/feature/hs/hs_common.c index 11efe078ae..d4736c2862 100644 --- a/src/feature/hs/hs_common.c +++ b/src/feature/hs/hs_common.c @@ -1590,7 +1590,7 @@ hs_purge_last_hid_serv_requests(void) * one that we should use to fetch a descriptor right now. Take into account * previous failed attempts at fetching this descriptor from HSDirs using the * string identifier req_key_str. We return whether we are rate limited - * into *is_rate_limited if it is not NULL. + * into *is_rate_limited_out if it is not NULL. * * Steals ownership of responsible_dirs. * @@ -1598,7 +1598,7 @@ hs_purge_last_hid_serv_requests(void) * NULL if no HSDirs are worth trying right now. */ routerstatus_t * hs_pick_hsdir(smartlist_t *responsible_dirs, const char *req_key_str, - bool *is_rate_limited) + bool *is_rate_limited_out) { smartlist_t *usable_responsible_dirs = smartlist_new(); const or_options_t *options = get_options(); @@ -1665,8 +1665,8 @@ hs_pick_hsdir(smartlist_t *responsible_dirs, const char *req_key_str, hs_lookup_last_hid_serv_request(hs_dir, req_key_str, now, 1); } - if (is_rate_limited != NULL) { - *is_rate_limited = rate_limited; + if (is_rate_limited_out != NULL) { + *is_rate_limited_out = rate_limited; } return hs_dir; diff --git a/src/feature/hs/hs_common.h b/src/feature/hs/hs_common.h index fc0c39f94f..3009780d90 100644 --- a/src/feature/hs/hs_common.h +++ b/src/feature/hs/hs_common.h @@ -241,7 +241,8 @@ void hs_get_responsible_hsdirs(const struct ed25519_public_key_t *blinded_pk, int use_second_hsdir_index, int for_fetching, smartlist_t *responsible_dirs); routerstatus_t *hs_pick_hsdir(smartlist_t *responsible_dirs, - const char *req_key_str, bool *is_rate_limited); + const char *req_key_str, + bool *is_rate_limited_out); time_t hs_hsdir_requery_period(const or_options_t *options); time_t hs_lookup_last_hid_serv_request(routerstatus_t *hs_dir, From da268e3b508fb5f9ab3b930deded331c387f7243 Mon Sep 17 00:00:00 2001 From: Neel Chauhan Date: Tue, 31 Jul 2018 08:17:59 -0400 Subject: [PATCH 0826/2557] Add function fascist_firewall_choose_address_ls() --- src/core/or/policies.c | 55 ++++++++++++++++++++++++++++++++++++++++++ src/core/or/policies.h | 3 +++ 2 files changed, 58 insertions(+) diff --git a/src/core/or/policies.c b/src/core/or/policies.c index f59894ea8f..26a053821a 100644 --- a/src/core/or/policies.c +++ b/src/core/or/policies.c @@ -31,6 +31,7 @@ #include "ht.h" #include "lib/crypt_ops/crypto_rand.h" #include "lib/encoding/confline.h" +#include "trunnel/ed25519_cert.h" #include "core/or/addr_policy_st.h" #include "feature/dirclient/dir_server_st.h" @@ -1015,6 +1016,60 @@ fascist_firewall_choose_address_rs(const routerstatus_t *rs, } } +/** Like fascist_firewall_choose_address_base(), but takes in a smartlist + * lspecs consisting of one or more link specifiers. + */ +void +fascist_firewall_choose_address_ls(const smartlist_t *lspecs, + int pref_only, tor_addr_port_t* ap, + int direct_conn) +{ + int have_v4 = 0, have_v6 = 0; + uint16_t port_v4 = 0, port_v6 = 0; + tor_addr_t addr_v4, addr_v6; + + SMARTLIST_FOREACH_BEGIN(lspecs, const link_specifier_t *, ls) { + switch (link_specifier_get_ls_type(ls)) { + case LS_IPV4: + /* Skip if we already seen a v4. */ + if (have_v4) continue; + tor_addr_from_ipv4h(&addr_v4, + link_specifier_get_un_ipv4_addr(ls)); + port_v4 = link_specifier_get_un_ipv4_port(ls); + have_v4 = 1; + break; + case LS_IPV6: + /* Skip if we already seen a v6, or deliberately skip it if we're not a + * direct connection. */ + if (have_v6 || !direct_conn) continue; + tor_addr_from_ipv6_bytes(&addr_v6, + (const char *) link_specifier_getconstarray_un_ipv6_addr(ls)); + port_v6 = link_specifier_get_un_ipv6_port(ls); + have_v6 = 1; + break; + default: + /* Ignore unknown. */ + break; + } + } SMARTLIST_FOREACH_END(ls); + + tor_assert(ap); + + tor_addr_make_null(&ap->addr, AF_UNSPEC); + ap->port = 0; + + /* Here, don't check for DirPorts as link specifiers are only used for + * ORPorts. */ + const or_options_t *options = get_options(); + int pref_ipv6 = fascist_firewall_prefer_ipv6_orport(options); + /* Assume that the DirPorts are zero as link specifiers only use ORPorts. */ + fascist_firewall_choose_address_base(&addr_v4, port_v4, 0, + &addr_v6, port_v6, 0, + FIREWALL_OR_CONNECTION, + pref_only, pref_ipv6, + ap); +} + /** Like fascist_firewall_choose_address_base(), but takes node, and * looks up the node's IPv6 preference rather than taking an argument * for pref_ipv6. */ diff --git a/src/core/or/policies.h b/src/core/or/policies.h index 324c1c2dd1..f4c68fd952 100644 --- a/src/core/or/policies.h +++ b/src/core/or/policies.h @@ -92,6 +92,9 @@ int fascist_firewall_allows_dir_server(const dir_server_t *ds, void fascist_firewall_choose_address_rs(const routerstatus_t *rs, firewall_connection_t fw_connection, int pref_only, tor_addr_port_t* ap); +void fascist_firewall_choose_address_ls(const smartlist_t *lspecs, + int pref_only, tor_addr_port_t* ap, + int direct_conn); void fascist_firewall_choose_address_node(const node_t *node, firewall_connection_t fw_connection, int pref_only, tor_addr_port_t* ap); From 26183476575abf0f8daccc2f4ca8be4ba0e2a5de Mon Sep 17 00:00:00 2001 From: Neel Chauhan Date: Tue, 31 Jul 2018 08:41:21 -0400 Subject: [PATCH 0827/2557] Use fascist_firewall_choose_address_ls() in hs_get_extend_info_from_lspecs() --- changes/bug23588 | 7 ++++ src/core/or/policies.c | 22 ++++++----- src/core/or/policies.h | 3 +- src/feature/hs/hs_common.c | 78 +++++++++++++++----------------------- 4 files changed, 52 insertions(+), 58 deletions(-) create mode 100644 changes/bug23588 diff --git a/changes/bug23588 b/changes/bug23588 new file mode 100644 index 0000000000..754d083614 --- /dev/null +++ b/changes/bug23588 @@ -0,0 +1,7 @@ + o Minor bugfixes (address selection): + - Introduce fascist_firewall_choose_address_ls() which chooses an + IPv6 or IPv4 address based on a smartlist of link specifiers of + what is available and what we prefer. We use this function in + hs_get_extend_info_from_lspecs(). Fixes bug 23588; bugfix on + 0.3.5.1-alpha. Patch by Neel Chauhan. + diff --git a/src/core/or/policies.c b/src/core/or/policies.c index 26a053821a..b58a5c9cb6 100644 --- a/src/core/or/policies.c +++ b/src/core/or/policies.c @@ -1017,17 +1017,26 @@ fascist_firewall_choose_address_rs(const routerstatus_t *rs, } /** Like fascist_firewall_choose_address_base(), but takes in a smartlist - * lspecs consisting of one or more link specifiers. + * lspecs consisting of one or more link specifiers. We assume + * fw_connection is FIREWALL_OR_CONNECTION as link specifiers cannot + * contain DirPorts. */ void fascist_firewall_choose_address_ls(const smartlist_t *lspecs, - int pref_only, tor_addr_port_t* ap, - int direct_conn) + int pref_only, tor_addr_port_t* ap) { int have_v4 = 0, have_v6 = 0; uint16_t port_v4 = 0, port_v6 = 0; tor_addr_t addr_v4, addr_v6; + tor_assert(ap); + + tor_addr_make_null(&ap->addr, AF_UNSPEC); + ap->port = 0; + + tor_addr_make_null(&addr_v4, AF_INET); + tor_addr_make_null(&addr_v6, AF_INET6); + SMARTLIST_FOREACH_BEGIN(lspecs, const link_specifier_t *, ls) { switch (link_specifier_get_ls_type(ls)) { case LS_IPV4: @@ -1041,7 +1050,7 @@ fascist_firewall_choose_address_ls(const smartlist_t *lspecs, case LS_IPV6: /* Skip if we already seen a v6, or deliberately skip it if we're not a * direct connection. */ - if (have_v6 || !direct_conn) continue; + if (have_v6) continue; tor_addr_from_ipv6_bytes(&addr_v6, (const char *) link_specifier_getconstarray_un_ipv6_addr(ls)); port_v6 = link_specifier_get_un_ipv6_port(ls); @@ -1053,11 +1062,6 @@ fascist_firewall_choose_address_ls(const smartlist_t *lspecs, } } SMARTLIST_FOREACH_END(ls); - tor_assert(ap); - - tor_addr_make_null(&ap->addr, AF_UNSPEC); - ap->port = 0; - /* Here, don't check for DirPorts as link specifiers are only used for * ORPorts. */ const or_options_t *options = get_options(); diff --git a/src/core/or/policies.h b/src/core/or/policies.h index f4c68fd952..3c46363c04 100644 --- a/src/core/or/policies.h +++ b/src/core/or/policies.h @@ -93,8 +93,7 @@ void fascist_firewall_choose_address_rs(const routerstatus_t *rs, firewall_connection_t fw_connection, int pref_only, tor_addr_port_t* ap); void fascist_firewall_choose_address_ls(const smartlist_t *lspecs, - int pref_only, tor_addr_port_t* ap, - int direct_conn); + int pref_only, tor_addr_port_t* ap); void fascist_firewall_choose_address_node(const node_t *node, firewall_connection_t fw_connection, int pref_only, tor_addr_port_t* ap); diff --git a/src/feature/hs/hs_common.c b/src/feature/hs/hs_common.c index d4736c2862..ca7e991b75 100644 --- a/src/feature/hs/hs_common.c +++ b/src/feature/hs/hs_common.c @@ -1672,14 +1672,16 @@ hs_pick_hsdir(smartlist_t *responsible_dirs, const char *req_key_str, return hs_dir; } -/* From a list of link specifier, an onion key and if we are requesting a - * direct connection (ex: single onion service), return a newly allocated - * extend_info_t object. This function always returns an extend info with - * an IPv4 address, or NULL. +/* Given a list of link specifiers lspecs, a curve 25519 onion_key, and + * a direct connection boolean direct_conn (true for single onion services), + * return a newly allocated extend_info_t object. + * + * This function always returns an extend info with a valid IP address and + * ORPort, or NULL. If direct_conn is false, the IP address is always IPv4. * * It performs the following checks: - * if either IPv4 or legacy ID is missing, return NULL. - * if direct_conn, and we can't reach the IPv4 address, return NULL. + * if there is no usable IP address, or legacy ID is missing, return NULL. + * if direct_conn, and we can't reach any IP address, return NULL. */ extend_info_t * hs_get_extend_info_from_lspecs(const smartlist_t *lspecs, @@ -1688,10 +1690,12 @@ hs_get_extend_info_from_lspecs(const smartlist_t *lspecs, { int have_v4 = 0, have_legacy_id = 0, have_ed25519_id = 0; char legacy_id[DIGEST_LEN] = {0}; - uint16_t port_v4 = 0; - tor_addr_t addr_v4; ed25519_public_key_t ed25519_pk; extend_info_t *info = NULL; + tor_addr_port_t ap; + + tor_addr_make_null(&ap.addr, AF_UNSPEC); + ap.port = 0; tor_assert(lspecs); @@ -1704,11 +1708,14 @@ hs_get_extend_info_from_lspecs(const smartlist_t *lspecs, SMARTLIST_FOREACH_BEGIN(lspecs, const link_specifier_t *, ls) { switch (link_specifier_get_ls_type(ls)) { case LS_IPV4: - /* Skip if we already seen a v4. */ - if (have_v4) continue; - tor_addr_from_ipv4h(&addr_v4, + /* Skip if we already seen a v4. If direct_conn is true, we skip this + * block because fascist_firewall_choose_address_ls() will set ap. If + * direct_conn is false, set ap to the first IPv4 address and port in + * the link specifiers.*/ + if (have_v4 || direct_conn) continue; + tor_addr_from_ipv4h(&ap.addr, link_specifier_get_un_ipv4_addr(ls)); - port_v4 = link_specifier_get_un_ipv4_port(ls); + ap.port = link_specifier_get_un_ipv4_port(ls); have_v4 = 1; break; case LS_LEGACY_ID: @@ -1732,55 +1739,32 @@ hs_get_extend_info_from_lspecs(const smartlist_t *lspecs, } } SMARTLIST_FOREACH_END(ls); - /* Legacy ID is mandatory, and we require IPv4. */ - if (!have_v4 || !have_legacy_id) { - bool both = !have_v4 && !have_legacy_id; - log_fn(LOG_PROTOCOL_WARN, LD_REND, "Missing %s%s%s link specifier%s.", - !have_v4 ? "IPv4" : "", - both ? " and " : "", - !have_legacy_id ? "legacy ID" : "", - both ? "s" : ""); + /* Choose a preferred address first, but fall back to an allowed address. */ + if (direct_conn) + fascist_firewall_choose_address_ls(lspecs, 0, &ap); + + /* Legacy ID is mandatory, and we require an IP address. */ + if (!tor_addr_port_is_valid_ap(&ap, 0) || !have_legacy_id) { + /* If we're missing the legacy ID or the IP address, return NULL. */ goto done; } - /* We know we have IPv4, because we just checked. */ - if (!direct_conn) { - /* All clients can extend to any IPv4 via a 3-hop path. */ - goto validate; - } else if (direct_conn && - fascist_firewall_allows_address_addr(&addr_v4, port_v4, - FIREWALL_OR_CONNECTION, - 0, 0)) { - /* Direct connection and we can reach it in IPv4 so go for it. */ - goto validate; + /* We will add support for falling back to a 3-hop path in a later + * release. */ - /* We will add support for falling back to a 3-hop path in a later - * release. */ - } else { - /* If we can't reach IPv4, return NULL. */ - log_fn(LOG_PROTOCOL_WARN, LD_REND, - "Received an IPv4 link specifier, " - "but the address is not reachable: %s:%u", - fmt_addr(&addr_v4), port_v4); - goto done; - } - - /* We will add support for IPv6 in a later release. */ - - validate: /* We'll validate now that the address we've picked isn't a private one. If * it is, are we allowed to extend to private addresses? */ - if (!extend_info_addr_is_allowed(&addr_v4)) { + if (!extend_info_addr_is_allowed(&ap.addr)) { log_fn(LOG_PROTOCOL_WARN, LD_REND, "Requested address is private and we are not allowed to extend to " - "it: %s:%u", fmt_addr(&addr_v4), port_v4); + "it: %s:%u", fmt_addr(&ap.addr), ap.port); goto done; } /* We do have everything for which we think we can connect successfully. */ info = extend_info_new(NULL, legacy_id, (have_ed25519_id) ? &ed25519_pk : NULL, NULL, - onion_key, &addr_v4, port_v4); + onion_key, &ap.addr, ap.port); done: return info; } From b65f8c419a4eb2608beecbf31af0b5bdc6cc38ec Mon Sep 17 00:00:00 2001 From: Neel Chauhan Date: Sun, 4 Nov 2018 20:08:57 -0500 Subject: [PATCH 0828/2557] Add firewall_choose_address_ls() and hs_get_extend_info_from_lspecs() tests --- src/core/or/policies.c | 19 +++ src/feature/hs/hs_common.c | 20 +++- src/test/test_policy.c | 235 +++++++++++++++++++++++++++++++++++++ 3 files changed, 271 insertions(+), 3 deletions(-) diff --git a/src/core/or/policies.c b/src/core/or/policies.c index b58a5c9cb6..83d9a53fc0 100644 --- a/src/core/or/policies.c +++ b/src/core/or/policies.c @@ -1031,6 +1031,15 @@ fascist_firewall_choose_address_ls(const smartlist_t *lspecs, tor_assert(ap); + if (lspecs == NULL) { + log_warn(LD_BUG, "Unknown or missing link specifiers"); + return; + } + if (smartlist_len(lspecs) == 0) { + log_warn(LD_PROTOCOL, "Link specifiers are empty"); + return; + } + tor_addr_make_null(&ap->addr, AF_UNSPEC); ap->port = 0; @@ -1062,6 +1071,16 @@ fascist_firewall_choose_address_ls(const smartlist_t *lspecs, } } SMARTLIST_FOREACH_END(ls); + /* If we don't have IPv4 or IPv6 in link specifiers, log a bug and return. */ + if (!have_v4 && !have_v6) { + if (!have_v6) { + log_warn(LD_PROTOCOL, "None of our link specifiers have IPv4 or IPv6"); + } else { + log_warn(LD_PROTOCOL, "None of our link specifiers have IPv4"); + } + return; + } + /* Here, don't check for DirPorts as link specifiers are only used for * ORPorts. */ const or_options_t *options = get_options(); diff --git a/src/feature/hs/hs_common.c b/src/feature/hs/hs_common.c index ca7e991b75..9b24129b63 100644 --- a/src/feature/hs/hs_common.c +++ b/src/feature/hs/hs_common.c @@ -1697,7 +1697,15 @@ hs_get_extend_info_from_lspecs(const smartlist_t *lspecs, tor_addr_make_null(&ap.addr, AF_UNSPEC); ap.port = 0; - tor_assert(lspecs); + if (lspecs == NULL) { + log_warn(LD_BUG, "Specified link specifiers is null"); + goto done; + } + + if (onion_key == NULL) { + log_warn(LD_BUG, "Specified onion key is null"); + goto done; + } if (smartlist_len(lspecs) == 0) { log_fn(LOG_PROTOCOL_WARN, LD_REND, "Empty link specifier list."); @@ -1744,8 +1752,14 @@ hs_get_extend_info_from_lspecs(const smartlist_t *lspecs, fascist_firewall_choose_address_ls(lspecs, 0, &ap); /* Legacy ID is mandatory, and we require an IP address. */ - if (!tor_addr_port_is_valid_ap(&ap, 0) || !have_legacy_id) { - /* If we're missing the legacy ID or the IP address, return NULL. */ + if (!tor_addr_port_is_valid_ap(&ap, 0)) { + /* If we're missing the IP address, log a warning and return NULL. */ + log_info(LD_NET, "Unreachable or invalid IP address in link state"); + goto done; + } + if (!have_legacy_id) { + /* If we're missing the legacy ID, log a warning and return NULL. */ + log_warn(LD_PROTOCOL, "Missing Legacy ID in link state"); goto done; } diff --git a/src/test/test_policy.c b/src/test/test_policy.c index 46d4a1b94a..e58bb3d174 100644 --- a/src/test/test_policy.c +++ b/src/test/test_policy.c @@ -6,13 +6,18 @@ #include "core/or/or.h" #include "app/config/config.h" +#include "core/or/circuitbuild.h" #include "core/or/policies.h" #include "feature/dirparse/policy_parse.h" +#include "feature/hs/hs_common.h" +#include "feature/hs/hs_descriptor.h" #include "feature/relay/router.h" #include "lib/encoding/confline.h" #include "test/test.h" +#include "test/log_test_helpers.h" #include "core/or/addr_policy_st.h" +#include "core/or/extend_info_st.h" #include "core/or/port_cfg_st.h" #include "feature/nodelist/node_st.h" #include "feature/nodelist/routerinfo_st.h" @@ -2024,6 +2029,101 @@ test_policies_fascist_firewall_allows_address(void *arg) expect_ap); \ STMT_END +/* Check that fascist_firewall_choose_address_ls() returns the expected + * results. */ +#define CHECK_CHOSEN_ADDR_NULL_LS() \ + STMT_BEGIN \ + tor_addr_port_t chosen_ls_ap; \ + tor_addr_make_null(&chosen_ls_ap.addr, AF_UNSPEC); \ + chosen_ls_ap.port = 0; \ + setup_full_capture_of_logs(LOG_WARN); \ + fascist_firewall_choose_address_ls(NULL, 1, &chosen_ls_ap); \ + expect_single_log_msg("Unknown or missing link specifiers"); \ + teardown_capture_of_logs(); \ + STMT_END + +#define CHECK_CHOSEN_ADDR_LS(fake_ls, pref_only, expect_rv, expect_ap) \ + STMT_BEGIN \ + tor_addr_port_t chosen_ls_ap; \ + tor_addr_make_null(&chosen_ls_ap.addr, AF_UNSPEC); \ + chosen_ls_ap.port = 0; \ + setup_full_capture_of_logs(LOG_WARN); \ + fascist_firewall_choose_address_ls(fake_ls, pref_only, &chosen_ls_ap); \ + if (smartlist_len(fake_ls) == 0) { \ + expect_single_log_msg("Link specifiers are empty"); \ + } else { \ + expect_no_log_entry(); \ + tt_assert(tor_addr_eq(&(expect_ap).addr, &chosen_ls_ap.addr)); \ + tt_int_op((expect_ap).port, OP_EQ, chosen_ls_ap.port); \ + } \ + teardown_capture_of_logs(); \ + STMT_END + +#define CHECK_LS_LEGACY_ONLY(fake_ls) \ + STMT_BEGIN \ + tor_addr_port_t chosen_ls_ap; \ + tor_addr_make_null(&chosen_ls_ap.addr, AF_UNSPEC); \ + chosen_ls_ap.port = 0; \ + setup_full_capture_of_logs(LOG_WARN); \ + fascist_firewall_choose_address_ls(fake_ls, 0, &chosen_ls_ap); \ + expect_single_log_msg("None of our link specifiers have IPv4 or IPv6"); \ + teardown_capture_of_logs(); \ + STMT_END + +#define CHECK_HS_EXTEND_INFO_ADDR_LS(fake_ls, direct_conn, expect_ap) \ + STMT_BEGIN \ + curve25519_secret_key_t seckey; \ + curve25519_secret_key_generate(&seckey, 0); \ + curve25519_public_key_t pubkey; \ + curve25519_public_key_generate(&pubkey, &seckey); \ + setup_full_capture_of_logs(LOG_WARN); \ + extend_info_t *ei = hs_get_extend_info_from_lspecs(fake_ls, &pubkey, \ + direct_conn); \ + if (fake_ls == NULL) { \ + tt_ptr_op(ei, OP_EQ, NULL); \ + expect_single_log_msg("Specified link specifiers is null"); \ + } else { \ + expect_no_log_entry(); \ + tt_assert(tor_addr_eq(&(expect_ap).addr, &ei->addr)); \ + tt_int_op((expect_ap).port, OP_EQ, ei->port); \ + extend_info_free(ei); \ + } \ + teardown_capture_of_logs(); \ + STMT_END + +#define CHECK_HS_EXTEND_INFO_ADDR_LS_NULL_KEY(fake_ls) \ + STMT_BEGIN \ + setup_full_capture_of_logs(LOG_WARN); \ + extend_info_t *ei = hs_get_extend_info_from_lspecs(fake_ls, NULL, 0); \ + tt_ptr_op(ei, OP_EQ, NULL); \ + expect_single_log_msg("Specified onion key is null"); \ + teardown_capture_of_logs(); \ + STMT_END + +#define CHECK_HS_EXTEND_INFO_ADDR_LS_EXPECT_NULL(fake_ls, direct_conn) \ + STMT_BEGIN \ + curve25519_secret_key_t seckey; \ + curve25519_secret_key_generate(&seckey, 0); \ + curve25519_public_key_t pubkey; \ + curve25519_public_key_generate(&pubkey, &seckey); \ + extend_info_t *ei = hs_get_extend_info_from_lspecs(fake_ls, &pubkey, \ + direct_conn); \ + tt_ptr_op(ei, OP_EQ, NULL); \ + STMT_END + +#define CHECK_HS_EXTEND_INFO_ADDR_LS_EXPECT_MSG(fake_ls, msg_level, msg) \ + STMT_BEGIN \ + curve25519_secret_key_t seckey; \ + curve25519_secret_key_generate(&seckey, 0); \ + curve25519_public_key_t pubkey; \ + curve25519_public_key_generate(&pubkey, &seckey); \ + setup_full_capture_of_logs(msg_level); \ + extend_info_t *ei = hs_get_extend_info_from_lspecs(fake_ls, &pubkey, 0); \ + tt_ptr_op(ei, OP_EQ, NULL); \ + expect_single_log_msg(msg); \ + teardown_capture_of_logs(); \ + STMT_END + /** Mock the preferred address function to return zero (prefer IPv4). */ static int mock_fascist_firewall_rand_prefer_ipv6_addr_use_ipv4(void) @@ -2472,6 +2572,141 @@ test_policies_fascist_firewall_choose_address(void *arg) UNMOCK(fascist_firewall_rand_prefer_ipv6_addr); + /* Test firewall_choose_address_ls(). To do this, we make a fake link + * specifier. */ + smartlist_t *lspecs = smartlist_new(), + *lspecs_blank = smartlist_new(), + *lspecs_v4 = smartlist_new(), + *lspecs_v6 = smartlist_new(), + *lspecs_no_legacy = smartlist_new(), + *lspecs_legacy_only = smartlist_new(); + link_specifier_t *fake_ls; + + /* IPv4 link specifier */ + fake_ls = link_specifier_new(); + link_specifier_set_ls_type(fake_ls, LS_IPV4); + link_specifier_set_un_ipv4_addr(fake_ls, + tor_addr_to_ipv4h(&ipv4_or_ap.addr)); + link_specifier_set_un_ipv4_port(fake_ls, ipv4_or_ap.port); + link_specifier_set_ls_len(fake_ls, sizeof(ipv4_or_ap.addr.addr.in_addr) + + sizeof(ipv4_or_ap.port)); + smartlist_add(lspecs, fake_ls); + smartlist_add(lspecs_v4, fake_ls); + smartlist_add(lspecs_no_legacy, fake_ls); + + /* IPv6 link specifier */ + fake_ls = link_specifier_new(); + link_specifier_set_ls_type(fake_ls, LS_IPV6); + size_t addr_len = link_specifier_getlen_un_ipv6_addr(fake_ls); + const uint8_t *in6_addr = tor_addr_to_in6_addr8(&ipv6_or_ap.addr); + uint8_t *ipv6_array = link_specifier_getarray_un_ipv6_addr(fake_ls); + memcpy(ipv6_array, in6_addr, addr_len); + link_specifier_set_un_ipv6_port(fake_ls, ipv6_or_ap.port); + link_specifier_set_ls_len(fake_ls, addr_len + sizeof(ipv6_or_ap.port)); + smartlist_add(lspecs, fake_ls); + smartlist_add(lspecs_v6, fake_ls); + + /* Legacy ID link specifier */ + fake_ls = link_specifier_new(); + link_specifier_set_ls_type(fake_ls, LS_LEGACY_ID); + uint8_t *legacy_id = link_specifier_getarray_un_legacy_id(fake_ls); + memset(legacy_id, 'A', sizeof(*legacy_id)); + link_specifier_set_ls_len(fake_ls, + link_specifier_getlen_un_legacy_id(fake_ls)); + smartlist_add(lspecs, fake_ls); + smartlist_add(lspecs_legacy_only, fake_ls); + smartlist_add(lspecs_v4, fake_ls); + smartlist_add(lspecs_v6, fake_ls); + + /* Check with bogus requests. */ + tor_addr_port_t null_ap; \ + tor_addr_make_null(&null_ap.addr, AF_UNSPEC); \ + null_ap.port = 0; \ + + /* Check for a null link state. */ + CHECK_CHOSEN_ADDR_NULL_LS(); + CHECK_HS_EXTEND_INFO_ADDR_LS(NULL, 1, null_ap); + + /* Check for a blank link state. */ + CHECK_CHOSEN_ADDR_LS(lspecs_blank, 0, 0, null_ap); + CHECK_HS_EXTEND_INFO_ADDR_LS_EXPECT_NULL(lspecs_blank, 0); + + /* Check for a link state with only a Legacy ID. */ + CHECK_LS_LEGACY_ONLY(lspecs_legacy_only); + CHECK_HS_EXTEND_INFO_ADDR_LS_EXPECT_NULL(lspecs_legacy_only, 0); + smartlist_free(lspecs_legacy_only); + + /* Check with a null onion_key. */ + CHECK_HS_EXTEND_INFO_ADDR_LS_NULL_KEY(lspecs_blank); + smartlist_free(lspecs_blank); + + /* Check with a null onion_key. */ + CHECK_HS_EXTEND_INFO_ADDR_LS_EXPECT_MSG(lspecs_no_legacy, LOG_WARN, + "Missing Legacy ID in link state"); + smartlist_free(lspecs_no_legacy); + + /* Enable both IPv4 and IPv6. */ + memset(&mock_options, 0, sizeof(or_options_t)); + mock_options.ClientUseIPv4 = 1; + mock_options.ClientUseIPv6 = 1; + + /* Prefer IPv4, enable both IPv4 and IPv6. */ + mock_options.ClientPreferIPv6ORPort = 0; + + CHECK_CHOSEN_ADDR_LS(lspecs, 0, 1, ipv4_or_ap); + CHECK_CHOSEN_ADDR_LS(lspecs, 1, 1, ipv4_or_ap); + + CHECK_HS_EXTEND_INFO_ADDR_LS(lspecs, 1, ipv4_or_ap); + CHECK_HS_EXTEND_INFO_ADDR_LS(lspecs, 0, ipv4_or_ap); + + /* Prefer IPv6, enable both IPv4 and IPv6. */ + mock_options.ClientPreferIPv6ORPort = 1; + + CHECK_CHOSEN_ADDR_LS(lspecs, 0, 1, ipv6_or_ap); + CHECK_CHOSEN_ADDR_LS(lspecs, 1, 1, ipv6_or_ap); + + CHECK_HS_EXTEND_INFO_ADDR_LS(lspecs, 1, ipv6_or_ap); + CHECK_HS_EXTEND_INFO_ADDR_LS(lspecs, 0, ipv4_or_ap); + + /* IPv4-only. */ + memset(&mock_options, 0, sizeof(or_options_t)); + mock_options.ClientUseIPv4 = 1; + mock_options.ClientUseIPv6 = 0; + + CHECK_CHOSEN_ADDR_LS(lspecs, 0, 1, ipv4_or_ap); + CHECK_CHOSEN_ADDR_LS(lspecs, 1, 1, ipv4_or_ap); + + CHECK_CHOSEN_ADDR_LS(lspecs_v6, 0, 0, null_ap); + + CHECK_HS_EXTEND_INFO_ADDR_LS(lspecs, 1, ipv4_or_ap); + CHECK_HS_EXTEND_INFO_ADDR_LS(lspecs, 0, ipv4_or_ap); + + CHECK_HS_EXTEND_INFO_ADDR_LS_EXPECT_NULL(lspecs_v6, 0); + CHECK_HS_EXTEND_INFO_ADDR_LS_EXPECT_NULL(lspecs_v6, 1); + + /* IPv6-only. */ + memset(&mock_options, 0, sizeof(or_options_t)); + mock_options.ClientUseIPv4 = 0; + mock_options.ClientUseIPv6 = 1; + + CHECK_CHOSEN_ADDR_LS(lspecs, 0, 1, ipv6_or_ap); + CHECK_CHOSEN_ADDR_LS(lspecs, 1, 1, ipv6_or_ap); + + CHECK_CHOSEN_ADDR_LS(lspecs_v4, 0, 0, null_ap); + + CHECK_HS_EXTEND_INFO_ADDR_LS(lspecs, 1, ipv6_or_ap); + CHECK_HS_EXTEND_INFO_ADDR_LS(lspecs, 0, ipv4_or_ap); + + CHECK_HS_EXTEND_INFO_ADDR_LS_EXPECT_NULL(lspecs_v4, 1); + CHECK_HS_EXTEND_INFO_ADDR_LS_EXPECT_NULL(lspecs_v6, 0); + + smartlist_free(lspecs_v4); + smartlist_free(lspecs_v6); + + SMARTLIST_FOREACH(lspecs, link_specifier_t *, lspec, \ + link_specifier_free(lspec)); \ + smartlist_free(lspecs); + done: UNMOCK(get_options); } From b19dd1bb11a59ab38dc1b45f2108a4a6f5ce2813 Mon Sep 17 00:00:00 2001 From: teor Date: Wed, 12 Dec 2018 12:43:48 +1000 Subject: [PATCH 0829/2557] Update 23588 changes file to say what the patch actually does Stop ignoring IPv6 link specifiers sent to v3 onion services. v3 onion service IPv6 support is still incomplete, see 23493 for details. Fixes bug 23588; bugfix on 0.3.2.1-alpha. Patch by Neel Chauhan. --- changes/bug23588 | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/changes/bug23588 b/changes/bug23588 index 754d083614..86064ab313 100644 --- a/changes/bug23588 +++ b/changes/bug23588 @@ -1,7 +1,5 @@ - o Minor bugfixes (address selection): - - Introduce fascist_firewall_choose_address_ls() which chooses an - IPv6 or IPv4 address based on a smartlist of link specifiers of - what is available and what we prefer. We use this function in - hs_get_extend_info_from_lspecs(). Fixes bug 23588; bugfix on - 0.3.5.1-alpha. Patch by Neel Chauhan. - + o Minor bugfixes (v3 onion services): + - Stop ignoring IPv6 link specifiers sent to v3 onion services. + v3 onion service IPv6 support is still incomplete, see 23493 for + details. Fixes bug 23588; bugfix on 0.3.2.1-alpha. + Patch by Neel Chauhan. From abe086dd976174ea52707f0ab27aae31228dfc2b Mon Sep 17 00:00:00 2001 From: teor Date: Wed, 24 Apr 2019 16:57:49 +1000 Subject: [PATCH 0830/2557] test/hs: Re-enable the IPv6 intro point in the HS tests Tests 23588, partially reverts 23576. Implements 29237. --- src/test/hs_test_helpers.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/test/hs_test_helpers.c b/src/test/hs_test_helpers.c index c57bdc730b..0a21fe576b 100644 --- a/src/test/hs_test_helpers.c +++ b/src/test/hs_test_helpers.c @@ -162,11 +162,8 @@ hs_helper_build_hs_desc_impl(unsigned int no_ip, /* Add four intro points. */ smartlist_add(desc->encrypted_data.intro_points, hs_helper_build_intro_point(signing_kp, now, "1.2.3.4", 0)); -/* IPv6-only introduction points are not supported yet, see #23588 */ -#if 0 smartlist_add(desc->encrypted_data.intro_points, hs_helper_build_intro_point(signing_kp, now, "[2600::1]", 0)); -#endif smartlist_add(desc->encrypted_data.intro_points, hs_helper_build_intro_point(signing_kp, now, "3.2.1.4", 1)); smartlist_add(desc->encrypted_data.intro_points, From f05e3f3c9af2cd05404a0646d4f7acc670c421da Mon Sep 17 00:00:00 2001 From: teor Date: Wed, 24 Apr 2019 17:23:00 +1000 Subject: [PATCH 0831/2557] test/hs: Correctly convert an IPv6 intro point to an extend_info Part of #23588. --- src/test/test_hs_client.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/test/test_hs_client.c b/src/test/test_hs_client.c index 8362b6cbda..aba0c937cc 100644 --- a/src/test/test_hs_client.c +++ b/src/test/test_hs_client.c @@ -479,6 +479,17 @@ test_client_pick_intro(void *arg) SMARTLIST_FOREACH_BEGIN(desc->encrypted_data.intro_points, hs_desc_intro_point_t *, ip) { extend_info_t *intro_ei = desc_intro_point_to_extend_info(ip); + /* desc_intro_point_to_extend_info() doesn't return IPv6 intro points + * yet, because we can't extend to them. See #24404, #24451, and #24181. + */ + if (intro_ei == NULL) { + /* Pretend we're making a direct connection, and that we can use IPv6 + */ + get_options_mutable()->ClientUseIPv6 = 1; + intro_ei = hs_get_extend_info_from_lspecs(ip->link_specifiers, + &ip->onion_key, 1); + tt_assert(tor_addr_family(&intro_ei->addr) == AF_INET6); + } tt_assert(intro_ei); if (intro_ei) { const char *ptr; From 3d89f0374a20571ca04068f10291ffefc9d76b40 Mon Sep 17 00:00:00 2001 From: teor Date: Wed, 24 Apr 2019 17:23:53 +1000 Subject: [PATCH 0832/2557] hs_config: Allow Tor to be configured as an IPv6-only v3 single onion service Part of #23588. --- src/feature/hs/hs_config.c | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/feature/hs/hs_config.c b/src/feature/hs/hs_config.c index ee4499ef5b..87f6257591 100644 --- a/src/feature/hs/hs_config.c +++ b/src/feature/hs/hs_config.c @@ -496,15 +496,6 @@ config_generic_service(const config_line_t *line_, * becomes a single onion service. */ if (rend_service_non_anonymous_mode_enabled(options)) { config->is_single_onion = 1; - /* We will add support for IPv6-only v3 single onion services in a future - * Tor version. This won't catch "ReachableAddresses reject *4", but that - * option doesn't work anyway. */ - if (options->ClientUseIPv4 == 0 && config->version == HS_VERSION_THREE) { - log_warn(LD_CONFIG, "IPv6-only v3 single onion services are not " - "supported. Set HiddenServiceSingleHopMode 0 and " - "HiddenServiceNonAnonymousMode 0, or set ClientUseIPv4 1."); - goto err; - } } /* Success */ From de91b838496a3646721608bdb7740696601f57c4 Mon Sep 17 00:00:00 2001 From: teor Date: Tue, 23 Apr 2019 12:36:06 +1000 Subject: [PATCH 0833/2557] practracker: Accept ~80 extra lines in src/core/or/policies.c Part of 23588. --- scripts/maint/practracker/exceptions.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/maint/practracker/exceptions.txt b/scripts/maint/practracker/exceptions.txt index 1992c563ac..0a5f3b1699 100644 --- a/scripts/maint/practracker/exceptions.txt +++ b/scripts/maint/practracker/exceptions.txt @@ -117,7 +117,7 @@ problem include-count /src/core/or/connection_or.c 51 problem function-size /src/core/or/connection_or.c:connection_or_group_set_badness_() 105 problem function-size /src/core/or/connection_or.c:connection_or_client_learned_peer_id() 144 problem function-size /src/core/or/connection_or.c:connection_or_compute_authenticate_cell_body() 235 -problem file-size /src/core/or/policies.c 3171 +problem file-size /src/core/or/policies.c 3249 problem function-size /src/core/or/policies.c:policy_summarize() 107 problem function-size /src/core/or/protover.c:protover_all_supported() 116 problem file-size /src/core/or/relay.c 3173 From f35bd3681466e58e525d95714ba630caa98f67ae Mon Sep 17 00:00:00 2001 From: teor Date: Wed, 24 Apr 2019 17:41:09 +1000 Subject: [PATCH 0834/2557] test-network-all: Test IPv6-only v3 single onion services In "make test-network-all", test IPv6-only v3 single onion services, using the chutney network single-onion-v23-ipv6-md. This test will not pass until 23588 has been merged. Closes ticket 27251. --- changes/ticket27251 | 4 ++++ src/test/include.am | 4 +--- 2 files changed, 5 insertions(+), 3 deletions(-) create mode 100644 changes/ticket27251 diff --git a/changes/ticket27251 b/changes/ticket27251 new file mode 100644 index 0000000000..7ce296e8da --- /dev/null +++ b/changes/ticket27251 @@ -0,0 +1,4 @@ + o Testing (chutney): + - In "make test-network-all", test IPv6-only v3 single onion services, + using the chutney network single-onion-v23-ipv6-md. This test will + not pass until 23588 has been merged. Closes ticket 27251. diff --git a/src/test/include.am b/src/test/include.am index 497aa320a4..fb4e9f4bc0 100644 --- a/src/test/include.am +++ b/src/test/include.am @@ -46,10 +46,8 @@ TESTS += src/test/test src/test/test-slow src/test/test-memwipe \ TEST_CHUTNEY_FLAVORS = basic-min bridges-min hs-v2-min hs-v3-min \ single-onion-v23 # only run if we can ping6 ::1 (localhost) -# IPv6-only v3 single onion services don't work yet, so we don't test the -# single-onion-v23-ipv6-md flavor TEST_CHUTNEY_FLAVORS_IPV6 = bridges+ipv6-min ipv6-exit-min hs-v23-ipv6-md \ - single-onion-ipv6-md + single-onion-v23-ipv6-md # only run if we can find a stable (or simply another) version of tor TEST_CHUTNEY_FLAVORS_MIXED = mixed+hs-v2 From d0a0f3e8cd31c035406b3f11c574cc3bbe8eb662 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 24 Apr 2019 14:15:18 -0400 Subject: [PATCH 0835/2557] Allow do_resolve() to be longer. --- scripts/maint/practracker/exceptions.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/maint/practracker/exceptions.txt b/scripts/maint/practracker/exceptions.txt index 1992c563ac..37f11bb440 100644 --- a/scripts/maint/practracker/exceptions.txt +++ b/scripts/maint/practracker/exceptions.txt @@ -287,5 +287,5 @@ problem function-size /src/lib/tls/tortls_openssl.c:tor_tls_context_new() 171 problem function-size /src/lib/tls/x509_nss.c:tor_tls_create_certificate_internal() 126 problem function-size /src/tools/tor-gencert.c:parse_commandline() 111 problem function-size /src/tools/tor-resolve.c:build_socks5_resolve_request() 104 -problem function-size /src/tools/tor-resolve.c:do_resolve() 173 +problem function-size /src/tools/tor-resolve.c:do_resolve() 175 problem function-size /src/tools/tor-resolve.c:main() 112 From 04290724957e14710646a5c28bd3230710466c92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Thu, 25 Apr 2019 01:50:13 +0200 Subject: [PATCH 0836/2557] Lower log level of unlink() errors in networkstatus_set_current_consensus(). In this patch we lower the log level of the failures for the three calls to unlink() in networkstatus_set_current_consensus(). These errors might trigger on Windows because the memory mapped consensus file keeps the file in open state even after we have close()'d it. Windows will then error on the unlink() call with a "Permission denied" error. The consequences of ignoring these errors is that we leave an unused file around on the file-system, which is an easier way to fix this problem right now than refactoring networkstatus_set_current_consensus(). See: https://bugs.torproject.org/29930 --- changes/bug29930 | 4 ++++ src/feature/nodelist/networkstatus.c | 18 +++++++++--------- 2 files changed, 13 insertions(+), 9 deletions(-) create mode 100644 changes/bug29930 diff --git a/changes/bug29930 b/changes/bug29930 new file mode 100644 index 0000000000..a99b11430b --- /dev/null +++ b/changes/bug29930 @@ -0,0 +1,4 @@ + o Minor bugfixes (UI): + - Lower log level of unlink() errors during bootstrap. Fixes bug 29930; + bugfix on 0.4.0.1-alpha. + diff --git a/src/feature/nodelist/networkstatus.c b/src/feature/nodelist/networkstatus.c index a988f700f3..24e3b212f0 100644 --- a/src/feature/nodelist/networkstatus.c +++ b/src/feature/nodelist/networkstatus.c @@ -2033,9 +2033,9 @@ networkstatus_set_current_consensus(const char *consensus, * latest consensus. */ if (was_waiting_for_certs && from_cache) if (unlink(unverified_fname) != 0) { - log_warn(LD_FS, - "Failed to unlink %s: %s", - unverified_fname, strerror(errno)); + log_debug(LD_FS, + "Failed to unlink %s: %s", + unverified_fname, strerror(errno)); } } goto done; @@ -2048,9 +2048,9 @@ networkstatus_set_current_consensus(const char *consensus, } if (was_waiting_for_certs && (r < -1) && from_cache) { if (unlink(unverified_fname) != 0) { - log_warn(LD_FS, - "Failed to unlink %s: %s", - unverified_fname, strerror(errno)); + log_debug(LD_FS, + "Failed to unlink %s: %s", + unverified_fname, strerror(errno)); } } goto done; @@ -2115,9 +2115,9 @@ networkstatus_set_current_consensus(const char *consensus, waiting->set_at = 0; waiting->dl_failed = 0; if (unlink(unverified_fname) != 0) { - log_warn(LD_FS, - "Failed to unlink %s: %s", - unverified_fname, strerror(errno)); + log_debug(LD_FS, + "Failed to unlink %s: %s", + unverified_fname, strerror(errno)); } } From 233835e14f2ea1878804ba3a81e6f24e3419eafc Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 25 Apr 2019 10:09:36 -0400 Subject: [PATCH 0837/2557] Move the responsibility for listing periodic events to periodic.c The end goal here is to move the periodic callback to their respective modules, so that mainloop.c doesn't have to include so many other things. This patch doesn't actually move any of the callbacks out of mainloop.c yet. --- src/core/mainloop/mainloop.c | 69 +++-------------- src/core/mainloop/mainloop.h | 2 +- src/core/mainloop/periodic.c | 137 ++++++++++++++++++++++++++++++++- src/core/mainloop/periodic.h | 7 ++ src/test/test_periodic_event.c | 36 ++++----- 5 files changed, 173 insertions(+), 78 deletions(-) diff --git a/src/core/mainloop/mainloop.c b/src/core/mainloop/mainloop.c index c9f2b0d896..b32532c762 100644 --- a/src/core/mainloop/mainloop.c +++ b/src/core/mainloop/mainloop.c @@ -1387,7 +1387,7 @@ CALLBACK(second_elapsed); PERIODIC_EVENT(name, PERIODIC_EVENT_ROLE_ ## r, f) #define FL(name) (PERIODIC_EVENT_FLAG_ ## name) -STATIC periodic_event_item_t periodic_events[] = { +STATIC periodic_event_item_t mainloop_periodic_events[] = { /* Everyone needs to run these. They need to have very long timeouts for * that to be safe. */ @@ -1485,24 +1485,7 @@ static periodic_event_item_t *prune_old_routers_event=NULL; void reset_all_main_loop_timers(void) { - int i; - for (i = 0; periodic_events[i].name; ++i) { - periodic_event_reschedule(&periodic_events[i]); - } -} - -/** Return the member of periodic_events[] whose name is name. - * Return NULL if no such event is found. - */ -static periodic_event_item_t * -find_periodic_event(const char *name) -{ - int i; - for (i = 0; periodic_events[i].name; ++i) { - if (strcmp(name, periodic_events[i].name) == 0) - return &periodic_events[i]; - } - return NULL; + periodic_events_reset_all(); } /** Return a bitmask of the roles this tor instance is configured for using @@ -1565,8 +1548,8 @@ initialize_periodic_events_cb(evutil_socket_t fd, short events, void *data) rescan_periodic_events(get_options()); } -/** Set up all the members of periodic_events[], and configure them all to be - * launched from a callback. */ +/** Set up all the members of mainloop_periodic_events[], and configure them + * all to be launched from a callback. */ STATIC void initialize_periodic_events(void) { @@ -1575,14 +1558,15 @@ initialize_periodic_events(void) periodic_events_initialized = 1; - /* Set up all periodic events. We'll launch them by roles. */ - int i; - for (i = 0; periodic_events[i].name; ++i) { - periodic_event_setup(&periodic_events[i]); + for (int i = 0; mainloop_periodic_events[i].name; ++i) { + periodic_events_add(&mainloop_periodic_events[i]); } + /* Set up all periodic events. We'll launch them by roles. */ + periodic_events_setup_all(); + #define NAMED_CALLBACK(name) \ - STMT_BEGIN name ## _event = find_periodic_event( #name ); STMT_END + STMT_BEGIN name ## _event = periodic_events_find( #name ); STMT_END NAMED_CALLBACK(check_descriptor); NAMED_CALLBACK(prune_old_routers); @@ -1602,10 +1586,7 @@ initialize_periodic_events(void) STATIC void teardown_periodic_events(void) { - int i; - for (i = 0; periodic_events[i].name; ++i) { - periodic_event_destroy(&periodic_events[i]); - } + periodic_events_destroy_all(); periodic_events_initialized = 0; } @@ -1647,33 +1628,7 @@ rescan_periodic_events(const or_options_t *options) return; } - int roles = get_my_roles(options); - - for (int i = 0; periodic_events[i].name; ++i) { - periodic_event_item_t *item = &periodic_events[i]; - - int enable = !!(item->roles & roles); - - /* Handle the event flags. */ - if (net_is_disabled() && - (item->flags & PERIODIC_EVENT_FLAG_NEED_NET)) { - enable = 0; - } - - /* Enable the event if needed. It is safe to enable an event that was - * already enabled. Same goes for disabling it. */ - if (enable) { - log_debug(LD_GENERAL, "Launching periodic event %s", item->name); - periodic_event_enable(item); - } else { - log_debug(LD_GENERAL, "Disabling periodic event %s", item->name); - if (item->flags & PERIODIC_EVENT_FLAG_RUN_ON_DISABLE) { - periodic_event_schedule_and_disable(item); - } else { - periodic_event_disable(item); - } - } - } + periodic_events_rescan_by_roles(get_my_roles(options), net_is_disabled()); } /* We just got new options globally set, see if we need to enabled or disable diff --git a/src/core/mainloop/mainloop.h b/src/core/mainloop/mainloop.h index 6ed93fa900..850918c35e 100644 --- a/src/core/mainloop/mainloop.h +++ b/src/core/mainloop/mainloop.h @@ -113,7 +113,7 @@ extern smartlist_t *connection_array; /* We need the periodic_event_item_t definition. */ #include "core/mainloop/periodic.h" -extern periodic_event_item_t periodic_events[]; +extern periodic_event_item_t mainloop_periodic_events[]; #endif #endif /* defined(MAIN_PRIVATE) */ diff --git a/src/core/mainloop/periodic.c b/src/core/mainloop/periodic.c index c0363b15ea..706dbc1b5e 100644 --- a/src/core/mainloop/periodic.c +++ b/src/core/mainloop/periodic.c @@ -6,9 +6,17 @@ * * \brief Generic backend for handling periodic events. * - * The events in this module are used by main.c to track items that need + * The events in this module are used to track items that need * to fire once every N seconds, possibly picking a new interval each time - * that they fire. See periodic_events[] in main.c for examples. + * that they fire. See periodic_events[] in mainloop.c for examples. + * + * This module manages a global list of periodic_event_item_t objects, + * each corresponding to a single event. To register an event, pass it to + * periodic_events_add() when initializing your subsystem. + * + * We expect that periodic_event_item_t objects will be statically allocated; + * we set them up and tear them down here, but we don't take ownership of + * them. */ #include "core/or/or.h" @@ -24,6 +32,11 @@ */ static const int MAX_INTERVAL = 10 * 365 * 86400; +/** + * + **/ +static smartlist_t *the_periodic_events = NULL; + /** Set the event event to run in next_interval seconds from * now. */ static void @@ -184,3 +197,123 @@ periodic_event_schedule_and_disable(periodic_event_item_t *event) mainloop_event_activate(event->ev); } + +/** + * Add item to the list of periodic events. + * + * Note that item should be statically allocated: we do not + * take ownership of it. + **/ +void +periodic_events_add(periodic_event_item_t *item) +{ + if (!the_periodic_events) + the_periodic_events = smartlist_new(); + + if (BUG(smartlist_contains(the_periodic_events, item))) + return; + + smartlist_add(the_periodic_events, item); +} + +/** Set up all not-previously setup periodic events. */ +void +periodic_events_setup_all(void) +{ + if (! the_periodic_events) + return; + + SMARTLIST_FOREACH_BEGIN(the_periodic_events, periodic_event_item_t *, item) { + if (item->ev) + continue; + periodic_event_setup(item); + } SMARTLIST_FOREACH_END(item); +} + +/** Reset all the registered periodic events so we'll do all our actions again + * as if we just started up. + * + * Useful if our clock just moved back a long time from the future, + * so we don't wait until that future arrives again before acting. + */ +void +periodic_events_reset_all(void) +{ + if (! the_periodic_events) + return; + + SMARTLIST_FOREACH_BEGIN(the_periodic_events, periodic_event_item_t *, item) { + periodic_event_reschedule(item); + } SMARTLIST_FOREACH_END(item); +} + +/** + * Return the registered periodic event whose name is name. + * Return NULL if no such event is found. + */ +periodic_event_item_t * +periodic_events_find(const char *name) +{ + if (! the_periodic_events) + return NULL; + + SMARTLIST_FOREACH_BEGIN(the_periodic_events, periodic_event_item_t *, item) { + if (strcmp(name, item->name) == 0) + return item; + } SMARTLIST_FOREACH_END(item); + return NULL; +} + +/** + * Start or stop registered periodic events, depending on our current set of + * roles. + * + * Invoked when our list of roles, or the net_disabled flag has changed. + **/ +void +periodic_events_rescan_by_roles(int roles, bool net_disabled) +{ + if (! the_periodic_events) + return; + + SMARTLIST_FOREACH_BEGIN(the_periodic_events, periodic_event_item_t *, item) { + int enable = !!(item->roles & roles); + + /* Handle the event flags. */ + if (net_disabled && + (item->flags & PERIODIC_EVENT_FLAG_NEED_NET)) { + enable = 0; + } + + /* Enable the event if needed. It is safe to enable an event that was + * already enabled. Same goes for disabling it. */ + if (enable) { + log_debug(LD_GENERAL, "Launching periodic event %s", item->name); + periodic_event_enable(item); + } else { + log_debug(LD_GENERAL, "Disabling periodic event %s", item->name); + if (item->flags & PERIODIC_EVENT_FLAG_RUN_ON_DISABLE) { + periodic_event_schedule_and_disable(item); + } else { + periodic_event_disable(item); + } + } + } SMARTLIST_FOREACH_END(item); +} + +/** Invoked at shutdown: free resources used in this module. + * + * Does not free the periodic_event_item_t object themselves, because we do + * not own them. */ +void +periodic_events_destroy_all(void) +{ + if (! the_periodic_events) + return; + + SMARTLIST_FOREACH_BEGIN(the_periodic_events, periodic_event_item_t *, item) { + periodic_event_destroy(item); + } SMARTLIST_FOREACH_END(item); + + smartlist_free(the_periodic_events); +} diff --git a/src/core/mainloop/periodic.h b/src/core/mainloop/periodic.h index 344fc9ad25..a021a141db 100644 --- a/src/core/mainloop/periodic.h +++ b/src/core/mainloop/periodic.h @@ -90,4 +90,11 @@ void periodic_event_enable(periodic_event_item_t *event); void periodic_event_disable(periodic_event_item_t *event); void periodic_event_schedule_and_disable(periodic_event_item_t *event); +void periodic_events_add(periodic_event_item_t *item); +void periodic_events_setup_all(void); +void periodic_events_reset_all(void); +periodic_event_item_t *periodic_events_find(const char *name); +void periodic_events_rescan_by_roles(int roles, bool net_disabled); +void periodic_events_destroy_all(void); + #endif /* !defined(TOR_PERIODIC_H) */ diff --git a/src/test/test_periodic_event.c b/src/test/test_periodic_event.c index ebac20838f..645274d371 100644 --- a/src/test/test_periodic_event.c +++ b/src/test/test_periodic_event.c @@ -55,8 +55,8 @@ test_pe_initialize(void *arg) rescan_periodic_events(get_options()); /* Validate that all events have been set up. */ - for (int i = 0; periodic_events[i].name; ++i) { - periodic_event_item_t *item = &periodic_events[i]; + for (int i = 0; mainloop_periodic_events[i].name; ++i) { + periodic_event_item_t *item = &mainloop_periodic_events[i]; tt_assert(item->ev); tt_assert(item->fn); tt_u64_op(item->last_action_time, OP_EQ, 0); @@ -89,8 +89,8 @@ test_pe_launch(void *arg) /* Hack: We'll set a dumb fn() of each events so they don't get called when * dispatching them. We just want to test the state of the callbacks, not * the whole code path. */ - for (int i = 0; periodic_events[i].name; ++i) { - periodic_event_item_t *item = &periodic_events[i]; + for (int i = 0; mainloop_periodic_events[i].name; ++i) { + periodic_event_item_t *item = &mainloop_periodic_events[i]; item->fn = dumb_event_fn; } @@ -116,8 +116,8 @@ test_pe_launch(void *arg) int mask = PERIODIC_EVENT_ROLE_CLIENT|PERIODIC_EVENT_ROLE_ALL| PERIODIC_EVENT_ROLE_NET_PARTICIPANT; - for (int i = 0; periodic_events[i].name; ++i) { - periodic_event_item_t *item = &periodic_events[i]; + for (int i = 0; mainloop_periodic_events[i].name; ++i) { + periodic_event_item_t *item = &mainloop_periodic_events[i]; int should_be_enabled = !!(item->roles & mask); tt_int_op(periodic_event_is_enabled(item), OP_EQ, should_be_enabled); // enabled or not, the event has not yet been run. @@ -134,8 +134,8 @@ test_pe_launch(void *arg) PERIODIC_EVENT_ROLE_RELAY|PERIODIC_EVENT_ROLE_DIRSERVER| PERIODIC_EVENT_ROLE_ALL|PERIODIC_EVENT_ROLE_NET_PARTICIPANT); - for (int i = 0; periodic_events[i].name; ++i) { - periodic_event_item_t *item = &periodic_events[i]; + for (int i = 0; mainloop_periodic_events[i].name; ++i) { + periodic_event_item_t *item = &mainloop_periodic_events[i]; /* Only Client role should be disabled. */ if (item->roles == PERIODIC_EVENT_ROLE_CLIENT) { tt_int_op(periodic_event_is_enabled(item), OP_EQ, 0); @@ -156,8 +156,8 @@ test_pe_launch(void *arg) set_network_participation(false); periodic_events_on_new_options(options); - for (int i = 0; periodic_events[i].name; ++i) { - periodic_event_item_t *item = &periodic_events[i]; + for (int i = 0; mainloop_periodic_events[i].name; ++i) { + periodic_event_item_t *item = &mainloop_periodic_events[i]; int should_be_enabled = (item->roles & PERIODIC_EVENT_ROLE_ALL) && !(item->flags & PERIODIC_EVENT_FLAG_NEED_NET); tt_int_op(periodic_event_is_enabled(item), OP_EQ, should_be_enabled); @@ -177,8 +177,8 @@ test_pe_launch(void *arg) * trigger a rescan of the event disabling the HS service event. */ to_remove = &service; - for (int i = 0; periodic_events[i].name; ++i) { - periodic_event_item_t *item = &periodic_events[i]; + for (int i = 0; mainloop_periodic_events[i].name; ++i) { + periodic_event_item_t *item = &mainloop_periodic_events[i]; tt_int_op(periodic_event_is_enabled(item), OP_EQ, (item->roles != PERIODIC_EVENT_ROLE_CONTROLEV)); } @@ -304,8 +304,8 @@ test_pe_hs_service(void *arg) /* Hack: We'll set a dumb fn() of each events so they don't get called when * dispatching them. We just want to test the state of the callbacks, not * the whole code path. */ - for (int i = 0; periodic_events[i].name; ++i) { - periodic_event_item_t *item = &periodic_events[i]; + for (int i = 0; mainloop_periodic_events[i].name; ++i) { + periodic_event_item_t *item = &mainloop_periodic_events[i]; item->fn = dumb_event_fn; } @@ -318,8 +318,8 @@ test_pe_hs_service(void *arg) * trigger a rescan of the event disabling the HS service event. */ to_remove = &service; - for (int i = 0; periodic_events[i].name; ++i) { - periodic_event_item_t *item = &periodic_events[i]; + for (int i = 0; mainloop_periodic_events[i].name; ++i) { + periodic_event_item_t *item = &mainloop_periodic_events[i]; if (item->roles & PERIODIC_EVENT_ROLE_HS_SERVICE) { tt_int_op(periodic_event_is_enabled(item), OP_EQ, 1); } @@ -329,8 +329,8 @@ test_pe_hs_service(void *arg) /* Remove the service from the global map, it should trigger a rescan and * disable the HS service events. */ remove_service(get_hs_service_map(), &service); - for (int i = 0; periodic_events[i].name; ++i) { - periodic_event_item_t *item = &periodic_events[i]; + for (int i = 0; mainloop_periodic_events[i].name; ++i) { + periodic_event_item_t *item = &mainloop_periodic_events[i]; if (item->roles & PERIODIC_EVENT_ROLE_HS_SERVICE) { tt_int_op(periodic_event_is_enabled(item), OP_EQ, 0); } From 0d650e7958d011dc165fbad7477e2eff842598ea Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 25 Apr 2019 13:23:18 -0400 Subject: [PATCH 0838/2557] Move responsibility for checking if events are setup into periodic.c We have checks in various places in mainlook.c to make sure that events are initialized before we invoke any periodic_foo() functions on them. But now that each subsystem will own its own periodic events, it will be cleaner if we don't assume that they are all setup or not. --- src/core/mainloop/mainloop.c | 27 ++++++++++----------------- src/core/mainloop/periodic.c | 10 ++++++++-- 2 files changed, 18 insertions(+), 19 deletions(-) diff --git a/src/core/mainloop/mainloop.c b/src/core/mainloop/mainloop.c index b32532c762..ddcc3bcd76 100644 --- a/src/core/mainloop/mainloop.c +++ b/src/core/mainloop/mainloop.c @@ -1587,6 +1587,13 @@ STATIC void teardown_periodic_events(void) { periodic_events_destroy_all(); + check_descriptor_event = NULL; + dirvote_event = NULL; + fetch_networkstatus_event = NULL; + launch_descriptor_fetches_event = NULL; + check_dns_honesty_event = NULL; + save_state_event = NULL; + prune_old_routers_event = NULL; periodic_events_initialized = 0; } @@ -1621,13 +1628,6 @@ rescan_periodic_events(const or_options_t *options) { tor_assert(options); - /* Avoid scanning the event list if we haven't initialized it yet. This is - * particularly useful for unit tests in order to avoid initializing main - * loop events everytime. */ - if (!periodic_events_initialized) { - return; - } - periodic_events_rescan_by_roles(get_my_roles(options), net_is_disabled()); } @@ -1636,13 +1636,7 @@ rescan_periodic_events(const or_options_t *options) void periodic_events_on_new_options(const or_options_t *options) { - /* Only if we've already initialized the events, rescan the list which will - * enable or disable events depending on our roles. This will be called at - * bootup and we don't want this function to initialize the events because - * they aren't set up at this stage. */ - if (periodic_events_initialized) { - rescan_periodic_events(options); - } + rescan_periodic_events(options); } /** @@ -2107,7 +2101,7 @@ dirvote_callback(time_t now, const or_options_t *options) void reschedule_dirvote(const or_options_t *options) { - if (periodic_events_initialized && authdir_mode_v3(options)) { + if (authdir_mode_v3(options) && dirvote_event) { periodic_event_reschedule(dirvote_event); } } @@ -2753,8 +2747,7 @@ dns_servers_relaunch_checks(void) { if (server_mode(get_options())) { dns_reset_correctness_checks(); - if (periodic_events_initialized) { - tor_assert(check_dns_honesty_event); + if (check_dns_honesty_event) { periodic_event_reschedule(check_dns_honesty_event); } } diff --git a/src/core/mainloop/periodic.c b/src/core/mainloop/periodic.c index 706dbc1b5e..a4a03ed297 100644 --- a/src/core/mainloop/periodic.c +++ b/src/core/mainloop/periodic.c @@ -100,8 +100,8 @@ periodic_event_dispatch(mainloop_event_t *ev, void *data) void periodic_event_reschedule(periodic_event_item_t *event) { - /* Don't reschedule a disabled event. */ - if (periodic_event_is_enabled(event)) { + /* Don't reschedule a disabled or uninitialized event. */ + if (event->ev && periodic_event_is_enabled(event)) { periodic_event_set_interval(event, 1); } } @@ -243,6 +243,9 @@ periodic_events_reset_all(void) return; SMARTLIST_FOREACH_BEGIN(the_periodic_events, periodic_event_item_t *, item) { + if (!item->ev) + continue; + periodic_event_reschedule(item); } SMARTLIST_FOREACH_END(item); } @@ -277,6 +280,9 @@ periodic_events_rescan_by_roles(int roles, bool net_disabled) return; SMARTLIST_FOREACH_BEGIN(the_periodic_events, periodic_event_item_t *, item) { + if (!item->ev) + continue; + int enable = !!(item->roles & roles); /* Handle the event flags. */ From de70eebc65d40d50f877b0f82df4d05ce670faa5 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 2 Apr 2019 08:17:22 -0400 Subject: [PATCH 0839/2557] Start on a command-parsing tool for controller commands. There _is_ an underlying logic to these commands, but it isn't wholly uniform, given years of tweaks and changes. Fortunately I think there is a superset that will work. This commit adds a parser for some of the most basic cases -- the ones currently handled by getargs_helper() and some of the object-taking ones. Soon will come initial tests; then I'll start using the parser. After that, I'll expand the parser to handle the other cases that come up in the controller protocol. --- src/core/include.am | 1 + src/feature/control/control_cmd.c | 145 ++++++++++++++++++++-- src/feature/control/control_cmd.h | 39 ++++++ src/feature/control/control_cmd_args_st.h | 44 +++++++ 4 files changed, 221 insertions(+), 8 deletions(-) create mode 100644 src/feature/control/control_cmd_args_st.h diff --git a/src/core/include.am b/src/core/include.am index 9824601725..b927f17a9c 100644 --- a/src/core/include.am +++ b/src/core/include.am @@ -298,6 +298,7 @@ noinst_HEADERS += \ src/feature/control/control.h \ src/feature/control/control_auth.h \ src/feature/control/control_cmd.h \ + src/feature/control/control_cmd_args_st.h \ src/feature/control/control_connection_st.h \ src/feature/control/control_events.h \ src/feature/control/control_fmt.h \ diff --git a/src/feature/control/control_cmd.c b/src/feature/control/control_cmd.c index 930276b104..aa0bef5135 100644 --- a/src/feature/control/control_cmd.c +++ b/src/feature/control/control_cmd.c @@ -45,6 +45,7 @@ #include "core/or/entry_connection_st.h" #include "core/or/origin_circuit_st.h" #include "core/or/socks_request_st.h" +#include "feature/control/control_cmd_args_st.h" #include "feature/control/control_connection_st.h" #include "feature/nodelist/node_st.h" #include "feature/nodelist/routerinfo_st.h" @@ -60,6 +61,87 @@ static int control_setconf_helper(control_connection_t *conn, uint32_t len, * finished authentication and is accepting commands. */ #define STATE_IS_OPEN(s) ((s) == CONTROL_CONN_STATE_OPEN) +/** + * Release all storage held in args + **/ +void +control_cmd_args_free_(control_cmd_args_t *args) +{ + if (! args) + return; + + if (args->args) { + SMARTLIST_FOREACH(args->args, char *, c, tor_free(c)); + smartlist_free(args->args); + } + tor_free(args->object); + + tor_free(args); +} + +/** + * Helper: parse the arguments to a command according to syntax. On + * success, set *error_out to NULL and return a newly allocated + * control_cmd_args_t. On failure, set *error_out to newly allocated + * error string, and return NULL. + **/ +STATIC control_cmd_args_t * +control_cmd_parse_args(const char *command, + const control_cmd_syntax_t *syntax, + size_t body_len, + const char *body, + char **error_out) +{ + *error_out = NULL; + control_cmd_args_t *result = tor_malloc_zero(sizeof(control_cmd_args_t)); + const char *cmdline; + char *cmdline_alloc = NULL; + + result->command = command; + + const char *eol = memchr(body, '\n', body_len); + if (syntax->want_object) { + if (! eol || (eol+1) == body+body_len) { + *error_out = tor_strdup("Empty body"); + goto err; + } + cmdline_alloc = tor_memdup_nulterm(body, eol-body); + cmdline = cmdline_alloc; + ++eol; + result->object_len = read_escaped_data(eol, (body+body_len)-eol, + &result->object); + } else { + if (eol && (eol+1) != body+body_len) { + *error_out = tor_strdup("Unexpected body"); + goto err; + } + cmdline = body; + } + + result->args = smartlist_new(); + smartlist_split_string(result->args, cmdline, " ", + SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); + size_t n_args = smartlist_len(result->args); + if (n_args < syntax->min_args) { + tor_asprintf(error_out, "Need at least %u argument(s)", + syntax->min_args); + goto err; + } else if (n_args > syntax->max_args) { + tor_asprintf(error_out, "Cannot accept more than %u argument(s)", + syntax->max_args); + goto err; + } + + tor_assert_nonfatal(*error_out == NULL); + goto done; + err: + tor_assert_nonfatal(*error_out != NULL); + control_cmd_args_free(result); + done: + tor_free(cmdline_alloc); + return result; +} + /** Called when we receive a SETCONF message: parse the body and try * to update our configuration. Reply with a DONE or ERROR message. * Modifies the contents of body.*/ @@ -2230,7 +2312,8 @@ handle_control_obsolete(control_connection_t *conn, **/ typedef enum handler_type_t { hnd_legacy, - hnd_legacy_mut + hnd_legacy_mut, + hnd_parsed, } handler_type_t; /** @@ -2257,6 +2340,13 @@ typedef union handler_fn_t { int (*legacy_mut)(control_connection_t *conn, uint32_t arg_len, char *args); + + /** + * A "parsed" handler expects its arguments in a pre-parsed format, in + * an immutable control_cmd_args_t *object. + **/ + int (*parsed)(control_connection_t *conn, + const control_cmd_args_t *args); } handler_fn_t; /** @@ -2279,6 +2369,10 @@ typedef struct control_cmd_def_t { * Zero or more CMD_FL_* flags, or'd together. */ unsigned flags; + /** + * For parsed command: a syntax description. + */ + const control_cmd_syntax_t *syntax; } control_cmd_def_t; /** @@ -2287,16 +2381,27 @@ typedef struct control_cmd_def_t { */ #define CMD_FL_WIPE (1u<<0) -/** - * Macro: declare a command with a one-line argument and a given set of - * flags. +#define SYNTAX_IGNORE { 0, UINT_MAX, false } + +/** Macro: declare a command with a one-line argument, a given set of flags, + * and a syntax definition. **/ -#define ONE_LINE(name, htype, flags) \ +#define ONE_LINE_(name, htype, flags, syntax) \ { #name, \ hnd_ ##htype, \ { .htype = handle_control_ ##name }, \ - flags \ + flags, \ + syntax, \ } + +/** Macro: declare a parsed command with a one-line argument, a given set of + * flags, and a syntax definition. + **/ +#define ONE_LINE(name, htype, flags) \ + ONE_LINE_(name, htype, flags, NULL) +#define ONE_LINE_PARSED(name, flags, syntax) \ + ONE_LINE_(name, parsed, flags, syntax) + /** * Macro: declare a command with a multi-line argument and a given set of * flags. @@ -2305,7 +2410,8 @@ typedef struct control_cmd_def_t { { "+"#name, \ hnd_ ##htype, \ { .htype = handle_control_ ##name }, \ - flags \ + flags, \ + NULL \ } /** * Macro: declare an obsolete command. (Obsolete commands give a different @@ -2315,7 +2421,8 @@ typedef struct control_cmd_def_t { { #name, \ hnd_legacy, \ { .legacy = handle_control_obsolete }, \ - 0 \ + 0, \ + NULL, \ } /** @@ -2379,6 +2486,28 @@ handle_single_control_command(const control_cmd_def_t *def, if (def->handler.legacy_mut(conn, cmd_data_len, args)) rv = -1; break; + case hnd_parsed: { + control_cmd_args_t *parsed_args; + char *err=NULL; + tor_assert(def->syntax); + parsed_args = control_cmd_parse_args(conn->incoming_cmd, + def->syntax, + cmd_data_len, args, + &err); + if (!parsed_args) { + connection_printf_to_buf(conn, + "512 Bad arguments to %s: %s\r\n", + conn->incoming_cmd, err?err:""); + tor_free(err); + } else { + if (BUG(err)) + tor_free(err); + if (def->handler.parsed(conn, parsed_args)) + rv = 0; + control_cmd_args_free(parsed_args); + } + break; + } default: tor_assert_unreached(); } diff --git a/src/feature/control/control_cmd.h b/src/feature/control/control_cmd.h index a417e10da3..1070a9edb7 100644 --- a/src/feature/control/control_cmd.h +++ b/src/feature/control/control_cmd.h @@ -12,11 +12,19 @@ #ifndef TOR_CONTROL_CMD_H #define TOR_CONTROL_CMD_H +#include "lib/malloc/malloc.h" + int handle_control_command(control_connection_t *conn, uint32_t cmd_data_len, char *args); void control_cmd_free_all(void); +typedef struct control_cmd_args_t control_cmd_args_t; +void control_cmd_args_free_(control_cmd_args_t *args); + +#define control_cmd_args_free(v) \ + FREE_AND_NULL(control_cmd_args_t, control_cmd_args_free_, (v)) + #ifdef CONTROL_CMD_PRIVATE #include "lib/crypt_ops/crypto_ed25519.h" @@ -39,6 +47,37 @@ STATIC int add_onion_helper_keyarg(const char *arg, int discard_pk, STATIC rend_authorized_client_t *add_onion_helper_clientauth(const char *arg, int *created, char **err_msg_out); +/** + * Definition for the syntax of a controller command, as parsed by + * control_cmd_parse_args. + * + * WORK IN PROGRESS: This structure is going to get more complex as this + * branch goes on. + **/ +typedef struct control_cmd_syntax_t { + /** + * Lowest number of positional arguments that this command accepts. + * 0 for "it's okay not to have positional arguments." + **/ + unsigned int min_args; + /** + * Highest number of positional arguments that this command accepts. + * UINT_MAX for no limit. + **/ + unsigned int max_args; + /** + * True iff this command wants to be followed by a multiline object. + **/ + bool want_object; +} control_cmd_syntax_t; + +STATIC control_cmd_args_t *control_cmd_parse_args( + const char *command, + const control_cmd_syntax_t *syntax, + size_t body_len, + const char *body, + char **error_out); + #endif /* defined(CONTROL_CMD_PRIVATE) */ #ifdef CONTROL_MODULE_PRIVATE diff --git a/src/feature/control/control_cmd_args_st.h b/src/feature/control/control_cmd_args_st.h new file mode 100644 index 0000000000..9ecc5d750e --- /dev/null +++ b/src/feature/control/control_cmd_args_st.h @@ -0,0 +1,44 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file control_cmd_args_st.h + * \brief Definition for control_cmd_args_t + **/ + +#ifndef TOR_CONTROL_CMD_ST_H +#define TOR_CONTROL_CMD_ST_H + +struct smartlist_t; +struct config_line_t; + +/** + * Parsed arguments for a control command. + * + * WORK IN PROGRESS: This structure is going to get more complex as this + * branch goes on. + **/ +struct control_cmd_args_t { + /** + * The command itself, as provided by the controller. Not owned by this + * structure. + **/ + const char *command; + /** + * Positional arguments to the command. + **/ + struct smartlist_t *args; + /** + * Number of bytes in object; 0 if object is not set. + **/ + size_t object_len; + /** + * A multiline object passed with this command. + **/ + char *object; +}; + +#endif /* !defined(TOR_CONTROL_CMD_ST_H) */ From f18b7dc4731bcb853db92a0faaa4ec03d6ef5586 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 2 Apr 2019 10:29:07 -0400 Subject: [PATCH 0840/2557] Extract the argument-splitting part of control.c's parser This is preliminary work for fixing 29984; no behavior has changed. --- src/feature/control/control.c | 42 +++++++++++++++++++++++------------ src/feature/control/control.h | 5 +++++ 2 files changed, 33 insertions(+), 14 deletions(-) diff --git a/src/feature/control/control.c b/src/feature/control/control.c index 41e21c0a14..308f1936b9 100644 --- a/src/feature/control/control.c +++ b/src/feature/control/control.c @@ -33,6 +33,7 @@ **/ #define CONTROL_MODULE_PRIVATE +#define CONTROL_PRIVATE #include "core/or/or.h" #include "app/config/config.h" @@ -274,6 +275,31 @@ peek_connection_has_http_command(connection_t *conn) return peek_buf_has_http_command(conn->inbuf); } +/** + * Helper: take a nul-terminated command of given length, and find where + * the command starts and the argument begins. Separate them with a NUL, + * and return a pointer to the arguments. + **/ +STATIC char * +control_split_incoming_command(char *incoming_cmd, size_t *data_len) +{ + size_t cmd_len = 0; + while (cmd_len < *data_len + && !TOR_ISSPACE(incoming_cmd[cmd_len])) + ++cmd_len; + + incoming_cmd[cmd_len]='\0'; + char *args = incoming_cmd+cmd_len+1; + tor_assert(*data_len>cmd_len); + *data_len -= (cmd_len+1); /* skip the command and NUL we added after it */ + while (TOR_ISSPACE(*args)) { + ++args; + --*data_len; + } + + return args; +} + static const char CONTROLPORT_IS_NOT_AN_HTTP_PROXY_MSG[] = "HTTP/1.0 501 Tor ControlPort is not an HTTP proxy" "\r\nContent-Type: text/html; charset=iso-8859-1\r\n\r\n" @@ -308,7 +334,6 @@ connection_control_process_inbuf(control_connection_t *conn) { size_t data_len; uint32_t cmd_data_len; - int cmd_len; char *args; tor_assert(conn); @@ -400,22 +425,11 @@ connection_control_process_inbuf(control_connection_t *conn) /* Otherwise, read another line. */ } data_len = conn->incoming_cmd_cur_len; + /* Okay, we now have a command sitting on conn->incoming_cmd. See if we * recognize it. */ - cmd_len = 0; - while ((size_t)cmd_len < data_len - && !TOR_ISSPACE(conn->incoming_cmd[cmd_len])) - ++cmd_len; - - conn->incoming_cmd[cmd_len]='\0'; - args = conn->incoming_cmd+cmd_len+1; - tor_assert(data_len>(size_t)cmd_len); - data_len -= (cmd_len+1); /* skip the command and NUL we added after it */ - while (TOR_ISSPACE(*args)) { - ++args; - --data_len; - } + args = control_split_incoming_command(conn->incoming_cmd, &data_len); /* If the connection is already closing, ignore further commands */ if (TO_CONN(conn)->marked_for_close) { diff --git a/src/feature/control/control.h b/src/feature/control/control.h index 3083837931..6fc1c40cac 100644 --- a/src/feature/control/control.h +++ b/src/feature/control/control.h @@ -60,4 +60,9 @@ int get_cached_network_liveness(void); void set_cached_network_liveness(int liveness); #endif /* defined(CONTROL_MODULE_PRIVATE) */ +#ifdef CONTROL_PRIVATE +STATIC char *control_split_incoming_command(char *incoming_cmd, + size_t *data_len); +#endif + #endif /* !defined(TOR_CONTROL_H) */ From dbfe1a14e44647a4d5f27f8d495f3468208d75dd Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 2 Apr 2019 10:41:12 -0400 Subject: [PATCH 0841/2557] When parsing a multiline controller command, be careful with linebreaks The first line break in particular was mishandled: it was discarded if no arguments came before it, which made it impossible to distinguish arguments from the first line of the body. To solve this, we need to allocate a copy of the command rather than using NUL to separate it, since we might have "COMMAND\n" as our input. Fixes ticket 29984. --- changes/ticket29984 | 5 +++ scripts/maint/practracker/exceptions.txt | 2 +- src/core/mainloop/connection.c | 1 + src/feature/control/control.c | 45 ++++++++++++++------- src/feature/control/control.h | 3 +- src/feature/control/control_cmd.c | 12 +++--- src/feature/control/control_connection_st.h | 3 +- 7 files changed, 48 insertions(+), 23 deletions(-) create mode 100644 changes/ticket29984 diff --git a/changes/ticket29984 b/changes/ticket29984 new file mode 100644 index 0000000000..8631dff27b --- /dev/null +++ b/changes/ticket29984 @@ -0,0 +1,5 @@ + o Minor bugfixes (controller protocol): + - Teach the controller parser to correctly distinguish an object + preceded by an argument list from one without. Previously, it + couldn't distinguish an argument list from the first line of a + multiline object. Fixes bug 29984; bugfix on 0.2.3.8-alpha. diff --git a/scripts/maint/practracker/exceptions.txt b/scripts/maint/practracker/exceptions.txt index 7d03bf27d6..7582395fea 100644 --- a/scripts/maint/practracker/exceptions.txt +++ b/scripts/maint/practracker/exceptions.txt @@ -56,7 +56,7 @@ problem function-size /src/app/main/ntmain.c:nt_service_install() 125 problem include-count /src/app/main/shutdown.c 52 problem file-size /src/core/mainloop/connection.c 5558 problem include-count /src/core/mainloop/connection.c 61 -problem function-size /src/core/mainloop/connection.c:connection_free_minimal() 184 +problem function-size /src/core/mainloop/connection.c:connection_free_minimal() 185 problem function-size /src/core/mainloop/connection.c:connection_listener_new() 328 problem function-size /src/core/mainloop/connection.c:connection_handle_listener_read() 161 problem function-size /src/core/mainloop/connection.c:connection_connect_sockaddr() 103 diff --git a/src/core/mainloop/connection.c b/src/core/mainloop/connection.c index 51c19b4c4c..30504e4edb 100644 --- a/src/core/mainloop/connection.c +++ b/src/core/mainloop/connection.c @@ -697,6 +697,7 @@ connection_free_minimal(connection_t *conn) control_connection_t *control_conn = TO_CONTROL_CONN(conn); tor_free(control_conn->safecookie_client_hash); tor_free(control_conn->incoming_cmd); + tor_free(control_conn->current_cmd); if (control_conn->ephemeral_onion_services) { SMARTLIST_FOREACH(control_conn->ephemeral_onion_services, char *, cp, { memwipe(cp, 0, strlen(cp)); diff --git a/src/feature/control/control.c b/src/feature/control/control.c index 308f1936b9..23ef83ef95 100644 --- a/src/feature/control/control.c +++ b/src/feature/control/control.c @@ -276,25 +276,38 @@ peek_connection_has_http_command(connection_t *conn) } /** - * Helper: take a nul-terminated command of given length, and find where - * the command starts and the argument begins. Separate them with a NUL, - * and return a pointer to the arguments. + * Helper: take a nul-terminated command of given length, and find where the + * command starts and the arguments begin. Separate them, allocate a new + * string in current_cmd_out for the command, and return a pointer + * to the arguments. **/ STATIC char * -control_split_incoming_command(char *incoming_cmd, size_t *data_len) +control_split_incoming_command(char *incoming_cmd, + size_t *data_len, + char **current_cmd_out) { + const bool is_multiline = *data_len && incoming_cmd[0] == '+'; size_t cmd_len = 0; while (cmd_len < *data_len && !TOR_ISSPACE(incoming_cmd[cmd_len])) ++cmd_len; - incoming_cmd[cmd_len]='\0'; - char *args = incoming_cmd+cmd_len+1; - tor_assert(*data_len>cmd_len); - *data_len -= (cmd_len+1); /* skip the command and NUL we added after it */ - while (TOR_ISSPACE(*args)) { - ++args; - --*data_len; + *current_cmd_out = tor_memdup_nulterm(incoming_cmd, cmd_len); + char *args = incoming_cmd+cmd_len; + tor_assert(*data_len>=cmd_len); + *data_len -= cmd_len; + if (is_multiline) { + // Only match horizontal space: any line after the first is data, + // not arguments. + while ((*args == '\t' || *args == ' ') && *data_len) { + ++args; + --*data_len; + } + } else { + while (TOR_ISSPACE(*args) && *data_len) { + ++args; + --*data_len; + } } return args; @@ -429,7 +442,11 @@ connection_control_process_inbuf(control_connection_t *conn) /* Okay, we now have a command sitting on conn->incoming_cmd. See if we * recognize it. */ - args = control_split_incoming_command(conn->incoming_cmd, &data_len); + tor_free(conn->current_cmd); + args = control_split_incoming_command(conn->incoming_cmd, &data_len, + &conn->current_cmd); + if (BUG(!conn->current_cmd)) + return -1; /* If the connection is already closing, ignore further commands */ if (TO_CONN(conn)->marked_for_close) { @@ -437,14 +454,14 @@ connection_control_process_inbuf(control_connection_t *conn) } /* Otherwise, Quit is always valid. */ - if (!strcasecmp(conn->incoming_cmd, "QUIT")) { + if (!strcasecmp(conn->current_cmd, "QUIT")) { connection_write_str_to_buf("250 closing connection\r\n", conn); connection_mark_and_flush(TO_CONN(conn)); return 0; } if (conn->base_.state == CONTROL_CONN_STATE_NEEDAUTH && - !is_valid_initial_command(conn, conn->incoming_cmd)) { + !is_valid_initial_command(conn, conn->current_cmd)) { connection_write_str_to_buf("514 Authentication required.\r\n", conn); connection_mark_for_close(TO_CONN(conn)); return 0; diff --git a/src/feature/control/control.h b/src/feature/control/control.h index 6fc1c40cac..8d3595d2ed 100644 --- a/src/feature/control/control.h +++ b/src/feature/control/control.h @@ -62,7 +62,8 @@ void set_cached_network_liveness(int liveness); #ifdef CONTROL_PRIVATE STATIC char *control_split_incoming_command(char *incoming_cmd, - size_t *data_len); + size_t *data_len, + char **current_cmd_out); #endif #endif /* !defined(TOR_CONTROL_H) */ diff --git a/src/feature/control/control_cmd.c b/src/feature/control/control_cmd.c index aa0bef5135..53cbf2bd0f 100644 --- a/src/feature/control/control_cmd.c +++ b/src/feature/control/control_cmd.c @@ -2299,7 +2299,7 @@ handle_control_obsolete(control_connection_t *conn, { (void)arg_len; (void)args; - char *command = tor_strdup(conn->incoming_cmd); + char *command = tor_strdup(conn->current_cmd); tor_strupper(command); connection_printf_to_buf(conn, "511 %s is obsolete.\r\n", command); tor_free(command); @@ -2490,14 +2490,14 @@ handle_single_control_command(const control_cmd_def_t *def, control_cmd_args_t *parsed_args; char *err=NULL; tor_assert(def->syntax); - parsed_args = control_cmd_parse_args(conn->incoming_cmd, + parsed_args = control_cmd_parse_args(conn->current_cmd, def->syntax, cmd_data_len, args, &err); if (!parsed_args) { connection_printf_to_buf(conn, "512 Bad arguments to %s: %s\r\n", - conn->incoming_cmd, err?err:""); + conn->current_cmd, err?err:""); tor_free(err); } else { if (BUG(err)) @@ -2519,7 +2519,7 @@ handle_single_control_command(const control_cmd_def_t *def, } /** - * Run a given controller command, as selected by the incoming_cmd field of + * Run a given controller command, as selected by the current_cmd field of * conn. */ int @@ -2533,13 +2533,13 @@ handle_control_command(control_connection_t *conn, for (unsigned i = 0; i < N_CONTROL_COMMANDS; ++i) { const control_cmd_def_t *def = &CONTROL_COMMANDS[i]; - if (!strcasecmp(conn->incoming_cmd, def->name)) { + if (!strcasecmp(conn->current_cmd, def->name)) { return handle_single_control_command(def, conn, cmd_data_len, args); } } connection_printf_to_buf(conn, "510 Unrecognized command \"%s\"\r\n", - conn->incoming_cmd); + conn->current_cmd); return 0; } diff --git a/src/feature/control/control_connection_st.h b/src/feature/control/control_connection_st.h index 177a916257..cace6bb36f 100644 --- a/src/feature/control/control_connection_st.h +++ b/src/feature/control/control_connection_st.h @@ -40,7 +40,8 @@ struct control_connection_t { /** A control command that we're reading from the inbuf, but which has not * yet arrived completely. */ char *incoming_cmd; + /** The control command that we are currently processing. */ + char *current_cmd; }; #endif - From cbd1a7e0534d6f5e693c65c8a8b239e35e6f3b2c Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 2 Apr 2019 11:08:22 -0400 Subject: [PATCH 0842/2557] Unit tests for current control-command parser logic --- src/test/test_controller.c | 134 +++++++++++++++++++++++++++++++++++++ 1 file changed, 134 insertions(+) diff --git a/src/test/test_controller.c b/src/test/test_controller.c index f3af6d2ec0..ca853367ae 100644 --- a/src/test/test_controller.c +++ b/src/test/test_controller.c @@ -20,10 +20,138 @@ #include "lib/net/resolve.h" #include "feature/control/control_connection_st.h" +#include "feature/control/control_cmd_args_st.h" #include "feature/dirclient/download_status_st.h" #include "feature/nodelist/microdesc_st.h" #include "feature/nodelist/node_st.h" +typedef struct { + const char *input; + const char *expected_parse; + const char *expected_error; +} parser_testcase_t; + +typedef struct { + const control_cmd_syntax_t *syntax; + size_t n_testcases; + const parser_testcase_t *testcases; +} parse_test_params_t; + +static char * +control_cmd_dump_args(const control_cmd_args_t *result) +{ + buf_t *buf = buf_new(); + buf_add_string(buf, "{ args=["); + if (result->args) { + if (smartlist_len(result->args)) { + buf_add_string(buf, " "); + } + SMARTLIST_FOREACH_BEGIN(result->args, const char *, s) { + const bool last = (s_sl_idx == smartlist_len(result->args)-1); + buf_add_printf(buf, "%s%s ", + escaped(s), + last ? "" : ","); + } SMARTLIST_FOREACH_END(s); + } + buf_add_string(buf, "]"); + if (result->object) { + buf_add_string(buf, ", obj="); + buf_add_string(buf, escaped(result->object)); + } + buf_add_string(buf, " }"); + + char *encoded = buf_extract(buf, NULL); + buf_free(buf); + return encoded; +} + +static void +test_controller_parse_cmd(void *arg) +{ + const parse_test_params_t *params = arg; + control_cmd_args_t *result = NULL; + char *error = NULL; + char *encoded = NULL; + + for (size_t i = 0; i < params->n_testcases; ++i) { + const parser_testcase_t *t = ¶ms->testcases[i]; + result = control_cmd_parse_args("EXAMPLE", + params->syntax, + strlen(t->input), + t->input, + &error); + // A valid test should expect exactly one parse or error. + tt_int_op((t->expected_parse == NULL), OP_NE, + (t->expected_error == NULL)); + // We get a result or an error, not both. + tt_int_op((result == NULL), OP_EQ, (error != NULL)); + // We got the one we expected. + tt_int_op((result == NULL), OP_EQ, (t->expected_parse == NULL)); + + if (result) { + encoded = control_cmd_dump_args(result); + tt_str_op(encoded, OP_EQ, t->expected_parse); + } else { + tt_str_op(error, OP_EQ, t->expected_error); + } + + tor_free(error); + tor_free(encoded); + control_cmd_args_free(result); + } + + done: + tor_free(error); + tor_free(encoded); + control_cmd_args_free(result); +} + +#define OK(inp, out) \ + { inp "\r\n", out, NULL } +#define ERR(inp, err) \ + { inp "\r\n", NULL, err } + +#define TESTPARAMS(syntax, array) \ + { &syntax, \ + ARRAY_LENGTH(array), \ + array } + +static const parser_testcase_t one_to_three_tests[] = { + ERR("", "Need at least 1 argument(s)"), + ERR(" \t", "Need at least 1 argument(s)"), + OK("hello", "{ args=[ \"hello\" ] }"), + OK("hello world", "{ args=[ \"hello\", \"world\" ] }"), + OK("hello world", "{ args=[ \"hello\", \"world\" ] }"), + OK(" hello world", "{ args=[ \"hello\", \"world\" ] }"), + OK(" hello world ", "{ args=[ \"hello\", \"world\" ] }"), + OK("hello there world", "{ args=[ \"hello\", \"there\", \"world\" ] }"), + ERR("why hello there world", "Cannot accept more than 3 argument(s)"), + ERR("hello\r\nworld.\r\n.", "Unexpected body"), +}; + +static const control_cmd_syntax_t one_to_three_syntax = { + .min_args=1, .max_args=3 +}; + +static const parse_test_params_t parse_one_to_three_params = + TESTPARAMS( one_to_three_syntax, one_to_three_tests ); + +static const parser_testcase_t no_args_one_obj_tests[] = { + ERR("Hi there!\r\n.", "Cannot accept more than 0 argument(s)"), + ERR("", "Empty body"), + OK("\r\n", "{ args=[], obj=\"\\n\" }"), + OK("\r\nHello world\r\n", "{ args=[], obj=\"Hello world\\n\\n\" }"), + OK("\r\nHello\r\nworld\r\n", "{ args=[], obj=\"Hello\\nworld\\n\\n\" }"), + OK("\r\nHello\r\n..\r\nworld\r\n", + "{ args=[], obj=\"Hello\\n.\\nworld\\n\\n\" }"), +}; +static const control_cmd_syntax_t no_args_one_obj_syntax = { + .min_args=0, .max_args=0, + .want_object=true, +}; +static const parse_test_params_t parse_no_args_one_obj_params = + TESTPARAMS( no_args_one_obj_syntax, no_args_one_obj_tests ); + static void test_add_onion_helper_keyarg_v3(void *arg) { @@ -1617,7 +1745,13 @@ test_getinfo_md_all(void *arg) return; } +#define PARSER_TEST(type) \ + { "parse/" #type, test_controller_parse_cmd, 0, &passthrough_setup, \ + (void*)&parse_ ## type ## _params } + struct testcase_t controller_tests[] = { + PARSER_TEST(one_to_three), + PARSER_TEST(no_args_one_obj), { "add_onion_helper_keyarg_v2", test_add_onion_helper_keyarg_v2, 0, NULL, NULL }, { "add_onion_helper_keyarg_v3", test_add_onion_helper_keyarg_v3, 0, From 01b07c548b93bcc58adac612a02c69dcb4b63b28 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 2 Apr 2019 19:00:23 -0400 Subject: [PATCH 0843/2557] Use parsing code for the simpler controller commands. (This should be all of the command that work nicely with positional arguments only.) Some of these commands should probably treat extra arguments as incorrect, but for now I'm trying to be careful not to break any existing users. --- scripts/maint/practracker/exceptions.txt | 4 +- src/feature/control/control_cmd.c | 239 +++++++++++------------ src/feature/control/control_cmd.h | 44 ++--- src/feature/control/control_getinfo.c | 16 +- src/feature/control/control_getinfo.h | 8 +- 5 files changed, 154 insertions(+), 157 deletions(-) diff --git a/scripts/maint/practracker/exceptions.txt b/scripts/maint/practracker/exceptions.txt index 7582395fea..438c1c582a 100644 --- a/scripts/maint/practracker/exceptions.txt +++ b/scripts/maint/practracker/exceptions.txt @@ -54,7 +54,7 @@ problem function-size /src/app/main/main.c:sandbox_init_filter() 291 problem function-size /src/app/main/main.c:run_tor_main_loop() 105 problem function-size /src/app/main/ntmain.c:nt_service_install() 125 problem include-count /src/app/main/shutdown.c 52 -problem file-size /src/core/mainloop/connection.c 5558 +problem file-size /src/core/mainloop/connection.c 5559 problem include-count /src/core/mainloop/connection.c 61 problem function-size /src/core/mainloop/connection.c:connection_free_minimal() 185 problem function-size /src/core/mainloop/connection.c:connection_listener_new() 328 @@ -152,7 +152,7 @@ problem function-size /src/feature/control/control_cmd.c:handle_control_add_onio problem function-size /src/feature/control/control_cmd.c:add_onion_helper_keyarg() 125 problem function-size /src/feature/control/control_cmd.c:handle_control_command() 104 problem function-size /src/feature/control/control_events.c:control_event_stream_status() 119 -problem include-count /src/feature/control/control_getinfo.c 52 +problem include-count /src/feature/control/control_getinfo.c 53 problem function-size /src/feature/control/control_getinfo.c:getinfo_helper_misc() 109 problem function-size /src/feature/control/control_getinfo.c:getinfo_helper_dir() 304 problem function-size /src/feature/control/control_getinfo.c:getinfo_helper_events() 236 diff --git a/src/feature/control/control_cmd.c b/src/feature/control/control_cmd.c index 53cbf2bd0f..f457e9fa54 100644 --- a/src/feature/control/control_cmd.c +++ b/src/feature/control/control_cmd.c @@ -160,13 +160,17 @@ handle_control_resetconf(control_connection_t *conn, uint32_t len, char *body) return control_setconf_helper(conn, len, body, 1); } +static const control_cmd_syntax_t getconf_syntax = { + .max_args=UINT_MAX +}; + /** Called when we receive a GETCONF message. Parse the request, and * reply with a CONFVALUE or an ERROR message */ static int -handle_control_getconf(control_connection_t *conn, uint32_t body_len, - char *body) +handle_control_getconf(control_connection_t *conn, + const control_cmd_args_t *args) { - smartlist_t *questions = smartlist_new(); + const smartlist_t *questions = args->args; smartlist_t *answers = smartlist_new(); smartlist_t *unrecognized = smartlist_new(); char *msg = NULL; @@ -174,9 +178,6 @@ handle_control_getconf(control_connection_t *conn, uint32_t body_len, const or_options_t *options = get_options(); int i, len; - (void) body_len; /* body is NUL-terminated; so we can ignore len. */ - smartlist_split_string(questions, body, " ", - SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); SMARTLIST_FOREACH_BEGIN(questions, const char *, q) { if (!option_is_recognized(q)) { smartlist_add(unrecognized, (char*) q); @@ -221,8 +222,6 @@ handle_control_getconf(control_connection_t *conn, uint32_t body_len, SMARTLIST_FOREACH(answers, char *, cp, tor_free(cp)); smartlist_free(answers); - SMARTLIST_FOREACH(questions, char *, cp, tor_free(cp)); - smartlist_free(questions); smartlist_free(unrecognized); tor_free(msg); @@ -230,17 +229,21 @@ handle_control_getconf(control_connection_t *conn, uint32_t body_len, return 0; } +static const control_cmd_syntax_t loadconf_syntax = { + .want_object = true +}; + /** Called when we get a +LOADCONF message. */ static int -handle_control_loadconf(control_connection_t *conn, uint32_t len, - const char *body) +handle_control_loadconf(control_connection_t *conn, + const control_cmd_args_t *args) { setopt_err_t retval; char *errstring = NULL; const char *msg = NULL; - (void) len; - retval = options_init_from_string(NULL, body, CMD_RUN_TOR, NULL, &errstring); + retval = options_init_from_string(NULL, args->object, + CMD_RUN_TOR, NULL, &errstring); if (retval != SETOPT_OK) log_warn(LD_CONTROL, @@ -276,20 +279,20 @@ handle_control_loadconf(control_connection_t *conn, uint32_t len, return 0; } +static const control_cmd_syntax_t setevents_syntax = { + .max_args = UINT_MAX +}; + /** Called when we get a SETEVENTS message: update conn->event_mask, * and reply with DONE or ERROR. */ static int -handle_control_setevents(control_connection_t *conn, uint32_t len, - const char *body) +handle_control_setevents(control_connection_t *conn, + const control_cmd_args_t *args) { int event_code; event_mask_t event_mask = 0; - smartlist_t *events = smartlist_new(); + const smartlist_t *events = args->args; - (void) len; - - smartlist_split_string(events, body, " ", - SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); SMARTLIST_FOREACH_BEGIN(events, const char *, ev) { if (!strcasecmp(ev, "EXTENDED") || @@ -311,16 +314,12 @@ handle_control_setevents(control_connection_t *conn, uint32_t len, if (event_code == -1) { connection_printf_to_buf(conn, "552 Unrecognized event \"%s\"\r\n", ev); - SMARTLIST_FOREACH(events, char *, e, tor_free(e)); - smartlist_free(events); return 0; } } event_mask |= (((event_mask_t)1) << event_code); } SMARTLIST_FOREACH_END(ev); - SMARTLIST_FOREACH(events, char *, e, tor_free(e)); - smartlist_free(events); conn->event_mask = event_mask; @@ -348,23 +347,23 @@ handle_control_saveconf(control_connection_t *conn, uint32_t len, return 0; } +static const control_cmd_syntax_t signal_syntax = { + .min_args = 1, + .max_args = 1, +}; + /** Called when we get a SIGNAL command. React to the provided signal, and * report success or failure. (If the signal results in a shutdown, success * may not be reported.) */ static int -handle_control_signal(control_connection_t *conn, uint32_t len, - const char *body) +handle_control_signal(control_connection_t *conn, + const control_cmd_args_t *args) { int sig = -1; int i; - int n = 0; - char *s; - (void) len; - - while (body[n] && ! TOR_ISSPACE(body[n])) - ++n; - s = tor_strndup(body, n); + tor_assert(smartlist_len(args->args) == 1); + const char *s = smartlist_get(args->args, 0); for (i = 0; signal_table[i].signal_name != NULL; ++i) { if (!strcasecmp(s, signal_table[i].signal_name)) { @@ -376,7 +375,6 @@ handle_control_signal(control_connection_t *conn, uint32_t len, if (sig < 0) connection_printf_to_buf(conn, "552 Unrecognized signal code \"%s\"\r\n", s); - tor_free(s); if (sig < 0) return 0; @@ -390,15 +388,18 @@ handle_control_signal(control_connection_t *conn, uint32_t len, return 0; } +static const control_cmd_syntax_t takeownership_syntax = { + .max_args = UINT_MAX, // This should probably become zero. XXXXX +}; + /** Called when we get a TAKEOWNERSHIP command. Mark this connection * as an owning connection, so that we will exit if the connection * closes. */ static int -handle_control_takeownership(control_connection_t *conn, uint32_t len, - const char *body) +handle_control_takeownership(control_connection_t *conn, + const control_cmd_args_t *args) { - (void)len; - (void)body; + (void)args; conn->is_owning_control_connection = 1; @@ -410,15 +411,18 @@ handle_control_takeownership(control_connection_t *conn, uint32_t len, return 0; } +static const control_cmd_syntax_t dropownership_syntax = { + .max_args = UINT_MAX, // This should probably become zero. XXXXX +}; + /** Called when we get a DROPOWNERSHIP command. Mark this connection * as a non-owning connection, so that we will not exit if the connection * closes. */ static int -handle_control_dropownership(control_connection_t *conn, uint32_t len, - const char *body) +handle_control_dropownership(control_connection_t *conn, + const control_cmd_args_t *args) { - (void)len; - (void)body; + (void)args; conn->is_owning_control_connection = 0; @@ -1099,21 +1103,21 @@ handle_control_postdescriptor(control_connection_t *conn, uint32_t len, return 0; } +static const control_cmd_syntax_t redirectstream_syntax = { + .min_args = 2, + .max_args = UINT_MAX, // XXX should be 3. +}; + /** Called when we receive a REDIRECTSTERAM command. Try to change the target * address of the named AP stream, and report success or failure. */ static int -handle_control_redirectstream(control_connection_t *conn, uint32_t len, - const char *body) +handle_control_redirectstream(control_connection_t *conn, + const control_cmd_args_t *cmd_args) { entry_connection_t *ap_conn = NULL; char *new_addr = NULL; uint16_t new_port = 0; - smartlist_t *args; - (void) len; - - args = getargs_helper("REDIRECTSTREAM", conn, body, 2, -1); - if (!args) - return 0; + const smartlist_t *args = cmd_args->args; if (!(ap_conn = get_stream(smartlist_get(args, 0))) || !ap_conn->socks_request) { @@ -1133,8 +1137,6 @@ handle_control_redirectstream(control_connection_t *conn, uint32_t len, } } - SMARTLIST_FOREACH(args, char *, cp, tor_free(cp)); - smartlist_free(args); if (!new_addr) return 0; @@ -1147,23 +1149,26 @@ handle_control_redirectstream(control_connection_t *conn, uint32_t len, return 0; } +static const control_cmd_syntax_t closestream_syntax = { + .min_args = 2, + .max_args = UINT_MAX, /* XXXX This is the original behavior, but + * maybe we should change the spec. */ +}; + /** Called when we get a CLOSESTREAM command; try to close the named stream * and report success or failure. */ static int -handle_control_closestream(control_connection_t *conn, uint32_t len, - const char *body) +handle_control_closestream(control_connection_t *conn, + const control_cmd_args_t *cmd_args) { entry_connection_t *ap_conn=NULL; uint8_t reason=0; - smartlist_t *args; int ok; - (void) len; + const smartlist_t *args = cmd_args->args; - args = getargs_helper("CLOSESTREAM", conn, body, 2, -1); - if (!args) - return 0; + tor_assert(smartlist_len(args) >= 2); - else if (!(ap_conn = get_stream(smartlist_get(args, 0)))) + if (!(ap_conn = get_stream(smartlist_get(args, 0)))) connection_printf_to_buf(conn, "552 Unknown stream \"%s\"\r\n", (char*)smartlist_get(args, 0)); else { @@ -1175,8 +1180,6 @@ handle_control_closestream(control_connection_t *conn, uint32_t len, ap_conn = NULL; } } - SMARTLIST_FOREACH(args, char *, cp, tor_free(cp)); - smartlist_free(args); if (!ap_conn) return 0; @@ -1269,19 +1272,20 @@ handle_control_resolve(control_connection_t *conn, uint32_t len, return 0; } +static const control_cmd_syntax_t protocolinfo_syntax = { + .max_args = UINT_MAX +}; + /** Called when we get a PROTOCOLINFO command: send back a reply. */ static int -handle_control_protocolinfo(control_connection_t *conn, uint32_t len, - const char *body) +handle_control_protocolinfo(control_connection_t *conn, + const control_cmd_args_t *cmd_args) { const char *bad_arg = NULL; - smartlist_t *args; - (void)len; + const smartlist_t *args = cmd_args->args; conn->have_sent_protocolinfo = 1; - args = smartlist_new(); - smartlist_split_string(args, body, " ", - SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); + SMARTLIST_FOREACH(args, const char *, arg, { int ok; tor_parse_long(arg, 10, 0, LONG_MAX, &ok, NULL); @@ -1337,24 +1341,21 @@ handle_control_protocolinfo(control_connection_t *conn, uint32_t len, tor_free(esc_cfile); } done: - SMARTLIST_FOREACH(args, char *, cp, tor_free(cp)); - smartlist_free(args); return 0; } +static const control_cmd_syntax_t usefeature_syntax = { + .max_args = UINT_MAX +}; + /** Called when we get a USEFEATURE command: parse the feature list, and * set up the control_connection's options properly. */ static int handle_control_usefeature(control_connection_t *conn, - uint32_t len, - const char *body) + const control_cmd_args_t *cmd_args) { - smartlist_t *args; + const smartlist_t *args = cmd_args->args; int bad = 0; - (void) len; /* body is nul-terminated; it's safe to ignore the length */ - args = smartlist_new(); - smartlist_split_string(args, body, " ", - SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); SMARTLIST_FOREACH_BEGIN(args, const char *, arg) { if (!strcasecmp(arg, "VERBOSE_NAMES")) ; @@ -1372,22 +1373,19 @@ handle_control_usefeature(control_connection_t *conn, send_control_done(conn); } - SMARTLIST_FOREACH(args, char *, cp, tor_free(cp)); - smartlist_free(args); return 0; } +static const control_cmd_syntax_t dropguards_syntax = { + .max_args = 0, +}; + /** Implementation for the DROPGUARDS command. */ static int handle_control_dropguards(control_connection_t *conn, - uint32_t len, - const char *body) + const control_cmd_args_t *args) { - smartlist_t *args; - (void) len; /* body is nul-terminated; it's safe to ignore the length */ - args = smartlist_new(); - smartlist_split_string(args, body, " ", - SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); + (void) args; /* We don't take arguments. */ static int have_warned = 0; if (! have_warned) { @@ -1397,15 +1395,9 @@ handle_control_dropguards(control_connection_t *conn, have_warned = 1; } - if (smartlist_len(args)) { - connection_printf_to_buf(conn, "512 Too many arguments to DROPGUARDS\r\n"); - } else { - remove_all_entry_guards(); - send_control_done(conn); - } + remove_all_entry_guards(); + send_control_done(conn); - SMARTLIST_FOREACH(args, char *, cp, tor_free(cp)); - smartlist_free(args); return 0; } @@ -2202,19 +2194,19 @@ add_onion_helper_clientauth(const char *arg, int *created, char **err_msg) return client; } +static const control_cmd_syntax_t del_onion_syntax = { + .min_args = 1, .max_args = 1, +}; + /** Called when we get a DEL_ONION command; parse the body, and remove * the existing ephemeral Onion Service. */ static int handle_control_del_onion(control_connection_t *conn, - uint32_t len, - const char *body) + const control_cmd_args_t *cmd_args) { int hs_version = 0; - smartlist_t *args; - (void) len; /* body is nul-terminated; it's safe to ignore the length */ - args = getargs_helper("DEL_ONION", conn, body, 1, 1); - if (!args) - return 0; + smartlist_t *args = cmd_args->args; + tor_assert(smartlist_len(args) == 1); const char *service_id = smartlist_get(args, 0); if (rend_valid_v2_service_id(service_id)) { @@ -2280,11 +2272,6 @@ handle_control_del_onion(control_connection_t *conn, } out: - SMARTLIST_FOREACH(args, char *, cp, { - memwipe(cp, 0, strlen(cp)); - tor_free(cp); - }); - smartlist_free(args); return 0; } @@ -2399,20 +2386,26 @@ typedef struct control_cmd_def_t { **/ #define ONE_LINE(name, htype, flags) \ ONE_LINE_(name, htype, flags, NULL) -#define ONE_LINE_PARSED(name, flags, syntax) \ - ONE_LINE_(name, parsed, flags, syntax) +#define ONE_LINE_PARSED(name, flags) \ + ONE_LINE_(name, parsed, flags, &name ##_syntax) /** * Macro: declare a command with a multi-line argument and a given set of * flags. **/ -#define MULTLINE(name, htype, flags) \ +#define MULTLINE_(name, htype, flags, syntax) \ { "+"#name, \ hnd_ ##htype, \ { .htype = handle_control_ ##name }, \ flags, \ - NULL \ + syntax \ } + +#define MULTLINE(name, htype, flags) \ + MULTLINE_(name, htype, flags, NULL) +#define MULTLINE_PARSED(name, flags) \ + MULTLINE_(name, parsed, flags, &name##_syntax) + /** * Macro: declare an obsolete command. (Obsolete commands give a different * error than non-existent ones.) @@ -2432,33 +2425,33 @@ static const control_cmd_def_t CONTROL_COMMANDS[] = { ONE_LINE(setconf, legacy_mut, 0), ONE_LINE(resetconf, legacy_mut, 0), - ONE_LINE(getconf, legacy_mut, 0), - MULTLINE(loadconf, legacy, 0), - ONE_LINE(setevents, legacy, 0), + ONE_LINE_PARSED(getconf, 0), + MULTLINE_PARSED(loadconf, 0), + ONE_LINE_PARSED(setevents, 0), ONE_LINE(authenticate, legacy, CMD_FL_WIPE), ONE_LINE(saveconf, legacy, 0), - ONE_LINE(signal, legacy, 0), - ONE_LINE(takeownership, legacy, 0), - ONE_LINE(dropownership, legacy, 0), + ONE_LINE_PARSED(signal, 0), + ONE_LINE_PARSED(takeownership, 0), + ONE_LINE_PARSED(dropownership, 0), ONE_LINE(mapaddress, legacy, 0), - ONE_LINE(getinfo, legacy, 0), + ONE_LINE_PARSED(getinfo, 0), ONE_LINE(extendcircuit, legacy, 0), ONE_LINE(setcircuitpurpose, legacy, 0), OBSOLETE(setrouterpurpose), ONE_LINE(attachstream, legacy, 0), MULTLINE(postdescriptor, legacy, 0), - ONE_LINE(redirectstream, legacy, 0), - ONE_LINE(closestream, legacy, 0), + ONE_LINE_PARSED(redirectstream, 0), + ONE_LINE_PARSED(closestream, 0), ONE_LINE(closecircuit, legacy, 0), - ONE_LINE(usefeature, legacy, 0), + ONE_LINE_PARSED(usefeature, 0), ONE_LINE(resolve, legacy, 0), - ONE_LINE(protocolinfo, legacy, 0), + ONE_LINE_PARSED(protocolinfo, 0), ONE_LINE(authchallenge, legacy, CMD_FL_WIPE), - ONE_LINE(dropguards, legacy, 0), + ONE_LINE_PARSED(dropguards, 0), ONE_LINE(hsfetch, legacy, 0), MULTLINE(hspost, legacy, 0), ONE_LINE(add_onion, legacy, CMD_FL_WIPE), - ONE_LINE(del_onion, legacy, CMD_FL_WIPE), + ONE_LINE_PARSED(del_onion, CMD_FL_WIPE), }; /** diff --git a/src/feature/control/control_cmd.h b/src/feature/control/control_cmd.h index 1070a9edb7..801bf43709 100644 --- a/src/feature/control/control_cmd.h +++ b/src/feature/control/control_cmd.h @@ -25,28 +25,6 @@ void control_cmd_args_free_(control_cmd_args_t *args); #define control_cmd_args_free(v) \ FREE_AND_NULL(control_cmd_args_t, control_cmd_args_free_, (v)) -#ifdef CONTROL_CMD_PRIVATE -#include "lib/crypt_ops/crypto_ed25519.h" - -/* ADD_ONION secret key to create an ephemeral service. The command supports - * multiple versions so this union stores the key and passes it to the HS - * subsystem depending on the requested version. */ -typedef union add_onion_secret_key_t { - /* Hidden service v2 secret key. */ - crypto_pk_t *v2; - /* Hidden service v3 secret key. */ - ed25519_secret_key_t *v3; -} add_onion_secret_key_t; - -STATIC int add_onion_helper_keyarg(const char *arg, int discard_pk, - const char **key_new_alg_out, - char **key_new_blob_out, - add_onion_secret_key_t *decoded_key, - int *hs_version, char **err_msg_out); - -STATIC rend_authorized_client_t *add_onion_helper_clientauth(const char *arg, - int *created, char **err_msg_out); - /** * Definition for the syntax of a controller command, as parsed by * control_cmd_parse_args. @@ -71,6 +49,28 @@ typedef struct control_cmd_syntax_t { bool want_object; } control_cmd_syntax_t; +#ifdef CONTROL_CMD_PRIVATE +#include "lib/crypt_ops/crypto_ed25519.h" + +/* ADD_ONION secret key to create an ephemeral service. The command supports + * multiple versions so this union stores the key and passes it to the HS + * subsystem depending on the requested version. */ +typedef union add_onion_secret_key_t { + /* Hidden service v2 secret key. */ + crypto_pk_t *v2; + /* Hidden service v3 secret key. */ + ed25519_secret_key_t *v3; +} add_onion_secret_key_t; + +STATIC int add_onion_helper_keyarg(const char *arg, int discard_pk, + const char **key_new_alg_out, + char **key_new_blob_out, + add_onion_secret_key_t *decoded_key, + int *hs_version, char **err_msg_out); + +STATIC rend_authorized_client_t *add_onion_helper_clientauth(const char *arg, + int *created, char **err_msg_out); + STATIC control_cmd_args_t *control_cmd_parse_args( const char *command, const control_cmd_syntax_t *syntax, diff --git a/src/feature/control/control_getinfo.c b/src/feature/control/control_getinfo.c index a7a85f2fdf..5c6a0d4aa2 100644 --- a/src/feature/control/control_getinfo.c +++ b/src/feature/control/control_getinfo.c @@ -55,6 +55,7 @@ #include "core/or/origin_circuit_st.h" #include "core/or/socks_request_st.h" #include "feature/control/control_connection_st.h" +#include "feature/control/control_cmd_args_st.h" #include "feature/dircache/cached_dir_st.h" #include "feature/nodelist/extrainfo_st.h" #include "feature/nodelist/microdesc_st.h" @@ -1584,21 +1585,22 @@ handle_getinfo_helper(control_connection_t *control_conn, return 0; /* unrecognized */ } +const control_cmd_syntax_t getinfo_syntax = { + .max_args = UINT_MAX, +}; + /** Called when we receive a GETINFO command. Try to fetch all requested * information, and reply with information or error message. */ int -handle_control_getinfo(control_connection_t *conn, uint32_t len, - const char *body) +handle_control_getinfo(control_connection_t *conn, + const control_cmd_args_t *args) { - smartlist_t *questions = smartlist_new(); + const smartlist_t *questions = args->args; smartlist_t *answers = smartlist_new(); smartlist_t *unrecognized = smartlist_new(); char *ans = NULL; int i; - (void) len; /* body is NUL-terminated, so it's safe to ignore the length. */ - smartlist_split_string(questions, body, " ", - SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); SMARTLIST_FOREACH_BEGIN(questions, const char *, q) { const char *errmsg = NULL; @@ -1653,8 +1655,6 @@ handle_control_getinfo(control_connection_t *conn, uint32_t len, done: SMARTLIST_FOREACH(answers, char *, cp, tor_free(cp)); smartlist_free(answers); - SMARTLIST_FOREACH(questions, char *, cp, tor_free(cp)); - smartlist_free(questions); SMARTLIST_FOREACH(unrecognized, char *, cp, tor_free(cp)); smartlist_free(unrecognized); diff --git a/src/feature/control/control_getinfo.h b/src/feature/control/control_getinfo.h index d5a2feb3e0..2d56586f6d 100644 --- a/src/feature/control/control_getinfo.h +++ b/src/feature/control/control_getinfo.h @@ -12,8 +12,12 @@ #ifndef TOR_CONTROL_GETINFO_H #define TOR_CONTROL_GETINFO_H -int handle_control_getinfo(control_connection_t *conn, uint32_t len, - const char *body); +struct control_cmd_syntax_t; +struct control_cmd_args_t; +extern const struct control_cmd_syntax_t getinfo_syntax; + +int handle_control_getinfo(control_connection_t *conn, + const struct control_cmd_args_t *args); #ifdef CONTROL_GETINFO_PRIVATE STATIC int getinfo_helper_onions( From 73df91bbb55498b05faae16b49ab49545fdffa8f Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 5 Apr 2019 15:29:37 -0400 Subject: [PATCH 0844/2557] kvline: handle empty alues as well as empty keys The two options are mutually exclusive, since otherwise an entry like "Foo" would be ambiguous. We want to have the ability to treat entries like this as keys, though, since some controller commands interpret them as flags. --- src/lib/encoding/kvline.c | 48 +++++++++++++++++++++++++++++----- src/lib/encoding/kvline.h | 1 + src/test/test_config.c | 55 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 97 insertions(+), 7 deletions(-) diff --git a/src/lib/encoding/kvline.c b/src/lib/encoding/kvline.c index 307adc3f12..806f9d3db0 100644 --- a/src/lib/encoding/kvline.c +++ b/src/lib/encoding/kvline.c @@ -53,6 +53,15 @@ line_has_no_key(const config_line_t *line) return line->key == NULL || strlen(line->key) == 0; } +/** + * Return true iff the value in line is not set. + **/ +static bool +line_has_no_val(const config_line_t *line) +{ + return line->value == NULL || strlen(line->value) == 0; +} + /** * Return true iff the all the lines in line can be encoded * using flags. @@ -98,6 +107,10 @@ kvline_can_encode_lines(const config_line_t *line, unsigned flags) * If KV_OMIT_KEYS is set in flags, then pairs with empty keys are * allowed, and are encoded as 'Value'. Otherwise, such pairs are not * allowed. + * + * If KV_OMIT_VALS is set in flags, then an empty value is + * encoded as 'Key', not as 'Key=' or 'Key=""'. Mutually exclusive with + * KV_OMIT_KEYS. */ char * kvline_encode(const config_line_t *line, @@ -106,6 +119,9 @@ kvline_encode(const config_line_t *line, if (!kvline_can_encode_lines(line, flags)) return NULL; + tor_assert((flags & (KV_OMIT_KEYS|KV_OMIT_VALS)) != + (KV_OMIT_KEYS|KV_OMIT_VALS)); + smartlist_t *elements = smartlist_new(); for (; line; line = line->next) { @@ -126,7 +142,10 @@ kvline_encode(const config_line_t *line, } } - if (esc) { + if ((flags & KV_OMIT_VALS) && line_has_no_val(line)) { + eq = ""; + v = ""; + } else if (esc) { tmp = esc_for_log(line->value); v = tmp; } else { @@ -155,13 +174,21 @@ kvline_encode(const config_line_t *line, * * If KV_OMIT_KEYS is set in flags, then values without keys are * allowed. Otherwise, such values are not allowed. + * + * If KV_OMIT_VALS is set in flags, then keys without values are + * allowed. Otherwise, such keys are not allowed. Mutually exclusive with + * KV_OMIT_KEYS. */ config_line_t * kvline_parse(const char *line, unsigned flags) { + tor_assert((flags & (KV_OMIT_KEYS|KV_OMIT_VALS)) != + (KV_OMIT_KEYS|KV_OMIT_VALS)); + const char *cp = line, *cplast = NULL; - bool omit_keys = (flags & KV_OMIT_KEYS) != 0; - bool quoted = (flags & KV_QUOTED) != 0; + const bool omit_keys = (flags & KV_OMIT_KEYS) != 0; + const bool omit_vals = (flags & KV_OMIT_VALS) != 0; + const bool quoted = (flags & KV_QUOTED) != 0; config_line_t *result = NULL; config_line_t **next_line = &result; @@ -171,27 +198,33 @@ kvline_parse(const char *line, unsigned flags) while (*cp) { key = val = NULL; + /* skip all spaces */ { size_t idx = strspn(cp, " \t\r\v\n"); cp += idx; } if (BUG(cp == cplast)) { - /* If we didn't parse anything, this code is broken. */ + /* If we didn't parse anything since the last loop, this code is + * broken. */ goto err; // LCOV_EXCL_LINE } cplast = cp; if (! *cp) break; /* End of string; we're done. */ - /* Possible formats are K=V, K="V", V, and "V", depending on flags. */ + /* Possible formats are K=V, K="V", K, V, and "V", depending on flags. */ - /* Find the key. */ + /* Find where the key ends */ if (*cp != '\"') { size_t idx = strcspn(cp, " \t\r\v\n="); if (cp[idx] == '=') { key = tor_memdup_nulterm(cp, idx); cp += idx + 1; + } else if (omit_vals) { + key = tor_memdup_nulterm(cp, idx); + cp += idx; + goto commit; } else { if (!omit_keys) goto err; @@ -214,6 +247,7 @@ kvline_parse(const char *line, unsigned flags) cp += idx; } + commit: if (key && strlen(key) == 0) { /* We don't allow empty keys. */ goto err; @@ -221,7 +255,7 @@ kvline_parse(const char *line, unsigned flags) *next_line = tor_malloc_zero(sizeof(config_line_t)); (*next_line)->key = key ? key : tor_strdup(""); - (*next_line)->value = val; + (*next_line)->value = val ? val : tor_strdup(""); next_line = &(*next_line)->next; key = val = NULL; } diff --git a/src/lib/encoding/kvline.h b/src/lib/encoding/kvline.h index 4eed30a223..6740f81d54 100644 --- a/src/lib/encoding/kvline.h +++ b/src/lib/encoding/kvline.h @@ -17,6 +17,7 @@ struct config_line_t; #define KV_QUOTED (1u<<0) #define KV_OMIT_KEYS (1u<<1) +#define KV_OMIT_VALS (1u<<2) struct config_line_t *kvline_parse(const char *line, unsigned flags); char *kvline_encode(const struct config_line_t *line, unsigned flags); diff --git a/src/test/test_config.c b/src/test/test_config.c index 72649dd9b1..6cfb7b764b 100644 --- a/src/test/test_config.c +++ b/src/test/test_config.c @@ -5886,6 +5886,61 @@ test_config_kvline_parse(void *arg) tt_assert(lines); tt_str_op(lines->key, OP_EQ, "AB"); tt_str_op(lines->value, OP_EQ, ""); + config_free_lines(lines); + + lines = kvline_parse("AB=", KV_OMIT_VALS); + tt_assert(lines); + tt_str_op(lines->key, OP_EQ, "AB"); + tt_str_op(lines->value, OP_EQ, ""); + config_free_lines(lines); + + lines = kvline_parse(" AB ", KV_OMIT_VALS); + tt_assert(lines); + tt_str_op(lines->key, OP_EQ, "AB"); + tt_str_op(lines->value, OP_EQ, ""); + config_free_lines(lines); + + lines = kvline_parse("AB", KV_OMIT_VALS); + tt_assert(lines); + tt_str_op(lines->key, OP_EQ, "AB"); + tt_str_op(lines->value, OP_EQ, ""); + enc = kvline_encode(lines, KV_OMIT_VALS); + tt_str_op(enc, OP_EQ, "AB"); + tor_free(enc); + config_free_lines(lines); + + lines = kvline_parse("AB=CD", KV_OMIT_VALS); + tt_assert(lines); + tt_str_op(lines->key, OP_EQ, "AB"); + tt_str_op(lines->value, OP_EQ, "CD"); + enc = kvline_encode(lines, KV_OMIT_VALS); + tt_str_op(enc, OP_EQ, "AB=CD"); + tor_free(enc); + config_free_lines(lines); + + lines = kvline_parse("AB=CD DE FGH=I", KV_OMIT_VALS); + tt_assert(lines); + tt_str_op(lines->key, OP_EQ, "AB"); + tt_str_op(lines->value, OP_EQ, "CD"); + tt_str_op(lines->next->key, OP_EQ, "DE"); + tt_str_op(lines->next->value, OP_EQ, ""); + tt_str_op(lines->next->next->key, OP_EQ, "FGH"); + tt_str_op(lines->next->next->value, OP_EQ, "I"); + enc = kvline_encode(lines, KV_OMIT_VALS); + tt_str_op(enc, OP_EQ, "AB=CD DE FGH=I"); + tor_free(enc); + config_free_lines(lines); + + lines = kvline_parse("AB=\"CD E\" DE FGH=\"I\"", KV_OMIT_VALS|KV_QUOTED); + tt_assert(lines); + tt_str_op(lines->key, OP_EQ, "AB"); + tt_str_op(lines->value, OP_EQ, "CD E"); + tt_str_op(lines->next->key, OP_EQ, "DE"); + tt_str_op(lines->next->value, OP_EQ, ""); + tt_str_op(lines->next->next->key, OP_EQ, "FGH"); + tt_str_op(lines->next->next->value, OP_EQ, "I"); + enc = kvline_encode(lines, KV_OMIT_VALS|KV_QUOTED); + tt_str_op(enc, OP_EQ, "AB=\"CD E\" DE FGH=I"); done: config_free_lines(lines); From bb37ad695729984553275880cce131c47361345f Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 5 Apr 2019 15:55:52 -0400 Subject: [PATCH 0845/2557] Add fuzzing support for several more groups of kvlines flags --- src/test/fuzz/fuzz_strops.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/test/fuzz/fuzz_strops.c b/src/test/fuzz/fuzz_strops.c index a37cbb5be8..459b4e21aa 100644 --- a/src/test/fuzz/fuzz_strops.c +++ b/src/test/fuzz/fuzz_strops.c @@ -235,6 +235,18 @@ fuzz_main(const uint8_t *stdin_buf, size_t data_size) kv_flags = 0; ENCODE_ROUNDTRIP(kv_enc, kv_dec, config_free_lines_); break; + case 7: + kv_flags = KV_OMIT_VALS; + ENCODE_ROUNDTRIP(kv_enc, kv_dec, config_free_lines_); + break; + case 8: + kv_flags = KV_QUOTED; + ENCODE_ROUNDTRIP(kv_enc, kv_dec, config_free_lines_); + break; + case 9: + kv_flags = KV_QUOTED|KV_OMIT_VALS; + ENCODE_ROUNDTRIP(kv_enc, kv_dec, config_free_lines_); + break; } return 0; From 0841a69357d73353905f8012f455ec6128201131 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 5 Apr 2019 15:29:56 -0400 Subject: [PATCH 0846/2557] Allow kvlines in control commands. --- src/feature/control/control_cmd.c | 46 ++++++++++++++++++++++- src/feature/control/control_cmd.h | 17 +++++++++ src/feature/control/control_cmd_args_st.h | 4 ++ src/test/test_controller.c | 24 ++++++++++++ 4 files changed, 89 insertions(+), 2 deletions(-) diff --git a/src/feature/control/control_cmd.c b/src/feature/control/control_cmd.c index f457e9fa54..727950a938 100644 --- a/src/feature/control/control_cmd.c +++ b/src/feature/control/control_cmd.c @@ -40,6 +40,7 @@ #include "lib/crypt_ops/crypto_rand.h" #include "lib/crypt_ops/crypto_util.h" #include "lib/encoding/confline.h" +#include "lib/encoding/kvline.h" #include "core/or/cpath_build_state_st.h" #include "core/or/entry_connection_st.h" @@ -74,11 +75,27 @@ control_cmd_args_free_(control_cmd_args_t *args) SMARTLIST_FOREACH(args->args, char *, c, tor_free(c)); smartlist_free(args->args); } + config_free_lines(args->kwargs); tor_free(args->object); tor_free(args); } +/** + * Return true iff any element of the NULL-terminated array matches + * kwd. Case-insensitive. + **/ +static bool +string_array_contains_keyword(const char **array, const char *kwd) +{ + for (unsigned i = 0; array[i]; ++i) { + if (! strcasecmp(array[i], kwd)) + return true; + } + return false; +} + + /** * Helper: parse the arguments to a command according to syntax. On * success, set *error_out to NULL and return a newly allocated @@ -96,6 +113,7 @@ control_cmd_parse_args(const char *command, control_cmd_args_t *result = tor_malloc_zero(sizeof(control_cmd_args_t)); const char *cmdline; char *cmdline_alloc = NULL; + tor_assert(syntax->max_args < INT_MAX || syntax->max_args == UINT_MAX); result->command = command; @@ -120,18 +138,42 @@ control_cmd_parse_args(const char *command, result->args = smartlist_new(); smartlist_split_string(result->args, cmdline, " ", - SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); + SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, + (int)(syntax->max_args+1)); size_t n_args = smartlist_len(result->args); if (n_args < syntax->min_args) { tor_asprintf(error_out, "Need at least %u argument(s)", syntax->min_args); goto err; - } else if (n_args > syntax->max_args) { + } else if (n_args > syntax->max_args && ! syntax->accept_keywords) { tor_asprintf(error_out, "Cannot accept more than %u argument(s)", syntax->max_args); goto err; } + if (n_args > syntax->max_args) { + tor_assert(n_args == syntax->max_args + 1); + tor_assert(syntax->accept_keywords); + char *remainder = smartlist_pop_last(result->args); + result->kwargs = kvline_parse(remainder, syntax->kvline_flags); + tor_free(remainder); + if (result->kwargs == NULL) { + tor_asprintf(error_out, "Cannot parse keyword argument(s)"); + goto err; + } + if (syntax->allowed_keywords) { + /* Check for unpermitted arguments */ + const config_line_t *line; + for (line = result->kwargs; line; line = line->next) { + if (! string_array_contains_keyword(syntax->allowed_keywords, + line->key)) { + tor_asprintf(error_out, "Unrecognized keyword argument %s", + escaped(line->key)); + } + } + } + } + tor_assert_nonfatal(*error_out == NULL); goto done; err: diff --git a/src/feature/control/control_cmd.h b/src/feature/control/control_cmd.h index 801bf43709..6f35c74de0 100644 --- a/src/feature/control/control_cmd.h +++ b/src/feature/control/control_cmd.h @@ -43,6 +43,23 @@ typedef struct control_cmd_syntax_t { * UINT_MAX for no limit. **/ unsigned int max_args; + /** + * If true, we should parse options after the positional arguments + * as a set of unordered flags and key=value arguments. + * + * Requires that max_args is not UINT_MAX. + **/ + bool accept_keywords; + /** + * If accept_keywords is true, then only the keywords listed in this + * (NULL-terminated) array are valid keywords for this command. + **/ + const char **allowed_keywords; + /** + * If accept_keywords is true, this option is passed to kvline_parse() as + * its flags. + **/ + unsigned kvline_flags; /** * True iff this command wants to be followed by a multiline object. **/ diff --git a/src/feature/control/control_cmd_args_st.h b/src/feature/control/control_cmd_args_st.h index 9ecc5d750e..06e2a183ba 100644 --- a/src/feature/control/control_cmd_args_st.h +++ b/src/feature/control/control_cmd_args_st.h @@ -31,6 +31,10 @@ struct control_cmd_args_t { * Positional arguments to the command. **/ struct smartlist_t *args; + /** + * Keyword arguments to the command. + **/ + struct config_line_t *kwargs; /** * Number of bytes in object; 0 if object is not set. **/ diff --git a/src/test/test_controller.c b/src/test/test_controller.c index ca853367ae..dc286daccb 100644 --- a/src/test/test_controller.c +++ b/src/test/test_controller.c @@ -18,6 +18,8 @@ #include "test/test.h" #include "test/test_helpers.h" #include "lib/net/resolve.h" +#include "lib/encoding/confline.h" +#include "lib/encoding/kvline.h" #include "feature/control/control_connection_st.h" #include "feature/control/control_cmd_args_st.h" @@ -58,6 +60,16 @@ control_cmd_dump_args(const control_cmd_args_t *result) buf_add_string(buf, ", obj="); buf_add_string(buf, escaped(result->object)); } + if (result->kwargs) { + buf_add_string(buf, ", { "); + const config_line_t *line; + for (line = result->kwargs; line; line = line->next) { + const bool last = (line->next == NULL); + buf_add_printf(buf, "%s=%s%s ", line->key, escaped(line->value), + last ? "" : ","); + } + buf_add_string(buf, "}"); + } buf_add_string(buf, " }"); char *encoded = buf_extract(buf, NULL); @@ -152,6 +164,17 @@ static const control_cmd_syntax_t no_args_one_obj_syntax = { static const parse_test_params_t parse_no_args_one_obj_params = TESTPARAMS( no_args_one_obj_syntax, no_args_one_obj_tests ); +static const parser_testcase_t no_args_kwargs_tests[] = { + OK("", "{ args=[] }"), +}; +static const control_cmd_syntax_t no_args_kwargs_syntax = { + .min_args=0, .max_args=0, + .accept_keywords=true, + .kvline_flags=KV_OMIT_VALS +}; +static const parse_test_params_t parse_no_args_kwargs_params = + TESTPARAMS( no_args_kwargs_syntax, no_args_kwargs_tests ); + static void test_add_onion_helper_keyarg_v3(void *arg) { @@ -1752,6 +1775,7 @@ test_getinfo_md_all(void *arg) struct testcase_t controller_tests[] = { PARSER_TEST(one_to_three), PARSER_TEST(no_args_one_obj), + PARSER_TEST(no_args_kwargs), { "add_onion_helper_keyarg_v2", test_add_onion_helper_keyarg_v2, 0, NULL, NULL }, { "add_onion_helper_keyarg_v3", test_add_onion_helper_keyarg_v3, 0, From 9471391694168f3d82365ced07b14e466b32b612 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 8 Apr 2019 09:37:39 -0400 Subject: [PATCH 0847/2557] Add kvline support to controller command parser. This should let us handle all (or nearly all) of the remaining commands. --- src/feature/control/control_cmd.c | 2 +- src/test/test_controller.c | 27 +++++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/src/feature/control/control_cmd.c b/src/feature/control/control_cmd.c index 727950a938..4f67e31637 100644 --- a/src/feature/control/control_cmd.c +++ b/src/feature/control/control_cmd.c @@ -95,7 +95,6 @@ string_array_contains_keyword(const char **array, const char *kwd) return false; } - /** * Helper: parse the arguments to a command according to syntax. On * success, set *error_out to NULL and return a newly allocated @@ -169,6 +168,7 @@ control_cmd_parse_args(const char *command, line->key)) { tor_asprintf(error_out, "Unrecognized keyword argument %s", escaped(line->key)); + goto err; } } } diff --git a/src/test/test_controller.c b/src/test/test_controller.c index dc286daccb..fd4f26f086 100644 --- a/src/test/test_controller.c +++ b/src/test/test_controller.c @@ -148,6 +148,7 @@ static const control_cmd_syntax_t one_to_three_syntax = { static const parse_test_params_t parse_one_to_three_params = TESTPARAMS( one_to_three_syntax, one_to_three_tests ); +// = static const parser_testcase_t no_args_one_obj_tests[] = { ERR("Hi there!\r\n.", "Cannot accept more than 0 argument(s)"), ERR("", "Empty body"), @@ -166,6 +167,11 @@ static const parse_test_params_t parse_no_args_one_obj_params = static const parser_testcase_t no_args_kwargs_tests[] = { OK("", "{ args=[] }"), + OK(" ", "{ args=[] }"), + OK("hello there=world", "{ args=[], { hello=\"\", there=\"world\" } }"), + OK("hello there=world today", + "{ args=[], { hello=\"\", there=\"world\", today=\"\" } }"), + ERR("=Foo", "Cannot parse keyword argument(s)"), }; static const control_cmd_syntax_t no_args_kwargs_syntax = { .min_args=0, .max_args=0, @@ -175,6 +181,26 @@ static const control_cmd_syntax_t no_args_kwargs_syntax = { static const parse_test_params_t parse_no_args_kwargs_params = TESTPARAMS( no_args_kwargs_syntax, no_args_kwargs_tests ); +static const char *one_arg_kwargs_allow_keywords[] = { + "Hello", "world", NULL +}; +static const parser_testcase_t one_arg_kwargs_tests[] = { + ERR("", "Need at least 1 argument(s)"), + OK("Hi", "{ args=[ \"Hi\" ] }"), + ERR("hello there=world", "Unrecognized keyword argument \"there\""), + OK("Hi HELLO=foo", "{ args=[ \"Hi\" ], { HELLO=\"foo\" } }"), + OK("Hi world=\"bar baz\" hello ", + "{ args=[ \"Hi\" ], { world=\"bar baz\", hello=\"\" } }"), +}; +static const control_cmd_syntax_t one_arg_kwargs_syntax = { + .min_args=1, .max_args=1, + .accept_keywords=true, + .allowed_keywords=one_arg_kwargs_allow_keywords, + .kvline_flags=KV_OMIT_VALS|KV_QUOTED, +}; +static const parse_test_params_t parse_one_arg_kwargs_params = + TESTPARAMS( one_arg_kwargs_syntax, one_arg_kwargs_tests ); + static void test_add_onion_helper_keyarg_v3(void *arg) { @@ -1776,6 +1802,7 @@ struct testcase_t controller_tests[] = { PARSER_TEST(one_to_three), PARSER_TEST(no_args_one_obj), PARSER_TEST(no_args_kwargs), + PARSER_TEST(one_arg_kwargs), { "add_onion_helper_keyarg_v2", test_add_onion_helper_keyarg_v2, 0, NULL, NULL }, { "add_onion_helper_keyarg_v3", test_add_onion_helper_keyarg_v3, 0, From dab35386cafc837c29fd251213337dec092043fe Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 8 Apr 2019 10:28:56 -0400 Subject: [PATCH 0848/2557] Add a case-insensitive variant to config_line_find() --- src/lib/encoding/confline.c | 13 +++++++++++++ src/lib/encoding/confline.h | 2 ++ 2 files changed, 15 insertions(+) diff --git a/src/lib/encoding/confline.c b/src/lib/encoding/confline.c index 8110f3dd9c..fdb575e03f 100644 --- a/src/lib/encoding/confline.c +++ b/src/lib/encoding/confline.c @@ -82,6 +82,19 @@ config_line_find(const config_line_t *lines, return NULL; } +/** As config_line_find(), but perform a case-insensitive comparison. */ +const config_line_t * +config_line_find_case(const config_line_t *lines, + const char *key) +{ + const config_line_t *cl; + for (cl = lines; cl; cl = cl->next) { + if (!strcasecmp(cl->key, key)) + return cl; + } + return NULL; +} + /** Auxiliary function that does all the work of config_get_lines. * recursion_level is the count of how many nested %includes we have. * opened_lst will have a list of opened files if provided. diff --git a/src/lib/encoding/confline.h b/src/lib/encoding/confline.h index 3d9ae8a662..56ea36bf61 100644 --- a/src/lib/encoding/confline.h +++ b/src/lib/encoding/confline.h @@ -48,6 +48,8 @@ config_line_t *config_lines_dup_and_filter(const config_line_t *inp, const char *key); const config_line_t *config_line_find(const config_line_t *lines, const char *key); +const config_line_t *config_line_find_case(const config_line_t *lines, + const char *key); int config_lines_eq(config_line_t *a, config_line_t *b); int config_count_key(const config_line_t *a, const char *key); void config_free_lines_(config_line_t *front); From d8b3ec865de2738144bd6bbf9f6355662e64eb25 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 8 Apr 2019 10:23:21 -0400 Subject: [PATCH 0849/2557] Update more controller commands, now that we have kvline support --- src/feature/control/control_cmd.c | 547 +++++++++++++----------------- 1 file changed, 241 insertions(+), 306 deletions(-) diff --git a/src/feature/control/control_cmd.c b/src/feature/control/control_cmd.c index 4f67e31637..1848555351 100644 --- a/src/feature/control/control_cmd.c +++ b/src/feature/control/control_cmd.c @@ -184,6 +184,17 @@ control_cmd_parse_args(const char *command, return result; } +/** + * Return true iff lines contains flags as a no-value + * (keyword-only) entry. + **/ +static bool +config_lines_contain_flag(const config_line_t *lines, const char *flag) +{ + const config_line_t *line = config_line_find_case(lines, flag); + return line && !strcmp(line->value, ""); +} + /** Called when we receive a SETCONF message: parse the body and try * to update our configuration. Reply with a DONE or ERROR message. * Modifies the contents of body.*/ @@ -370,15 +381,19 @@ handle_control_setevents(control_connection_t *conn, return 0; } +static const control_cmd_syntax_t saveconf_syntax = { + .max_args = 0, + .accept_keywords = true, + .kvline_flags=KV_OMIT_VALS, +}; + /** Called when we get a SAVECONF command. Try to flush the current options to * disk, and report success or failure. */ static int -handle_control_saveconf(control_connection_t *conn, uint32_t len, - const char *body) +handle_control_saveconf(control_connection_t *conn, + const control_cmd_args_t *args) { - (void) len; - - int force = !strcmpstart(body, "FORCE"); + bool force = config_lines_contain_flag(args->kwargs, "FORCE"); const or_options_t *options = get_options(); if ((!force && options->IncludeUsed) || options_save_current() < 0) { connection_write_str_to_buf( @@ -620,30 +635,27 @@ address_is_invalid_mapaddress_target(const char *addr) return address_is_invalid_destination(addr, 1); } +static const control_cmd_syntax_t mapaddress_syntax = { + .max_args=0, + .accept_keywords=true, +}; + /** Called when we get a MAPADDRESS command; try to bind all listed addresses, * and report success or failure. */ static int -handle_control_mapaddress(control_connection_t *conn, uint32_t len, - const char *body) +handle_control_mapaddress(control_connection_t *conn, + const control_cmd_args_t *args) { - smartlist_t *elts; - smartlist_t *lines; smartlist_t *reply; char *r; size_t sz; - (void) len; /* body is NUL-terminated, so it's safe to ignore the length. */ - lines = smartlist_new(); - elts = smartlist_new(); reply = smartlist_new(); - smartlist_split_string(lines, body, " ", - SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); - SMARTLIST_FOREACH_BEGIN(lines, char *, line) { - tor_strlower(line); - smartlist_split_string(elts, line, "=", 0, 2); - if (smartlist_len(elts) == 2) { - const char *from = smartlist_get(elts,0); - const char *to = smartlist_get(elts,1); + const config_line_t *line; + for (line = args->kwargs; line; line = line->next) { + const char *from = line->key; + const char *to = line->value; + { if (address_is_invalid_mapaddress_target(to)) { smartlist_add_asprintf(reply, "512-syntax error: invalid address '%s'", to); @@ -658,10 +670,10 @@ handle_control_mapaddress(control_connection_t *conn, uint32_t len, type, tor_strdup(to)); if (!address) { smartlist_add_asprintf(reply, - "451-resource exhausted: skipping '%s'", line); + "451-resource exhausted: skipping '%s=%s'", from,to); log_warn(LD_CONTROL, "Unable to allocate address for '%s' in MapAddress msg", - safe_str_client(line)); + safe_str_client(to)); } else { smartlist_add_asprintf(reply, "250-%s=%s", address, to); } @@ -671,27 +683,16 @@ handle_control_mapaddress(control_connection_t *conn, uint32_t len, ADDRMAPSRC_CONTROLLER, &msg) < 0) { smartlist_add_asprintf(reply, "512-syntax error: invalid address mapping " - " '%s': %s", line, msg); + " '%s=%s': %s", from, to, msg); log_warn(LD_CONTROL, - "Skipping invalid argument '%s' in MapAddress msg: %s", - line, msg); + "Skipping invalid argument '%s=%s' in MapAddress msg: %s", + from, to, msg); } else { - smartlist_add_asprintf(reply, "250-%s", line); + smartlist_add_asprintf(reply, "250-%s=%s", from, to); } } - } else { - smartlist_add_asprintf(reply, "512-syntax error: mapping '%s' is " - "not of expected form 'foo=bar'.", line); - log_info(LD_CONTROL, "Skipping MapAddress '%s': wrong " - "number of items.", - safe_str_client(line)); } - SMARTLIST_FOREACH(elts, char *, cp, tor_free(cp)); - smartlist_clear(elts); - } SMARTLIST_FOREACH_END(line); - SMARTLIST_FOREACH(lines, char *, cp, tor_free(cp)); - smartlist_free(lines); - smartlist_free(elts); + } if (smartlist_len(reply)) { ((char*)smartlist_get(reply,smartlist_len(reply)-1))[3] = ' '; @@ -931,36 +932,36 @@ handle_control_extendcircuit(control_connection_t *conn, uint32_t len, return 0; } +static const control_cmd_syntax_t setcircuitpurpose_syntax = { + .max_args=1, + .accept_keywords=true, +}; + /** Called when we get a SETCIRCUITPURPOSE message. If we can find the * circuit and it's a valid purpose, change it. */ static int handle_control_setcircuitpurpose(control_connection_t *conn, - uint32_t len, const char *body) + const control_cmd_args_t *args) { origin_circuit_t *circ = NULL; uint8_t new_purpose; - smartlist_t *args; - (void) len; /* body is NUL-terminated, so it's safe to ignore the length. */ + const char *circ_id = smartlist_get(args->args,0); - args = getargs_helper("SETCIRCUITPURPOSE", conn, body, 2, -1); - if (!args) - goto done; - - if (!(circ = get_circ(smartlist_get(args,0)))) { - connection_printf_to_buf(conn, "552 Unknown circuit \"%s\"\r\n", - (char*)smartlist_get(args, 0)); + if (!(circ = get_circ(circ_id))) { + connection_printf_to_buf(conn, "552 Unknown circuit \"%s\"\r\n", circ_id); goto done; } { - const char *purp = find_element_starting_with(args,1,"PURPOSE="); + const config_line_t *purp = config_line_find_case(args->kwargs, "PURPOSE"); if (!purp) { connection_write_str_to_buf("552 No purpose given\r\n", conn); goto done; } - new_purpose = circuit_purpose_from_string(purp); + new_purpose = circuit_purpose_from_string(purp->value); if (new_purpose == CIRCUIT_PURPOSE_UNKNOWN) { - connection_printf_to_buf(conn, "552 Unknown purpose \"%s\"\r\n", purp); + connection_printf_to_buf(conn, "552 Unknown purpose \"%s\"\r\n", + purp->value); goto done; } } @@ -969,54 +970,50 @@ handle_control_setcircuitpurpose(control_connection_t *conn, connection_write_str_to_buf("250 OK\r\n", conn); done: - if (args) { - SMARTLIST_FOREACH(args, char *, cp, tor_free(cp)); - smartlist_free(args); - } return 0; } +static const char *attachstream_keywords[] = { + "HOP", NULL +}; +static const control_cmd_syntax_t attachstream_syntax = { + .min_args=2, .max_args=2, + .accept_keywords=true, + .allowed_keywords=attachstream_keywords +}; + /** Called when we get an ATTACHSTREAM message. Try to attach the requested * stream, and report success or failure. */ static int -handle_control_attachstream(control_connection_t *conn, uint32_t len, - const char *body) +handle_control_attachstream(control_connection_t *conn, + const control_cmd_args_t *args) { entry_connection_t *ap_conn = NULL; origin_circuit_t *circ = NULL; - int zero_circ; - smartlist_t *args; crypt_path_t *cpath=NULL; int hop=0, hop_line_ok=1; - (void) len; + const char *stream_id = smartlist_get(args->args, 0); + const char *circ_id = smartlist_get(args->args, 1); + int zero_circ = !strcmp(circ_id, "0"); + const config_line_t *hoparg = config_line_find_case(args->kwargs, "HOP"); - args = getargs_helper("ATTACHSTREAM", conn, body, 2, -1); - if (!args) + if (!(ap_conn = get_stream(stream_id))) { + connection_printf_to_buf(conn, "552 Unknown stream \"%s\"\r\n", stream_id); + return 0; + } else if (!zero_circ && !(circ = get_circ(circ_id))) { + connection_printf_to_buf(conn, "552 Unknown circuit \"%s\"\r\n", circ_id); return 0; - - zero_circ = !strcmp("0", (char*)smartlist_get(args,1)); - - if (!(ap_conn = get_stream(smartlist_get(args, 0)))) { - connection_printf_to_buf(conn, "552 Unknown stream \"%s\"\r\n", - (char*)smartlist_get(args, 0)); - } else if (!zero_circ && !(circ = get_circ(smartlist_get(args, 1)))) { - connection_printf_to_buf(conn, "552 Unknown circuit \"%s\"\r\n", - (char*)smartlist_get(args, 1)); } else if (circ) { - const char *hopstring = find_element_starting_with(args,2,"HOP="); - if (hopstring) { - hopstring += strlen("HOP="); - hop = (int) tor_parse_ulong(hopstring, 10, 0, INT_MAX, + if (hoparg) { + hop = (int) tor_parse_ulong(hoparg->value, 10, 0, INT_MAX, &hop_line_ok, NULL); if (!hop_line_ok) { /* broken hop line */ - connection_printf_to_buf(conn, "552 Bad value hop=%s\r\n", hopstring); + connection_printf_to_buf(conn, "552 Bad value hop=%s\r\n", + hoparg->value); + return 0; } } } - SMARTLIST_FOREACH(args, char *, cp, tor_free(cp)); - smartlist_free(args); - if (!ap_conn || (!zero_circ && !circ) || !hop_line_ok) - return 0; if (ENTRY_TO_CONN(ap_conn)->state != AP_CONN_STATE_CONTROLLER_WAIT && ENTRY_TO_CONN(ap_conn)->state != AP_CONN_STATE_CONNECT_WAIT && @@ -1071,59 +1068,49 @@ handle_control_attachstream(control_connection_t *conn, uint32_t len, return 0; } +static const char *postdescriptor_keywords[] = { + "cache", "purpose", NULL, +}; + +static const control_cmd_syntax_t postdescriptor_syntax = { + .max_args = 0, + .accept_keywords = true, + .allowed_keywords = postdescriptor_keywords, + .want_object = true, +}; + /** Called when we get a POSTDESCRIPTOR message. Try to learn the provided * descriptor, and report success or failure. */ static int -handle_control_postdescriptor(control_connection_t *conn, uint32_t len, - const char *body) +handle_control_postdescriptor(control_connection_t *conn, + const control_cmd_args_t *args) { - char *desc; const char *msg=NULL; uint8_t purpose = ROUTER_PURPOSE_GENERAL; int cache = 0; /* eventually, we may switch this to 1 */ + const config_line_t *line; - const char *cp = memchr(body, '\n', len); - - if (cp == NULL) { - connection_printf_to_buf(conn, "251 Empty body\r\n"); - return 0; + line = config_line_find_case(args->kwargs, "purpose"); + if (line) { + purpose = router_purpose_from_string(line->value); + connection_printf_to_buf(conn, "552 Unknown purpose \"%s\"\r\n", + line->value); + goto done; } - ++cp; - - char *cmdline = tor_memdup_nulterm(body, cp-body); - smartlist_t *args = smartlist_new(); - smartlist_split_string(args, cmdline, " ", - SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); - SMARTLIST_FOREACH_BEGIN(args, char *, option) { - if (!strcasecmpstart(option, "purpose=")) { - option += strlen("purpose="); - purpose = router_purpose_from_string(option); - if (purpose == ROUTER_PURPOSE_UNKNOWN) { - connection_printf_to_buf(conn, "552 Unknown purpose \"%s\"\r\n", - option); - goto done; - } - } else if (!strcasecmpstart(option, "cache=")) { - option += strlen("cache="); - if (!strcasecmp(option, "no")) - cache = 0; - else if (!strcasecmp(option, "yes")) - cache = 1; - else { - connection_printf_to_buf(conn, "552 Unknown cache request \"%s\"\r\n", - option); - goto done; - } - } else { /* unrecognized argument? */ - connection_printf_to_buf(conn, - "512 Unexpected argument \"%s\" to postdescriptor\r\n", option); + line = config_line_find_case(args->kwargs, "cache"); + if (line) { + if (!strcasecmp(line->value, "no")) + cache = 0; + else if (!strcasecmp(line->value, "yes")) + cache = 1; + else { + connection_printf_to_buf(conn, "552 Unknown cache request \"%s\"\r\n", + line->value); goto done; } - } SMARTLIST_FOREACH_END(option); + } - read_escaped_data(cp, len-(cp-body), &desc); - - switch (router_load_single_router(desc, purpose, cache, &msg)) { + switch (router_load_single_router(args->object, purpose, cache, &msg)) { case -1: if (!msg) msg = "Could not parse descriptor"; connection_printf_to_buf(conn, "554 %s\r\n", msg); @@ -1137,11 +1124,7 @@ handle_control_postdescriptor(control_connection_t *conn, uint32_t len, break; } - tor_free(desc); done: - SMARTLIST_FOREACH(args, char *, arg, tor_free(arg)); - smartlist_free(args); - tor_free(cmdline); return 0; } @@ -1230,38 +1213,29 @@ handle_control_closestream(control_connection_t *conn, return 0; } +static const control_cmd_syntax_t closecircuit_syntax = { + .min_args=1, .max_args=1, + .accept_keywords=true, + .kvline_flags=KV_OMIT_VALS, + // XXXX we might want to exclude unrecognized flags, but for now we + // XXXX just ignore them for backward compatibility. +}; + /** Called when we get a CLOSECIRCUIT command; try to close the named circuit * and report success or failure. */ static int -handle_control_closecircuit(control_connection_t *conn, uint32_t len, - const char *body) +handle_control_closecircuit(control_connection_t *conn, + const control_cmd_args_t *args) { + const char *circ_id = smartlist_get(args->args, 0); origin_circuit_t *circ = NULL; - int safe = 0; - smartlist_t *args; - (void) len; - args = getargs_helper("CLOSECIRCUIT", conn, body, 1, -1); - if (!args) + if (!(circ=get_circ(circ_id))) { + connection_printf_to_buf(conn, "552 Unknown circuit \"%s\"\r\n", circ_id); return 0; - - if (!(circ=get_circ(smartlist_get(args, 0)))) - connection_printf_to_buf(conn, "552 Unknown circuit \"%s\"\r\n", - (char*)smartlist_get(args, 0)); - else { - int i; - for (i=1; i < smartlist_len(args); ++i) { - if (!strcasecmp(smartlist_get(args, i), "IfUnused")) - safe = 1; - else - log_info(LD_CONTROL, "Skipping unknown option %s", - (char*)smartlist_get(args,i)); - } } - SMARTLIST_FOREACH(args, char *, cp, tor_free(cp)); - smartlist_free(args); - if (!circ) - return 0; + + bool safe = config_lines_contain_flag(args->kwargs, "IfUnused"); if (!safe || !circ->p_streams) { circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_REQUESTED); @@ -1271,36 +1245,43 @@ handle_control_closecircuit(control_connection_t *conn, uint32_t len, return 0; } +static const control_cmd_syntax_t resolve_syntax = { + .max_args=0, + .accept_keywords=true, + .kvline_flags=KV_OMIT_VALS, +}; + /** Called when we get a RESOLVE command: start trying to resolve * the listed addresses. */ static int -handle_control_resolve(control_connection_t *conn, uint32_t len, - const char *body) +handle_control_resolve(control_connection_t *conn, + const control_cmd_args_t *args) { - smartlist_t *args, *failed; + smartlist_t *failed; int is_reverse = 0; - (void) len; /* body is nul-terminated; it's safe to ignore the length */ if (!(conn->event_mask & (((event_mask_t)1)<kwargs, "mode"); + if (modearg && !strcasecmp(modearg->value, "reverse")) is_reverse = 1; } failed = smartlist_new(); - SMARTLIST_FOREACH(args, const char *, arg, { - if (!is_keyval_pair(arg)) { - if (dnsserv_launch_request(arg, is_reverse, conn)<0) - smartlist_add(failed, (char*)arg); - } - }); + for (const config_line_t *line = args->kwargs; line; line = line->next) { + if (!strlen(line->value)) { + const char *addr = line->key; + if (dnsserv_launch_request(addr, is_reverse, conn)<0) + smartlist_add(failed, (char*)addr); + } else { + // XXXX arguably we should reject unrecognized keyword arguments, + // XXXX but the old implementation didn't do that. + } + } send_control_done(conn); SMARTLIST_FOREACH(failed, const char *, arg, { @@ -1308,8 +1289,6 @@ handle_control_resolve(control_connection_t *conn, uint32_t len, "internal", 0); }); - SMARTLIST_FOREACH(args, char *, cp, tor_free(cp)); - smartlist_free(args); smartlist_free(failed); return 0; } @@ -1443,30 +1422,33 @@ handle_control_dropguards(control_connection_t *conn, return 0; } +static const char *hsfetch_keywords[] = { + "SERVER", NULL, +}; +static const control_cmd_syntax_t hsfetch_syntax = { + .min_args = 1, .max_args = 1, + .accept_keywords = true, + .allowed_keywords = hsfetch_keywords, + .want_object = true, +}; + /** Implementation for the HSFETCH command. */ static int -handle_control_hsfetch(control_connection_t *conn, uint32_t len, - const char *body) +handle_control_hsfetch(control_connection_t *conn, + const control_cmd_args_t *args) + { - int i; - char digest[DIGEST_LEN], *hsaddress = NULL, *arg1 = NULL, *desc_id = NULL; - smartlist_t *args = NULL, *hsdirs = NULL; - (void) len; /* body is nul-terminated; it's safe to ignore the length */ - static const char *hsfetch_command = "HSFETCH"; + char digest[DIGEST_LEN], *desc_id = NULL; + smartlist_t *hsdirs = NULL; static const char *v2_str = "v2-"; const size_t v2_str_len = strlen(v2_str); rend_data_t *rend_query = NULL; ed25519_public_key_t v3_pk; uint32_t version; - - /* Make sure we have at least one argument, the HSAddress. */ - args = getargs_helper(hsfetch_command, conn, body, 1, -1); - if (!args) { - goto exit; - } + const char *hsaddress = NULL; /* Extract the first argument (either HSAddress or DescID). */ - arg1 = smartlist_get(args, 0); + const char *arg1 = smartlist_get(args->args, 0); /* Test if it's an HS address without the .onion part. */ if (rend_valid_v2_service_id(arg1)) { hsaddress = arg1; @@ -1490,18 +1472,11 @@ handle_control_hsfetch(control_connection_t *conn, uint32_t len, goto done; } - static const char *opt_server = "SERVER="; + for (const config_line_t *line = args->kwargs; line; line = line->next) { + if (!strcasecmp(line->key, "SERVER")) { + const char *server = line->value; - /* Skip first argument because it's the HSAddress or DescID. */ - for (i = 1; i < smartlist_len(args); ++i) { - const char *arg = smartlist_get(args, i); - const node_t *node; - - if (!strcasecmpstart(arg, opt_server)) { - const char *server; - - server = arg + strlen(opt_server); - node = node_get_by_hex_id(server, 0); + const node_t *node = node_get_by_hex_id(server, 0); if (!node) { connection_printf_to_buf(conn, "552 Server \"%s\" not found\r\n", server); @@ -1514,9 +1489,7 @@ handle_control_hsfetch(control_connection_t *conn, uint32_t len, /* Valid server, add it to our local list. */ smartlist_add(hsdirs, node->rs); } else { - connection_printf_to_buf(conn, "513 Unexpected argument \"%s\"\r\n", - arg); - goto done; + tor_assert_nonfatal_unreached(); } } @@ -1532,9 +1505,8 @@ handle_control_hsfetch(control_connection_t *conn, uint32_t len, /* Using a descriptor ID, we force the user to provide at least one * hsdir server using the SERVER= option. */ if (desc_id && (!hsdirs || !smartlist_len(hsdirs))) { - connection_printf_to_buf(conn, "512 %s option is required\r\n", - opt_server); - goto done; + connection_printf_to_buf(conn, "512 SERVER option is required\r\n"); + goto done; } /* We are about to trigger HSDir fetch so send the OK now because after @@ -1552,96 +1524,75 @@ handle_control_hsfetch(control_connection_t *conn, uint32_t len, } done: - SMARTLIST_FOREACH(args, char *, cp, tor_free(cp)); - smartlist_free(args); /* Contains data pointer that we don't own thus no cleanup. */ smartlist_free(hsdirs); rend_data_free(rend_query); - exit: return 0; } +static const char *hspost_keywords[] = { + "SERVER", "HSADDRESS", NULL +}; +static const control_cmd_syntax_t hspost_syntax = { + .min_args = 0, .max_args = 0, + .accept_keywords = true, + .want_object = true, + .allowed_keywords = hspost_keywords +}; + /** Implementation for the HSPOST command. */ static int handle_control_hspost(control_connection_t *conn, - uint32_t len, - const char *body) + const control_cmd_args_t *args) { - static const char *opt_server = "SERVER="; - static const char *opt_hsaddress = "HSADDRESS="; smartlist_t *hs_dirs = NULL; - const char *encoded_desc = body; - size_t encoded_desc_len = len; + const char *encoded_desc = args->object; + size_t encoded_desc_len = args->object_len; const char *onion_address = NULL; + const config_line_t *line; - char *cp = memchr(body, '\n', len); - if (cp == NULL) { - connection_printf_to_buf(conn, "251 Empty body\r\n"); - return 0; - } - char *argline = tor_strndup(body, cp-body); + for (line = args->kwargs; line; line = line->next) { + if (!strcasecmpstart(line->key, "SERVER")) { + const char *server = line->value; + const node_t *node = node_get_by_hex_id(server, 0); - smartlist_t *args = smartlist_new(); - - /* If any SERVER= or HSADDRESS= options were specified, try to parse - * the options line. */ - if (!strcasecmpstart(argline, opt_server) || - !strcasecmpstart(argline, opt_hsaddress)) { - /* encoded_desc begins after a newline character */ - cp = cp + 1; - encoded_desc = cp; - encoded_desc_len = len-(cp-body); - - smartlist_split_string(args, argline, " ", - SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); - SMARTLIST_FOREACH_BEGIN(args, const char *, arg) { - if (!strcasecmpstart(arg, opt_server)) { - const char *server = arg + strlen(opt_server); - const node_t *node = node_get_by_hex_id(server, 0); - - if (!node || !node->rs) { - connection_printf_to_buf(conn, "552 Server \"%s\" not found\r\n", - server); - goto done; - } - /* Valid server, add it to our local list. */ - if (!hs_dirs) - hs_dirs = smartlist_new(); - smartlist_add(hs_dirs, node->rs); - } else if (!strcasecmpstart(arg, opt_hsaddress)) { - const char *address = arg + strlen(opt_hsaddress); - if (!hs_address_is_valid(address)) { - connection_printf_to_buf(conn, "512 Malformed onion address\r\n"); - goto done; - } - onion_address = address; - } else { - connection_printf_to_buf(conn, "512 Unexpected argument \"%s\"\r\n", - arg); + if (!node || !node->rs) { + connection_printf_to_buf(conn, "552 Server \"%s\" not found\r\n", + server); goto done; } - } SMARTLIST_FOREACH_END(arg); + /* Valid server, add it to our local list. */ + if (!hs_dirs) + hs_dirs = smartlist_new(); + smartlist_add(hs_dirs, node->rs); + } else if (!strcasecmpstart(line->key, "HSADDRESS")) { + const char *address = line->value; + if (!hs_address_is_valid(address)) { + connection_printf_to_buf(conn, "512 Malformed onion address\r\n"); + goto done; + } + onion_address = address; + } else { + tor_assert_nonfatal_unreached(); + } } /* Handle the v3 case. */ if (onion_address) { - char *desc_str = NULL; - read_escaped_data(encoded_desc, encoded_desc_len, &desc_str); - if (hs_control_hspost_command(desc_str, onion_address, hs_dirs) < 0) { + if (hs_control_hspost_command(encoded_desc, onion_address, hs_dirs) < 0) { connection_printf_to_buf(conn, "554 Invalid descriptor\r\n"); } else { send_control_done(conn); } - tor_free(desc_str); goto done; } /* From this point on, it is only v2. */ - /* Read the dot encoded descriptor, and parse it. */ + /* parse it. */ rend_encoded_v2_service_descriptor_t *desc = tor_malloc_zero(sizeof(rend_encoded_v2_service_descriptor_t)); - read_escaped_data(encoded_desc, encoded_desc_len, &desc->desc_str); + desc->desc_str = tor_memdup_nulterm(encoded_desc, encoded_desc_len); rend_service_descriptor_t *parsed = NULL; char *intro_content = NULL; @@ -1675,10 +1626,7 @@ handle_control_hspost(control_connection_t *conn, tor_free(intro_content); rend_encoded_v2_service_descriptor_free(desc); done: - tor_free(argline); smartlist_free(hs_dirs); /* Contents belong to the rend service code. */ - SMARTLIST_FOREACH(args, char *, arg, tor_free(arg)); - smartlist_free(args); return 0; } @@ -1742,21 +1690,21 @@ get_detached_onion_services(void) return detached_onion_services; } +static const char *add_onion_keywords[] = { + "Port", "Flags", "MaxStreams", "ClientAuth", NULL +}; +static const control_cmd_syntax_t add_onion_syntax = { + .min_args = 1, .max_args = 1, + .accept_keywords = true, + .allowed_keywords = add_onion_keywords +}; + /** Called when we get a ADD_ONION command; parse the body, and set up * the new ephemeral Onion Service. */ static int handle_control_add_onion(control_connection_t *conn, - uint32_t len, - const char *body) + const control_cmd_args_t *args) { - smartlist_t *args; - int arg_len; - (void) len; /* body is nul-terminated; it's safe to ignore the length */ - args = getargs_helper("ADD_ONION", conn, body, 2, -1); - if (!args) - return 0; - arg_len = smartlist_len(args); - /* Parse all of the arguments that do not involve handling cryptographic * material first, since there's no reason to touch that at all if any of * the other arguments are malformed. @@ -1769,36 +1717,28 @@ handle_control_add_onion(control_connection_t *conn, int max_streams = 0; int max_streams_close_circuit = 0; rend_auth_type_t auth_type = REND_NO_AUTH; - /* Default to adding an anonymous hidden service if no flag is given */ int non_anonymous = 0; - for (int i = 1; i < arg_len; i++) { - static const char *port_prefix = "Port="; - static const char *flags_prefix = "Flags="; - static const char *max_s_prefix = "MaxStreams="; - static const char *auth_prefix = "ClientAuth="; + const config_line_t *arg; - const char *arg = smartlist_get(args, (int)i); - if (!strcasecmpstart(arg, port_prefix)) { + for (arg = args->kwargs; arg; arg = arg->next) { + if (!strcasecmp(arg->key, "Port")) { /* "Port=VIRTPORT[,TARGET]". */ - const char *port_str = arg + strlen(port_prefix); - rend_service_port_config_t *cfg = - rend_service_parse_port_config(port_str, ",", NULL); + rend_service_parse_port_config(arg->value, ",", NULL); if (!cfg) { connection_printf_to_buf(conn, "512 Invalid VIRTPORT/TARGET\r\n"); goto out; } smartlist_add(port_cfgs, cfg); - } else if (!strcasecmpstart(arg, max_s_prefix)) { + } else if (!strcasecmp(arg->key, "MaxStreams")) { /* "MaxStreams=[0..65535]". */ - const char *max_s_str = arg + strlen(max_s_prefix); int ok = 0; - max_streams = (int)tor_parse_long(max_s_str, 10, 0, 65535, &ok, NULL); + max_streams = (int)tor_parse_long(arg->value, 10, 0, 65535, &ok, NULL); if (!ok) { connection_printf_to_buf(conn, "512 Invalid MaxStreams\r\n"); goto out; } - } else if (!strcasecmpstart(arg, flags_prefix)) { + } else if (!strcasecmp(arg->key, "Flags")) { /* "Flags=Flag[,Flag]", where Flag can be: * * 'DiscardPK' - If tor generates the keypair, do not include it in * the response. @@ -1821,8 +1761,7 @@ handle_control_add_onion(control_connection_t *conn, smartlist_t *flags = smartlist_new(); int bad = 0; - smartlist_split_string(flags, arg + strlen(flags_prefix), ",", - SPLIT_IGNORE_BLANK, 0); + smartlist_split_string(flags, arg->value, ",", SPLIT_IGNORE_BLANK, 0); if (smartlist_len(flags) < 1) { connection_printf_to_buf(conn, "512 Invalid 'Flags' argument\r\n"); bad = 1; @@ -1851,11 +1790,12 @@ handle_control_add_onion(control_connection_t *conn, smartlist_free(flags); if (bad) goto out; - } else if (!strcasecmpstart(arg, auth_prefix)) { + + } else if (!strcasecmp(arg->key, "ClientAuth")) { char *err_msg = NULL; int created = 0; rend_authorized_client_t *client = - add_onion_helper_clientauth(arg + strlen(auth_prefix), + add_onion_helper_clientauth(arg->value, &created, &err_msg); if (!client) { if (err_msg) { @@ -1888,7 +1828,7 @@ handle_control_add_onion(control_connection_t *conn, smartlist_add(auth_created_clients, client); } } else { - connection_printf_to_buf(conn, "513 Invalid argument\r\n"); + tor_assert_nonfatal_unreached(); goto out; } } @@ -1929,7 +1869,8 @@ handle_control_add_onion(control_connection_t *conn, char *key_new_blob = NULL; char *err_msg = NULL; - if (add_onion_helper_keyarg(smartlist_get(args, 0), discard_pk, + const char *onionkey = smartlist_get(args->args, 0); + if (add_onion_helper_keyarg(onionkey, discard_pk, &key_new_alg, &key_new_blob, &pk, &hs_version, &err_msg) < 0) { if (err_msg) { @@ -2031,12 +1972,6 @@ handle_control_add_onion(control_connection_t *conn, // Do not free entries; they are the same as auth_clients smartlist_free(auth_created_clients); } - - SMARTLIST_FOREACH(args, char *, cp, { - memwipe(cp, 0, strlen(cp)); - tor_free(cp); - }); - smartlist_free(args); return 0; } @@ -2471,28 +2406,28 @@ static const control_cmd_def_t CONTROL_COMMANDS[] = MULTLINE_PARSED(loadconf, 0), ONE_LINE_PARSED(setevents, 0), ONE_LINE(authenticate, legacy, CMD_FL_WIPE), - ONE_LINE(saveconf, legacy, 0), + ONE_LINE_PARSED(saveconf, 0), ONE_LINE_PARSED(signal, 0), ONE_LINE_PARSED(takeownership, 0), ONE_LINE_PARSED(dropownership, 0), - ONE_LINE(mapaddress, legacy, 0), + ONE_LINE_PARSED(mapaddress, 0), ONE_LINE_PARSED(getinfo, 0), ONE_LINE(extendcircuit, legacy, 0), - ONE_LINE(setcircuitpurpose, legacy, 0), + ONE_LINE_PARSED(setcircuitpurpose, 0), OBSOLETE(setrouterpurpose), - ONE_LINE(attachstream, legacy, 0), - MULTLINE(postdescriptor, legacy, 0), + ONE_LINE_PARSED(attachstream, 0), + MULTLINE_PARSED(postdescriptor, 0), ONE_LINE_PARSED(redirectstream, 0), ONE_LINE_PARSED(closestream, 0), - ONE_LINE(closecircuit, legacy, 0), + ONE_LINE_PARSED(closecircuit, 0), ONE_LINE_PARSED(usefeature, 0), - ONE_LINE(resolve, legacy, 0), + ONE_LINE_PARSED(resolve, 0), ONE_LINE_PARSED(protocolinfo, 0), ONE_LINE(authchallenge, legacy, CMD_FL_WIPE), ONE_LINE_PARSED(dropguards, 0), - ONE_LINE(hsfetch, legacy, 0), - MULTLINE(hspost, legacy, 0), - ONE_LINE(add_onion, legacy, CMD_FL_WIPE), + ONE_LINE_PARSED(hsfetch, 0), + MULTLINE_PARSED(hspost, 0), + ONE_LINE_PARSED(add_onion, CMD_FL_WIPE), ONE_LINE_PARSED(del_onion, CMD_FL_WIPE), }; From 95afdb005cce04cfb87df6a75980944173c15ed7 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 8 Apr 2019 16:19:09 -0400 Subject: [PATCH 0850/2557] Use new parser logic for SETCONF/RESETCONF code. Here we get to throw away a LOT of unused code, since most of the old parsing was redundant with kvline. --- src/feature/control/control_cmd.c | 108 +++++++----------------------- src/feature/control/control_fmt.c | 14 ---- src/feature/control/control_fmt.h | 2 - 3 files changed, 26 insertions(+), 98 deletions(-) diff --git a/src/feature/control/control_cmd.c b/src/feature/control/control_cmd.c index 1848555351..cfd35e8096 100644 --- a/src/feature/control/control_cmd.c +++ b/src/feature/control/control_cmd.c @@ -54,8 +54,8 @@ #include "feature/rend/rend_encoded_v2_service_descriptor_st.h" #include "feature/rend/rend_service_descriptor_st.h" -static int control_setconf_helper(control_connection_t *conn, uint32_t len, - char *body, +static int control_setconf_helper(control_connection_t *conn, + const control_cmd_args_t *args, int use_defaults); /** Yield true iff s is the state of a control_connection_t that has @@ -195,22 +195,36 @@ config_lines_contain_flag(const config_line_t *lines, const char *flag) return line && !strcmp(line->value, ""); } +static const control_cmd_syntax_t setconf_syntax = { + .max_args=0, + .accept_keywords=true, + .kvline_flags=KV_OMIT_VALS|KV_QUOTED, +}; + /** Called when we receive a SETCONF message: parse the body and try * to update our configuration. Reply with a DONE or ERROR message. * Modifies the contents of body.*/ static int -handle_control_setconf(control_connection_t *conn, uint32_t len, char *body) +handle_control_setconf(control_connection_t *conn, + const control_cmd_args_t *args) { - return control_setconf_helper(conn, len, body, 0); + return control_setconf_helper(conn, args, 0); } +static const control_cmd_syntax_t resetconf_syntax = { + .max_args=0, + .accept_keywords=true, + .kvline_flags=KV_OMIT_VALS|KV_QUOTED, +}; + /** Called when we receive a RESETCONF message: parse the body and try * to update our configuration. Reply with a DONE or ERROR message. * Modifies the contents of body. */ static int -handle_control_resetconf(control_connection_t *conn, uint32_t len, char *body) +handle_control_resetconf(control_connection_t *conn, + const control_cmd_args_t *args) { - return control_setconf_helper(conn, len, body, 1); + return control_setconf_helper(conn, args, 1); } static const control_cmd_syntax_t getconf_syntax = { @@ -524,73 +538,17 @@ get_stream(const char *id) * contents of body. */ static int -control_setconf_helper(control_connection_t *conn, uint32_t len, char *body, +control_setconf_helper(control_connection_t *conn, + const control_cmd_args_t *args, int use_defaults) { setopt_err_t opt_err; - config_line_t *lines=NULL; - char *start = body; char *errstring = NULL; const unsigned flags = CAL_CLEAR_FIRST | (use_defaults ? CAL_USE_DEFAULTS : 0); - char *config; - smartlist_t *entries = smartlist_new(); - - /* We have a string, "body", of the format '(key(=val|="val")?)' entries - * separated by space. break it into a list of configuration entries. */ - while (*body) { - char *eq = body; - char *key; - char *entry; - while (!TOR_ISSPACE(*eq) && *eq != '=') - ++eq; - key = tor_strndup(body, eq-body); - body = eq+1; - if (*eq == '=') { - char *val=NULL; - size_t val_len=0; - if (*body != '\"') { - char *val_start = body; - while (!TOR_ISSPACE(*body)) - body++; - val = tor_strndup(val_start, body-val_start); - val_len = strlen(val); - } else { - body = (char*)extract_escaped_string(body, (len - (body-start)), - &val, &val_len); - if (!body) { - connection_write_str_to_buf("551 Couldn't parse string\r\n", conn); - SMARTLIST_FOREACH(entries, char *, cp, tor_free(cp)); - smartlist_free(entries); - tor_free(key); - return 0; - } - } - tor_asprintf(&entry, "%s %s", key, val); - tor_free(key); - tor_free(val); - } else { - entry = key; - } - smartlist_add(entries, entry); - while (TOR_ISSPACE(*body)) - ++body; - } - - smartlist_add_strdup(entries, ""); - config = smartlist_join_strings(entries, "\n", 0, NULL); - SMARTLIST_FOREACH(entries, char *, cp, tor_free(cp)); - smartlist_free(entries); - - if (config_get_lines(config, &lines, 0) < 0) { - log_warn(LD_CONTROL,"Controller gave us config lines we can't parse."); - connection_write_str_to_buf("551 Couldn't parse configuration\r\n", - conn); - tor_free(config); - return 0; - } - tor_free(config); + // We need a copy here, since confparse.c wants to canonicalize cases. + config_line_t *lines = config_lines_dup(args->kwargs); opt_err = options_trial_assign(lines, flags, &errstring); { @@ -2276,7 +2234,6 @@ handle_control_obsolete(control_connection_t *conn, **/ typedef enum handler_type_t { hnd_legacy, - hnd_legacy_mut, hnd_parsed, } handler_type_t; @@ -2296,15 +2253,6 @@ typedef union handler_fn_t { int (*legacy)(control_connection_t *conn, uint32_t arg_len, const char *args); - /** - * A "legacy_mut" handler is the same as a "legacy" one, except that it may - * change the contents of the command's arguments -- for example, by - * inserting NULs. It may not deallocate them. - */ - int (*legacy_mut)(control_connection_t *conn, - uint32_t arg_len, - char *args); - /** * A "parsed" handler expects its arguments in a pre-parsed format, in * an immutable control_cmd_args_t *object. @@ -2400,8 +2348,8 @@ typedef struct control_cmd_def_t { **/ static const control_cmd_def_t CONTROL_COMMANDS[] = { - ONE_LINE(setconf, legacy_mut, 0), - ONE_LINE(resetconf, legacy_mut, 0), + ONE_LINE_PARSED(setconf, 0), + ONE_LINE_PARSED(resetconf, 0), ONE_LINE_PARSED(getconf, 0), MULTLINE_PARSED(loadconf, 0), ONE_LINE_PARSED(setevents, 0), @@ -2452,10 +2400,6 @@ handle_single_control_command(const control_cmd_def_t *def, if (def->handler.legacy(conn, cmd_data_len, args)) rv = -1; break; - case hnd_legacy_mut: - if (def->handler.legacy_mut(conn, cmd_data_len, args)) - rv = -1; - break; case hnd_parsed: { control_cmd_args_t *parsed_args; char *err=NULL; diff --git a/src/feature/control/control_fmt.c b/src/feature/control/control_fmt.c index 71f9d82163..427f4288fc 100644 --- a/src/feature/control/control_fmt.c +++ b/src/feature/control/control_fmt.c @@ -342,20 +342,6 @@ get_escaped_string_length(const char *start, size_t in_len_max, return (int)(cp - start+1); } -/** As decode_escaped_string, but does not decode the string: copies the - * entire thing, including quotation marks. */ -const char * -extract_escaped_string(const char *start, size_t in_len_max, - char **out, size_t *out_len) -{ - int length = get_escaped_string_length(start, in_len_max, NULL); - if (length<0) - return NULL; - *out_len = length; - *out = tor_strndup(start, *out_len); - return start+length; -} - /** Given a pointer to a string starting at start containing * in_len_max characters, decode a string beginning with one double * quote, containing any number of non-quote characters or characters escaped diff --git a/src/feature/control/control_fmt.h b/src/feature/control/control_fmt.h index 74545eb309..08acf85181 100644 --- a/src/feature/control/control_fmt.h +++ b/src/feature/control/control_fmt.h @@ -25,8 +25,6 @@ char *circuit_describe_status_for_controller(origin_circuit_t *circ); size_t write_escaped_data(const char *data, size_t len, char **out); size_t read_escaped_data(const char *data, size_t len, char **out); -const char *extract_escaped_string(const char *start, size_t in_len_max, - char **out, size_t *out_len); const char *decode_escaped_string(const char *start, size_t in_len_max, char **out, size_t *out_len); void send_control_done(control_connection_t *conn); From 0c0b869ba450363e36e8dd0bdacb4a197e0f0019 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 9 Apr 2019 09:28:57 -0400 Subject: [PATCH 0851/2557] Use the new controller command parser for EXTENDCIRCUIT. This command does not fit perfectly with the others, since its second argument is optional and may contain equal signs. Still, it's probably better to squeeze it into the new metaformat, since doing so allows us to remove several pieces of the old command-parsing machinery. --- src/feature/control/control_cmd.c | 152 +++++++++++------------------- 1 file changed, 54 insertions(+), 98 deletions(-) diff --git a/src/feature/control/control_cmd.c b/src/feature/control/control_cmd.c index cfd35e8096..0b4f3555ff 100644 --- a/src/feature/control/control_cmd.c +++ b/src/feature/control/control_cmd.c @@ -594,7 +594,7 @@ address_is_invalid_mapaddress_target(const char *addr) } static const control_cmd_syntax_t mapaddress_syntax = { - .max_args=0, + .max_args=1, .accept_keywords=true, }; @@ -683,94 +683,61 @@ circuit_purpose_from_string(const char *string) return CIRCUIT_PURPOSE_UNKNOWN; } -/** Return a newly allocated smartlist containing the arguments to the command - * waiting in body. If there are fewer than min_args arguments, - * or if max_args is nonnegative and there are more than - * max_args arguments, send a 512 error to the controller, using - * command as the command name in the error message. */ -static smartlist_t * -getargs_helper(const char *command, control_connection_t *conn, - const char *body, int min_args, int max_args) -{ - smartlist_t *args = smartlist_new(); - smartlist_split_string(args, body, " ", - SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); - if (smartlist_len(args) < min_args) { - connection_printf_to_buf(conn, "512 Missing argument to %s\r\n",command); - goto err; - } else if (max_args >= 0 && smartlist_len(args) > max_args) { - connection_printf_to_buf(conn, "512 Too many arguments to %s\r\n",command); - goto err; - } - return args; - err: - SMARTLIST_FOREACH(args, char *, s, tor_free(s)); - smartlist_free(args); - return NULL; -} - -/** Helper. Return the first element of sl at index start_at or - * higher that starts with prefix, case-insensitive. Return NULL if no - * such element exists. */ -static const char * -find_element_starting_with(smartlist_t *sl, int start_at, const char *prefix) -{ - int i; - for (i = start_at; i < smartlist_len(sl); ++i) { - const char *elt = smartlist_get(sl, i); - if (!strcasecmpstart(elt, prefix)) - return elt; - } - return NULL; -} - -/** Helper. Return true iff s is an argument that we should treat as a - * key-value pair. */ -static int -is_keyval_pair(const char *s) -{ - /* An argument is a key-value pair if it has an =, and it isn't of the form - * $fingeprint=name */ - return strchr(s, '=') && s[0] != '$'; -} +static const control_cmd_syntax_t extendcircuit_syntax = { + .min_args=1, + .max_args=1, // see note in function + .accept_keywords=true, + .kvline_flags=KV_OMIT_VALS +}; /** Called when we get an EXTENDCIRCUIT message. Try to extend the listed * circuit, and report success or failure. */ static int -handle_control_extendcircuit(control_connection_t *conn, uint32_t len, - const char *body) +handle_control_extendcircuit(control_connection_t *conn, + const control_cmd_args_t *args) { - smartlist_t *router_nicknames=NULL, *nodes=NULL; + smartlist_t *router_nicknames=smartlist_new(), *nodes=NULL; origin_circuit_t *circ = NULL; - int zero_circ; uint8_t intended_purpose = CIRCUIT_PURPOSE_C_GENERAL; - smartlist_t *args; - (void) len; + const config_line_t *kwargs = args->kwargs; + const char *circ_id = smartlist_get(args->args, 0); + const char *path_str = NULL; + char *path_str_alloc = NULL; - router_nicknames = smartlist_new(); + /* The syntax for this command is unfortunate. The second argument is + optional, and is a comma-separated list long-format fingerprints, which + can (historically!) contain an equals sign. - args = getargs_helper("EXTENDCIRCUIT", conn, body, 1, -1); - if (!args) - goto done; + Here we check the second argument to see if it's a path, and if so we + remove it from the kwargs list and put it in path_str. + */ + if (kwargs) { + const config_line_t *arg1 = kwargs; + if (!strcmp(arg1->value, "")) { + path_str = arg1->key; + kwargs = kwargs->next; + } else if (arg1->key[0] == '$') { + tor_asprintf(&path_str_alloc, "%s=%s", arg1->key, arg1->value); + path_str = path_str_alloc; + kwargs = kwargs->next; + } + } - zero_circ = !strcmp("0", (char*)smartlist_get(args,0)); + const config_line_t *purpose_line = config_line_find_case(kwargs, "PURPOSE"); + bool zero_circ = !strcmp("0", circ_id); + + if (purpose_line) { + intended_purpose = circuit_purpose_from_string(purpose_line->value); + if (intended_purpose == CIRCUIT_PURPOSE_UNKNOWN) { + connection_printf_to_buf(conn, "552 Unknown purpose \"%s\"\r\n", + purpose_line->value); + goto done; + } + } if (zero_circ) { - const char *purp = find_element_starting_with(args, 1, "PURPOSE="); - - if (purp) { - intended_purpose = circuit_purpose_from_string(purp); - if (intended_purpose == CIRCUIT_PURPOSE_UNKNOWN) { - connection_printf_to_buf(conn, "552 Unknown purpose \"%s\"\r\n", purp); - SMARTLIST_FOREACH(args, char *, cp, tor_free(cp)); - smartlist_free(args); - goto done; - } - } - - if ((smartlist_len(args) == 1) || - (smartlist_len(args) >= 2 && is_keyval_pair(smartlist_get(args, 1)))) { - // "EXTENDCIRCUIT 0" || EXTENDCIRCUIT 0 foo=bar" + if (!path_str) { + // "EXTENDCIRCUIT 0" with no path. circ = circuit_launch(intended_purpose, CIRCLAUNCH_NEED_CAPACITY); if (!circ) { connection_write_str_to_buf("551 Couldn't start circuit\r\n", conn); @@ -778,37 +745,24 @@ handle_control_extendcircuit(control_connection_t *conn, uint32_t len, connection_printf_to_buf(conn, "250 EXTENDED %lu\r\n", (unsigned long)circ->global_identifier); } - SMARTLIST_FOREACH(args, char *, cp, tor_free(cp)); - smartlist_free(args); goto done; } - // "EXTENDCIRCUIT 0 router1,router2" || - // "EXTENDCIRCUIT 0 router1,router2 PURPOSE=foo" } - if (!zero_circ && !(circ = get_circ(smartlist_get(args,0)))) { - connection_printf_to_buf(conn, "552 Unknown circuit \"%s\"\r\n", - (char*)smartlist_get(args, 0)); - SMARTLIST_FOREACH(args, char *, cp, tor_free(cp)); - smartlist_free(args); + if (!zero_circ && !(circ = get_circ(circ_id))) { + connection_printf_to_buf(conn, "552 Unknown circuit \"%s\"\r\n", circ_id); goto done; } - if (smartlist_len(args) < 2) { - connection_printf_to_buf(conn, - "512 syntax error: not enough arguments.\r\n"); - SMARTLIST_FOREACH(args, char *, cp, tor_free(cp)); - smartlist_free(args); + if (!path_str) { + connection_printf_to_buf(conn, "512 syntax error: path required.\r\n"); goto done; } - smartlist_split_string(router_nicknames, smartlist_get(args,1), ",", 0, 0); - - SMARTLIST_FOREACH(args, char *, cp, tor_free(cp)); - smartlist_free(args); + smartlist_split_string(router_nicknames, path_str, ",", 0, 0); nodes = smartlist_new(); - int first_node = zero_circ; + bool first_node = zero_circ; SMARTLIST_FOREACH_BEGIN(router_nicknames, const char *, n) { const node_t *node = node_get_by_nickname(n, 0); if (!node) { @@ -820,8 +774,9 @@ handle_control_extendcircuit(control_connection_t *conn, uint32_t len, goto done; } smartlist_add(nodes, (void*)node); - first_node = 0; + first_node = false; } SMARTLIST_FOREACH_END(n); + if (!smartlist_len(nodes)) { connection_write_str_to_buf("512 No router names provided\r\n", conn); goto done; @@ -887,6 +842,7 @@ handle_control_extendcircuit(control_connection_t *conn, uint32_t len, SMARTLIST_FOREACH(router_nicknames, char *, n, tor_free(n)); smartlist_free(router_nicknames); smartlist_free(nodes); + tor_free(path_str_alloc); return 0; } @@ -2360,7 +2316,7 @@ static const control_cmd_def_t CONTROL_COMMANDS[] = ONE_LINE_PARSED(dropownership, 0), ONE_LINE_PARSED(mapaddress, 0), ONE_LINE_PARSED(getinfo, 0), - ONE_LINE(extendcircuit, legacy, 0), + ONE_LINE_PARSED(extendcircuit, 0), ONE_LINE_PARSED(setcircuitpurpose, 0), OBSOLETE(setrouterpurpose), ONE_LINE_PARSED(attachstream, 0), From ba05324242aebdbf646ebb8e8f3aaef45b1f29ec Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 9 Apr 2019 09:42:51 -0400 Subject: [PATCH 0852/2557] Move and rename decode_escaped_string() This function decodes something different from the usual c-escaped format. It is only used in controller authorization. --- src/feature/control/control_auth.c | 7 ++- src/feature/control/control_fmt.c | 74 ------------------------ src/feature/control/control_fmt.h | 2 - src/lib/encoding/include.am | 2 + src/lib/encoding/qstring.c | 90 ++++++++++++++++++++++++++++++ src/lib/encoding/qstring.h | 18 ++++++ 6 files changed, 114 insertions(+), 79 deletions(-) create mode 100644 src/lib/encoding/qstring.c create mode 100644 src/lib/encoding/qstring.h diff --git a/src/feature/control/control_auth.c b/src/feature/control/control_auth.c index 927115a308..8204290489 100644 --- a/src/feature/control/control_auth.c +++ b/src/feature/control/control_auth.c @@ -17,6 +17,7 @@ #include "lib/crypt_ops/crypto_rand.h" #include "lib/crypt_ops/crypto_util.h" #include "lib/encoding/confline.h" +#include "lib/encoding/qstring.h" #include "lib/crypt_ops/crypto_s2k.h" @@ -149,8 +150,8 @@ handle_control_authchallenge(control_connection_t *conn, uint32_t len, cp += strspn(cp, " \t\n\r"); if (*cp == '"') { const char *newcp = - decode_escaped_string(cp, len - (cp - body), - &client_nonce, &client_nonce_len); + decode_qstring(cp, len - (cp - body), + &client_nonce, &client_nonce_len); if (newcp == NULL) { connection_write_str_to_buf("513 Invalid quoted client nonce\r\n", conn); @@ -275,7 +276,7 @@ handle_control_authenticate(control_connection_t *conn, uint32_t len, return 0; } } else { - if (!decode_escaped_string(body, len, &password, &password_len)) { + if (!decode_qstring(body, len, &password, &password_len)) { connection_write_str_to_buf("551 Invalid quoted string. You need " "to put the password in double quotes.\r\n", conn); connection_mark_for_close(TO_CONN(conn)); diff --git a/src/feature/control/control_fmt.c b/src/feature/control/control_fmt.c index 427f4288fc..b2ab4f10bb 100644 --- a/src/feature/control/control_fmt.c +++ b/src/feature/control/control_fmt.c @@ -305,80 +305,6 @@ send_control_done(control_connection_t *conn) connection_write_str_to_buf("250 OK\r\n", conn); } -/** If the first in_len_max characters in start contain a - * double-quoted string with escaped characters, return the length of that - * string (as encoded, including quotes). Otherwise return -1. */ -static inline int -get_escaped_string_length(const char *start, size_t in_len_max, - int *chars_out) -{ - const char *cp, *end; - int chars = 0; - - if (*start != '\"') - return -1; - - cp = start+1; - end = start+in_len_max; - - /* Calculate length. */ - while (1) { - if (cp >= end) { - return -1; /* Too long. */ - } else if (*cp == '\\') { - if (++cp == end) - return -1; /* Can't escape EOS. */ - ++cp; - ++chars; - } else if (*cp == '\"') { - break; - } else { - ++cp; - ++chars; - } - } - if (chars_out) - *chars_out = chars; - return (int)(cp - start+1); -} - -/** Given a pointer to a string starting at start containing - * in_len_max characters, decode a string beginning with one double - * quote, containing any number of non-quote characters or characters escaped - * with a backslash, and ending with a final double quote. Place the resulting - * string (unquoted, unescaped) into a newly allocated string in *out; - * store its length in out_len. On success, return a pointer to the - * character immediately following the escaped string. On failure, return - * NULL. */ -const char * -decode_escaped_string(const char *start, size_t in_len_max, - char **out, size_t *out_len) -{ - const char *cp, *end; - char *outp; - int len, n_chars = 0; - - len = get_escaped_string_length(start, in_len_max, &n_chars); - if (len<0) - return NULL; - - end = start+len-1; /* Index of last quote. */ - tor_assert(*end == '\"'); - outp = *out = tor_malloc(len+1); - *out_len = n_chars; - - cp = start+1; - while (cp < end) { - if (*cp == '\\') - ++cp; - *outp++ = *cp++; - } - *outp = '\0'; - tor_assert((outp - *out) == (int)*out_len); - - return end+1; -} - /** Return a longname the node whose identity is id_digest. If * node_get_by_id() returns NULL, base 16 encoding of id_digest is * returned instead. diff --git a/src/feature/control/control_fmt.h b/src/feature/control/control_fmt.h index 08acf85181..8bbbaa95d0 100644 --- a/src/feature/control/control_fmt.h +++ b/src/feature/control/control_fmt.h @@ -25,8 +25,6 @@ char *circuit_describe_status_for_controller(origin_circuit_t *circ); size_t write_escaped_data(const char *data, size_t len, char **out); size_t read_escaped_data(const char *data, size_t len, char **out); -const char *decode_escaped_string(const char *start, size_t in_len_max, - char **out, size_t *out_len); void send_control_done(control_connection_t *conn); MOCK_DECL(const char *, node_describe_longname_by_id,(const char *id_digest)); diff --git a/src/lib/encoding/include.am b/src/lib/encoding/include.am index 83e9211b6f..8272e4e5fa 100644 --- a/src/lib/encoding/include.am +++ b/src/lib/encoding/include.am @@ -11,6 +11,7 @@ src_lib_libtor_encoding_a_SOURCES = \ src/lib/encoding/keyval.c \ src/lib/encoding/kvline.c \ src/lib/encoding/pem.c \ + src/lib/encoding/qstring.c \ src/lib/encoding/time_fmt.c src_lib_libtor_encoding_testing_a_SOURCES = \ @@ -25,4 +26,5 @@ noinst_HEADERS += \ src/lib/encoding/keyval.h \ src/lib/encoding/kvline.h \ src/lib/encoding/pem.h \ + src/lib/encoding/qstring.h \ src/lib/encoding/time_fmt.h diff --git a/src/lib/encoding/qstring.c b/src/lib/encoding/qstring.c new file mode 100644 index 0000000000..a92d28c706 --- /dev/null +++ b/src/lib/encoding/qstring.c @@ -0,0 +1,90 @@ +/* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file qstring.c + * \brief Implement QuotedString parsing. + * + * Note that this is only used for controller authentication; do not + * create new users for this. Instead, prefer the cstring.c functions. + **/ + +#include "orconfig.h" +#include "lib/encoding/qstring.h" +#include "lib/malloc/malloc.h" +#include "lib/log/util_bug.h" + +/** If the first in_len_max characters in start contain a + * QuotedString, return the length of that + * string (as encoded, including quotes). Otherwise return -1. */ +static inline int +get_qstring_length(const char *start, size_t in_len_max, + int *chars_out) +{ + const char *cp, *end; + int chars = 0; + + if (*start != '\"') + return -1; + + cp = start+1; + end = start+in_len_max; + + /* Calculate length. */ + while (1) { + if (cp >= end) { + return -1; /* Too long. */ + } else if (*cp == '\\') { + if (++cp == end) + return -1; /* Can't escape EOS. */ + ++cp; + ++chars; + } else if (*cp == '\"') { + break; + } else { + ++cp; + ++chars; + } + } + if (chars_out) + *chars_out = chars; + return (int)(cp - start+1); +} + +/** Given a pointer to a string starting at start containing + * in_len_max characters, decode a string beginning with one double + * quote, containing any number of non-quote characters or characters escaped + * with a backslash, and ending with a final double quote. Place the resulting + * string (unquoted, unescaped) into a newly allocated string in *out; + * store its length in out_len. On success, return a pointer to the + * character immediately following the escaped string. On failure, return + * NULL. */ +const char * +decode_qstring(const char *start, size_t in_len_max, + char **out, size_t *out_len) +{ + const char *cp, *end; + char *outp; + int len, n_chars = 0; + + len = get_qstring_length(start, in_len_max, &n_chars); + if (len<0) + return NULL; + + end = start+len-1; /* Index of last quote. */ + tor_assert(*end == '\"'); + outp = *out = tor_malloc(len+1); + *out_len = n_chars; + + cp = start+1; + while (cp < end) { + if (*cp == '\\') + ++cp; + *outp++ = *cp++; + } + *outp = '\0'; + tor_assert((outp - *out) == (int)*out_len); + + return end+1; +} diff --git a/src/lib/encoding/qstring.h b/src/lib/encoding/qstring.h new file mode 100644 index 0000000000..fe15b655f1 --- /dev/null +++ b/src/lib/encoding/qstring.h @@ -0,0 +1,18 @@ +/* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file qstring.h + * \brief Header for qstring.c + */ + +#ifndef TOR_ENCODING_QSTRING_H +#define TOR_ENCODING_QSTRING_H + +#include + +const char *decode_qstring(const char *start, size_t in_len_max, + char **out, size_t *out_len); + +#endif From 8799b4e805ed5495409b6036b82d08e4624bacd3 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 9 Apr 2019 09:49:03 -0400 Subject: [PATCH 0853/2557] Add rudimentary qstring support to kvline.c --- src/lib/encoding/kvline.c | 26 +++++++++++++++++++++----- src/lib/encoding/kvline.h | 1 + 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/src/lib/encoding/kvline.c b/src/lib/encoding/kvline.c index 806f9d3db0..d4a8f510ba 100644 --- a/src/lib/encoding/kvline.c +++ b/src/lib/encoding/kvline.c @@ -16,6 +16,7 @@ #include "lib/encoding/confline.h" #include "lib/encoding/cstring.h" #include "lib/encoding/kvline.h" +#include "lib/encoding/qstring.h" #include "lib/malloc/malloc.h" #include "lib/string/compat_ctype.h" #include "lib/string/printf.h" @@ -111,11 +112,15 @@ kvline_can_encode_lines(const config_line_t *line, unsigned flags) * If KV_OMIT_VALS is set in flags, then an empty value is * encoded as 'Key', not as 'Key=' or 'Key=""'. Mutually exclusive with * KV_OMIT_KEYS. + * + * KV_QUOTED_QSTRING is not supported. */ char * kvline_encode(const config_line_t *line, unsigned flags) { + tor_assert(! (flags & KV_QUOTED_QSTRING)); + if (!kvline_can_encode_lines(line, flags)) return NULL; @@ -170,7 +175,7 @@ kvline_encode(const config_line_t *line, * allocated list of pairs on success, or NULL on failure. * * If KV_QUOTED is set in flags, then (double-)quoted values are - * allowed. Otherwise, such values are not allowed. + * allowed and handled as C strings. Otherwise, such values are not allowed. * * If KV_OMIT_KEYS is set in flags, then values without keys are * allowed. Otherwise, such values are not allowed. @@ -178,6 +183,10 @@ kvline_encode(const config_line_t *line, * If KV_OMIT_VALS is set in flags, then keys without values are * allowed. Otherwise, such keys are not allowed. Mutually exclusive with * KV_OMIT_KEYS. + * + * If KV_QUOTED_QSTRING is set in flags, then double-quoted values + * are allowed and handled as QuotedStrings per qstring.c. Do not add + * new users of this flag. */ config_line_t * kvline_parse(const char *line, unsigned flags) @@ -188,7 +197,8 @@ kvline_parse(const char *line, unsigned flags) const char *cp = line, *cplast = NULL; const bool omit_keys = (flags & KV_OMIT_KEYS) != 0; const bool omit_vals = (flags & KV_OMIT_VALS) != 0; - const bool quoted = (flags & KV_QUOTED) != 0; + const bool quoted = (flags & (KV_QUOTED|KV_QUOTED_QSTRING)) != 0; + const bool c_quoted = (flags & (KV_QUOTED)) != 0; config_line_t *result = NULL; config_line_t **next_line = &result; @@ -236,7 +246,11 @@ kvline_parse(const char *line, unsigned flags) if (!quoted) goto err; size_t len=0; - cp = unescape_string(cp, &val, &len); + if (c_quoted) { + cp = unescape_string(cp, &val, &len); + } else { + cp = decode_qstring(cp, strlen(cp), &val, &len); + } if (cp == NULL || len != strlen(val)) { // The string contains a NUL or is badly coded. goto err; @@ -260,8 +274,10 @@ kvline_parse(const char *line, unsigned flags) key = val = NULL; } - if (!kvline_can_encode_lines(result, flags)) { - goto err; + if (! (flags & KV_QUOTED_QSTRING)) { + if (!kvline_can_encode_lines(result, flags)) { + goto err; + } } return result; diff --git a/src/lib/encoding/kvline.h b/src/lib/encoding/kvline.h index 6740f81d54..dea2ce1809 100644 --- a/src/lib/encoding/kvline.h +++ b/src/lib/encoding/kvline.h @@ -18,6 +18,7 @@ struct config_line_t; #define KV_QUOTED (1u<<0) #define KV_OMIT_KEYS (1u<<1) #define KV_OMIT_VALS (1u<<2) +#define KV_QUOTED_QSTRING (1u<<3) struct config_line_t *kvline_parse(const char *line, unsigned flags); char *kvline_encode(const struct config_line_t *line, unsigned flags); From ddd33d39c7e115045d296dd88fd4d310b932a4e1 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 9 Apr 2019 10:09:38 -0400 Subject: [PATCH 0854/2557] Port the authenticate and authchallenge commands to the new parser These two presented their own challenge, because of their use of QString, and their distinguished handling of quoted versus non-quoted values. --- src/feature/control/control_auth.c | 143 ++++++++++++---------- src/feature/control/control_auth.h | 13 +- src/feature/control/control_cmd.c | 9 +- src/feature/control/control_cmd.h | 7 ++ src/feature/control/control_cmd_args_st.h | 4 + 5 files changed, 102 insertions(+), 74 deletions(-) diff --git a/src/feature/control/control_auth.c b/src/feature/control/control_auth.c index 8204290489..a86442c21f 100644 --- a/src/feature/control/control_auth.c +++ b/src/feature/control/control_auth.c @@ -11,12 +11,15 @@ #include "app/config/config.h" #include "core/mainloop/connection.h" #include "feature/control/control.h" +#include "feature/control/control_cmd.h" #include "feature/control/control_auth.h" +#include "feature/control/control_cmd_args_st.h" #include "feature/control/control_connection_st.h" #include "feature/control/control_fmt.h" #include "lib/crypt_ops/crypto_rand.h" #include "lib/crypt_ops/crypto_util.h" #include "lib/encoding/confline.h" +#include "lib/encoding/kvline.h" #include "lib/encoding/qstring.h" #include "lib/crypt_ops/crypto_s2k.h" @@ -117,12 +120,19 @@ decode_hashed_passwords(config_line_t *passwords) return NULL; } +const control_cmd_syntax_t authchallenge_syntax = { + .min_args = 1, + .max_args = 1, + .accept_keywords=true, + .kvline_flags=KV_OMIT_KEYS|KV_QUOTED_QSTRING, + .store_raw_body=true +}; + /** Called when we get an AUTHCHALLENGE command. */ int -handle_control_authchallenge(control_connection_t *conn, uint32_t len, - const char *body) +handle_control_authchallenge(control_connection_t *conn, + const control_cmd_args_t *args) { - const char *cp = body; char *client_nonce; size_t client_nonce_len; char server_hash[DIGEST256_LEN]; @@ -130,63 +140,50 @@ handle_control_authchallenge(control_connection_t *conn, uint32_t len, char server_nonce[SAFECOOKIE_SERVER_NONCE_LEN]; char server_nonce_encoded[(2*SAFECOOKIE_SERVER_NONCE_LEN) + 1]; - cp += strspn(cp, " \t\n\r"); - if (!strcasecmpstart(cp, "SAFECOOKIE")) { - cp += strlen("SAFECOOKIE"); - } else { + if (strcasecmp(smartlist_get(args->args, 0), "SAFECOOKIE")) { connection_write_str_to_buf("513 AUTHCHALLENGE only supports SAFECOOKIE " "authentication\r\n", conn); - connection_mark_for_close(TO_CONN(conn)); - return -1; + goto fail; } - if (!authentication_cookie_is_set) { connection_write_str_to_buf("515 Cookie authentication is disabled\r\n", conn); - connection_mark_for_close(TO_CONN(conn)); - return -1; + goto fail; + } + if (args->kwargs == NULL || args->kwargs->next != NULL) { + /* connection_write_str_to_buf("512 AUTHCHALLENGE requires exactly " + "2 arguments.\r\n", conn); + */ + connection_printf_to_buf(conn, + "512 AUTHCHALLENGE dislikes argument list %s\r\n", + escaped(args->raw_body)); + goto fail; + } + if (strcmp(args->kwargs->key, "")) { + connection_write_str_to_buf("512 AUTHCHALLENGE does not accept keyword " + "arguments.\r\n", conn); + goto fail; } - cp += strspn(cp, " \t\n\r"); - if (*cp == '"') { - const char *newcp = - decode_qstring(cp, len - (cp - body), - &client_nonce, &client_nonce_len); - if (newcp == NULL) { - connection_write_str_to_buf("513 Invalid quoted client nonce\r\n", - conn); - connection_mark_for_close(TO_CONN(conn)); - return -1; - } - cp = newcp; + bool contains_quote = strchr(args->raw_body, '\"'); + if (contains_quote) { + /* The nonce was quoted */ + client_nonce = tor_strdup(args->kwargs->value); + client_nonce_len = strlen(client_nonce); } else { - size_t client_nonce_encoded_len = strspn(cp, "0123456789ABCDEFabcdef"); - - client_nonce_len = client_nonce_encoded_len / 2; - client_nonce = tor_malloc_zero(client_nonce_len); - - if (base16_decode(client_nonce, client_nonce_len, - cp, client_nonce_encoded_len) - != (int) client_nonce_len) { + /* The nonce was should be in hex. */ + const char *hex_nonce = args->kwargs->value; + client_nonce_len = strlen(hex_nonce) / 2; + client_nonce = tor_malloc(client_nonce_len); + if (base16_decode(client_nonce, client_nonce_len, hex_nonce, + strlen(hex_nonce)) != (int)client_nonce_len) { connection_write_str_to_buf("513 Invalid base16 client nonce\r\n", conn); - connection_mark_for_close(TO_CONN(conn)); tor_free(client_nonce); - return -1; + goto fail; } - - cp += client_nonce_encoded_len; } - cp += strspn(cp, " \t\n\r"); - if (*cp != '\0' || - cp != body + len) { - connection_write_str_to_buf("513 Junk at end of AUTHCHALLENGE command\r\n", - conn); - connection_mark_for_close(TO_CONN(conn)); - tor_free(client_nonce); - return -1; - } crypto_rand(server_nonce, SAFECOOKIE_SERVER_NONCE_LEN); /* Now compute and send the server-to-controller response, and the @@ -234,38 +231,56 @@ handle_control_authchallenge(control_connection_t *conn, uint32_t len, tor_free(client_nonce); return 0; + fail: + connection_mark_for_close(TO_CONN(conn)); + return -1; } +const control_cmd_syntax_t authenticate_syntax = { + .max_args = 0, + .accept_keywords=true, + .kvline_flags=KV_OMIT_KEYS|KV_QUOTED_QSTRING, + .store_raw_body=true +}; + /** Called when we get an AUTHENTICATE message. Check whether the * authentication is valid, and if so, update the connection's state to * OPEN. Reply with DONE or ERROR. */ int -handle_control_authenticate(control_connection_t *conn, uint32_t len, - const char *body) +handle_control_authenticate(control_connection_t *conn, + const control_cmd_args_t *args) { - int used_quoted_string = 0; + bool used_quoted_string = false; const or_options_t *options = get_options(); const char *errstr = "Unknown error"; char *password; size_t password_len; - const char *cp; - int i; int bad_cookie=0, bad_password=0; smartlist_t *sl = NULL; - if (!len) { + if (args->kwargs == NULL) { password = tor_strdup(""); password_len = 0; - } else if (TOR_ISXDIGIT(body[0])) { - cp = body; - while (TOR_ISXDIGIT(*cp)) - ++cp; - i = (int)(cp - body); - tor_assert(i>0); - password_len = i/2; - password = tor_malloc(password_len + 1); - if (base16_decode(password, password_len+1, body, i) + } else if (args->kwargs->next) { + connection_write_str_to_buf( + "512 Too many arguments to AUTHENTICATE.\r\n", conn); + connection_mark_for_close(TO_CONN(conn)); + return 0; + } else if (strcmp(args->kwargs->key, "")) { + connection_write_str_to_buf( + "512 AUTHENTICATE does not accept keyword arguments.\r\n", conn); + connection_mark_for_close(TO_CONN(conn)); + return 0; + } else if (strchr(args->raw_body, '\"')) { + used_quoted_string = true; + password = tor_strdup(args->kwargs->value); + password_len = strlen(password); + } else { + const char *hex_passwd = args->kwargs->value; + password_len = strlen(hex_passwd) / 2; + password = tor_malloc(password_len+1); + if (base16_decode(password, password_len+1, hex_passwd, strlen(hex_passwd)) != (int) password_len) { connection_write_str_to_buf( "551 Invalid hexadecimal encoding. Maybe you tried a plain text " @@ -275,14 +290,6 @@ handle_control_authenticate(control_connection_t *conn, uint32_t len, tor_free(password); return 0; } - } else { - if (!decode_qstring(body, len, &password, &password_len)) { - connection_write_str_to_buf("551 Invalid quoted string. You need " - "to put the password in double quotes.\r\n", conn); - connection_mark_for_close(TO_CONN(conn)); - return 0; - } - used_quoted_string = 1; } if (conn->safecookie_client_hash != NULL) { diff --git a/src/feature/control/control_auth.h b/src/feature/control/control_auth.h index f436482e4a..246e18ccbc 100644 --- a/src/feature/control/control_auth.h +++ b/src/feature/control/control_auth.h @@ -12,16 +12,21 @@ #ifndef TOR_CONTROL_AUTH_H #define TOR_CONTROL_AUTH_H +struct control_cmd_args_t; +struct control_cmd_syntax_t; + int init_control_cookie_authentication(int enabled); char *get_controller_cookie_file_name(void); struct config_line_t; smartlist_t *decode_hashed_passwords(struct config_line_t *passwords); -int handle_control_authchallenge(control_connection_t *conn, uint32_t len, - const char *body); +int handle_control_authchallenge(control_connection_t *conn, + const struct control_cmd_args_t *args); int handle_control_authenticate(control_connection_t *conn, - uint32_t cmd_data_len, - const char *args); + const struct control_cmd_args_t *args); void control_auth_free_all(void); +extern const struct control_cmd_syntax_t authchallenge_syntax; +extern const struct control_cmd_syntax_t authenticate_syntax; + #endif /* !defined(TOR_CONTROL_AUTH_H) */ diff --git a/src/feature/control/control_cmd.c b/src/feature/control/control_cmd.c index 0b4f3555ff..a29ab72ffa 100644 --- a/src/feature/control/control_cmd.c +++ b/src/feature/control/control_cmd.c @@ -116,6 +116,11 @@ control_cmd_parse_args(const char *command, result->command = command; + if (syntax->store_raw_body) { + tor_assert(body[body_len] == 0); + result->raw_body = body; + } + const char *eol = memchr(body, '\n', body_len); if (syntax->want_object) { if (! eol || (eol+1) == body+body_len) { @@ -2309,7 +2314,7 @@ static const control_cmd_def_t CONTROL_COMMANDS[] = ONE_LINE_PARSED(getconf, 0), MULTLINE_PARSED(loadconf, 0), ONE_LINE_PARSED(setevents, 0), - ONE_LINE(authenticate, legacy, CMD_FL_WIPE), + ONE_LINE_PARSED(authenticate, CMD_FL_WIPE), ONE_LINE_PARSED(saveconf, 0), ONE_LINE_PARSED(signal, 0), ONE_LINE_PARSED(takeownership, 0), @@ -2327,7 +2332,7 @@ static const control_cmd_def_t CONTROL_COMMANDS[] = ONE_LINE_PARSED(usefeature, 0), ONE_LINE_PARSED(resolve, 0), ONE_LINE_PARSED(protocolinfo, 0), - ONE_LINE(authchallenge, legacy, CMD_FL_WIPE), + ONE_LINE_PARSED(authchallenge, CMD_FL_WIPE), ONE_LINE_PARSED(dropguards, 0), ONE_LINE_PARSED(hsfetch, 0), MULTLINE_PARSED(hspost, 0), diff --git a/src/feature/control/control_cmd.h b/src/feature/control/control_cmd.h index 6f35c74de0..b825d6da5c 100644 --- a/src/feature/control/control_cmd.h +++ b/src/feature/control/control_cmd.h @@ -64,6 +64,13 @@ typedef struct control_cmd_syntax_t { * True iff this command wants to be followed by a multiline object. **/ bool want_object; + /** + * True iff this command needs access to the raw body of the input. + * + * This should not be needed for pure commands; it is purely a legacy + * option. + **/ + bool store_raw_body; } control_cmd_syntax_t; #ifdef CONTROL_CMD_PRIVATE diff --git a/src/feature/control/control_cmd_args_st.h b/src/feature/control/control_cmd_args_st.h index 06e2a183ba..fb4fe64a08 100644 --- a/src/feature/control/control_cmd_args_st.h +++ b/src/feature/control/control_cmd_args_st.h @@ -43,6 +43,10 @@ struct control_cmd_args_t { * A multiline object passed with this command. **/ char *object; + /** + * If set, a nul-terminated string containing the raw unparsed arguments. + **/ + const char *raw_body; }; #endif /* !defined(TOR_CONTROL_CMD_ST_H) */ From 88d22b898efa5bfce82c614e454874d8807e2104 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 9 Apr 2019 10:58:42 -0400 Subject: [PATCH 0855/2557] Simplify handler logic in control_cmd.c Now that the legacy handlers are gone, we can simplify the structures and macros here. --- src/feature/control/control_cmd.c | 188 +++++++++++------------------- 1 file changed, 66 insertions(+), 122 deletions(-) diff --git a/src/feature/control/control_cmd.c b/src/feature/control/control_cmd.c index a29ab72ffa..c3863c6461 100644 --- a/src/feature/control/control_cmd.c +++ b/src/feature/control/control_cmd.c @@ -2171,16 +2171,18 @@ handle_control_del_onion(control_connection_t *conn, return 0; } +static const control_cmd_syntax_t obsolete_syntax = { + .max_args = UINT_MAX +}; + /** * Called when we get an obsolete command: tell the controller that it is * obsolete. */ static int handle_control_obsolete(control_connection_t *conn, - uint32_t arg_len, - const char *args) + const control_cmd_args_t *args) { - (void)arg_len; (void)args; char *command = tor_strdup(conn->current_cmd); tor_strupper(command); @@ -2190,37 +2192,10 @@ handle_control_obsolete(control_connection_t *conn, } /** - * Selects an API to a controller command. See handler_fn_t for the - * possible types. + * Function pointer to a handler function for a controller command. **/ -typedef enum handler_type_t { - hnd_legacy, - hnd_parsed, -} handler_type_t; - -/** - * Union: a function pointer to a handler function for a controller command. - * - * This needs to be a union (rather than just a single pointer) since not - * all controller commands have the same type. - **/ -typedef union handler_fn_t { - /** - * A "legacy" handler takes a command's arguments as a nul-terminated - * string, and their length. It may not change the contents of the - * arguments. If the command is a multiline one, then the arguments may - * extend across multiple lines. - */ - int (*legacy)(control_connection_t *conn, - uint32_t arg_len, - const char *args); - /** - * A "parsed" handler expects its arguments in a pre-parsed format, in - * an immutable control_cmd_args_t *object. - **/ - int (*parsed)(control_connection_t *conn, - const control_cmd_args_t *args); -} handler_fn_t; +typedef int (*handler_fn_t) (control_connection_t *conn, + const control_cmd_args_t *args); /** * Definition for a controller command. @@ -2230,10 +2205,6 @@ typedef struct control_cmd_def_t { * The name of the command. If the command is multiline, the name must * begin with "+". This is not case-sensitive. */ const char *name; - /** - * Which API to use when calling the handler function. - */ - handler_type_t handler_type; /** * A function to execute the command. */ @@ -2254,54 +2225,37 @@ typedef struct control_cmd_def_t { */ #define CMD_FL_WIPE (1u<<0) -#define SYNTAX_IGNORE { 0, UINT_MAX, false } - /** Macro: declare a command with a one-line argument, a given set of flags, * and a syntax definition. **/ -#define ONE_LINE_(name, htype, flags, syntax) \ - { #name, \ - hnd_ ##htype, \ - { .htype = handle_control_ ##name }, \ +#define ONE_LINE(name, flags) \ + { \ + #name, \ + handle_control_ ##name, \ flags, \ - syntax, \ - } - -/** Macro: declare a parsed command with a one-line argument, a given set of - * flags, and a syntax definition. - **/ -#define ONE_LINE(name, htype, flags) \ - ONE_LINE_(name, htype, flags, NULL) -#define ONE_LINE_PARSED(name, flags) \ - ONE_LINE_(name, parsed, flags, &name ##_syntax) + &name##_syntax, \ + } /** * Macro: declare a command with a multi-line argument and a given set of * flags. **/ -#define MULTLINE_(name, htype, flags, syntax) \ +#define MULTLINE(name, flags) \ { "+"#name, \ - hnd_ ##htype, \ - { .htype = handle_control_ ##name }, \ + handle_control_ ##name, \ flags, \ - syntax \ + &name##_syntax \ } -#define MULTLINE(name, htype, flags) \ - MULTLINE_(name, htype, flags, NULL) -#define MULTLINE_PARSED(name, flags) \ - MULTLINE_(name, parsed, flags, &name##_syntax) - /** * Macro: declare an obsolete command. (Obsolete commands give a different * error than non-existent ones.) **/ #define OBSOLETE(name) \ { #name, \ - hnd_legacy, \ - { .legacy = handle_control_obsolete }, \ + handle_control_obsolete, \ 0, \ - NULL, \ + &obsolete_syntax, \ } /** @@ -2309,35 +2263,35 @@ typedef struct control_cmd_def_t { **/ static const control_cmd_def_t CONTROL_COMMANDS[] = { - ONE_LINE_PARSED(setconf, 0), - ONE_LINE_PARSED(resetconf, 0), - ONE_LINE_PARSED(getconf, 0), - MULTLINE_PARSED(loadconf, 0), - ONE_LINE_PARSED(setevents, 0), - ONE_LINE_PARSED(authenticate, CMD_FL_WIPE), - ONE_LINE_PARSED(saveconf, 0), - ONE_LINE_PARSED(signal, 0), - ONE_LINE_PARSED(takeownership, 0), - ONE_LINE_PARSED(dropownership, 0), - ONE_LINE_PARSED(mapaddress, 0), - ONE_LINE_PARSED(getinfo, 0), - ONE_LINE_PARSED(extendcircuit, 0), - ONE_LINE_PARSED(setcircuitpurpose, 0), + ONE_LINE(setconf, 0), + ONE_LINE(resetconf, 0), + ONE_LINE(getconf, 0), + MULTLINE(loadconf, 0), + ONE_LINE(setevents, 0), + ONE_LINE(authenticate, CMD_FL_WIPE), + ONE_LINE(saveconf, 0), + ONE_LINE(signal, 0), + ONE_LINE(takeownership, 0), + ONE_LINE(dropownership, 0), + ONE_LINE(mapaddress, 0), + ONE_LINE(getinfo, 0), + ONE_LINE(extendcircuit, 0), + ONE_LINE(setcircuitpurpose, 0), OBSOLETE(setrouterpurpose), - ONE_LINE_PARSED(attachstream, 0), - MULTLINE_PARSED(postdescriptor, 0), - ONE_LINE_PARSED(redirectstream, 0), - ONE_LINE_PARSED(closestream, 0), - ONE_LINE_PARSED(closecircuit, 0), - ONE_LINE_PARSED(usefeature, 0), - ONE_LINE_PARSED(resolve, 0), - ONE_LINE_PARSED(protocolinfo, 0), - ONE_LINE_PARSED(authchallenge, CMD_FL_WIPE), - ONE_LINE_PARSED(dropguards, 0), - ONE_LINE_PARSED(hsfetch, 0), - MULTLINE_PARSED(hspost, 0), - ONE_LINE_PARSED(add_onion, CMD_FL_WIPE), - ONE_LINE_PARSED(del_onion, CMD_FL_WIPE), + ONE_LINE(attachstream, 0), + MULTLINE(postdescriptor, 0), + ONE_LINE(redirectstream, 0), + ONE_LINE(closestream, 0), + ONE_LINE(closecircuit, 0), + ONE_LINE(usefeature, 0), + ONE_LINE(resolve, 0), + ONE_LINE(protocolinfo, 0), + ONE_LINE(authchallenge, CMD_FL_WIPE), + ONE_LINE(dropguards, 0), + ONE_LINE(hsfetch, 0), + MULTLINE(hspost, 0), + ONE_LINE(add_onion, CMD_FL_WIPE), + ONE_LINE(del_onion, CMD_FL_WIPE), }; /** @@ -2356,35 +2310,25 @@ handle_single_control_command(const control_cmd_def_t *def, char *args) { int rv = 0; - switch (def->handler_type) { - case hnd_legacy: - if (def->handler.legacy(conn, cmd_data_len, args)) - rv = -1; - break; - case hnd_parsed: { - control_cmd_args_t *parsed_args; - char *err=NULL; - tor_assert(def->syntax); - parsed_args = control_cmd_parse_args(conn->current_cmd, - def->syntax, - cmd_data_len, args, - &err); - if (!parsed_args) { - connection_printf_to_buf(conn, - "512 Bad arguments to %s: %s\r\n", - conn->current_cmd, err?err:""); - tor_free(err); - } else { - if (BUG(err)) - tor_free(err); - if (def->handler.parsed(conn, parsed_args)) - rv = 0; - control_cmd_args_free(parsed_args); - } - break; - } - default: - tor_assert_unreached(); + + control_cmd_args_t *parsed_args; + char *err=NULL; + tor_assert(def->syntax); + parsed_args = control_cmd_parse_args(conn->current_cmd, + def->syntax, + cmd_data_len, args, + &err); + if (!parsed_args) { + connection_printf_to_buf(conn, + "512 Bad arguments to %s: %s\r\n", + conn->current_cmd, err?err:""); + tor_free(err); + } else { + if (BUG(err)) + tor_free(err); + if (def->handler(conn, parsed_args)) + rv = 0; + control_cmd_args_free(parsed_args); } if (def->flags & CMD_FL_WIPE) From ff9ba7d6c4ba7072b9a96d527959350ef90406b3 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 9 Apr 2019 11:10:26 -0400 Subject: [PATCH 0856/2557] expand CMD_FL_WIPE to wipe the parsed arguments too --- src/feature/control/control_cmd.c | 22 ++++++++++++++++++++++ src/feature/control/control_cmd.h | 1 + 2 files changed, 23 insertions(+) diff --git a/src/feature/control/control_cmd.c b/src/feature/control/control_cmd.c index c3863c6461..7a9af19cd0 100644 --- a/src/feature/control/control_cmd.c +++ b/src/feature/control/control_cmd.c @@ -81,6 +81,24 @@ control_cmd_args_free_(control_cmd_args_t *args) tor_free(args); } +/** Erase all memory held in args. */ +void +control_cmd_args_wipe(control_cmd_args_t *args) +{ + if (!args) + return; + + if (args->args) { + SMARTLIST_FOREACH(args->args, char *, c, memwipe(c, 0, strlen(c))); + } + for (config_line_t *line = args->kwargs; line; line = line->next) { + memwipe(line->key, 0, strlen(line->key)); + memwipe(line->value, 0, strlen(line->value)); + } + if (args->object) + memwipe(args->object, 0, args->object_len); +} + /** * Return true iff any element of the NULL-terminated array matches * kwd. Case-insensitive. @@ -2328,6 +2346,10 @@ handle_single_control_command(const control_cmd_def_t *def, tor_free(err); if (def->handler(conn, parsed_args)) rv = 0; + + if (def->flags & CMD_FL_WIPE) + control_cmd_args_wipe(parsed_args); + control_cmd_args_free(parsed_args); } diff --git a/src/feature/control/control_cmd.h b/src/feature/control/control_cmd.h index b825d6da5c..986718887c 100644 --- a/src/feature/control/control_cmd.h +++ b/src/feature/control/control_cmd.h @@ -21,6 +21,7 @@ void control_cmd_free_all(void); typedef struct control_cmd_args_t control_cmd_args_t; void control_cmd_args_free_(control_cmd_args_t *args); +void control_cmd_args_wipe(control_cmd_args_t *args); #define control_cmd_args_free(v) \ FREE_AND_NULL(control_cmd_args_t, control_cmd_args_free_, (v)) From 3ed7ceeb856b3c1fbe5edf3b8c6ffe3f5e875622 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 9 Apr 2019 11:18:18 -0400 Subject: [PATCH 0857/2557] changes file for ticket 30091 (controller parsing refactor) --- changes/ticket30091 | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 changes/ticket30091 diff --git a/changes/ticket30091 b/changes/ticket30091 new file mode 100644 index 0000000000..968ea01f4a --- /dev/null +++ b/changes/ticket30091 @@ -0,0 +1,4 @@ + o Major features (controller protocol): + - Controller commands are now parsed using a generalized parsing + subsystem. Previously, each controller command was responsible for + parsing its own input. Closes ticket 30091. From a0299cd240292d93b0f465bf1845a5e54889f61b Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 24 Apr 2019 11:32:14 -0400 Subject: [PATCH 0858/2557] In control command api, rename "object" to "cmddata" This makes it match control-spec.txt. --- src/feature/control/control_cmd.c | 30 +++++++++++------------ src/feature/control/control_cmd.h | 2 +- src/feature/control/control_cmd_args_st.h | 6 ++--- src/test/test_controller.c | 6 ++--- 4 files changed, 22 insertions(+), 22 deletions(-) diff --git a/src/feature/control/control_cmd.c b/src/feature/control/control_cmd.c index 7a9af19cd0..da9a95fc5d 100644 --- a/src/feature/control/control_cmd.c +++ b/src/feature/control/control_cmd.c @@ -76,7 +76,7 @@ control_cmd_args_free_(control_cmd_args_t *args) smartlist_free(args->args); } config_free_lines(args->kwargs); - tor_free(args->object); + tor_free(args->cmddata); tor_free(args); } @@ -95,8 +95,8 @@ control_cmd_args_wipe(control_cmd_args_t *args) memwipe(line->key, 0, strlen(line->key)); memwipe(line->value, 0, strlen(line->value)); } - if (args->object) - memwipe(args->object, 0, args->object_len); + if (args->cmddata) + memwipe(args->cmddata, 0, args->cmddata_len); } /** @@ -140,7 +140,7 @@ control_cmd_parse_args(const char *command, } const char *eol = memchr(body, '\n', body_len); - if (syntax->want_object) { + if (syntax->want_cmddata) { if (! eol || (eol+1) == body+body_len) { *error_out = tor_strdup("Empty body"); goto err; @@ -148,8 +148,8 @@ control_cmd_parse_args(const char *command, cmdline_alloc = tor_memdup_nulterm(body, eol-body); cmdline = cmdline_alloc; ++eol; - result->object_len = read_escaped_data(eol, (body+body_len)-eol, - &result->object); + result->cmddata_len = read_escaped_data(eol, (body+body_len)-eol, + &result->cmddata); } else { if (eol && (eol+1) != body+body_len) { *error_out = tor_strdup("Unexpected body"); @@ -320,7 +320,7 @@ handle_control_getconf(control_connection_t *conn, } static const control_cmd_syntax_t loadconf_syntax = { - .want_object = true + .want_cmddata = true }; /** Called when we get a +LOADCONF message. */ @@ -332,7 +332,7 @@ handle_control_loadconf(control_connection_t *conn, char *errstring = NULL; const char *msg = NULL; - retval = options_init_from_string(NULL, args->object, + retval = options_init_from_string(NULL, args->cmddata, CMD_RUN_TOR, NULL, &errstring); if (retval != SETOPT_OK) @@ -1013,7 +1013,7 @@ static const control_cmd_syntax_t postdescriptor_syntax = { .max_args = 0, .accept_keywords = true, .allowed_keywords = postdescriptor_keywords, - .want_object = true, + .want_cmddata = true, }; /** Called when we get a POSTDESCRIPTOR message. Try to learn the provided @@ -1047,7 +1047,7 @@ handle_control_postdescriptor(control_connection_t *conn, } } - switch (router_load_single_router(args->object, purpose, cache, &msg)) { + switch (router_load_single_router(args->cmddata, purpose, cache, &msg)) { case -1: if (!msg) msg = "Could not parse descriptor"; connection_printf_to_buf(conn, "554 %s\r\n", msg); @@ -1366,7 +1366,7 @@ static const control_cmd_syntax_t hsfetch_syntax = { .min_args = 1, .max_args = 1, .accept_keywords = true, .allowed_keywords = hsfetch_keywords, - .want_object = true, + .want_cmddata = true, }; /** Implementation for the HSFETCH command. */ @@ -1420,7 +1420,7 @@ handle_control_hsfetch(control_connection_t *conn, goto done; } if (!hsdirs) { - /* Stores routerstatus_t object for each specified server. */ + /* Stores routerstatus_t cmddata for each specified server. */ hsdirs = smartlist_new(); } /* Valid server, add it to our local list. */ @@ -1473,7 +1473,7 @@ static const char *hspost_keywords[] = { static const control_cmd_syntax_t hspost_syntax = { .min_args = 0, .max_args = 0, .accept_keywords = true, - .want_object = true, + .want_cmddata = true, .allowed_keywords = hspost_keywords }; @@ -1483,8 +1483,8 @@ handle_control_hspost(control_connection_t *conn, const control_cmd_args_t *args) { smartlist_t *hs_dirs = NULL; - const char *encoded_desc = args->object; - size_t encoded_desc_len = args->object_len; + const char *encoded_desc = args->cmddata; + size_t encoded_desc_len = args->cmddata_len; const char *onion_address = NULL; const config_line_t *line; diff --git a/src/feature/control/control_cmd.h b/src/feature/control/control_cmd.h index 986718887c..5c3d1a1cec 100644 --- a/src/feature/control/control_cmd.h +++ b/src/feature/control/control_cmd.h @@ -64,7 +64,7 @@ typedef struct control_cmd_syntax_t { /** * True iff this command wants to be followed by a multiline object. **/ - bool want_object; + bool want_cmddata; /** * True iff this command needs access to the raw body of the input. * diff --git a/src/feature/control/control_cmd_args_st.h b/src/feature/control/control_cmd_args_st.h index fb4fe64a08..8d7a4f55b3 100644 --- a/src/feature/control/control_cmd_args_st.h +++ b/src/feature/control/control_cmd_args_st.h @@ -36,13 +36,13 @@ struct control_cmd_args_t { **/ struct config_line_t *kwargs; /** - * Number of bytes in object; 0 if object is not set. + * Number of bytes in cmddata; 0 if cmddata is not set. **/ - size_t object_len; + size_t cmddata_len; /** * A multiline object passed with this command. **/ - char *object; + char *cmddata; /** * If set, a nul-terminated string containing the raw unparsed arguments. **/ diff --git a/src/test/test_controller.c b/src/test/test_controller.c index fd4f26f086..c130248592 100644 --- a/src/test/test_controller.c +++ b/src/test/test_controller.c @@ -56,9 +56,9 @@ control_cmd_dump_args(const control_cmd_args_t *result) } SMARTLIST_FOREACH_END(s); } buf_add_string(buf, "]"); - if (result->object) { + if (result->cmddata) { buf_add_string(buf, ", obj="); - buf_add_string(buf, escaped(result->object)); + buf_add_string(buf, escaped(result->cmddata)); } if (result->kwargs) { buf_add_string(buf, ", { "); @@ -160,7 +160,7 @@ static const parser_testcase_t no_args_one_obj_tests[] = { }; static const control_cmd_syntax_t no_args_one_obj_syntax = { .min_args=0, .max_args=0, - .want_object=true, + .want_cmddata=true, }; static const parse_test_params_t parse_no_args_one_obj_params = TESTPARAMS( no_args_one_obj_syntax, no_args_one_obj_tests ); From a5cced2b7acf4fb1818d6cd84c09a9ffe4dd1e89 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 24 Apr 2019 11:42:10 -0400 Subject: [PATCH 0859/2557] Extract keyword argument checking from argument parsing. --- src/feature/control/control_cmd.c | 53 +++++++++++++++++++++++-------- 1 file changed, 39 insertions(+), 14 deletions(-) diff --git a/src/feature/control/control_cmd.c b/src/feature/control/control_cmd.c index da9a95fc5d..9afa734d86 100644 --- a/src/feature/control/control_cmd.c +++ b/src/feature/control/control_cmd.c @@ -113,6 +113,41 @@ string_array_contains_keyword(const char **array, const char *kwd) return false; } +/** Helper for argument parsing: check whether the keyword arguments just + * parsed in result were well-formed according to syntax. + * + * On success, return 0. On failure, return -1 and set *error_out + * to a newly allocated error string. + **/ +static int +kvline_check_keyword_args(const control_cmd_args_t *result, + const control_cmd_syntax_t *syntax, + char **error_out) +{ + if (result->kwargs == NULL) { + tor_asprintf(error_out, "Cannot parse keyword argument(s)"); + return -1; + } + + if (! syntax->allowed_keywords) { + /* All keywords are permitted. */ + return 0; + } + + /* Check for unpermitted arguments */ + const config_line_t *line; + for (line = result->kwargs; line; line = line->next) { + if (! string_array_contains_keyword(syntax->allowed_keywords, + line->key)) { + tor_asprintf(error_out, "Unrecognized keyword argument %s", + escaped(line->key)); + return -1; + } + } + + return 0; +} + /** * Helper: parse the arguments to a command according to syntax. On * success, set *error_out to NULL and return a newly allocated @@ -174,27 +209,17 @@ control_cmd_parse_args(const char *command, } if (n_args > syntax->max_args) { + /* We have extra arguments after the positional arguments, and we didn't + treat them as an error, so they must count as keyword arguments: Either + K=V pairs, or flags, or both. */ tor_assert(n_args == syntax->max_args + 1); tor_assert(syntax->accept_keywords); char *remainder = smartlist_pop_last(result->args); result->kwargs = kvline_parse(remainder, syntax->kvline_flags); tor_free(remainder); - if (result->kwargs == NULL) { - tor_asprintf(error_out, "Cannot parse keyword argument(s)"); + if (kvline_check_keyword_args(result, syntax, error_out) < 0) { goto err; } - if (syntax->allowed_keywords) { - /* Check for unpermitted arguments */ - const config_line_t *line; - for (line = result->kwargs; line; line = line->next) { - if (! string_array_contains_keyword(syntax->allowed_keywords, - line->key)) { - tor_asprintf(error_out, "Unrecognized keyword argument %s", - escaped(line->key)); - goto err; - } - } - } } tor_assert_nonfatal(*error_out == NULL); From 1d44ac9acd6264141615b5fce6d537544dc6f52e Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 26 Apr 2019 10:36:49 -0400 Subject: [PATCH 0860/2557] Make nodelist_get_list() return a const pointer. --- scripts/maint/practracker/exceptions.txt | 2 +- src/core/or/circuitbuild.c | 5 +++-- src/core/or/circuitbuild.h | 4 ++-- src/feature/dirauth/voteflags.c | 2 +- src/feature/nodelist/nodelist.c | 4 ++-- src/feature/nodelist/nodelist.h | 2 +- src/feature/nodelist/routerset.c | 2 +- src/test/test_circuitbuild.c | 2 +- src/test/test_controller.c | 2 +- src/test/test_entrynodes.c | 2 +- src/test/test_helpers.c | 2 +- src/test/test_hs_common.c | 2 +- src/test/test_routerset.c | 8 ++++---- 13 files changed, 20 insertions(+), 19 deletions(-) diff --git a/scripts/maint/practracker/exceptions.txt b/scripts/maint/practracker/exceptions.txt index 37f11bb440..447284853b 100644 --- a/scripts/maint/practracker/exceptions.txt +++ b/scripts/maint/practracker/exceptions.txt @@ -78,7 +78,7 @@ problem function-size /src/core/or/channeltls.c:channel_tls_process_versions_cel problem function-size /src/core/or/channeltls.c:channel_tls_process_netinfo_cell() 214 problem function-size /src/core/or/channeltls.c:channel_tls_process_certs_cell() 246 problem function-size /src/core/or/channeltls.c:channel_tls_process_authenticate_cell() 202 -problem file-size /src/core/or/circuitbuild.c 3060 +problem file-size /src/core/or/circuitbuild.c 3061 problem include-count /src/core/or/circuitbuild.c 53 problem function-size /src/core/or/circuitbuild.c:get_unique_circ_id_by_chan() 128 problem function-size /src/core/or/circuitbuild.c:circuit_extend() 147 diff --git a/src/core/or/circuitbuild.c b/src/core/or/circuitbuild.c index f8e87bf026..cfe0a97bcf 100644 --- a/src/core/or/circuitbuild.c +++ b/src/core/or/circuitbuild.c @@ -1683,7 +1683,8 @@ route_len_for_purpose(uint8_t purpose, extend_info_t *exit_ei) * to handle the desired path length, return -1. */ STATIC int -new_route_len(uint8_t purpose, extend_info_t *exit_ei, smartlist_t *nodes) +new_route_len(uint8_t purpose, extend_info_t *exit_ei, + const smartlist_t *nodes) { int routelen; @@ -2345,7 +2346,7 @@ circuit_extend_to_new_exit(origin_circuit_t *circ, extend_info_t *exit_ei) * particular router. See bug #25885.) */ MOCK_IMPL(STATIC int, -count_acceptable_nodes, (smartlist_t *nodes, int direct)) +count_acceptable_nodes, (const smartlist_t *nodes, int direct)) { int num=0; diff --git a/src/core/or/circuitbuild.h b/src/core/or/circuitbuild.h index b19bc41235..b45bc816a3 100644 --- a/src/core/or/circuitbuild.h +++ b/src/core/or/circuitbuild.h @@ -83,8 +83,8 @@ void circuit_upgrade_circuits_from_guard_wait(void); #ifdef CIRCUITBUILD_PRIVATE STATIC circid_t get_unique_circ_id_by_chan(channel_t *chan); STATIC int new_route_len(uint8_t purpose, extend_info_t *exit_ei, - smartlist_t *nodes); -MOCK_DECL(STATIC int, count_acceptable_nodes, (smartlist_t *nodes, + const smartlist_t *nodes); +MOCK_DECL(STATIC int, count_acceptable_nodes, (const smartlist_t *nodes, int direct)); STATIC int onion_extend_cpath(origin_circuit_t *circ); diff --git a/src/feature/dirauth/voteflags.c b/src/feature/dirauth/voteflags.c index 4040f162fa..957ebe4a4f 100644 --- a/src/feature/dirauth/voteflags.c +++ b/src/feature/dirauth/voteflags.c @@ -239,7 +239,7 @@ dirserv_compute_performance_thresholds(digestmap_t *omit_as_sybil) uint32_t *uptimes, *bandwidths_kb, *bandwidths_excluding_exits_kb; long *tks; double *mtbfs, *wfus; - smartlist_t *nodelist; + const smartlist_t *nodelist; time_t now = time(NULL); const or_options_t *options = get_options(); diff --git a/src/feature/nodelist/nodelist.c b/src/feature/nodelist/nodelist.c index f878d47fd7..8aa4915107 100644 --- a/src/feature/nodelist/nodelist.c +++ b/src/feature/nodelist/nodelist.c @@ -944,7 +944,7 @@ nodelist_ensure_freshness(networkstatus_t *ns) /** Return a list of a node_t * for every node we know about. The caller * MUST NOT modify the list. (You can set and clear flags in the nodes if * you must, but you must not add or remove nodes.) */ -MOCK_IMPL(smartlist_t *, +MOCK_IMPL(const smartlist_t *, nodelist_get_list,(void)) { init_nodelist(); @@ -1939,7 +1939,7 @@ node_set_country(node_t *node) void nodelist_refresh_countries(void) { - smartlist_t *nodes = nodelist_get_list(); + const smartlist_t *nodes = nodelist_get_list(); SMARTLIST_FOREACH(nodes, node_t *, node, node_set_country(node)); } diff --git a/src/feature/nodelist/nodelist.h b/src/feature/nodelist/nodelist.h index a3d65347a8..84ab5f7a54 100644 --- a/src/feature/nodelist/nodelist.h +++ b/src/feature/nodelist/nodelist.h @@ -101,7 +101,7 @@ const struct curve25519_public_key_t *node_get_curve25519_onion_key( const node_t *node); crypto_pk_t *node_get_rsa_onion_key(const node_t *node); -MOCK_DECL(smartlist_t *, nodelist_get_list, (void)); +MOCK_DECL(const smartlist_t *, nodelist_get_list, (void)); /* Temporary during transition to multiple addresses. */ void node_get_addr(const node_t *node, tor_addr_t *addr_out); diff --git a/src/feature/nodelist/routerset.c b/src/feature/nodelist/routerset.c index 55e2756959..e801fd81b1 100644 --- a/src/feature/nodelist/routerset.c +++ b/src/feature/nodelist/routerset.c @@ -378,7 +378,7 @@ routerset_get_all_nodes(smartlist_t *out, const routerset_t *routerset, } else { /* We need to iterate over the routerlist to get all the ones of the * right kind. */ - smartlist_t *nodes = nodelist_get_list(); + const smartlist_t *nodes = nodelist_get_list(); SMARTLIST_FOREACH(nodes, const node_t *, node, { if (running_only && !node->is_running) continue; diff --git a/src/test/test_circuitbuild.c b/src/test/test_circuitbuild.c index 27f2cd1ca5..47218a559a 100644 --- a/src/test/test_circuitbuild.c +++ b/src/test/test_circuitbuild.c @@ -21,7 +21,7 @@ static smartlist_t dummy_nodes; static extend_info_t dummy_ei; static int -mock_count_acceptable_nodes(smartlist_t *nodes, int direct) +mock_count_acceptable_nodes(const smartlist_t *nodes, int direct) { (void)nodes; diff --git a/src/test/test_controller.c b/src/test/test_controller.c index f3af6d2ec0..00e2fb33cc 100644 --- a/src/test/test_controller.c +++ b/src/test/test_controller.c @@ -1546,7 +1546,7 @@ test_current_time(void *arg) static size_t n_nodelist_get_list = 0; static smartlist_t *nodes = NULL; -static smartlist_t * +static const smartlist_t * mock_nodelist_get_list(void) { n_nodelist_get_list++; diff --git a/src/test/test_entrynodes.c b/src/test/test_entrynodes.c index 729795b674..bdf057bb50 100644 --- a/src/test/test_entrynodes.c +++ b/src/test/test_entrynodes.c @@ -67,7 +67,7 @@ static networkstatus_t *dummy_consensus = NULL; static smartlist_t *big_fake_net_nodes = NULL; -static smartlist_t * +static const smartlist_t * bfn_mock_nodelist_get_list(void) { return big_fake_net_nodes; diff --git a/src/test/test_helpers.c b/src/test/test_helpers.c index 13de1e154b..489c257761 100644 --- a/src/test/test_helpers.c +++ b/src/test/test_helpers.c @@ -78,7 +78,7 @@ helper_setup_fake_routerlist(void) { int retval; routerlist_t *our_routerlist = NULL; - smartlist_t *our_nodelist = NULL; + const smartlist_t *our_nodelist = NULL; /* Read the file that contains our test descriptors. */ diff --git a/src/test/test_hs_common.c b/src/test/test_hs_common.c index eb7f3bfbb0..bb41f1f870 100644 --- a/src/test/test_hs_common.c +++ b/src/test/test_hs_common.c @@ -275,7 +275,7 @@ test_start_time_of_next_time_period(void *arg) static void cleanup_nodelist(void) { - smartlist_t *nodelist = nodelist_get_list(); + const smartlist_t *nodelist = nodelist_get_list(); SMARTLIST_FOREACH_BEGIN(nodelist, node_t *, node) { tor_free(node->md); node->md = NULL; diff --git a/src/test/test_routerset.c b/src/test/test_routerset.c index c45f0e1595..cc73e6c20a 100644 --- a/src/test/test_routerset.c +++ b/src/test/test_routerset.c @@ -1765,7 +1765,7 @@ NS(node_get_by_nickname)(const char *nickname, unsigned flags) * Structural test for routerset_get_all_nodes, when the nodelist has no nodes. */ -NS_DECL(smartlist_t *, nodelist_get_list, (void)); +NS_DECL(const smartlist_t *, nodelist_get_list, (void)); static smartlist_t *NS(mock_smartlist); @@ -1795,7 +1795,7 @@ NS(test_main)(void *arg) ; } -smartlist_t * +const smartlist_t * NS(nodelist_get_list)(void) { CALLED(nodelist_get_list)++; @@ -1811,7 +1811,7 @@ NS(nodelist_get_list)(void) * the running_only flag is set, but the nodes are not running. */ -NS_DECL(smartlist_t *, nodelist_get_list, (void)); +NS_DECL(const smartlist_t *, nodelist_get_list, (void)); static smartlist_t *NS(mock_smartlist); static node_t NS(mock_node); @@ -1844,7 +1844,7 @@ NS(test_main)(void *arg) ; } -smartlist_t * +const smartlist_t * NS(nodelist_get_list)(void) { CALLED(nodelist_get_list)++; From 650b94ebc157da01fcb34e08341ad1717c61f4fe Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 26 Apr 2019 11:03:22 -0400 Subject: [PATCH 0861/2557] Use a linear algorithm to subtract two nodelists. The nodelist_idx for each node_t serves as a unique identifier for the node, so we can use a bitarray to hold all the excluded nodes, and then remove them from the smartlist. Previously use used smartlist_subtract(sl, excluded), which is O(len(sl)*len(excluded)). We can use this function in other places too, but this is the one that showed up on the profiles of 30291. Closes ticket 30307. --- changes/ticket30307 | 4 ++ src/feature/nodelist/node_select.c | 74 ++++++++++++++++++++++++------ src/test/test_entrynodes.c | 1 + 3 files changed, 65 insertions(+), 14 deletions(-) create mode 100644 changes/ticket30307 diff --git a/changes/ticket30307 b/changes/ticket30307 new file mode 100644 index 0000000000..abcacb6085 --- /dev/null +++ b/changes/ticket30307 @@ -0,0 +1,4 @@ + o Major features (performance): + - Update our node selection algorithm to exclude nodes in linear time. + Previously, the algorithm was quadratic, which could slow down heavily + used onion services. Closes ticket 30307. diff --git a/src/feature/nodelist/node_select.c b/src/feature/nodelist/node_select.c index 93ddb066d4..719b4b1b27 100644 --- a/src/feature/nodelist/node_select.c +++ b/src/feature/nodelist/node_select.c @@ -30,6 +30,7 @@ #include "feature/nodelist/routerset.h" #include "feature/relay/router.h" #include "feature/relay/routermode.h" +#include "lib/container/bitarray.h" #include "lib/crypt_ops/crypto_rand.h" #include "lib/math/fp.h" @@ -826,6 +827,58 @@ routerlist_add_node_and_family(smartlist_t *sl, const routerinfo_t *router) nodelist_add_node_and_family(sl, node); } +/** + * Remove every node_t that appears in excluded from sl. + * + * Behaves like smartlist_subtract, but uses nodelist_idx values to deliver + * linear performance when smartlist_subtract would be quadratic. + **/ +static void +nodelist_subtract(smartlist_t *sl, const smartlist_t *excluded) +{ + const smartlist_t *nodelist = nodelist_get_list(); + const int nodelist_len = smartlist_len(nodelist); + bitarray_t *excluded_idx = bitarray_init_zero(nodelist_len); + + /* We haven't used nodelist_idx in this way previously, so I'm going to be + * paranoid in this code, and check that nodelist_idx is correct for every + * node before we use it. If we fail, we fall back to smartlist_subtract(). + */ + + /* Set the excluded_idx bit corresponding to every excluded node... + */ + SMARTLIST_FOREACH_BEGIN(excluded, const node_t *, node) { + const int idx = node->nodelist_idx; + if (BUG(idx < 0) || BUG(idx >= nodelist_len) || + BUG(node != smartlist_get(nodelist, idx))) { + goto internal_error; + } + bitarray_set(excluded_idx, idx); + } SMARTLIST_FOREACH_END(node); + + /* Then remove them from sl. + */ + SMARTLIST_FOREACH_BEGIN(sl, const node_t *, node) { + const int idx = node->nodelist_idx; + if (BUG(idx < 0) || BUG(idx >= nodelist_len) || + BUG(node != smartlist_get(nodelist, idx))) { + goto internal_error; + } + if (bitarray_is_set(excluded_idx, idx)) { + SMARTLIST_DEL_CURRENT(sl, node); + } + } SMARTLIST_FOREACH_END(node); + + bitarray_free(excluded_idx); + return; + + internal_error: + log_warn(LD_BUG, "Internal error prevented us from using the fast method " + "for subtracting nodelists. Falling back to the quadratic way."); + smartlist_subtract(sl, excluded); + bitarray_free(excluded_idx); +} + /** Return a random running node from the nodelist. Never * pick a node that is in * excludedsmartlist, or which matches excludedset, @@ -860,6 +913,7 @@ router_choose_random_node(smartlist_t *excludedsmartlist, const int direct_conn = (flags & CRN_DIRECT_CONN) != 0; const int rendezvous_v3 = (flags & CRN_RENDEZVOUS_V3) != 0; + const smartlist_t *node_list = nodelist_get_list(); smartlist_t *sl=smartlist_new(), *excludednodes=smartlist_new(); const node_t *choice = NULL; @@ -870,17 +924,17 @@ router_choose_random_node(smartlist_t *excludedsmartlist, rule = weight_for_exit ? WEIGHT_FOR_EXIT : (need_guard ? WEIGHT_FOR_GUARD : WEIGHT_FOR_MID); - SMARTLIST_FOREACH_BEGIN(nodelist_get_list(), node_t *, node) { + SMARTLIST_FOREACH_BEGIN(node_list, const node_t *, node) { if (node_allows_single_hop_exits(node)) { /* Exclude relays that allow single hop exit circuits. This is an * obsolete option since 0.2.9.2-alpha and done by default in * 0.3.1.0-alpha. */ - smartlist_add(excludednodes, node); + smartlist_add(excludednodes, (node_t*)node); } else if (rendezvous_v3 && !node_supports_v3_rendezvous_point(node)) { /* Exclude relays that do not support to rendezvous for a hidden service * version 3. */ - smartlist_add(excludednodes, node); + smartlist_add(excludednodes, (node_t*)node); } } SMARTLIST_FOREACH_END(node); @@ -897,19 +951,11 @@ router_choose_random_node(smartlist_t *excludedsmartlist, "We found %d running nodes.", smartlist_len(sl)); - smartlist_subtract(sl,excludednodes); - log_debug(LD_CIRC, - "We removed %d excludednodes, leaving %d nodes.", - smartlist_len(excludednodes), - smartlist_len(sl)); - if (excludedsmartlist) { - smartlist_subtract(sl,excludedsmartlist); - log_debug(LD_CIRC, - "We removed %d excludedsmartlist, leaving %d nodes.", - smartlist_len(excludedsmartlist), - smartlist_len(sl)); + smartlist_add_all(excludednodes, excludedsmartlist); } + nodelist_subtract(sl, excludednodes); + if (excludedset) { routerset_subtract_nodes(sl,excludedset); log_debug(LD_CIRC, diff --git a/src/test/test_entrynodes.c b/src/test/test_entrynodes.c index bdf057bb50..c43b21c673 100644 --- a/src/test/test_entrynodes.c +++ b/src/test/test_entrynodes.c @@ -197,6 +197,7 @@ big_fake_network_setup(const struct testcase_t *testcase) n->md->exit_policy = parse_short_policy("accept 443"); } + n->nodelist_idx = smartlist_len(big_fake_net_nodes); smartlist_add(big_fake_net_nodes, n); } From 806539b40a18dec15e1d3d108eb5aec9d9f3ca40 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 26 Apr 2019 11:19:46 -0400 Subject: [PATCH 0862/2557] Use fast check for missing id in node_is_a_configured_bridge() Fixes bug 30308; bugfix on 0.3.5.1-alpha. --- changes/ticket30308 | 5 +++++ src/feature/client/bridges.c | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 changes/ticket30308 diff --git a/changes/ticket30308 b/changes/ticket30308 new file mode 100644 index 0000000000..b78e6b3e9f --- /dev/null +++ b/changes/ticket30308 @@ -0,0 +1,5 @@ + o Minor bugfixes (performance): + - When checking a node for bridge status, use a fast check to make sure + that its identity is set. Previously, we used a constant-time check, + which is not necessary when verifying a BUG() condition that causes + a stack trace. Fixes bug 30308; bugfix on 0.3.5.1-alpha. diff --git a/src/feature/client/bridges.c b/src/feature/client/bridges.c index 05f89ad36c..ea1aff9519 100644 --- a/src/feature/client/bridges.c +++ b/src/feature/client/bridges.c @@ -348,7 +348,7 @@ int node_is_a_configured_bridge(const node_t *node) { /* First, let's try searching for a bridge with matching identity. */ - if (BUG(tor_digest_is_zero(node->identity))) + if (BUG(tor_mem_is_zero(node->identity, DIGEST_LEN))) return 0; if (find_bridge_by_digest(node->identity) != NULL) From 6d347fe329cdb2fdda4059ce36a6916225ab4c87 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Tue, 9 Apr 2019 11:23:03 +0300 Subject: [PATCH 0863/2557] Remove obsolete OpenSUSE initscript --- changes/ticket30076 | 2 + configure.ac | 1 - contrib/README | 2 - contrib/dist/suse/tor.sh.in | 118 ------------------------------------ contrib/include.am | 1 - 5 files changed, 2 insertions(+), 122 deletions(-) create mode 100644 changes/ticket30076 delete mode 100644 contrib/dist/suse/tor.sh.in diff --git a/changes/ticket30076 b/changes/ticket30076 new file mode 100644 index 0000000000..1334bc4603 --- /dev/null +++ b/changes/ticket30076 @@ -0,0 +1,2 @@ + o Removed features: + - Remove obsolete OpenSUSE initscript. Resolves issue 30076. diff --git a/configure.ac b/configure.ac index 3ea578bbba..e65e960d7f 100644 --- a/configure.ac +++ b/configure.ac @@ -2457,7 +2457,6 @@ AC_CONFIG_FILES([ Doxyfile Makefile config.rust - contrib/dist/suse/tor.sh contrib/operator-tools/tor.logrotate contrib/dist/torctl contrib/dist/tor.service diff --git a/contrib/README b/contrib/README index 3a94bb5016..735fcf4c9f 100644 --- a/contrib/README +++ b/contrib/README @@ -34,8 +34,6 @@ tools. Everybody likes to write init scripts differently, it seems. tor.service is a sample service file for use with systemd. -The suse/ subdirectory contains files used by the suse distribution. - operator-tools/ -- Tools for Tor relay operators ------------------------------------------------ diff --git a/contrib/dist/suse/tor.sh.in b/contrib/dist/suse/tor.sh.in deleted file mode 100644 index b7e9005eb5..0000000000 --- a/contrib/dist/suse/tor.sh.in +++ /dev/null @@ -1,118 +0,0 @@ -#!/bin/sh -# -# Copyright (c) 2006-2007 Andrew Lewman -# -# tor The Onion Router -# -# Startup/shutdown script for tor. This is a wrapper around torctl; -# torctl does the actual work in a relatively system-independent, or at least -# distribution-independent, way, and this script deals with fitting the -# whole thing into the conventions of the particular system at hand. -# -# These next couple of lines "declare" tor for the "chkconfig" program, -# originally from SGI, used on Red Hat/Fedora and probably elsewhere. -# -# chkconfig: 2345 90 10 -# description: Onion Router - A low-latency anonymous proxy -# - -### BEGIN INIT INFO -# Provides: tor -# Required-Start: $remote_fs $network -# Required-Stop: $remote_fs $network -# Default-Start: 3 5 -# Default-Stop: 0 1 2 6 -# Short-Description: Start the tor daemon -# Description: Start the tor daemon: the anon-proxy server -### END INIT INFO - -. /etc/rc.status - -# Shell functions sourced from /etc/rc.status: -# rc_check check and set local and overall rc status -# rc_status check and set local and overall rc status -# rc_status -v ditto but be verbose in local rc status -# rc_status -v -r ditto and clear the local rc status -# rc_failed set local and overall rc status to failed -# rc_reset clear local rc status (overall remains) -# rc_exit exit appropriate to overall rc status - -# First reset status of this service -rc_reset - -# Increase open file descriptors a reasonable amount -ulimit -n 8192 - -TORCTL=@BINDIR@/torctl - -# torctl will use these environment variables -TORUSER=@TORUSER@ -export TORUSER -TORGROUP=@TORGROUP@ -export TORGROUP - -TOR_DAEMON_PID_DIR="@LOCALSTATEDIR@/run/tor" - -if [ -x /bin/su ] ; then - SUPROG=/bin/su -elif [ -x /sbin/su ] ; then - SUPROG=/sbin/su -elif [ -x /usr/bin/su ] ; then - SUPROG=/usr/bin/su -elif [ -x /usr/sbin/su ] ; then - SUPROG=/usr/sbin/su -else - SUPROG=/bin/su -fi - -case "$1" in - - start) - echo "Starting tor daemon" - - if [ ! -d $TOR_DAEMON_PID_DIR ] ; then - mkdir -p $TOR_DAEMON_PID_DIR - chown $TORUSER:$TORGROUP $TOR_DAEMON_PID_DIR - fi - - ## Start daemon with startproc(8). If this fails - ## the echo return value is set appropriate. - - startproc -f $TORCTL start - # Remember status and be verbose - rc_status -v - ;; - - stop) - echo "Stopping tor daemon" - startproc -f $TORCTL stop - # Remember status and be verbose - rc_status -v - ;; - - restart) - echo "Restarting tor daemon" - startproc -f $TORCTL restart - # Remember status and be verbose - rc_status -v - ;; - - reload) - echo "Reloading tor daemon" - startproc -f $TORCTL reload - # Remember status and be verbose - rc_status -v - ;; - - status) - startproc -f $TORCTL status - # Remember status and be verbose - rc_status -v - ;; - - *) - echo "Usage: $0 (start|stop|restart|reload|status)" - RETVAL=1 -esac - -rc_exit diff --git a/contrib/include.am b/contrib/include.am index 9f4775632c..8dd8593304 100644 --- a/contrib/include.am +++ b/contrib/include.am @@ -3,7 +3,6 @@ EXTRA_DIST+= \ contrib/README \ contrib/client-tools/torify \ contrib/dist/rc.subr \ - contrib/dist/suse/tor.sh.in \ contrib/dist/torctl \ contrib/dist/tor.service.in \ contrib/operator-tools/tor-exit-notice.html \ From e5806dcea891cf9c6aa4d55c6d5deae9792792d7 Mon Sep 17 00:00:00 2001 From: David Goulet Date: Tue, 8 Jan 2019 11:31:32 -0500 Subject: [PATCH 0864/2557] sendme: Move code to the new files sendme.{c|h} Take apart the SENDME cell specific code and put it in sendme.{c|h}. This is part of prop289 that implements authenticated SENDMEs. Creating those new files allow for the already huge relay.c to not grow in LOC and makes it easier to handle and test the SENDME cells in an isolated way. This commit only moves code. No behavior change. Signed-off-by: David Goulet --- src/core/include.am | 2 + src/core/or/connection_edge.c | 5 ++- src/core/or/relay.c | 72 ++------------------------------- src/core/or/sendme.c | 76 +++++++++++++++++++++++++++++++++++ src/core/or/sendme.h | 20 +++++++++ 5 files changed, 105 insertions(+), 70 deletions(-) create mode 100644 src/core/or/sendme.c create mode 100644 src/core/or/sendme.h diff --git a/src/core/include.am b/src/core/include.am index 9824601725..067b6857ae 100644 --- a/src/core/include.am +++ b/src/core/include.am @@ -52,6 +52,7 @@ LIBTOR_APP_A_SOURCES = \ src/core/or/scheduler.c \ src/core/or/scheduler_kist.c \ src/core/or/scheduler_vanilla.c \ + src/core/or/sendme.c \ src/core/or/status.c \ src/core/or/versions.c \ src/core/proto/proto_cell.c \ @@ -272,6 +273,7 @@ noinst_HEADERS += \ src/core/or/relay.h \ src/core/or/relay_crypto_st.h \ src/core/or/scheduler.h \ + src/core/or/sendme.h \ src/core/or/server_port_cfg_st.h \ src/core/or/socks_request_st.h \ src/core/or/status.h \ diff --git a/src/core/or/connection_edge.c b/src/core/or/connection_edge.c index 33ba723971..fa72fb7716 100644 --- a/src/core/or/connection_edge.c +++ b/src/core/or/connection_edge.c @@ -73,6 +73,7 @@ #include "core/or/policies.h" #include "core/or/reasons.h" #include "core/or/relay.h" +#include "core/or/sendme.h" #include "core/proto/proto_http.h" #include "core/proto/proto_socks.h" #include "feature/client/addressmap.h" @@ -767,7 +768,7 @@ connection_edge_flushed_some(edge_connection_t *conn) /* falls through. */ case EXIT_CONN_STATE_OPEN: - connection_edge_consider_sending_sendme(conn); + sendme_connection_edge_consider_sending(conn); break; } return 0; @@ -791,7 +792,7 @@ connection_edge_finished_flushing(edge_connection_t *conn) switch (conn->base_.state) { case AP_CONN_STATE_OPEN: case EXIT_CONN_STATE_OPEN: - connection_edge_consider_sending_sendme(conn); + sendme_connection_edge_consider_sending(conn); return 0; case AP_CONN_STATE_SOCKS_WAIT: case AP_CONN_STATE_NATD_WAIT: diff --git a/src/core/or/relay.c b/src/core/or/relay.c index a166904a5f..7cfacf761a 100644 --- a/src/core/or/relay.c +++ b/src/core/or/relay.c @@ -93,13 +93,12 @@ #include "core/or/origin_circuit_st.h" #include "feature/nodelist/routerinfo_st.h" #include "core/or/socks_request_st.h" +#include "core/or/sendme.h" static edge_connection_t *relay_lookup_conn(circuit_t *circ, cell_t *cell, cell_direction_t cell_direction, crypt_path_t *layer_hint); -static void circuit_consider_sending_sendme(circuit_t *circ, - crypt_path_t *layer_hint); static void circuit_resume_edge_reading(circuit_t *circ, crypt_path_t *layer_hint); static int circuit_resume_edge_reading_helper(edge_connection_t *conn, @@ -1564,7 +1563,7 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, log_debug(domain,"circ deliver_window now %d.", layer_hint ? layer_hint->deliver_window : circ->deliver_window); - circuit_consider_sending_sendme(circ, layer_hint); + sendme_circuit_consider_sending(circ, layer_hint); if (rh.stream_id == 0) { log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, "Relay data cell with zero " @@ -1615,7 +1614,7 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, /* Only send a SENDME if we're not getting optimistic data; otherwise * a SENDME could arrive before the CONNECTED. */ - connection_edge_consider_sending_sendme(conn); + sendme_connection_edge_consider_sending(conn); } return 0; @@ -1869,7 +1868,7 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, /* Don't allow the other endpoint to request more than our maximum * (i.e. initial) stream SENDME window worth of data. Well-behaved * stock clients will not request more than this max (as per the check - * in the while loop of connection_edge_consider_sending_sendme()). + * in the while loop of sendme_connection_edge_consider_sending()). */ if (conn->package_window + STREAMWINDOW_INCREMENT > STREAMWINDOW_START_MAX) { @@ -2117,42 +2116,6 @@ connection_edge_package_raw_inbuf(edge_connection_t *conn, int package_partial, goto repeat_connection_edge_package_raw_inbuf; } -/** Called when we've just received a relay data cell, when - * we've just finished flushing all bytes to stream conn, - * or when we've flushed *some* bytes to the stream conn. - * - * If conn->outbuf is not too full, and our deliver window is - * low, send back a suitable number of stream-level sendme cells. - */ -void -connection_edge_consider_sending_sendme(edge_connection_t *conn) -{ - circuit_t *circ; - - if (connection_outbuf_too_full(TO_CONN(conn))) - return; - - circ = circuit_get_by_edge_conn(conn); - if (!circ) { - /* this can legitimately happen if the destroy has already - * arrived and torn down the circuit */ - log_info(LD_APP,"No circuit associated with conn. Skipping."); - return; - } - - while (conn->deliver_window <= STREAMWINDOW_START - STREAMWINDOW_INCREMENT) { - log_debug(conn->base_.type == CONN_TYPE_AP ?LD_APP:LD_EXIT, - "Outbuf %d, Queuing stream sendme.", - (int)conn->base_.outbuf_flushlen); - conn->deliver_window += STREAMWINDOW_INCREMENT; - if (connection_edge_send_command(conn, RELAY_COMMAND_SENDME, - NULL, 0) < 0) { - log_warn(LD_APP,"connection_edge_send_command failed. Skipping."); - return; /* the circuit's closed, don't continue */ - } - } -} - /** The circuit circ has received a circuit-level sendme * (on hop layer_hint, if we're the OP). Go through all the * attached streams and let them resume reading and packaging, if @@ -2369,33 +2332,6 @@ circuit_consider_stop_edge_reading(circuit_t *circ, crypt_path_t *layer_hint) return 0; } -/** Check if the deliver_window for circuit circ (at hop - * layer_hint if it's defined) is low enough that we should - * send a circuit-level sendme back down the circuit. If so, send - * enough sendmes that the window would be overfull if we sent any - * more. - */ -static void -circuit_consider_sending_sendme(circuit_t *circ, crypt_path_t *layer_hint) -{ -// log_fn(LOG_INFO,"Considering: layer_hint is %s", -// layer_hint ? "defined" : "null"); - while ((layer_hint ? layer_hint->deliver_window : circ->deliver_window) <= - CIRCWINDOW_START - CIRCWINDOW_INCREMENT) { - log_debug(LD_CIRC,"Queuing circuit sendme."); - if (layer_hint) - layer_hint->deliver_window += CIRCWINDOW_INCREMENT; - else - circ->deliver_window += CIRCWINDOW_INCREMENT; - if (relay_send_command_from_edge(0, circ, RELAY_COMMAND_SENDME, - NULL, 0, layer_hint) < 0) { - log_warn(LD_CIRC, - "relay_send_command_from_edge failed. Circuit's closed."); - return; /* the circuit's closed, don't continue */ - } - } -} - /** The total number of cells we have allocated. */ static size_t total_cells_allocated = 0; diff --git a/src/core/or/sendme.c b/src/core/or/sendme.c new file mode 100644 index 0000000000..3e00721d26 --- /dev/null +++ b/src/core/or/sendme.c @@ -0,0 +1,76 @@ +/* Copyright (c) 2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file sendme.c + * \brief Code that is related to SENDME cells both in terms of + * creating/parsing cells and handling the content. + */ + +#include "core/or/or.h" + +#include "core/mainloop/connection.h" +#include "core/or/circuitlist.h" +#include "core/or/relay.h" +#include "core/or/sendme.h" + +/** Called when we've just received a relay data cell, when + * we've just finished flushing all bytes to stream conn, + * or when we've flushed *some* bytes to the stream conn. + * + * If conn->outbuf is not too full, and our deliver window is + * low, send back a suitable number of stream-level sendme cells. + */ +void +sendme_connection_edge_consider_sending(edge_connection_t *conn) +{ + circuit_t *circ; + + if (connection_outbuf_too_full(TO_CONN(conn))) + return; + + circ = circuit_get_by_edge_conn(conn); + if (!circ) { + /* this can legitimately happen if the destroy has already + * arrived and torn down the circuit */ + log_info(LD_APP,"No circuit associated with conn. Skipping."); + return; + } + + while (conn->deliver_window <= STREAMWINDOW_START - STREAMWINDOW_INCREMENT) { + log_debug(conn->base_.type == CONN_TYPE_AP ?LD_APP:LD_EXIT, + "Outbuf %d, Queuing stream sendme.", + (int)conn->base_.outbuf_flushlen); + conn->deliver_window += STREAMWINDOW_INCREMENT; + if (connection_edge_send_command(conn, RELAY_COMMAND_SENDME, + NULL, 0) < 0) { + log_warn(LD_APP,"connection_edge_send_command failed. Skipping."); + return; /* the circuit's closed, don't continue */ + } + } +} + +/** Check if the deliver_window for circuit circ (at hop + * layer_hint if it's defined) is low enough that we should + * send a circuit-level sendme back down the circuit. If so, send + * enough sendmes that the window would be overfull if we sent any + * more. + */ +void +sendme_circuit_consider_sending(circuit_t *circ, crypt_path_t *layer_hint) +{ + while ((layer_hint ? layer_hint->deliver_window : circ->deliver_window) <= + CIRCWINDOW_START - CIRCWINDOW_INCREMENT) { + log_debug(LD_CIRC,"Queuing circuit sendme."); + if (layer_hint) + layer_hint->deliver_window += CIRCWINDOW_INCREMENT; + else + circ->deliver_window += CIRCWINDOW_INCREMENT; + if (relay_send_command_from_edge(0, circ, RELAY_COMMAND_SENDME, + NULL, 0, layer_hint) < 0) { + log_warn(LD_CIRC, + "relay_send_command_from_edge failed. Circuit's closed."); + return; /* the circuit's closed, don't continue */ + } + } +} diff --git a/src/core/or/sendme.h b/src/core/or/sendme.h new file mode 100644 index 0000000000..3ff1d98e64 --- /dev/null +++ b/src/core/or/sendme.h @@ -0,0 +1,20 @@ +/* Copyright (c) 2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file sendme.h + * \brief Header file for sendme.c. + **/ + +#ifndef TOR_SENDME_H +#define TOR_SENDME_H + +#include "core/or/edge_connection_st.h" +#include "core/or/crypt_path_st.h" +#include "core/or/circuit_st.h" + +void sendme_connection_edge_consider_sending(edge_connection_t *edge_conn); +void sendme_circuit_consider_sending(circuit_t *circ, + crypt_path_t *layer_hint); + +#endif /* !defined(TOR_SENDME_H) */ From ed8593b9e0838f694eeb6315db38f6fadbc5ab71 Mon Sep 17 00:00:00 2001 From: David Goulet Date: Tue, 8 Jan 2019 12:09:01 -0500 Subject: [PATCH 0865/2557] sendme: Modernize and cleanup old moved code Signed-off-by: David Goulet --- src/core/or/sendme.c | 48 ++++++++++++++++++++++++++------------------ 1 file changed, 28 insertions(+), 20 deletions(-) diff --git a/src/core/or/sendme.c b/src/core/or/sendme.c index 3e00721d26..f3acf47147 100644 --- a/src/core/or/sendme.c +++ b/src/core/or/sendme.c @@ -14,40 +14,48 @@ #include "core/or/relay.h" #include "core/or/sendme.h" -/** Called when we've just received a relay data cell, when - * we've just finished flushing all bytes to stream conn, - * or when we've flushed *some* bytes to the stream conn. +/** Called when we've just received a relay data cell, when we've just + * finished flushing all bytes to stream conn, or when we've flushed + * *some* bytes to the stream conn. * - * If conn->outbuf is not too full, and our deliver window is - * low, send back a suitable number of stream-level sendme cells. + * If conn->outbuf is not too full, and our deliver window is low, send back a + * suitable number of stream-level sendme cells. */ void sendme_connection_edge_consider_sending(edge_connection_t *conn) { - circuit_t *circ; + tor_assert(conn); - if (connection_outbuf_too_full(TO_CONN(conn))) - return; + int log_domain = TO_CONN(conn)->type == CONN_TYPE_AP ? LD_APP : LD_EXIT; - circ = circuit_get_by_edge_conn(conn); - if (!circ) { - /* this can legitimately happen if the destroy has already - * arrived and torn down the circuit */ - log_info(LD_APP,"No circuit associated with conn. Skipping."); - return; + /* Don't send it if we still have data to deliver. */ + if (connection_outbuf_too_full(TO_CONN(conn))) { + goto end; } - while (conn->deliver_window <= STREAMWINDOW_START - STREAMWINDOW_INCREMENT) { - log_debug(conn->base_.type == CONN_TYPE_AP ?LD_APP:LD_EXIT, - "Outbuf %d, Queuing stream sendme.", - (int)conn->base_.outbuf_flushlen); + if (circuit_get_by_edge_conn(conn) == NULL) { + /* This can legitimately happen if the destroy has already arrived and + * torn down the circuit. */ + log_info(log_domain, "No circuit associated with edge connection. " + "Skipping sending SENDME."); + goto end; + } + + while (conn->deliver_window <= + (STREAMWINDOW_START - STREAMWINDOW_INCREMENT)) { + log_debug(log_domain, "Outbuf %" TOR_PRIuSZ ", queuing stream SENDME.", + TO_CONN(conn)->outbuf_flushlen); conn->deliver_window += STREAMWINDOW_INCREMENT; if (connection_edge_send_command(conn, RELAY_COMMAND_SENDME, NULL, 0) < 0) { - log_warn(LD_APP,"connection_edge_send_command failed. Skipping."); - return; /* the circuit's closed, don't continue */ + log_warn(LD_BUG, "connection_edge_send_command failed while sending " + "a SENDME. Circuit probably closed, skipping."); + goto end; /* The circuit's closed, don't continue */ } } + + end: + return; } /** Check if the deliver_window for circuit circ (at hop From 9c42cc1eb22da8125ec9596920dfb9113912eac0 Mon Sep 17 00:00:00 2001 From: David Goulet Date: Tue, 8 Jan 2019 15:03:04 -0500 Subject: [PATCH 0866/2557] sendme: Refactor SENDME cell processing This is a bit of a complicated commit. It moves code but also refactors part of it. No behavior change, the idea is to split things up so we can better handle and understand how SENDME cells are processed where ultimately it will be easier to handle authenticated SENDMEs (prop289) using the intermediate functions added in this commit. The entry point for the cell arriving at the edge (Client or Exit), is connection_edge_process_relay_cell() for which we look if it is a circuit or stream level SENDME. This commit refactors that part where two new functions are introduced to process each of the SENDME types. The sendme_process_circuit_level() has basically two code paths. If we are a Client (the circuit is origin) or we are an Exit. Depending on which, the package window is updated accordingly. Then finally, we resume the reading on every edge streams on the circuit. The sendme_process_stream_level() applies on the edge connection which will update the package window if needed and then will try to empty the inbuf if need be because we can now deliver more cells. Again, no behavior change but in order to split that code properly into their own functions and outside the relay.c file, code modification was needed. Part of #26840. Signed-off-by: David Goulet --- src/core/or/relay.c | 88 ++++++++++++--------------------------- src/core/or/sendme.c | 98 ++++++++++++++++++++++++++++++++++++++++++++ src/core/or/sendme.h | 5 +++ 3 files changed, 130 insertions(+), 61 deletions(-) diff --git a/src/core/or/relay.c b/src/core/or/relay.c index 7cfacf761a..132da21466 100644 --- a/src/core/or/relay.c +++ b/src/core/or/relay.c @@ -1807,87 +1807,52 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, (unsigned)circ->n_circ_id, rh.stream_id); return 0; case RELAY_COMMAND_SENDME: + { + int ret; + if (!rh.stream_id) { - if (layer_hint) { - if (layer_hint->package_window + CIRCWINDOW_INCREMENT > - CIRCWINDOW_START_MAX) { - static struct ratelim_t exit_warn_ratelim = RATELIM_INIT(600); - log_fn_ratelim(&exit_warn_ratelim, LOG_WARN, LD_PROTOCOL, - "Unexpected sendme cell from exit relay. " - "Closing circ."); - return -END_CIRC_REASON_TORPROTOCOL; - } - layer_hint->package_window += CIRCWINDOW_INCREMENT; - log_debug(LD_APP,"circ-level sendme at origin, packagewindow %d.", - layer_hint->package_window); - circuit_resume_edge_reading(circ, layer_hint); - - /* We count circuit-level sendme's as valid delivered data because - * they are rate limited. - */ - if (CIRCUIT_IS_ORIGIN(circ)) { - circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), - rh.length); - } - - } else { - if (circ->package_window + CIRCWINDOW_INCREMENT > - CIRCWINDOW_START_MAX) { - static struct ratelim_t client_warn_ratelim = RATELIM_INIT(600); - log_fn_ratelim(&client_warn_ratelim,LOG_PROTOCOL_WARN, LD_PROTOCOL, - "Unexpected sendme cell from client. " - "Closing circ (window %d).", - circ->package_window); - return -END_CIRC_REASON_TORPROTOCOL; - } - circ->package_window += CIRCWINDOW_INCREMENT; - log_debug(LD_APP, - "circ-level sendme at non-origin, packagewindow %d.", - circ->package_window); - circuit_resume_edge_reading(circ, layer_hint); + /* Circuit level SENDME cell. */ + ret = sendme_process_circuit_level(layer_hint, circ, rh.length); + if (ret < 0) { + return ret; } + /* Resume reading on any streams now that we've processed a valid + * SENDME cell that updated our package window. */ + circuit_resume_edge_reading(circ, layer_hint); + /* We are done, the rest of the code is for the stream level. */ return 0; } + + /* No connection, might be half edge state. We are done if so. */ if (!conn) { if (CIRCUIT_IS_ORIGIN(circ)) { origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ); if (connection_half_edge_is_valid_sendme(ocirc->half_streams, rh.stream_id)) { circuit_read_valid_data(ocirc, rh.length); - log_info(domain, - "sendme cell on circ %u valid on half-closed " - "stream id %d", ocirc->global_identifier, rh.stream_id); + log_info(domain, "Sendme cell on circ %u valid on half-closed " + "stream id %d", + ocirc->global_identifier, rh.stream_id); } } - log_info(domain,"sendme cell dropped, unknown stream (streamid %d).", + log_info(domain, "SENDME cell dropped, unknown stream (streamid %d).", rh.stream_id); return 0; } - /* Don't allow the other endpoint to request more than our maximum - * (i.e. initial) stream SENDME window worth of data. Well-behaved - * stock clients will not request more than this max (as per the check - * in the while loop of sendme_connection_edge_consider_sending()). - */ - if (conn->package_window + STREAMWINDOW_INCREMENT > - STREAMWINDOW_START_MAX) { - static struct ratelim_t stream_warn_ratelim = RATELIM_INIT(600); - log_fn_ratelim(&stream_warn_ratelim, LOG_PROTOCOL_WARN, LD_PROTOCOL, - "Unexpected stream sendme cell. Closing circ (window %d).", - conn->package_window); - return -END_CIRC_REASON_TORPROTOCOL; + /* Stream level SENDME cell. */ + ret = sendme_process_stream_level(conn, circ, rh.length); + if (ret < 0) { + /* Means we need to close the circuit with reason ret. */ + return ret; } - /* At this point, the stream sendme is valid */ - if (CIRCUIT_IS_ORIGIN(circ)) { - circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), - rh.length); - } + /* We've now processed properly a SENDME cell, all windows have been + * properly updated, we'll read on the edge connection to see if we can + * get data out towards the end point (Exit or client) since we are now + * allowed to deliver more cells. */ - conn->package_window += STREAMWINDOW_INCREMENT; - log_debug(domain,"stream-level sendme, packagewindow now %d.", - conn->package_window); if (circuit_queue_streams_are_blocked(circ)) { /* Still waiting for queue to flush; don't touch conn */ return 0; @@ -1900,6 +1865,7 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, return 0; } return 0; + } case RELAY_COMMAND_RESOLVE: if (layer_hint) { log_fn(LOG_PROTOCOL_WARN, LD_APP, diff --git a/src/core/or/sendme.c b/src/core/or/sendme.c index f3acf47147..df9fc57bd6 100644 --- a/src/core/or/sendme.c +++ b/src/core/or/sendme.c @@ -9,8 +9,10 @@ #include "core/or/or.h" +#include "app/config/config.h" #include "core/mainloop/connection.h" #include "core/or/circuitlist.h" +#include "core/or/circuituse.h" #include "core/or/relay.h" #include "core/or/sendme.h" @@ -82,3 +84,99 @@ sendme_circuit_consider_sending(circuit_t *circ, crypt_path_t *layer_hint) } } } + +/* Process a circuit-level SENDME cell that we just received. The layer_hint, + * if not NULL, is the Exit hop of the connection which means that we are a + * client. In that case, circ must be an origin circuit. The cell_body_len is + * the length of the SENDME cell payload (excluding the header). + * + * Return 0 on success that is the SENDME is valid and the package window has + * been updated properly. + * + * On error, a negative value is returned which indicate that the circuit must + * be closed using the value as the reason for it. */ +int +sendme_process_circuit_level(crypt_path_t *layer_hint, + circuit_t *circ, uint16_t cell_body_len) +{ + tor_assert(circ); + + /* If we are the origin of the circuit, we are the Client so we use the + * layer hint (the Exit hop) for the package window tracking. */ + if (CIRCUIT_IS_ORIGIN(circ)) { + if ((layer_hint->package_window + CIRCWINDOW_INCREMENT) > + CIRCWINDOW_START_MAX) { + static struct ratelim_t exit_warn_ratelim = RATELIM_INIT(600); + log_fn_ratelim(&exit_warn_ratelim, LOG_WARN, LD_PROTOCOL, + "Unexpected sendme cell from exit relay. " + "Closing circ."); + return -END_CIRC_REASON_TORPROTOCOL; + } + layer_hint->package_window += CIRCWINDOW_INCREMENT; + log_debug(LD_APP, "circ-level sendme at origin, packagewindow %d.", + layer_hint->package_window); + + /* We count circuit-level sendme's as valid delivered data because they + * are rate limited. */ + circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), cell_body_len); + } else { + /* We aren't the origin of this circuit so we are the Exit and thus we + * track the package window with the circuit object. */ + if ((circ->package_window + CIRCWINDOW_INCREMENT) > + CIRCWINDOW_START_MAX) { + static struct ratelim_t client_warn_ratelim = RATELIM_INIT(600); + log_fn_ratelim(&client_warn_ratelim, LOG_PROTOCOL_WARN, LD_PROTOCOL, + "Unexpected sendme cell from client. " + "Closing circ (window %d).", circ->package_window); + return -END_CIRC_REASON_TORPROTOCOL; + } + circ->package_window += CIRCWINDOW_INCREMENT; + log_debug(LD_EXIT, "circ-level sendme at non-origin, packagewindow %d.", + circ->package_window); + } + + return 0; +} + +/* Process a stream-level SENDME cell that we just received. The conn is the + * edge connection (stream) that the circuit circ is associated with. The + * cell_body_len is the length of the payload (excluding the header). + * + * Return 0 on success that is the SENDME is valid and the package window has + * been updated properly. + * + * On error, a negative value is returned which indicate that the circuit must + * be closed using the value as the reason for it. */ +int +sendme_process_stream_level(edge_connection_t *conn, circuit_t *circ, + uint16_t cell_body_len) +{ + tor_assert(conn); + tor_assert(circ); + + /* Don't allow the other endpoint to request more than our maximum (i.e. + * initial) stream SENDME window worth of data. Well-behaved stock clients + * will not request more than this max (as per the check in the while loop + * of sendme_connection_edge_consider_sending()). */ + if ((conn->package_window + STREAMWINDOW_INCREMENT) > + STREAMWINDOW_START_MAX) { + static struct ratelim_t stream_warn_ratelim = RATELIM_INIT(600); + log_fn_ratelim(&stream_warn_ratelim, LOG_PROTOCOL_WARN, LD_PROTOCOL, + "Unexpected stream sendme cell. Closing circ (window %d).", + conn->package_window); + return -END_CIRC_REASON_TORPROTOCOL; + } + /* At this point, the stream sendme is valid */ + conn->package_window += STREAMWINDOW_INCREMENT; + + /* We count circuit-level sendme's as valid delivered data because they are + * rate limited. */ + if (CIRCUIT_IS_ORIGIN(circ)) { + circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), cell_body_len); + } + + log_debug(CIRCUIT_IS_ORIGIN(circ) ? LD_APP : LD_EXIT, + "stream-level sendme, package_window now %d.", + conn->package_window); + return 0; +} diff --git a/src/core/or/sendme.h b/src/core/or/sendme.h index 3ff1d98e64..97748cf65e 100644 --- a/src/core/or/sendme.h +++ b/src/core/or/sendme.h @@ -17,4 +17,9 @@ void sendme_connection_edge_consider_sending(edge_connection_t *edge_conn); void sendme_circuit_consider_sending(circuit_t *circ, crypt_path_t *layer_hint); +int sendme_process_circuit_level(crypt_path_t *layer_hint, + circuit_t *circ, uint16_t cell_body_len); +int sendme_process_stream_level(edge_connection_t *conn, circuit_t *circ, + uint16_t cell_body_len); + #endif /* !defined(TOR_SENDME_H) */ From 2d3c600915c22f1e9a7e9dcbba8358556ef64505 Mon Sep 17 00:00:00 2001 From: David Goulet Date: Wed, 9 Jan 2019 10:39:58 -0500 Subject: [PATCH 0867/2557] sendme: Add helper functions for DATA cell delivery When we get a relay DATA cell delivered, we have to decrement the deliver window on both the circuit and stream level. This commit adds helper functions to handle the deliver window decrement. Part of #26840 Signed-off-by: David Goulet --- src/core/or/relay.c | 16 +++++++++++----- src/core/or/sendme.c | 33 +++++++++++++++++++++++++++++++++ src/core/or/sendme.h | 6 ++++++ 3 files changed, 50 insertions(+), 5 deletions(-) diff --git a/src/core/or/relay.c b/src/core/or/relay.c index 132da21466..0eb2ba3f67 100644 --- a/src/core/or/relay.c +++ b/src/core/or/relay.c @@ -1548,8 +1548,11 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, return connection_exit_begin_conn(cell, circ); case RELAY_COMMAND_DATA: ++stats_n_data_cells_received; - if (( layer_hint && --layer_hint->deliver_window < 0) || - (!layer_hint && --circ->deliver_window < 0)) { + + /* Update our circuit-level deliver window that we received a DATA cell. + * If the deliver window goes below 0, we end the connection due to a + * protocol failure. */ + if (sendme_circuit_data_received(circ, layer_hint) < 0) { log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, "(relay data) circ deliver_window below 0. Killing."); if (conn) { @@ -1560,9 +1563,8 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, } return -END_CIRC_REASON_TORPROTOCOL; } - log_debug(domain,"circ deliver_window now %d.", layer_hint ? - layer_hint->deliver_window : circ->deliver_window); + /* Consider sending a circuit-level SENDME cell. */ sendme_circuit_consider_sending(circ, layer_hint); if (rh.stream_id == 0) { @@ -1586,7 +1588,11 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, return 0; } - if (--conn->deliver_window < 0) { /* is it below 0 after decrement? */ + /* Update our stream-level deliver window that we just received a DATA + * cell. Going below 0 means we have a protocol level error so the + * circuit is closed. */ + + if (sendme_stream_data_received(conn) < 0) { log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, "(relay data) conn deliver_window below 0. Killing."); return -END_CIRC_REASON_TORPROTOCOL; diff --git a/src/core/or/sendme.c b/src/core/or/sendme.c index df9fc57bd6..6d9f6f7d05 100644 --- a/src/core/or/sendme.c +++ b/src/core/or/sendme.c @@ -180,3 +180,36 @@ sendme_process_stream_level(edge_connection_t *conn, circuit_t *circ, conn->package_window); return 0; } + +/* Called when a relay DATA cell is received on the given circuit. If + * layer_hint is NULL, this means we are the Exit end point else we are the + * Client. Update the deliver window and return its new value. */ +int +sendme_circuit_data_received(circuit_t *circ, crypt_path_t *layer_hint) +{ + int deliver_window, domain; + + if (CIRCUIT_IS_ORIGIN(circ)) { + tor_assert(layer_hint); + --layer_hint->deliver_window; + deliver_window = layer_hint->deliver_window; + domain = LD_APP; + } else { + tor_assert(!layer_hint); + --circ->deliver_window; + deliver_window = circ->deliver_window; + domain = LD_EXIT; + } + + log_debug(domain, "Circuit deliver_window now %d.", deliver_window); + return deliver_window; +} + +/* Called when a relay DATA cell is received for the given edge connection + * conn. Update the deliver window and return its new value. */ +int +sendme_stream_data_received(edge_connection_t *conn) +{ + tor_assert(conn); + return --conn->deliver_window; +} diff --git a/src/core/or/sendme.h b/src/core/or/sendme.h index 97748cf65e..6313e91216 100644 --- a/src/core/or/sendme.h +++ b/src/core/or/sendme.h @@ -13,13 +13,19 @@ #include "core/or/crypt_path_st.h" #include "core/or/circuit_st.h" +/* Sending SENDME cell. */ void sendme_connection_edge_consider_sending(edge_connection_t *edge_conn); void sendme_circuit_consider_sending(circuit_t *circ, crypt_path_t *layer_hint); +/* Processing SENDME cell. */ int sendme_process_circuit_level(crypt_path_t *layer_hint, circuit_t *circ, uint16_t cell_body_len); int sendme_process_stream_level(edge_connection_t *conn, circuit_t *circ, uint16_t cell_body_len); +/* Update deliver window functions. */ +int sendme_stream_data_received(edge_connection_t *conn); +int sendme_circuit_data_received(circuit_t *circ, crypt_path_t *layer_hint); + #endif /* !defined(TOR_SENDME_H) */ From 8e38791baf48ca2a4c865f3b7fc264392e63f426 Mon Sep 17 00:00:00 2001 From: David Goulet Date: Fri, 11 Jan 2019 11:36:08 -0500 Subject: [PATCH 0868/2557] sendme: Add helper functions for DATA cell packaging When we are about to send a DATA cell, we have to decrement the package window for both the circuit and stream level. This commit adds helper functions to handle the package window decrement. Part of #26288 Signed-off-by: David Goulet --- src/core/or/relay.c | 16 +++++++++------- src/core/or/sendme.c | 37 +++++++++++++++++++++++++++++++++++++ src/core/or/sendme.h | 4 ++++ 3 files changed, 50 insertions(+), 7 deletions(-) diff --git a/src/core/or/relay.c b/src/core/or/relay.c index 0eb2ba3f67..06e201f20d 100644 --- a/src/core/or/relay.c +++ b/src/core/or/relay.c @@ -2062,15 +2062,17 @@ connection_edge_package_raw_inbuf(edge_connection_t *conn, int package_partial, return 0; } - if (!cpath_layer) { /* non-rendezvous exit */ - tor_assert(circ->package_window > 0); - circ->package_window--; - } else { /* we're an AP, or an exit on a rendezvous circ */ - tor_assert(cpath_layer->package_window > 0); - cpath_layer->package_window--; + /* Handle the circuit-level SENDME package window. */ + if (sendme_circuit_data_packaged(circ, cpath_layer) < 0) { + /* Package window has gone under 0. Protocol issue. */ + log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, + "Circuit package window is below 0. Closing circuit."); + conn->end_reason = END_STREAM_REASON_TORPROTOCOL; + return -1; } - if (--conn->package_window <= 0) { /* is it 0 after decrement? */ + /* Handle the stream-level SENDME package window. */ + if (sendme_stream_data_packaged(conn) < 0) { connection_stop_reading(TO_CONN(conn)); log_debug(domain,"conn->package_window reached 0."); circuit_consider_stop_edge_reading(circ, cpath_layer); diff --git a/src/core/or/sendme.c b/src/core/or/sendme.c index 6d9f6f7d05..d7feb6bfc6 100644 --- a/src/core/or/sendme.c +++ b/src/core/or/sendme.c @@ -213,3 +213,40 @@ sendme_stream_data_received(edge_connection_t *conn) tor_assert(conn); return --conn->deliver_window; } + +/* Called when a relay DATA cell is packaged on the given circuit. If + * layer_hint is NULL, this means we are the Exit end point else we are the + * Client. Update the package window and return its new value. */ +int +sendme_circuit_data_packaged(circuit_t *circ, crypt_path_t *layer_hint) +{ + int package_window, domain; + + tor_assert(circ); + + if (CIRCUIT_IS_ORIGIN(circ)) { + /* Client side. */ + tor_assert(layer_hint); + --layer_hint->package_window; + package_window = layer_hint->package_window; + domain = LD_APP; + } else { + /* Exit side. */ + tor_assert(!layer_hint); + --circ->package_window; + package_window = circ->package_window; + domain = LD_EXIT; + } + + log_debug(domain, "Circuit package_window now %d.", package_window); + return package_window; +} + +/* Called when a relay DATA cell is packaged for the given edge connection + * conn. Update the package window and return its new value. */ +int +sendme_stream_data_packaged(edge_connection_t *conn) +{ + tor_assert(conn); + return --conn->package_window; +} diff --git a/src/core/or/sendme.h b/src/core/or/sendme.h index 6313e91216..f50ccfbfe2 100644 --- a/src/core/or/sendme.h +++ b/src/core/or/sendme.h @@ -28,4 +28,8 @@ int sendme_process_stream_level(edge_connection_t *conn, circuit_t *circ, int sendme_stream_data_received(edge_connection_t *conn); int sendme_circuit_data_received(circuit_t *circ, crypt_path_t *layer_hint); +/* Update package window functions. */ +int sendme_circuit_data_packaged(circuit_t *circ, crypt_path_t *layer_hint); +int sendme_stream_data_packaged(edge_connection_t *conn); + #endif /* !defined(TOR_SENDME_H) */ From 0e6e800c89c7cfef255491179473e13de5f72d03 Mon Sep 17 00:00:00 2001 From: David Goulet Date: Wed, 9 Jan 2019 11:03:49 -0500 Subject: [PATCH 0869/2557] sendme: Always close stream if deliver window is negative Previously, we would only close the stream when our deliver window was negative at the circuit-level but _not_ at the stream-level when receiving a DATA cell. This commit adds an helper function connection_edge_end_close() which sends an END and then mark the stream for close for a given reason. That function is now used both in case the deliver window goes below zero for both circuit and stream level. Part of #26840 Signed-off-by: David Goulet --- src/core/or/connection_edge.c | 19 +++++++++++++++++++ src/core/or/connection_edge.h | 1 + src/core/or/relay.c | 14 +++++--------- 3 files changed, 25 insertions(+), 9 deletions(-) diff --git a/src/core/or/connection_edge.c b/src/core/or/connection_edge.c index fa72fb7716..8ed4034560 100644 --- a/src/core/or/connection_edge.c +++ b/src/core/or/connection_edge.c @@ -4565,6 +4565,25 @@ circuit_clear_isolation(origin_circuit_t *circ) circ->socks_username_len = circ->socks_password_len = 0; } +/** Send an END and mark for close the given edge connection conn using the + * given reason that has to be a stream reason. + * + * Note: We don't unattached the AP connection (if applicable) because we + * don't want to flush the remaining data. This function aims at ending + * everything quickly regardless of the connection state. + * + * This function can't fail and does nothing if conn is NULL. */ +void +connection_edge_end_close(edge_connection_t *conn, uint8_t reason) +{ + if (!conn) { + return; + } + + connection_edge_end(conn, reason); + connection_mark_for_close(TO_CONN(conn)); +} + /** Free all storage held in module-scoped variables for connection_edge.c */ void connection_edge_free_all(void) diff --git a/src/core/or/connection_edge.h b/src/core/or/connection_edge.h index 68d8b19a11..e82b6bd765 100644 --- a/src/core/or/connection_edge.h +++ b/src/core/or/connection_edge.h @@ -80,6 +80,7 @@ int connection_edge_process_inbuf(edge_connection_t *conn, int connection_edge_destroy(circid_t circ_id, edge_connection_t *conn); int connection_edge_end(edge_connection_t *conn, uint8_t reason); int connection_edge_end_errno(edge_connection_t *conn); +void connection_edge_end_close(edge_connection_t *conn, uint8_t reason); int connection_edge_flushed_some(edge_connection_t *conn); int connection_edge_finished_flushing(edge_connection_t *conn); int connection_edge_finished_connecting(edge_connection_t *conn); diff --git a/src/core/or/relay.c b/src/core/or/relay.c index 06e201f20d..6ff053d8a0 100644 --- a/src/core/or/relay.c +++ b/src/core/or/relay.c @@ -1550,17 +1550,12 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, ++stats_n_data_cells_received; /* Update our circuit-level deliver window that we received a DATA cell. - * If the deliver window goes below 0, we end the connection due to a - * protocol failure. */ + * If the deliver window goes below 0, we end the circuit and stream due + * to a protocol failure. */ if (sendme_circuit_data_received(circ, layer_hint) < 0) { log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, "(relay data) circ deliver_window below 0. Killing."); - if (conn) { - /* XXXX Do we actually need to do this? Will killing the circuit - * not send an END and mark the stream for close as appropriate? */ - connection_edge_end(conn, END_STREAM_REASON_TORPROTOCOL); - connection_mark_for_close(TO_CONN(conn)); - } + connection_edge_end_close(conn, END_STREAM_REASON_TORPROTOCOL); return -END_CIRC_REASON_TORPROTOCOL; } @@ -1590,11 +1585,12 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, /* Update our stream-level deliver window that we just received a DATA * cell. Going below 0 means we have a protocol level error so the - * circuit is closed. */ + * stream and circuit are closed. */ if (sendme_stream_data_received(conn) < 0) { log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, "(relay data) conn deliver_window below 0. Killing."); + connection_edge_end_close(conn, END_STREAM_REASON_TORPROTOCOL); return -END_CIRC_REASON_TORPROTOCOL; } /* Total all valid application bytes delivered */ From c38d46bf4adf4107d39a2ce46aeacb630f3f112a Mon Sep 17 00:00:00 2001 From: David Goulet Date: Wed, 9 Jan 2019 12:02:01 -0500 Subject: [PATCH 0870/2557] prop289: Add two consensus parameters In order to be able to deploy the authenticated SENDMEs, these two consensus parameters are needed to control the minimum version that we can emit and accept. See section 4 in prop289 for more details. Note that at this commit, the functions that return the values aren't used so compilation fails if warnings are set to errors. Closes #26842 Signed-off-by: David Goulet --- src/core/or/sendme.c | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/src/core/or/sendme.c b/src/core/or/sendme.c index d7feb6bfc6..bba760ae98 100644 --- a/src/core/or/sendme.c +++ b/src/core/or/sendme.c @@ -15,6 +15,39 @@ #include "core/or/circuituse.h" #include "core/or/relay.h" #include "core/or/sendme.h" +#include "feature/nodelist/networkstatus.h" + +/* The cell version constants for when emitting a cell. */ +#define SENDME_EMIT_MIN_VERSION_DEFAULT 0 +#define SENDME_EMIT_MIN_VERSION_MIN 0 +#define SENDME_EMIT_MIN_VERSION_MAX UINT8_MAX + +/* The cell version constants for when accepting a cell. */ +#define SENDME_ACCEPT_MIN_VERSION_DEFAULT 0 +#define SENDME_ACCEPT_MIN_VERSION_MIN 0 +#define SENDME_ACCEPT_MIN_VERSION_MAX UINT8_MAX + +/* Return the minimum version given by the consensus (if any) that should be + * used when emitting a SENDME cell. */ +static int +get_emit_min_version(void) +{ + return networkstatus_get_param(NULL, "sendme_emit_min_version", + SENDME_EMIT_MIN_VERSION_DEFAULT, + SENDME_EMIT_MIN_VERSION_MIN, + SENDME_EMIT_MIN_VERSION_MAX); +} + +/* Return the minimum version given by the consensus (if any) that should be + * accepted when receiving a SENDME cell. */ +static int +get_accept_min_version(void) +{ + return networkstatus_get_param(NULL, "sendme_accept_min_version", + SENDME_ACCEPT_MIN_VERSION_DEFAULT, + SENDME_ACCEPT_MIN_VERSION_MIN, + SENDME_ACCEPT_MIN_VERSION_MAX); +} /** Called when we've just received a relay data cell, when we've just * finished flushing all bytes to stream conn, or when we've flushed From eef78ac0b07096a6925ae42e8b5d526304fa54a8 Mon Sep 17 00:00:00 2001 From: David Goulet Date: Tue, 8 Jan 2019 11:13:37 -0500 Subject: [PATCH 0871/2557] prop289: Add SENDME trunnel declaration Signed-off-by: David Goulet --- src/trunnel/include.am | 3 + src/trunnel/sendme.c | 492 +++++++++++++++++++++++++++++++++++++ src/trunnel/sendme.h | 170 +++++++++++++ src/trunnel/sendme.trunnel | 18 ++ 4 files changed, 683 insertions(+) create mode 100644 src/trunnel/sendme.c create mode 100644 src/trunnel/sendme.h create mode 100644 src/trunnel/sendme.trunnel diff --git a/src/trunnel/include.am b/src/trunnel/include.am index 4f4f1d3624..82e7a66959 100644 --- a/src/trunnel/include.am +++ b/src/trunnel/include.am @@ -11,6 +11,7 @@ TRUNNELINPUTS = \ src/trunnel/link_handshake.trunnel \ src/trunnel/pwbox.trunnel \ src/trunnel/channelpadding_negotiation.trunnel \ + src/trunnel/sendme.trunnel \ src/trunnel/socks5.trunnel \ src/trunnel/circpad_negotiation.trunnel @@ -24,6 +25,7 @@ TRUNNELSOURCES = \ src/trunnel/hs/cell_introduce1.c \ src/trunnel/hs/cell_rendezvous.c \ src/trunnel/channelpadding_negotiation.c \ + src/trunnel/sendme.c \ src/trunnel/socks5.c \ src/trunnel/netinfo.c \ src/trunnel/circpad_negotiation.c @@ -40,6 +42,7 @@ TRUNNELHEADERS = \ src/trunnel/hs/cell_introduce1.h \ src/trunnel/hs/cell_rendezvous.h \ src/trunnel/channelpadding_negotiation.h \ + src/trunnel/sendme.h \ src/trunnel/socks5.h \ src/trunnel/netinfo.h \ src/trunnel/circpad_negotiation.h diff --git a/src/trunnel/sendme.c b/src/trunnel/sendme.c new file mode 100644 index 0000000000..08f9ed5e91 --- /dev/null +++ b/src/trunnel/sendme.c @@ -0,0 +1,492 @@ +/* sendme.c -- generated by Trunnel v1.5.2. + * https://gitweb.torproject.org/trunnel.git + * You probably shouldn't edit this file. + */ +#include +#include "trunnel-impl.h" + +#include "sendme.h" + +#define TRUNNEL_SET_ERROR_CODE(obj) \ + do { \ + (obj)->trunnel_error_code_ = 1; \ + } while (0) + +#if defined(__COVERITY__) || defined(__clang_analyzer__) +/* If we're running a static analysis tool, we don't want it to complain + * that some of our remaining-bytes checks are dead-code. */ +int sendme_deadcode_dummy__ = 0; +#define OR_DEADCODE_DUMMY || sendme_deadcode_dummy__ +#else +#define OR_DEADCODE_DUMMY +#endif + +#define CHECK_REMAINING(nbytes, label) \ + do { \ + if (remaining < (nbytes) OR_DEADCODE_DUMMY) { \ + goto label; \ + } \ + } while (0) + +sendme_cell_t * +sendme_cell_new(void) +{ + sendme_cell_t *val = trunnel_calloc(1, sizeof(sendme_cell_t)); + if (NULL == val) + return NULL; + return val; +} + +/** Release all storage held inside 'obj', but do not free 'obj'. + */ +static void +sendme_cell_clear(sendme_cell_t *obj) +{ + (void) obj; + TRUNNEL_DYNARRAY_WIPE(&obj->data); + TRUNNEL_DYNARRAY_CLEAR(&obj->data); +} + +void +sendme_cell_free(sendme_cell_t *obj) +{ + if (obj == NULL) + return; + sendme_cell_clear(obj); + trunnel_memwipe(obj, sizeof(sendme_cell_t)); + trunnel_free_(obj); +} + +uint8_t +sendme_cell_get_version(const sendme_cell_t *inp) +{ + return inp->version; +} +int +sendme_cell_set_version(sendme_cell_t *inp, uint8_t val) +{ + if (! ((val == 0 || val == 1))) { + TRUNNEL_SET_ERROR_CODE(inp); + return -1; + } + inp->version = val; + return 0; +} +uint16_t +sendme_cell_get_data_len(const sendme_cell_t *inp) +{ + return inp->data_len; +} +int +sendme_cell_set_data_len(sendme_cell_t *inp, uint16_t val) +{ + inp->data_len = val; + return 0; +} +size_t +sendme_cell_getlen_data(const sendme_cell_t *inp) +{ + return TRUNNEL_DYNARRAY_LEN(&inp->data); +} + +uint8_t +sendme_cell_get_data(sendme_cell_t *inp, size_t idx) +{ + return TRUNNEL_DYNARRAY_GET(&inp->data, idx); +} + +uint8_t +sendme_cell_getconst_data(const sendme_cell_t *inp, size_t idx) +{ + return sendme_cell_get_data((sendme_cell_t*)inp, idx); +} +int +sendme_cell_set_data(sendme_cell_t *inp, size_t idx, uint8_t elt) +{ + TRUNNEL_DYNARRAY_SET(&inp->data, idx, elt); + return 0; +} +int +sendme_cell_add_data(sendme_cell_t *inp, uint8_t elt) +{ +#if SIZE_MAX >= UINT16_MAX + if (inp->data.n_ == UINT16_MAX) + goto trunnel_alloc_failed; +#endif + TRUNNEL_DYNARRAY_ADD(uint8_t, &inp->data, elt, {}); + return 0; + trunnel_alloc_failed: + TRUNNEL_SET_ERROR_CODE(inp); + return -1; +} + +uint8_t * +sendme_cell_getarray_data(sendme_cell_t *inp) +{ + return inp->data.elts_; +} +const uint8_t * +sendme_cell_getconstarray_data(const sendme_cell_t *inp) +{ + return (const uint8_t *)sendme_cell_getarray_data((sendme_cell_t*)inp); +} +int +sendme_cell_setlen_data(sendme_cell_t *inp, size_t newlen) +{ + uint8_t *newptr; +#if UINT16_MAX < SIZE_MAX + if (newlen > UINT16_MAX) + goto trunnel_alloc_failed; +#endif + newptr = trunnel_dynarray_setlen(&inp->data.allocated_, + &inp->data.n_, inp->data.elts_, newlen, + sizeof(inp->data.elts_[0]), (trunnel_free_fn_t) NULL, + &inp->trunnel_error_code_); + if (newlen != 0 && newptr == NULL) + goto trunnel_alloc_failed; + inp->data.elts_ = newptr; + return 0; + trunnel_alloc_failed: + TRUNNEL_SET_ERROR_CODE(inp); + return -1; +} +const char * +sendme_cell_check(const sendme_cell_t *obj) +{ + if (obj == NULL) + return "Object was NULL"; + if (obj->trunnel_error_code_) + return "A set function failed on this object"; + if (! (obj->version == 0 || obj->version == 1)) + return "Integer out of bounds"; + if (TRUNNEL_DYNARRAY_LEN(&obj->data) != obj->data_len) + return "Length mismatch for data"; + return NULL; +} + +ssize_t +sendme_cell_encoded_len(const sendme_cell_t *obj) +{ + ssize_t result = 0; + + if (NULL != sendme_cell_check(obj)) + return -1; + + + /* Length of u8 version IN [0, 1] */ + result += 1; + + /* Length of u16 data_len */ + result += 2; + + /* Length of u8 data[data_len] */ + result += TRUNNEL_DYNARRAY_LEN(&obj->data); + return result; +} +int +sendme_cell_clear_errors(sendme_cell_t *obj) +{ + int r = obj->trunnel_error_code_; + obj->trunnel_error_code_ = 0; + return r; +} +ssize_t +sendme_cell_encode(uint8_t *output, const size_t avail, const sendme_cell_t *obj) +{ + ssize_t result = 0; + size_t written = 0; + uint8_t *ptr = output; + const char *msg; +#ifdef TRUNNEL_CHECK_ENCODED_LEN + const ssize_t encoded_len = sendme_cell_encoded_len(obj); +#endif + + if (NULL != (msg = sendme_cell_check(obj))) + goto check_failed; + +#ifdef TRUNNEL_CHECK_ENCODED_LEN + trunnel_assert(encoded_len >= 0); +#endif + + /* Encode u8 version IN [0, 1] */ + trunnel_assert(written <= avail); + if (avail - written < 1) + goto truncated; + trunnel_set_uint8(ptr, (obj->version)); + written += 1; ptr += 1; + + /* Encode u16 data_len */ + trunnel_assert(written <= avail); + if (avail - written < 2) + goto truncated; + trunnel_set_uint16(ptr, trunnel_htons(obj->data_len)); + written += 2; ptr += 2; + + /* Encode u8 data[data_len] */ + { + size_t elt_len = TRUNNEL_DYNARRAY_LEN(&obj->data); + trunnel_assert(obj->data_len == elt_len); + trunnel_assert(written <= avail); + if (avail - written < elt_len) + goto truncated; + if (elt_len) + memcpy(ptr, obj->data.elts_, elt_len); + written += elt_len; ptr += elt_len; + } + + + trunnel_assert(ptr == output + written); +#ifdef TRUNNEL_CHECK_ENCODED_LEN + { + trunnel_assert(encoded_len >= 0); + trunnel_assert((size_t)encoded_len == written); + } + +#endif + + return written; + + truncated: + result = -2; + goto fail; + check_failed: + (void)msg; + result = -1; + goto fail; + fail: + trunnel_assert(result < 0); + return result; +} + +/** As sendme_cell_parse(), but do not allocate the output object. + */ +static ssize_t +sendme_cell_parse_into(sendme_cell_t *obj, const uint8_t *input, const size_t len_in) +{ + const uint8_t *ptr = input; + size_t remaining = len_in; + ssize_t result = 0; + (void)result; + + /* Parse u8 version IN [0, 1] */ + CHECK_REMAINING(1, truncated); + obj->version = (trunnel_get_uint8(ptr)); + remaining -= 1; ptr += 1; + if (! (obj->version == 0 || obj->version == 1)) + goto fail; + + /* Parse u16 data_len */ + CHECK_REMAINING(2, truncated); + obj->data_len = trunnel_ntohs(trunnel_get_uint16(ptr)); + remaining -= 2; ptr += 2; + + /* Parse u8 data[data_len] */ + CHECK_REMAINING(obj->data_len, truncated); + TRUNNEL_DYNARRAY_EXPAND(uint8_t, &obj->data, obj->data_len, {}); + obj->data.n_ = obj->data_len; + if (obj->data_len) + memcpy(obj->data.elts_, ptr, obj->data_len); + ptr += obj->data_len; remaining -= obj->data_len; + trunnel_assert(ptr + remaining == input + len_in); + return len_in - remaining; + + truncated: + return -2; + trunnel_alloc_failed: + return -1; + fail: + result = -1; + return result; +} + +ssize_t +sendme_cell_parse(sendme_cell_t **output, const uint8_t *input, const size_t len_in) +{ + ssize_t result; + *output = sendme_cell_new(); + if (NULL == *output) + return -1; + result = sendme_cell_parse_into(*output, input, len_in); + if (result < 0) { + sendme_cell_free(*output); + *output = NULL; + } + return result; +} +sendme_data_v1_t * +sendme_data_v1_new(void) +{ + sendme_data_v1_t *val = trunnel_calloc(1, sizeof(sendme_data_v1_t)); + if (NULL == val) + return NULL; + return val; +} + +/** Release all storage held inside 'obj', but do not free 'obj'. + */ +static void +sendme_data_v1_clear(sendme_data_v1_t *obj) +{ + (void) obj; +} + +void +sendme_data_v1_free(sendme_data_v1_t *obj) +{ + if (obj == NULL) + return; + sendme_data_v1_clear(obj); + trunnel_memwipe(obj, sizeof(sendme_data_v1_t)); + trunnel_free_(obj); +} + +size_t +sendme_data_v1_getlen_digest(const sendme_data_v1_t *inp) +{ + (void)inp; return 4; +} + +uint8_t +sendme_data_v1_get_digest(sendme_data_v1_t *inp, size_t idx) +{ + trunnel_assert(idx < 4); + return inp->digest[idx]; +} + +uint8_t +sendme_data_v1_getconst_digest(const sendme_data_v1_t *inp, size_t idx) +{ + return sendme_data_v1_get_digest((sendme_data_v1_t*)inp, idx); +} +int +sendme_data_v1_set_digest(sendme_data_v1_t *inp, size_t idx, uint8_t elt) +{ + trunnel_assert(idx < 4); + inp->digest[idx] = elt; + return 0; +} + +uint8_t * +sendme_data_v1_getarray_digest(sendme_data_v1_t *inp) +{ + return inp->digest; +} +const uint8_t * +sendme_data_v1_getconstarray_digest(const sendme_data_v1_t *inp) +{ + return (const uint8_t *)sendme_data_v1_getarray_digest((sendme_data_v1_t*)inp); +} +const char * +sendme_data_v1_check(const sendme_data_v1_t *obj) +{ + if (obj == NULL) + return "Object was NULL"; + if (obj->trunnel_error_code_) + return "A set function failed on this object"; + return NULL; +} + +ssize_t +sendme_data_v1_encoded_len(const sendme_data_v1_t *obj) +{ + ssize_t result = 0; + + if (NULL != sendme_data_v1_check(obj)) + return -1; + + + /* Length of u8 digest[4] */ + result += 4; + return result; +} +int +sendme_data_v1_clear_errors(sendme_data_v1_t *obj) +{ + int r = obj->trunnel_error_code_; + obj->trunnel_error_code_ = 0; + return r; +} +ssize_t +sendme_data_v1_encode(uint8_t *output, const size_t avail, const sendme_data_v1_t *obj) +{ + ssize_t result = 0; + size_t written = 0; + uint8_t *ptr = output; + const char *msg; +#ifdef TRUNNEL_CHECK_ENCODED_LEN + const ssize_t encoded_len = sendme_data_v1_encoded_len(obj); +#endif + + if (NULL != (msg = sendme_data_v1_check(obj))) + goto check_failed; + +#ifdef TRUNNEL_CHECK_ENCODED_LEN + trunnel_assert(encoded_len >= 0); +#endif + + /* Encode u8 digest[4] */ + trunnel_assert(written <= avail); + if (avail - written < 4) + goto truncated; + memcpy(ptr, obj->digest, 4); + written += 4; ptr += 4; + + + trunnel_assert(ptr == output + written); +#ifdef TRUNNEL_CHECK_ENCODED_LEN + { + trunnel_assert(encoded_len >= 0); + trunnel_assert((size_t)encoded_len == written); + } + +#endif + + return written; + + truncated: + result = -2; + goto fail; + check_failed: + (void)msg; + result = -1; + goto fail; + fail: + trunnel_assert(result < 0); + return result; +} + +/** As sendme_data_v1_parse(), but do not allocate the output object. + */ +static ssize_t +sendme_data_v1_parse_into(sendme_data_v1_t *obj, const uint8_t *input, const size_t len_in) +{ + const uint8_t *ptr = input; + size_t remaining = len_in; + ssize_t result = 0; + (void)result; + + /* Parse u8 digest[4] */ + CHECK_REMAINING(4, truncated); + memcpy(obj->digest, ptr, 4); + remaining -= 4; ptr += 4; + trunnel_assert(ptr + remaining == input + len_in); + return len_in - remaining; + + truncated: + return -2; +} + +ssize_t +sendme_data_v1_parse(sendme_data_v1_t **output, const uint8_t *input, const size_t len_in) +{ + ssize_t result; + *output = sendme_data_v1_new(); + if (NULL == *output) + return -1; + result = sendme_data_v1_parse_into(*output, input, len_in); + if (result < 0) { + sendme_data_v1_free(*output); + *output = NULL; + } + return result; +} diff --git a/src/trunnel/sendme.h b/src/trunnel/sendme.h new file mode 100644 index 0000000000..8207cb56f7 --- /dev/null +++ b/src/trunnel/sendme.h @@ -0,0 +1,170 @@ +/* sendme.h -- generated by Trunnel v1.5.2. + * https://gitweb.torproject.org/trunnel.git + * You probably shouldn't edit this file. + */ +#ifndef TRUNNEL_SENDME_H +#define TRUNNEL_SENDME_H + +#include +#include "trunnel.h" + +#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_SENDME_CELL) +struct sendme_cell_st { + uint8_t version; + uint16_t data_len; + TRUNNEL_DYNARRAY_HEAD(, uint8_t) data; + uint8_t trunnel_error_code_; +}; +#endif +typedef struct sendme_cell_st sendme_cell_t; +#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_SENDME_DATA_V1) +struct sendme_data_v1_st { + uint8_t digest[4]; + uint8_t trunnel_error_code_; +}; +#endif +typedef struct sendme_data_v1_st sendme_data_v1_t; +/** Return a newly allocated sendme_cell with all elements set to + * zero. + */ +sendme_cell_t *sendme_cell_new(void); +/** Release all storage held by the sendme_cell in 'victim'. (Do + * nothing if 'victim' is NULL.) + */ +void sendme_cell_free(sendme_cell_t *victim); +/** Try to parse a sendme_cell from the buffer in 'input', using up to + * 'len_in' bytes from the input buffer. On success, return the number + * of bytes consumed and set *output to the newly allocated + * sendme_cell_t. On failure, return -2 if the input appears + * truncated, and -1 if the input is otherwise invalid. + */ +ssize_t sendme_cell_parse(sendme_cell_t **output, const uint8_t *input, const size_t len_in); +/** Return the number of bytes we expect to need to encode the + * sendme_cell in 'obj'. On failure, return a negative value. Note + * that this value may be an overestimate, and can even be an + * underestimate for certain unencodeable objects. + */ +ssize_t sendme_cell_encoded_len(const sendme_cell_t *obj); +/** Try to encode the sendme_cell from 'input' into the buffer at + * 'output', using up to 'avail' bytes of the output buffer. On + * success, return the number of bytes used. On failure, return -2 if + * the buffer was not long enough, and -1 if the input was invalid. + */ +ssize_t sendme_cell_encode(uint8_t *output, size_t avail, const sendme_cell_t *input); +/** Check whether the internal state of the sendme_cell in 'obj' is + * consistent. Return NULL if it is, and a short message if it is not. + */ +const char *sendme_cell_check(const sendme_cell_t *obj); +/** Clear any errors that were set on the object 'obj' by its setter + * functions. Return true iff errors were cleared. + */ +int sendme_cell_clear_errors(sendme_cell_t *obj); +/** Return the value of the version field of the sendme_cell_t in + * 'inp' + */ +uint8_t sendme_cell_get_version(const sendme_cell_t *inp); +/** Set the value of the version field of the sendme_cell_t in 'inp' + * to 'val'. Return 0 on success; return -1 and set the error code on + * 'inp' on failure. + */ +int sendme_cell_set_version(sendme_cell_t *inp, uint8_t val); +/** Return the value of the data_len field of the sendme_cell_t in + * 'inp' + */ +uint16_t sendme_cell_get_data_len(const sendme_cell_t *inp); +/** Set the value of the data_len field of the sendme_cell_t in 'inp' + * to 'val'. Return 0 on success; return -1 and set the error code on + * 'inp' on failure. + */ +int sendme_cell_set_data_len(sendme_cell_t *inp, uint16_t val); +/** Return the length of the dynamic array holding the data field of + * the sendme_cell_t in 'inp'. + */ +size_t sendme_cell_getlen_data(const sendme_cell_t *inp); +/** Return the element at position 'idx' of the dynamic array field + * data of the sendme_cell_t in 'inp'. + */ +uint8_t sendme_cell_get_data(sendme_cell_t *inp, size_t idx); +/** As sendme_cell_get_data, but take and return a const pointer + */ +uint8_t sendme_cell_getconst_data(const sendme_cell_t *inp, size_t idx); +/** Change the element at position 'idx' of the dynamic array field + * data of the sendme_cell_t in 'inp', so that it will hold the value + * 'elt'. + */ +int sendme_cell_set_data(sendme_cell_t *inp, size_t idx, uint8_t elt); +/** Append a new element 'elt' to the dynamic array field data of the + * sendme_cell_t in 'inp'. + */ +int sendme_cell_add_data(sendme_cell_t *inp, uint8_t elt); +/** Return a pointer to the variable-length array field data of 'inp'. + */ +uint8_t * sendme_cell_getarray_data(sendme_cell_t *inp); +/** As sendme_cell_get_data, but take and return a const pointer + */ +const uint8_t * sendme_cell_getconstarray_data(const sendme_cell_t *inp); +/** Change the length of the variable-length array field data of 'inp' + * to 'newlen'.Fill extra elements with 0. Return 0 on success; return + * -1 and set the error code on 'inp' on failure. + */ +int sendme_cell_setlen_data(sendme_cell_t *inp, size_t newlen); +/** Return a newly allocated sendme_data_v1 with all elements set to + * zero. + */ +sendme_data_v1_t *sendme_data_v1_new(void); +/** Release all storage held by the sendme_data_v1 in 'victim'. (Do + * nothing if 'victim' is NULL.) + */ +void sendme_data_v1_free(sendme_data_v1_t *victim); +/** Try to parse a sendme_data_v1 from the buffer in 'input', using up + * to 'len_in' bytes from the input buffer. On success, return the + * number of bytes consumed and set *output to the newly allocated + * sendme_data_v1_t. On failure, return -2 if the input appears + * truncated, and -1 if the input is otherwise invalid. + */ +ssize_t sendme_data_v1_parse(sendme_data_v1_t **output, const uint8_t *input, const size_t len_in); +/** Return the number of bytes we expect to need to encode the + * sendme_data_v1 in 'obj'. On failure, return a negative value. Note + * that this value may be an overestimate, and can even be an + * underestimate for certain unencodeable objects. + */ +ssize_t sendme_data_v1_encoded_len(const sendme_data_v1_t *obj); +/** Try to encode the sendme_data_v1 from 'input' into the buffer at + * 'output', using up to 'avail' bytes of the output buffer. On + * success, return the number of bytes used. On failure, return -2 if + * the buffer was not long enough, and -1 if the input was invalid. + */ +ssize_t sendme_data_v1_encode(uint8_t *output, size_t avail, const sendme_data_v1_t *input); +/** Check whether the internal state of the sendme_data_v1 in 'obj' is + * consistent. Return NULL if it is, and a short message if it is not. + */ +const char *sendme_data_v1_check(const sendme_data_v1_t *obj); +/** Clear any errors that were set on the object 'obj' by its setter + * functions. Return true iff errors were cleared. + */ +int sendme_data_v1_clear_errors(sendme_data_v1_t *obj); +/** Return the (constant) length of the array holding the digest field + * of the sendme_data_v1_t in 'inp'. + */ +size_t sendme_data_v1_getlen_digest(const sendme_data_v1_t *inp); +/** Return the element at position 'idx' of the fixed array field + * digest of the sendme_data_v1_t in 'inp'. + */ +uint8_t sendme_data_v1_get_digest(sendme_data_v1_t *inp, size_t idx); +/** As sendme_data_v1_get_digest, but take and return a const pointer + */ +uint8_t sendme_data_v1_getconst_digest(const sendme_data_v1_t *inp, size_t idx); +/** Change the element at position 'idx' of the fixed array field + * digest of the sendme_data_v1_t in 'inp', so that it will hold the + * value 'elt'. + */ +int sendme_data_v1_set_digest(sendme_data_v1_t *inp, size_t idx, uint8_t elt); +/** Return a pointer to the 4-element array field digest of 'inp'. + */ +uint8_t * sendme_data_v1_getarray_digest(sendme_data_v1_t *inp); +/** As sendme_data_v1_get_digest, but take and return a const pointer + */ +const uint8_t * sendme_data_v1_getconstarray_digest(const sendme_data_v1_t *inp); + + +#endif diff --git a/src/trunnel/sendme.trunnel b/src/trunnel/sendme.trunnel new file mode 100644 index 0000000000..7294a09b47 --- /dev/null +++ b/src/trunnel/sendme.trunnel @@ -0,0 +1,18 @@ +/* This file contains the SENDME cell definition. */ + +struct sendme_cell { + /* Version field. */ + u8 version IN [0x00, 0x01]; + + /* The data content depends on the version. */ + u16 data_len; + u8 data[data_len]; +} + +/* SENDME version 0. No data. */ + +/* SENDME version 1. Authenticated with digest. */ +struct sendme_data_v1 { + /* A 4 bytes digest. */ + u8 digest[4]; +} From 023a70da841182fbbbe11389929c8fbbc7490365 Mon Sep 17 00:00:00 2001 From: David Goulet Date: Wed, 9 Jan 2019 12:22:35 -0500 Subject: [PATCH 0872/2557] prop289: Support sending SENDME version 1 This code will obey the consensus parameter "sendme_emit_min_version" to know which SENDME version it should send. For now, the default is 0 and the parameter is not yet used in the consensus. This commit adds the support to send version 1 SENDMEs but aren't sent on the wire at this commit. Closes #26840 Signed-off-by: David Goulet --- src/core/or/relay.c | 2 +- src/core/or/sendme.c | 106 ++++++++++++++++++++++++++++++++++++++++--- src/core/or/sendme.h | 3 +- 3 files changed, 103 insertions(+), 8 deletions(-) diff --git a/src/core/or/relay.c b/src/core/or/relay.c index 6ff053d8a0..6f69ed999b 100644 --- a/src/core/or/relay.c +++ b/src/core/or/relay.c @@ -1560,7 +1560,7 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, } /* Consider sending a circuit-level SENDME cell. */ - sendme_circuit_consider_sending(circ, layer_hint); + sendme_circuit_consider_sending(circ, layer_hint, NULL); if (rh.stream_id == 0) { log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, "Relay data cell with zero " diff --git a/src/core/or/sendme.c b/src/core/or/sendme.c index bba760ae98..f22e7027db 100644 --- a/src/core/or/sendme.c +++ b/src/core/or/sendme.c @@ -16,6 +16,7 @@ #include "core/or/relay.h" #include "core/or/sendme.h" #include "feature/nodelist/networkstatus.h" +#include "trunnel/sendme.h" /* The cell version constants for when emitting a cell. */ #define SENDME_EMIT_MIN_VERSION_DEFAULT 0 @@ -38,6 +39,7 @@ get_emit_min_version(void) SENDME_EMIT_MIN_VERSION_MAX); } +#if 0 /* Return the minimum version given by the consensus (if any) that should be * accepted when receiving a SENDME cell. */ static int @@ -48,6 +50,98 @@ get_accept_min_version(void) SENDME_ACCEPT_MIN_VERSION_MIN, SENDME_ACCEPT_MIN_VERSION_MAX); } +#endif + +/* Build and encode a version 1 SENDME cell into payload, which must be at + * least of RELAY_PAYLOAD_SIZE bytes, using the digest for the cell data. + * + * Return the size in bytes of the encoded cell in payload. A negative value + * is returned on encoding failure. */ +static ssize_t +build_cell_payload_v1(crypto_digest_t *cell_digest, uint8_t *payload) +{ + ssize_t len = -1; + sendme_cell_t *cell = NULL; + sendme_data_v1_t *data = NULL; + + tor_assert(cell_digest); + tor_assert(payload); + + cell = sendme_cell_new(); + data = sendme_data_v1_new(); + + /* Building a payload for version 1. */ + sendme_cell_set_version(cell, 0x01); + + /* Copy the digest into the data payload. */ + crypto_digest_get_digest(cell_digest, + (char *) sendme_data_v1_getarray_digest(data), + sendme_data_v1_getlen_digest(data)); + + /* Set the length of the data in the cell payload. It is the encoded length + * of the v1 data object. */ + sendme_cell_setlen_data(cell, sendme_data_v1_encoded_len(data)); + /* Encode into the cell's data field using its current length just set. */ + if (sendme_data_v1_encode(sendme_cell_getarray_data(cell), + sendme_cell_getlen_data(cell), data) < 0) { + goto end; + } + /* Set the DATA_LEN field to what we've just encoded. */ + sendme_cell_set_data_len(cell, sendme_cell_getlen_data(cell)); + + /* Finally, encode the cell into the payload. */ + len = sendme_cell_encode(payload, RELAY_PAYLOAD_SIZE, cell); + + end: + sendme_cell_free(cell); + sendme_data_v1_free(data); + return len; +} + +/* Send a circuit-level SENDME on the given circuit using the layer_hint if + * not NULL. The digest is only used for version 1. + * + * Return 0 on success else a negative value and the circuit will be closed + * because we failed to send the cell on it. */ +static int +send_circuit_level_sendme(circuit_t *circ, crypt_path_t *layer_hint, + crypto_digest_t *cell_digest) +{ + uint8_t emit_version; + uint8_t payload[RELAY_PAYLOAD_SIZE]; + ssize_t payload_len; + + tor_assert(circ); + tor_assert(cell_digest); + + emit_version = get_emit_min_version(); + switch (emit_version) { + case 0x01: + payload_len = build_cell_payload_v1(cell_digest, payload); + if (BUG(payload_len < 0)) { + /* Unable to encode the cell, abort. We can recover from this by closing + * the circuit but in theory it should never happen. */ + return -1; + } + log_debug(LD_PROTOCOL, "Emitting SENDME version 1 cell."); + break; + case 0x00: + /* Fallthrough because default is to use v0. */ + default: + /* Unknown version, fallback to version 0 meaning no payload. */ + payload_len = 0; + break; + } + + if (relay_send_command_from_edge(0, circ, RELAY_COMMAND_SENDME, + (char *) payload, payload_len, + layer_hint) < 0) { + log_warn(LD_CIRC, + "SENDME relay_send_command_from_edge failed. Circuit's closed."); + return -1; /* the circuit's closed, don't continue */ + } + return 0; +} /** Called when we've just received a relay data cell, when we've just * finished flushing all bytes to stream conn, or when we've flushed @@ -100,8 +194,11 @@ sendme_connection_edge_consider_sending(edge_connection_t *conn) * more. */ void -sendme_circuit_consider_sending(circuit_t *circ, crypt_path_t *layer_hint) +sendme_circuit_consider_sending(circuit_t *circ, crypt_path_t *layer_hint, + crypto_digest_t *digest) { + tor_assert(digest); + while ((layer_hint ? layer_hint->deliver_window : circ->deliver_window) <= CIRCWINDOW_START - CIRCWINDOW_INCREMENT) { log_debug(LD_CIRC,"Queuing circuit sendme."); @@ -109,11 +206,8 @@ sendme_circuit_consider_sending(circuit_t *circ, crypt_path_t *layer_hint) layer_hint->deliver_window += CIRCWINDOW_INCREMENT; else circ->deliver_window += CIRCWINDOW_INCREMENT; - if (relay_send_command_from_edge(0, circ, RELAY_COMMAND_SENDME, - NULL, 0, layer_hint) < 0) { - log_warn(LD_CIRC, - "relay_send_command_from_edge failed. Circuit's closed."); - return; /* the circuit's closed, don't continue */ + if (send_circuit_level_sendme(circ, layer_hint, digest) < 0) { + return; /* The circuit's closed, don't continue */ } } } diff --git a/src/core/or/sendme.h b/src/core/or/sendme.h index f50ccfbfe2..ba01ecfaf8 100644 --- a/src/core/or/sendme.h +++ b/src/core/or/sendme.h @@ -16,7 +16,8 @@ /* Sending SENDME cell. */ void sendme_connection_edge_consider_sending(edge_connection_t *edge_conn); void sendme_circuit_consider_sending(circuit_t *circ, - crypt_path_t *layer_hint); + crypt_path_t *layer_hint, + crypto_digest_t *digest); /* Processing SENDME cell. */ int sendme_process_circuit_level(crypt_path_t *layer_hint, From 81706d84279f0a2870f8b1789403188fd933b32a Mon Sep 17 00:00:00 2001 From: David Goulet Date: Wed, 9 Jan 2019 14:03:32 -0500 Subject: [PATCH 0873/2557] prop289: Support SENDME v1 cell parsing This commit makes tor able to parse and handle a SENDME version 1. It will look at the consensus parameter "sendme_accept_min_version" to know what is the minimum version it should look at. IMPORTANT: At this commit, the validation of the cell is not fully implemented. For this, we need #26839 to be completed that is to match the SENDME digest with the last cell digest. Closes #26841 Signed-off-by: David Goulet --- src/core/or/relay.c | 4 +- src/core/or/sendme.c | 140 +++++++++++++++++++++++++++++++++++++++++-- src/core/or/sendme.h | 3 +- 3 files changed, 140 insertions(+), 7 deletions(-) diff --git a/src/core/or/relay.c b/src/core/or/relay.c index 6f69ed999b..76f2203a9a 100644 --- a/src/core/or/relay.c +++ b/src/core/or/relay.c @@ -1814,7 +1814,9 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, if (!rh.stream_id) { /* Circuit level SENDME cell. */ - ret = sendme_process_circuit_level(layer_hint, circ, rh.length); + ret = sendme_process_circuit_level(layer_hint, circ, + cell->payload + RELAY_HEADER_SIZE, + rh.length); if (ret < 0) { return ret; } diff --git a/src/core/or/sendme.c b/src/core/or/sendme.c index f22e7027db..64497055e1 100644 --- a/src/core/or/sendme.c +++ b/src/core/or/sendme.c @@ -18,6 +18,10 @@ #include "feature/nodelist/networkstatus.h" #include "trunnel/sendme.h" +/* The maximum supported version. Above that value, the cell can't be + * recognized as a valid SENDME. */ +#define SENDME_MAX_SUPPORTED_VERSION 1 + /* The cell version constants for when emitting a cell. */ #define SENDME_EMIT_MIN_VERSION_DEFAULT 0 #define SENDME_EMIT_MIN_VERSION_MIN 0 @@ -39,7 +43,6 @@ get_emit_min_version(void) SENDME_EMIT_MIN_VERSION_MAX); } -#if 0 /* Return the minimum version given by the consensus (if any) that should be * accepted when receiving a SENDME cell. */ static int @@ -50,7 +53,124 @@ get_accept_min_version(void) SENDME_ACCEPT_MIN_VERSION_MIN, SENDME_ACCEPT_MIN_VERSION_MAX); } -#endif + +/* Return true iff the given decoded SENDME version 1 cell is valid. + * + * Validation is done by comparing the digest in the cell from the previous + * cell we saw which tells us that the other side has in fact seen that cell. + * See proposal 289 for more details. */ +static bool +cell_v1_is_valid(const sendme_cell_t *cell) +{ + sendme_data_v1_t *data = NULL; + + tor_assert(cell); + + if (sendme_data_v1_parse(&data, sendme_cell_getconstarray_data(cell), + sendme_cell_getlen_data(cell)) < 0) { + goto invalid; + } + + /* XXX: Match the digest in the cell to the previous cell. Needs to be + * implemented that is passed to this function and compared. For this, we + * need #26839 that is making tor remember the last digest(s). */ + + /* Validated SENDME v1 cell. */ + sendme_data_v1_free(data); + return 1; + invalid: + sendme_data_v1_free(data); + return 0; +} + +/* Return true iff the given cell version can be handled or if the minimum + * accepted version from the consensus is known to us. */ +static bool +cell_version_is_valid(uint8_t cell_version) +{ + int accept_version = get_accept_min_version(); + + /* Can we handle this version? */ + if (accept_version > SENDME_MAX_SUPPORTED_VERSION) { + log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, + "Unable to handle SENDME version %u. We only support <= %d " + "(from consensus). Probably your tor is too old?", + accept_version, cell_version); + goto invalid; + } + + /* We only accept a SENDME cell from what the consensus tells us. */ + if (cell_version < accept_version) { + log_info(LD_PROTOCOL, "Unacceptable SENDME version %d. Only " + "accepting %u (taken from the consensus). " + "Closing circuit.", + cell_version, accept_version); + goto invalid; + } + + return 1; + invalid: + return 0; +} + +/* Return true iff the encoded SENDME cell in cell_payload of length + * cell_payload_len is valid. For each version: + * + * 0: No validation + * 1: Authenticated with last cell digest. + * + * This is the main critical function to make sure we can continue to + * send/recv cells on a circuit. If the SENDME is invalid, the circuit should + * be mark for close. */ +static bool +sendme_is_valid(const uint8_t *cell_payload, size_t cell_payload_len) +{ + uint8_t cell_version; + sendme_cell_t *cell = NULL; + + tor_assert(cell_payload); + + /* An empty payload means version 0 so skip trunnel parsing. We won't be + * able to parse a 0 length buffer into a valid SENDME cell. */ + if (cell_payload_len == 0) { + cell_version = 0; + } else { + /* First we'll decode the cell so we can get the version. */ + if (sendme_cell_parse(&cell, cell_payload, cell_payload_len) < 0) { + log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, + "Unparseable SENDME cell received. Closing circuit."); + goto invalid; + } + cell_version = sendme_cell_get_version(cell); + } + + /* Validate that we can handle this cell version. */ + if (!cell_version_is_valid(cell_version)) { + goto invalid; + } + + /* Validate depending on the version now. */ + switch (cell_version) { + case 0x01: + if (!cell_v1_is_valid(cell)) { + goto invalid; + } + break; + case 0x00: + /* Fallthrough. Version 0, there is no work to be done on the payload so + * it is necessarily valid if we pass the version validation. */ + default: + /* Unknown version means we can't handle it so fallback to version 0. */ + break; + } + + /* Valid cell. */ + sendme_cell_free(cell); + return 1; + invalid: + sendme_cell_free(cell); + return 0; +} /* Build and encode a version 1 SENDME cell into payload, which must be at * least of RELAY_PAYLOAD_SIZE bytes, using the digest for the cell data. @@ -215,7 +335,8 @@ sendme_circuit_consider_sending(circuit_t *circ, crypt_path_t *layer_hint, /* Process a circuit-level SENDME cell that we just received. The layer_hint, * if not NULL, is the Exit hop of the connection which means that we are a * client. In that case, circ must be an origin circuit. The cell_body_len is - * the length of the SENDME cell payload (excluding the header). + * the length of the SENDME cell payload (excluding the header). The + * cell_payload is the payload. * * Return 0 on success that is the SENDME is valid and the package window has * been updated properly. @@ -224,9 +345,11 @@ sendme_circuit_consider_sending(circuit_t *circ, crypt_path_t *layer_hint, * be closed using the value as the reason for it. */ int sendme_process_circuit_level(crypt_path_t *layer_hint, - circuit_t *circ, uint16_t cell_body_len) + circuit_t *circ, const uint8_t *cell_payload, + uint16_t cell_payload_len) { tor_assert(circ); + tor_assert(cell_payload); /* If we are the origin of the circuit, we are the Client so we use the * layer hint (the Exit hop) for the package window tracking. */ @@ -245,8 +368,15 @@ sendme_process_circuit_level(crypt_path_t *layer_hint, /* We count circuit-level sendme's as valid delivered data because they * are rate limited. */ - circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), cell_body_len); + circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), cell_payload_len); } else { + /* Validate the SENDME cell. Depending on the version, different + * validation can be done. An invalid SENDME requires us to close the + * circuit. It is only done if we are the Exit of the circuit. */ + if (!sendme_is_valid(cell_payload, cell_payload_len)) { + return -END_CIRC_REASON_TORPROTOCOL; + } + /* We aren't the origin of this circuit so we are the Exit and thus we * track the package window with the circuit object. */ if ((circ->package_window + CIRCWINDOW_INCREMENT) > diff --git a/src/core/or/sendme.h b/src/core/or/sendme.h index ba01ecfaf8..033bc6ff75 100644 --- a/src/core/or/sendme.h +++ b/src/core/or/sendme.h @@ -21,7 +21,8 @@ void sendme_circuit_consider_sending(circuit_t *circ, /* Processing SENDME cell. */ int sendme_process_circuit_level(crypt_path_t *layer_hint, - circuit_t *circ, uint16_t cell_body_len); + circuit_t *circ, const uint8_t *cell_payload, + uint16_t cell_payload_len); int sendme_process_stream_level(edge_connection_t *conn, circuit_t *circ, uint16_t cell_body_len); From 93f9fbbd34f03aca68c8c64a7e39b64548462eeb Mon Sep 17 00:00:00 2001 From: David Goulet Date: Wed, 9 Jan 2019 15:08:14 -0500 Subject: [PATCH 0874/2557] prop289: Keep track of the last seen cell digests This makes tor remember the last seen digest of a cell if that cell is the last one before a SENDME on the Exit side. Closes #26839 Signed-off-by: David Goulet --- src/core/or/circuit_st.h | 6 ++++++ src/core/or/circuitlist.c | 6 ++++++ src/core/or/relay.c | 8 ++++++++ src/core/or/sendme.c | 29 +++++++++++++++++++++++++++++ src/core/or/sendme.h | 3 +++ 5 files changed, 52 insertions(+) diff --git a/src/core/or/circuit_st.h b/src/core/or/circuit_st.h index cc21cf62f7..5adb158935 100644 --- a/src/core/or/circuit_st.h +++ b/src/core/or/circuit_st.h @@ -104,6 +104,12 @@ struct circuit_t { * circuit-level sendme cells to indicate that we're willing to accept * more. */ int deliver_window; + /** FIFO containing the digest of the cells that are just before a SENDME is + * sent by the client. It is done at the last cell before our package_window + * goes down to 0 which is when we expect a SENDME. The protocol doesn't + * allow more than 10 outstanding SENDMEs worth of data meaning this list + * should only contain at most 10 digests of 4 bytes each. */ + smartlist_t *sendme_last_digests; /** Temporary field used during circuits_handle_oom. */ uint32_t age_tmp; diff --git a/src/core/or/circuitlist.c b/src/core/or/circuitlist.c index afbde06434..6428cdb8a7 100644 --- a/src/core/or/circuitlist.c +++ b/src/core/or/circuitlist.c @@ -1227,6 +1227,12 @@ circuit_free_(circuit_t *circ) * "active" checks will be violated. */ cell_queue_clear(&circ->n_chan_cells); + /* Cleanup possible SENDME state. */ + if (circ->sendme_last_digests) { + SMARTLIST_FOREACH(circ->sendme_last_digests, uint8_t *, d, tor_free(d)); + smartlist_free(circ->sendme_last_digests); + } + log_info(LD_CIRC, "Circuit %u (id: %" PRIu32 ") has been freed.", n_circ_id, CIRCUIT_IS_ORIGIN(circ) ? diff --git a/src/core/or/relay.c b/src/core/or/relay.c index 76f2203a9a..b26360b245 100644 --- a/src/core/or/relay.c +++ b/src/core/or/relay.c @@ -639,6 +639,14 @@ relay_send_command_from_edge_,(streamid_t stream_id, circuit_t *circ, circuit_mark_for_close(circ, END_CIRC_REASON_INTERNAL); return -1; } + + /* If applicable, note the cell digest for the SENDME version 1 purpose if + * we need to. This call needs to be after the circuit_package_relay_cell() + * because the cell digest is set within that function. */ + if (relay_command == RELAY_COMMAND_DATA) { + sendme_note_cell_digest(circ); + } + return 0; } diff --git a/src/core/or/sendme.c b/src/core/or/sendme.c index 64497055e1..69bcac4680 100644 --- a/src/core/or/sendme.c +++ b/src/core/or/sendme.c @@ -11,6 +11,7 @@ #include "app/config/config.h" #include "core/mainloop/connection.h" +#include "core/or/cell_st.h" #include "core/or/circuitlist.h" #include "core/or/circuituse.h" #include "core/or/relay.h" @@ -507,3 +508,31 @@ sendme_stream_data_packaged(edge_connection_t *conn) tor_assert(conn); return --conn->package_window; } + +/* Note the cell digest in the circuit sendme last digests FIFO if applicable. + * It is safe to pass a circuit that isn't meant to track those digests. */ +void +sendme_note_cell_digest(circuit_t *circ) +{ + uint8_t *digest; + + tor_assert(circ); + + /* We only keep the cell digest if we are the Exit on that circuit and if + * this cell is the last one before the client should send a SENDME. */ + if (CIRCUIT_IS_ORIGIN(circ)) { + return; + } + /* Is this the last cell before a SENDME? The idea is that if the + * package_window reaches a multiple of the increment, after this cell, we + * should expect a SENDME. */ + if (((circ->package_window - 1) % CIRCWINDOW_INCREMENT) != 0) { + return; + } + + digest = tor_malloc_zero(4); + if (circ->sendme_last_digests == NULL) { + circ->sendme_last_digests = smartlist_new(); + } + smartlist_add(circ->sendme_last_digests, digest); +} diff --git a/src/core/or/sendme.h b/src/core/or/sendme.h index 033bc6ff75..300fb25d96 100644 --- a/src/core/or/sendme.h +++ b/src/core/or/sendme.h @@ -34,4 +34,7 @@ int sendme_circuit_data_received(circuit_t *circ, crypt_path_t *layer_hint); int sendme_circuit_data_packaged(circuit_t *circ, crypt_path_t *layer_hint); int sendme_stream_data_packaged(edge_connection_t *conn); +/* Track cell digest. */ +void sendme_note_cell_digest(circuit_t *circ); + #endif /* !defined(TOR_SENDME_H) */ From bb473a807ae94a1e6c45a069db6ddf213413940a Mon Sep 17 00:00:00 2001 From: David Goulet Date: Wed, 9 Jan 2019 15:27:51 -0500 Subject: [PATCH 0875/2557] prop289: Match the SENDME digest Now that we keep the last seen cell digests on the Exit side on the circuit object, use that to match the SENDME v1 transforming this whole process into a real authenticated SENDME mechanism. Part of #26841 Signed-off-by: David Goulet --- src/core/or/sendme.c | 41 ++++++++++++++++++++++++++++++++++------- 1 file changed, 34 insertions(+), 7 deletions(-) diff --git a/src/core/or/sendme.c b/src/core/or/sendme.c index 69bcac4680..afade43f74 100644 --- a/src/core/or/sendme.c +++ b/src/core/or/sendme.c @@ -17,6 +17,7 @@ #include "core/or/relay.h" #include "core/or/sendme.h" #include "feature/nodelist/networkstatus.h" +#include "lib/ctime/di_ops.h" #include "trunnel/sendme.h" /* The maximum supported version. Above that value, the cell can't be @@ -61,7 +62,7 @@ get_accept_min_version(void) * cell we saw which tells us that the other side has in fact seen that cell. * See proposal 289 for more details. */ static bool -cell_v1_is_valid(const sendme_cell_t *cell) +cell_v1_is_valid(const sendme_cell_t *cell, const circuit_t *circ) { sendme_data_v1_t *data = NULL; @@ -72,9 +73,33 @@ cell_v1_is_valid(const sendme_cell_t *cell) goto invalid; } - /* XXX: Match the digest in the cell to the previous cell. Needs to be - * implemented that is passed to this function and compared. For this, we - * need #26839 that is making tor remember the last digest(s). */ + /* We shouldn't have received this SENDME if we have no digests. Log at + * protocol warning because it can be tricked by sending many SENDMEs + * without prior data cell. */ + if (circ->sendme_last_digests == NULL || + smartlist_len(circ->sendme_last_digests) == 0) { + log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, + "We received a SENDME but we have no cell digests to match. " + "Closing circuit."); + goto invalid; + } + + /* Pop the first element that was added (FIFO) and compare it. */ + { + uint8_t *digest = smartlist_get(circ->sendme_last_digests, 0); + smartlist_del_keeporder(circ->sendme_last_digests, 0); + + /* Compare the digest with the one in the SENDME. This cell is invalid + * without a perfect match. */ + if (tor_memcmp(digest, sendme_data_v1_getconstarray_digest(data), + sendme_data_v1_getlen_digest(data))) { + tor_free(digest); + log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, + "SENDME v1 cell digest do not match."); + goto invalid; + } + tor_free(digest); + } /* Validated SENDME v1 cell. */ sendme_data_v1_free(data); @@ -124,11 +149,13 @@ cell_version_is_valid(uint8_t cell_version) * send/recv cells on a circuit. If the SENDME is invalid, the circuit should * be mark for close. */ static bool -sendme_is_valid(const uint8_t *cell_payload, size_t cell_payload_len) +sendme_is_valid(const circuit_t *circ, const uint8_t *cell_payload, + size_t cell_payload_len) { uint8_t cell_version; sendme_cell_t *cell = NULL; + tor_assert(circ); tor_assert(cell_payload); /* An empty payload means version 0 so skip trunnel parsing. We won't be @@ -153,7 +180,7 @@ sendme_is_valid(const uint8_t *cell_payload, size_t cell_payload_len) /* Validate depending on the version now. */ switch (cell_version) { case 0x01: - if (!cell_v1_is_valid(cell)) { + if (!cell_v1_is_valid(cell, circ)) { goto invalid; } break; @@ -374,7 +401,7 @@ sendme_process_circuit_level(crypt_path_t *layer_hint, /* Validate the SENDME cell. Depending on the version, different * validation can be done. An invalid SENDME requires us to close the * circuit. It is only done if we are the Exit of the circuit. */ - if (!sendme_is_valid(cell_payload, cell_payload_len)) { + if (!sendme_is_valid(circ, cell_payload, cell_payload_len)) { return -END_CIRC_REASON_TORPROTOCOL; } From 402f0a4f5d70bee128130f4dbd0ea18de1747410 Mon Sep 17 00:00:00 2001 From: David Goulet Date: Wed, 23 Jan 2019 14:39:04 -0500 Subject: [PATCH 0876/2557] prop289: Remember the last cell digest for v1 SENDMEs In order to do so, depending on where the cell is going, we'll keep the last cell digest that is either received inbound or sent outbound. Then it can be used for validation. Part of #26288 Signed-off-by: David Goulet --- src/core/crypto/relay_crypto.c | 13 +++++++++++++ src/core/or/relay.c | 2 +- src/core/or/relay_crypto_st.h | 2 ++ src/core/or/sendme.c | 28 +++++++++++++++++++--------- src/core/or/sendme.h | 3 +-- 5 files changed, 36 insertions(+), 12 deletions(-) diff --git a/src/core/crypto/relay_crypto.c b/src/core/crypto/relay_crypto.c index 0b83b2d0a5..d4116d47ab 100644 --- a/src/core/crypto/relay_crypto.c +++ b/src/core/crypto/relay_crypto.c @@ -142,6 +142,13 @@ relay_decrypt_cell(circuit_t *circ, cell_t *cell, if (relay_digest_matches(thishop->crypto.b_digest, cell)) { *recognized = 1; *layer_hint = thishop; + /* Keep current digest of this cell for the possible SENDME. */ + if (thishop->crypto.sendme_digest) { + crypto_digest_free(thishop->crypto.sendme_digest); + } + thishop->crypto.sendme_digest = + crypto_digest_dup(thishop->crypto.b_digest); + return 0; } } @@ -212,6 +219,11 @@ relay_encrypt_cell_inbound(cell_t *cell, or_circuit_t *or_circ) { relay_set_digest(or_circ->crypto.b_digest, cell); + /* Keep a record of this cell, we might use it for validating the SENDME. */ + if (or_circ->crypto.sendme_digest) { + crypto_digest_free(or_circ->crypto.sendme_digest); + } + or_circ->crypto.sendme_digest = crypto_digest_dup(or_circ->crypto.b_digest); /* encrypt one layer */ relay_crypt_one_payload(or_circ->crypto.b_crypto, cell->payload); } @@ -229,6 +241,7 @@ relay_crypto_clear(relay_crypto_t *crypto) crypto_cipher_free(crypto->b_crypto); crypto_digest_free(crypto->f_digest); crypto_digest_free(crypto->b_digest); + crypto_digest_free(crypto->sendme_digest); } /** Initialize crypto from the key material in key_data. diff --git a/src/core/or/relay.c b/src/core/or/relay.c index b26360b245..47275a811e 100644 --- a/src/core/or/relay.c +++ b/src/core/or/relay.c @@ -1568,7 +1568,7 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, } /* Consider sending a circuit-level SENDME cell. */ - sendme_circuit_consider_sending(circ, layer_hint, NULL); + sendme_circuit_consider_sending(circ, layer_hint); if (rh.stream_id == 0) { log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, "Relay data cell with zero " diff --git a/src/core/or/relay_crypto_st.h b/src/core/or/relay_crypto_st.h index dafce257c7..dbdf1599dc 100644 --- a/src/core/or/relay_crypto_st.h +++ b/src/core/or/relay_crypto_st.h @@ -25,6 +25,8 @@ struct relay_crypto_t { /** Digest state for cells heading away from the OR at this step. */ struct crypto_digest_t *b_digest; + /** Digest used for the next SENDME cell if any. */ + struct crypto_digest_t *sendme_digest; }; #undef crypto_cipher_t diff --git a/src/core/or/sendme.c b/src/core/or/sendme.c index afade43f74..76f551a929 100644 --- a/src/core/or/sendme.c +++ b/src/core/or/sendme.c @@ -14,6 +14,7 @@ #include "core/or/cell_st.h" #include "core/or/circuitlist.h" #include "core/or/circuituse.h" +#include "core/or/or_circuit_st.h" #include "core/or/relay.h" #include "core/or/sendme.h" #include "feature/nodelist/networkstatus.h" @@ -342,18 +343,20 @@ sendme_connection_edge_consider_sending(edge_connection_t *conn) * more. */ void -sendme_circuit_consider_sending(circuit_t *circ, crypt_path_t *layer_hint, - crypto_digest_t *digest) +sendme_circuit_consider_sending(circuit_t *circ, crypt_path_t *layer_hint) { - tor_assert(digest); + crypto_digest_t *digest; while ((layer_hint ? layer_hint->deliver_window : circ->deliver_window) <= CIRCWINDOW_START - CIRCWINDOW_INCREMENT) { log_debug(LD_CIRC,"Queuing circuit sendme."); - if (layer_hint) + if (layer_hint) { layer_hint->deliver_window += CIRCWINDOW_INCREMENT; - else + digest = layer_hint->crypto.sendme_digest; + } else { circ->deliver_window += CIRCWINDOW_INCREMENT; + digest = TO_OR_CIRCUIT(circ)->crypto.sendme_digest; + } if (send_circuit_level_sendme(circ, layer_hint, digest) < 0) { return; /* The circuit's closed, don't continue */ } @@ -557,9 +560,16 @@ sendme_note_cell_digest(circuit_t *circ) return; } - digest = tor_malloc_zero(4); - if (circ->sendme_last_digests == NULL) { - circ->sendme_last_digests = smartlist_new(); + /* Only note the digest if we actually have the digest of the previous cell + * recorded. It should never happen in theory as we always record the last + * digest for the v1 SENDME. */ + if (TO_OR_CIRCUIT(circ)->crypto.sendme_digest) { + digest = tor_malloc_zero(4); + crypto_digest_get_digest(TO_OR_CIRCUIT(circ)->crypto.sendme_digest, + (char *) digest, 4); + if (circ->sendme_last_digests == NULL) { + circ->sendme_last_digests = smartlist_new(); + } + smartlist_add(circ->sendme_last_digests, digest); } - smartlist_add(circ->sendme_last_digests, digest); } diff --git a/src/core/or/sendme.h b/src/core/or/sendme.h index 300fb25d96..e7cf718bb1 100644 --- a/src/core/or/sendme.h +++ b/src/core/or/sendme.h @@ -16,8 +16,7 @@ /* Sending SENDME cell. */ void sendme_connection_edge_consider_sending(edge_connection_t *edge_conn); void sendme_circuit_consider_sending(circuit_t *circ, - crypt_path_t *layer_hint, - crypto_digest_t *digest); + crypt_path_t *layer_hint); /* Processing SENDME cell. */ int sendme_process_circuit_level(crypt_path_t *layer_hint, From a6e012508e5b0d676cdf204fcbd7942e3cc21419 Mon Sep 17 00:00:00 2001 From: David Goulet Date: Tue, 19 Feb 2019 15:02:11 -0500 Subject: [PATCH 0877/2557] prop289: Add random bytes to the unused portion of the cell Signed-off-by: David Goulet --- src/core/or/relay.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/core/or/relay.c b/src/core/or/relay.c index 47275a811e..63c406d8af 100644 --- a/src/core/or/relay.c +++ b/src/core/or/relay.c @@ -572,6 +572,14 @@ relay_send_command_from_edge_,(streamid_t stream_id, circuit_t *circ, if (payload_len) memcpy(cell.payload+RELAY_HEADER_SIZE, payload, payload_len); + /* Add random bytes to the unused portion of the payload, to foil attacks + * where the other side can predict all of the bytes in the payload and thus + * compute authenticated sendme cells without seeing the traffic. See + * proposal 289. */ + crypto_fast_rng_getbytes(get_thread_fast_rng(), + cell.payload + RELAY_HEADER_SIZE + payload_len, + RELAY_PAYLOAD_SIZE - payload_len); + log_debug(LD_OR,"delivering %d cell %s.", relay_command, cell_direction == CELL_DIRECTION_OUT ? "forward" : "backward"); From cede93b2d83fb810ec8b2152882732ed0a7481dc Mon Sep 17 00:00:00 2001 From: David Goulet Date: Tue, 19 Feb 2019 14:49:38 -0500 Subject: [PATCH 0878/2557] tests: Implement unit tests for SENDME v1 Part of #26288 Signed-off-by: David Goulet --- src/core/or/sendme.c | 12 +- src/core/or/sendme.h | 23 ++++ src/test/include.am | 1 + src/test/test.c | 1 + src/test/test.h | 1 + src/test/test_relaycell.c | 1 + src/test/test_sendme.c | 223 ++++++++++++++++++++++++++++++++++++++ 7 files changed, 257 insertions(+), 5 deletions(-) create mode 100644 src/test/test_sendme.c diff --git a/src/core/or/sendme.c b/src/core/or/sendme.c index 76f551a929..980684c827 100644 --- a/src/core/or/sendme.c +++ b/src/core/or/sendme.c @@ -7,6 +7,8 @@ * creating/parsing cells and handling the content. */ +#define SENDME_PRIVATE + #include "core/or/or.h" #include "app/config/config.h" @@ -37,7 +39,7 @@ /* Return the minimum version given by the consensus (if any) that should be * used when emitting a SENDME cell. */ -static int +STATIC int get_emit_min_version(void) { return networkstatus_get_param(NULL, "sendme_emit_min_version", @@ -48,7 +50,7 @@ get_emit_min_version(void) /* Return the minimum version given by the consensus (if any) that should be * accepted when receiving a SENDME cell. */ -static int +STATIC int get_accept_min_version(void) { return networkstatus_get_param(NULL, "sendme_accept_min_version", @@ -112,7 +114,7 @@ cell_v1_is_valid(const sendme_cell_t *cell, const circuit_t *circ) /* Return true iff the given cell version can be handled or if the minimum * accepted version from the consensus is known to us. */ -static bool +STATIC bool cell_version_is_valid(uint8_t cell_version) { int accept_version = get_accept_min_version(); @@ -149,7 +151,7 @@ cell_version_is_valid(uint8_t cell_version) * This is the main critical function to make sure we can continue to * send/recv cells on a circuit. If the SENDME is invalid, the circuit should * be mark for close. */ -static bool +STATIC bool sendme_is_valid(const circuit_t *circ, const uint8_t *cell_payload, size_t cell_payload_len) { @@ -206,7 +208,7 @@ sendme_is_valid(const circuit_t *circ, const uint8_t *cell_payload, * * Return the size in bytes of the encoded cell in payload. A negative value * is returned on encoding failure. */ -static ssize_t +STATIC ssize_t build_cell_payload_v1(crypto_digest_t *cell_digest, uint8_t *payload) { ssize_t len = -1; diff --git a/src/core/or/sendme.h b/src/core/or/sendme.h index e7cf718bb1..c2e2518da8 100644 --- a/src/core/or/sendme.h +++ b/src/core/or/sendme.h @@ -36,4 +36,27 @@ int sendme_stream_data_packaged(edge_connection_t *conn); /* Track cell digest. */ void sendme_note_cell_digest(circuit_t *circ); +/* Private section starts. */ +#ifdef SENDME_PRIVATE + +/* + * Unit tests declaractions. + */ +#ifdef TOR_UNIT_TESTS + +STATIC int get_emit_min_version(void); +STATIC int get_accept_min_version(void); + +STATIC bool cell_version_is_valid(uint8_t cell_version); + +STATIC ssize_t build_cell_payload_v1(crypto_digest_t *cell_digest, + uint8_t *payload); +STATIC bool sendme_is_valid(const circuit_t *circ, + const uint8_t *cell_payload, + size_t cell_payload_len); + +#endif /* TOR_UNIT_TESTS */ + +#endif /* SENDME_PRIVATE */ + #endif /* !defined(TOR_SENDME_H) */ diff --git a/src/test/include.am b/src/test/include.am index 497aa320a4..5f6b05fa95 100644 --- a/src/test/include.am +++ b/src/test/include.am @@ -179,6 +179,7 @@ src_test_test_SOURCES += \ src/test/test_routerlist.c \ src/test/test_routerset.c \ src/test/test_scheduler.c \ + src/test/test_sendme.c \ src/test/test_shared_random.c \ src/test/test_socks.c \ src/test/test_status.c \ diff --git a/src/test/test.c b/src/test/test.c index fbc30fb64e..17159b71c5 100644 --- a/src/test/test.c +++ b/src/test/test.c @@ -923,6 +923,7 @@ struct testgroup_t testgroups[] = { { "routerlist/", routerlist_tests }, { "routerset/" , routerset_tests }, { "scheduler/", scheduler_tests }, + { "sendme/", sendme_tests }, { "shared-random/", sr_tests }, { "socks/", socks_tests }, { "status/" , status_tests }, diff --git a/src/test/test.h b/src/test/test.h index 7d19af9b20..167fd090ac 100644 --- a/src/test/test.h +++ b/src/test/test.h @@ -266,6 +266,7 @@ extern struct testcase_t routerkeys_tests[]; extern struct testcase_t routerlist_tests[]; extern struct testcase_t routerset_tests[]; extern struct testcase_t scheduler_tests[]; +extern struct testcase_t sendme_tests[]; extern struct testcase_t socks_tests[]; extern struct testcase_t sr_tests[]; extern struct testcase_t status_tests[]; diff --git a/src/test/test_relaycell.c b/src/test/test_relaycell.c index 0623583511..c4ed215c7c 100644 --- a/src/test/test_relaycell.c +++ b/src/test/test_relaycell.c @@ -705,6 +705,7 @@ test_circbw_relay(void *arg) circ = helper_create_origin_circuit(CIRCUIT_PURPOSE_C_GENERAL, 0); circ->cpath->state = CPATH_STATE_AWAITING_KEYS; circ->cpath->deliver_window = CIRCWINDOW_START; + circ->cpath->crypto.sendme_digest = crypto_digest_new(); entryconn1 = fake_entry_conn(circ, 1); edgeconn = ENTRY_TO_EDGE_CONN(entryconn1); diff --git a/src/test/test_sendme.c b/src/test/test_sendme.c new file mode 100644 index 0000000000..ad6aac6c0c --- /dev/null +++ b/src/test/test_sendme.c @@ -0,0 +1,223 @@ +/* Copyright (c) 2014-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/* Unit tests for handling different kinds of relay cell */ + +#define CIRCUITLIST_PRIVATE +#define NETWORKSTATUS_PRIVATE +#define SENDME_PRIVATE + +#include "core/or/circuit_st.h" +#include "core/or/or_circuit_st.h" +#include "core/or/origin_circuit_st.h" +#include "core/or/circuitlist.h" +#include "core/or/sendme.h" + +#include "feature/nodelist/networkstatus.h" +#include "feature/nodelist/networkstatus_st.h" + +#include "test/test.h" +#include "test/log_test_helpers.h" + +static void +setup_mock_consensus(void) +{ + current_md_consensus = current_ns_consensus = + tor_malloc_zero(sizeof(networkstatus_t)); + current_md_consensus->net_params = smartlist_new(); + current_md_consensus->routerstatus_list = smartlist_new(); +} + +static void +free_mock_consensus(void) +{ + SMARTLIST_FOREACH(current_md_consensus->routerstatus_list, void *, r, + tor_free(r)); + smartlist_free(current_md_consensus->routerstatus_list); + smartlist_free(current_ns_consensus->net_params); + tor_free(current_ns_consensus); +} + +static void +test_v1_note_digest(void *arg) +{ + or_circuit_t *or_circ = NULL; + origin_circuit_t *orig_circ = NULL; + circuit_t *circ = NULL; + + (void) arg; + + /* Create our dummy circuits. */ + orig_circ = origin_circuit_new(); + tt_assert(orig_circ); + or_circ = or_circuit_new(1, NULL); + + /* Start by pointing to the origin circuit. */ + circ = TO_CIRCUIT(orig_circ); + circ->purpose = CIRCUIT_PURPOSE_S_REND_JOINED; + + /* We should never note SENDME digest on origin circuit. */ + sendme_note_cell_digest(circ); + tt_assert(!circ->sendme_last_digests); + /* We do not need the origin circuit for now. */ + orig_circ = NULL; + circuit_free_(circ); + /* Points it to the OR circuit now. */ + circ = TO_CIRCUIT(or_circ); + or_circ->crypto.sendme_digest = crypto_digest_new(); + + /* The package window has to be a multiple of CIRCWINDOW_INCREMENT minus 1 + * in order to catched the CIRCWINDOW_INCREMENT-nth cell. Try something that + * shouldn't be noted. */ + circ->package_window = CIRCWINDOW_INCREMENT; + sendme_note_cell_digest(circ); + tt_assert(!circ->sendme_last_digests); + + /* This should work now. Package window at CIRCWINDOW_INCREMENT + 1. */ + circ->package_window++; + sendme_note_cell_digest(circ); + tt_assert(circ->sendme_last_digests); + tt_int_op(smartlist_len(circ->sendme_last_digests), OP_EQ, 1); + + /* Next cell in the package window shouldn't do anything. */ + circ->package_window++; + sendme_note_cell_digest(circ); + tt_int_op(smartlist_len(circ->sendme_last_digests), OP_EQ, 1); + + /* The next CIRCWINDOW_INCREMENT should add one more digest. */ + circ->package_window = (CIRCWINDOW_INCREMENT * 2) + 1; + sendme_note_cell_digest(circ); + tt_int_op(smartlist_len(circ->sendme_last_digests), OP_EQ, 2); + + done: + circuit_free_(circ); +} + +static void +test_v1_consensus_params(void *arg) +{ + (void) arg; + + setup_mock_consensus(); + tt_assert(current_md_consensus); + + /* Both zeroes. */ + smartlist_add(current_md_consensus->net_params, + (void *) "sendme_emit_min_version=0"); + smartlist_add(current_md_consensus->net_params, + (void *) "sendme_accept_min_version=0"); + tt_int_op(get_emit_min_version(), OP_EQ, 0); + tt_int_op(get_accept_min_version(), OP_EQ, 0); + smartlist_clear(current_md_consensus->net_params); + + /* Both ones. */ + smartlist_add(current_md_consensus->net_params, + (void *) "sendme_emit_min_version=1"); + smartlist_add(current_md_consensus->net_params, + (void *) "sendme_accept_min_version=1"); + tt_int_op(get_emit_min_version(), OP_EQ, 1); + tt_int_op(get_accept_min_version(), OP_EQ, 1); + smartlist_clear(current_md_consensus->net_params); + + /* Different values from each other. */ + smartlist_add(current_md_consensus->net_params, + (void *) "sendme_emit_min_version=1"); + smartlist_add(current_md_consensus->net_params, + (void *) "sendme_accept_min_version=0"); + tt_int_op(get_emit_min_version(), OP_EQ, 1); + tt_int_op(get_accept_min_version(), OP_EQ, 0); + smartlist_clear(current_md_consensus->net_params); + + /* Validate is the cell version is coherent with our internal default value + * and the one in the consensus. */ + smartlist_add(current_md_consensus->net_params, + (void *) "sendme_accept_min_version=1"); + /* Minimum acceptable value is 1. */ + tt_int_op(cell_version_is_valid(1), OP_EQ, true); + /* Minimum acceptable value is 1 so a cell version of 0 is refused. */ + tt_int_op(cell_version_is_valid(0), OP_EQ, false); + + done: + free_mock_consensus(); +} + +static void +test_v1_build_cell(void *arg) +{ + uint8_t payload[RELAY_PAYLOAD_SIZE]; + ssize_t ret; + crypto_digest_t *cell_digest = NULL; + or_circuit_t *or_circ = NULL; + circuit_t *circ = NULL; + + (void) arg; + + or_circ = or_circuit_new(1, NULL); + circ = TO_CIRCUIT(or_circ); + + cell_digest = crypto_digest_new(); + crypto_digest_add_bytes(cell_digest, "AAAA", 4); + tt_assert(cell_digest); + + /* SENDME v1 payload is 7 bytes. See spec. */ + ret = build_cell_payload_v1(cell_digest, payload); + tt_int_op(ret, OP_EQ, 7); + + /* Validation. */ + + /* An empty payload means SENDME version 0 thus valid. */ + tt_int_op(sendme_is_valid(circ, payload, 0), OP_EQ, true); + + /* An unparseable cell means invalid. */ + setup_full_capture_of_logs(LOG_INFO); + tt_int_op(sendme_is_valid(circ, (const uint8_t *) "A", 1), OP_EQ, false); + expect_log_msg_containing("Unparseable SENDME cell received. " + "Closing circuit."); + teardown_capture_of_logs(); + + /* No cell digest recorded for this. */ + setup_full_capture_of_logs(LOG_INFO); + tt_int_op(sendme_is_valid(circ, payload, sizeof(payload)), OP_EQ, false); + expect_log_msg_containing("We received a SENDME but we have no cell digests " + "to match. Closing circuit."); + teardown_capture_of_logs(); + + /* Note the wrong digest in the circuit, cell should fail validation. */ + or_circ->crypto.sendme_digest = crypto_digest_new(); + circ->package_window = CIRCWINDOW_INCREMENT + 1; + sendme_note_cell_digest(circ); + tt_int_op(smartlist_len(circ->sendme_last_digests), OP_EQ, 1); + setup_full_capture_of_logs(LOG_INFO); + tt_int_op(sendme_is_valid(circ, payload, sizeof(payload)), OP_EQ, false); + /* After a validation, the last digests is always popped out. */ + tt_int_op(smartlist_len(circ->sendme_last_digests), OP_EQ, 0); + expect_log_msg_containing("SENDME v1 cell digest do not match."); + teardown_capture_of_logs(); + + /* Cleanup */ + crypto_digest_free(or_circ->crypto.sendme_digest); + + /* Record the cell digest into the circuit, cell should validate. */ + or_circ->crypto.sendme_digest = crypto_digest_dup(cell_digest); + circ->package_window = CIRCWINDOW_INCREMENT + 1; + sendme_note_cell_digest(circ); + tt_int_op(smartlist_len(circ->sendme_last_digests), OP_EQ, 1); + tt_int_op(sendme_is_valid(circ, payload, sizeof(payload)), OP_EQ, true); + /* After a validation, the last digests is always popped out. */ + tt_int_op(smartlist_len(circ->sendme_last_digests), OP_EQ, 0); + + done: + crypto_digest_free(cell_digest); + circuit_free_(circ); +} + +struct testcase_t sendme_tests[] = { + { "v1_note_digest", test_v1_note_digest, TT_FORK, + NULL, NULL }, + { "v1_consensus_params", test_v1_consensus_params, TT_FORK, + NULL, NULL }, + { "v1_build_cell", test_v1_build_cell, TT_FORK, + NULL, NULL }, + + END_OF_TESTCASES +}; From 504e05b02999afb6a58ebe4af5770ca8dc136233 Mon Sep 17 00:00:00 2001 From: David Goulet Date: Thu, 7 Mar 2019 11:20:23 -0500 Subject: [PATCH 0879/2557] prop289: Use a 20 bytes digest instead of 4 To achieve such, this commit also changes the trunnel declaration to use a union instead of a seperate object for the v1 data. A constant is added for the digest length so we can use it within the SENDME code giving us a single reference. Part of #26288 Signed-off-by: David Goulet --- src/core/or/sendme.c | 41 ++--- src/test/test_sendme.c | 6 +- src/trunnel/sendme.c | 347 +++++++++++-------------------------- src/trunnel/sendme.h | 107 ++---------- src/trunnel/sendme.trunnel | 19 +- 5 files changed, 145 insertions(+), 375 deletions(-) diff --git a/src/core/or/sendme.c b/src/core/or/sendme.c index 980684c827..a333b02b60 100644 --- a/src/core/or/sendme.c +++ b/src/core/or/sendme.c @@ -59,7 +59,8 @@ get_accept_min_version(void) SENDME_ACCEPT_MIN_VERSION_MAX); } -/* Return true iff the given decoded SENDME version 1 cell is valid. +/* Return true iff the given decoded SENDME version 1 cell is valid and + * matches the expected digest on the circuit. * * Validation is done by comparing the digest in the cell from the previous * cell we saw which tells us that the other side has in fact seen that cell. @@ -67,14 +68,12 @@ get_accept_min_version(void) static bool cell_v1_is_valid(const sendme_cell_t *cell, const circuit_t *circ) { - sendme_data_v1_t *data = NULL; + const uint8_t *cell_digest = NULL; tor_assert(cell); + tor_assert(circ); - if (sendme_data_v1_parse(&data, sendme_cell_getconstarray_data(cell), - sendme_cell_getlen_data(cell)) < 0) { - goto invalid; - } + cell_digest = sendme_cell_getconstarray_data_v1_digest(cell); /* We shouldn't have received this SENDME if we have no digests. Log at * protocol warning because it can be tricked by sending many SENDMEs @@ -94,8 +93,7 @@ cell_v1_is_valid(const sendme_cell_t *cell, const circuit_t *circ) /* Compare the digest with the one in the SENDME. This cell is invalid * without a perfect match. */ - if (tor_memcmp(digest, sendme_data_v1_getconstarray_digest(data), - sendme_data_v1_getlen_digest(data))) { + if (tor_memcmp(digest, cell_digest, TRUNNEL_SENDME_V1_DIGEST_LEN)) { tor_free(digest); log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, "SENDME v1 cell digest do not match."); @@ -105,10 +103,8 @@ cell_v1_is_valid(const sendme_cell_t *cell, const circuit_t *circ) } /* Validated SENDME v1 cell. */ - sendme_data_v1_free(data); return 1; invalid: - sendme_data_v1_free(data); return 0; } @@ -213,39 +209,26 @@ build_cell_payload_v1(crypto_digest_t *cell_digest, uint8_t *payload) { ssize_t len = -1; sendme_cell_t *cell = NULL; - sendme_data_v1_t *data = NULL; tor_assert(cell_digest); tor_assert(payload); cell = sendme_cell_new(); - data = sendme_data_v1_new(); /* Building a payload for version 1. */ sendme_cell_set_version(cell, 0x01); + /* Set the data length field for v1. */ + sendme_cell_set_data_len(cell, TRUNNEL_SENDME_V1_DIGEST_LEN); /* Copy the digest into the data payload. */ crypto_digest_get_digest(cell_digest, - (char *) sendme_data_v1_getarray_digest(data), - sendme_data_v1_getlen_digest(data)); - - /* Set the length of the data in the cell payload. It is the encoded length - * of the v1 data object. */ - sendme_cell_setlen_data(cell, sendme_data_v1_encoded_len(data)); - /* Encode into the cell's data field using its current length just set. */ - if (sendme_data_v1_encode(sendme_cell_getarray_data(cell), - sendme_cell_getlen_data(cell), data) < 0) { - goto end; - } - /* Set the DATA_LEN field to what we've just encoded. */ - sendme_cell_set_data_len(cell, sendme_cell_getlen_data(cell)); + (char *) sendme_cell_getarray_data_v1_digest(cell), + sendme_cell_get_data_len(cell)); /* Finally, encode the cell into the payload. */ len = sendme_cell_encode(payload, RELAY_PAYLOAD_SIZE, cell); - end: sendme_cell_free(cell); - sendme_data_v1_free(data); return len; } @@ -566,9 +549,9 @@ sendme_note_cell_digest(circuit_t *circ) * recorded. It should never happen in theory as we always record the last * digest for the v1 SENDME. */ if (TO_OR_CIRCUIT(circ)->crypto.sendme_digest) { - digest = tor_malloc_zero(4); + digest = tor_malloc_zero(TRUNNEL_SENDME_V1_DIGEST_LEN); crypto_digest_get_digest(TO_OR_CIRCUIT(circ)->crypto.sendme_digest, - (char *) digest, 4); + (char *) digest, TRUNNEL_SENDME_V1_DIGEST_LEN); if (circ->sendme_last_digests == NULL) { circ->sendme_last_digests = smartlist_new(); } diff --git a/src/test/test_sendme.c b/src/test/test_sendme.c index ad6aac6c0c..92f478df0e 100644 --- a/src/test/test_sendme.c +++ b/src/test/test_sendme.c @@ -156,12 +156,12 @@ test_v1_build_cell(void *arg) circ = TO_CIRCUIT(or_circ); cell_digest = crypto_digest_new(); - crypto_digest_add_bytes(cell_digest, "AAAA", 4); + crypto_digest_add_bytes(cell_digest, "AAAAAAAAAAAAAAAAAAAA", 20); tt_assert(cell_digest); - /* SENDME v1 payload is 7 bytes. See spec. */ + /* SENDME v1 payload is 3 bytes + 20 bytes digest. See spec. */ ret = build_cell_payload_v1(cell_digest, payload); - tt_int_op(ret, OP_EQ, 7); + tt_int_op(ret, OP_EQ, 23); /* Validation. */ diff --git a/src/trunnel/sendme.c b/src/trunnel/sendme.c index 08f9ed5e91..262b915234 100644 --- a/src/trunnel/sendme.c +++ b/src/trunnel/sendme.c @@ -43,8 +43,6 @@ static void sendme_cell_clear(sendme_cell_t *obj) { (void) obj; - TRUNNEL_DYNARRAY_WIPE(&obj->data); - TRUNNEL_DYNARRAY_CLEAR(&obj->data); } void @@ -84,71 +82,40 @@ sendme_cell_set_data_len(sendme_cell_t *inp, uint16_t val) return 0; } size_t -sendme_cell_getlen_data(const sendme_cell_t *inp) +sendme_cell_getlen_data_v1_digest(const sendme_cell_t *inp) { - return TRUNNEL_DYNARRAY_LEN(&inp->data); + (void)inp; return TRUNNEL_SENDME_V1_DIGEST_LEN; } uint8_t -sendme_cell_get_data(sendme_cell_t *inp, size_t idx) +sendme_cell_get_data_v1_digest(sendme_cell_t *inp, size_t idx) { - return TRUNNEL_DYNARRAY_GET(&inp->data, idx); + trunnel_assert(idx < TRUNNEL_SENDME_V1_DIGEST_LEN); + return inp->data_v1_digest[idx]; } uint8_t -sendme_cell_getconst_data(const sendme_cell_t *inp, size_t idx) +sendme_cell_getconst_data_v1_digest(const sendme_cell_t *inp, size_t idx) { - return sendme_cell_get_data((sendme_cell_t*)inp, idx); + return sendme_cell_get_data_v1_digest((sendme_cell_t*)inp, idx); } int -sendme_cell_set_data(sendme_cell_t *inp, size_t idx, uint8_t elt) +sendme_cell_set_data_v1_digest(sendme_cell_t *inp, size_t idx, uint8_t elt) { - TRUNNEL_DYNARRAY_SET(&inp->data, idx, elt); + trunnel_assert(idx < TRUNNEL_SENDME_V1_DIGEST_LEN); + inp->data_v1_digest[idx] = elt; return 0; } -int -sendme_cell_add_data(sendme_cell_t *inp, uint8_t elt) -{ -#if SIZE_MAX >= UINT16_MAX - if (inp->data.n_ == UINT16_MAX) - goto trunnel_alloc_failed; -#endif - TRUNNEL_DYNARRAY_ADD(uint8_t, &inp->data, elt, {}); - return 0; - trunnel_alloc_failed: - TRUNNEL_SET_ERROR_CODE(inp); - return -1; -} uint8_t * -sendme_cell_getarray_data(sendme_cell_t *inp) +sendme_cell_getarray_data_v1_digest(sendme_cell_t *inp) { - return inp->data.elts_; + return inp->data_v1_digest; } const uint8_t * -sendme_cell_getconstarray_data(const sendme_cell_t *inp) +sendme_cell_getconstarray_data_v1_digest(const sendme_cell_t *inp) { - return (const uint8_t *)sendme_cell_getarray_data((sendme_cell_t*)inp); -} -int -sendme_cell_setlen_data(sendme_cell_t *inp, size_t newlen) -{ - uint8_t *newptr; -#if UINT16_MAX < SIZE_MAX - if (newlen > UINT16_MAX) - goto trunnel_alloc_failed; -#endif - newptr = trunnel_dynarray_setlen(&inp->data.allocated_, - &inp->data.n_, inp->data.elts_, newlen, - sizeof(inp->data.elts_[0]), (trunnel_free_fn_t) NULL, - &inp->trunnel_error_code_); - if (newlen != 0 && newptr == NULL) - goto trunnel_alloc_failed; - inp->data.elts_ = newptr; - return 0; - trunnel_alloc_failed: - TRUNNEL_SET_ERROR_CODE(inp); - return -1; + return (const uint8_t *)sendme_cell_getarray_data_v1_digest((sendme_cell_t*)inp); } const char * sendme_cell_check(const sendme_cell_t *obj) @@ -159,8 +126,18 @@ sendme_cell_check(const sendme_cell_t *obj) return "A set function failed on this object"; if (! (obj->version == 0 || obj->version == 1)) return "Integer out of bounds"; - if (TRUNNEL_DYNARRAY_LEN(&obj->data) != obj->data_len) - return "Length mismatch for data"; + switch (obj->version) { + + case 0: + break; + + case 1: + break; + + default: + return "Bad tag for union"; + break; + } return NULL; } @@ -178,9 +155,21 @@ sendme_cell_encoded_len(const sendme_cell_t *obj) /* Length of u16 data_len */ result += 2; + switch (obj->version) { - /* Length of u8 data[data_len] */ - result += TRUNNEL_DYNARRAY_LEN(&obj->data); + case 0: + break; + + case 1: + + /* Length of u8 data_v1_digest[TRUNNEL_SENDME_V1_DIGEST_LEN] */ + result += TRUNNEL_SENDME_V1_DIGEST_LEN; + break; + + default: + trunnel_assert(0); + break; + } return result; } int @@ -201,6 +190,8 @@ sendme_cell_encode(uint8_t *output, const size_t avail, const sendme_cell_t *obj const ssize_t encoded_len = sendme_cell_encoded_len(obj); #endif + uint8_t *backptr_data_len = NULL; + if (NULL != (msg = sendme_cell_check(obj))) goto check_failed; @@ -216,22 +207,43 @@ sendme_cell_encode(uint8_t *output, const size_t avail, const sendme_cell_t *obj written += 1; ptr += 1; /* Encode u16 data_len */ + backptr_data_len = ptr; trunnel_assert(written <= avail); if (avail - written < 2) goto truncated; trunnel_set_uint16(ptr, trunnel_htons(obj->data_len)); written += 2; ptr += 2; - - /* Encode u8 data[data_len] */ { - size_t elt_len = TRUNNEL_DYNARRAY_LEN(&obj->data); - trunnel_assert(obj->data_len == elt_len); + size_t written_before_union = written; + + /* Encode union data[version] */ trunnel_assert(written <= avail); - if (avail - written < elt_len) - goto truncated; - if (elt_len) - memcpy(ptr, obj->data.elts_, elt_len); - written += elt_len; ptr += elt_len; + switch (obj->version) { + + case 0: + break; + + case 1: + + /* Encode u8 data_v1_digest[TRUNNEL_SENDME_V1_DIGEST_LEN] */ + trunnel_assert(written <= avail); + if (avail - written < TRUNNEL_SENDME_V1_DIGEST_LEN) + goto truncated; + memcpy(ptr, obj->data_v1_digest, TRUNNEL_SENDME_V1_DIGEST_LEN); + written += TRUNNEL_SENDME_V1_DIGEST_LEN; ptr += TRUNNEL_SENDME_V1_DIGEST_LEN; + break; + + default: + trunnel_assert(0); + break; + } + /* Write the length field back to data_len */ + trunnel_assert(written >= written_before_union); +#if UINT16_MAX < SIZE_MAX + if (written - written_before_union > UINT16_MAX) + goto check_failed; +#endif + trunnel_set_uint16(backptr_data_len, trunnel_htons(written - written_before_union)); } @@ -279,21 +291,41 @@ sendme_cell_parse_into(sendme_cell_t *obj, const uint8_t *input, const size_t le CHECK_REMAINING(2, truncated); obj->data_len = trunnel_ntohs(trunnel_get_uint16(ptr)); remaining -= 2; ptr += 2; + { + size_t remaining_after; + CHECK_REMAINING(obj->data_len, truncated); + remaining_after = remaining - obj->data_len; + remaining = obj->data_len; - /* Parse u8 data[data_len] */ - CHECK_REMAINING(obj->data_len, truncated); - TRUNNEL_DYNARRAY_EXPAND(uint8_t, &obj->data, obj->data_len, {}); - obj->data.n_ = obj->data_len; - if (obj->data_len) - memcpy(obj->data.elts_, ptr, obj->data_len); - ptr += obj->data_len; remaining -= obj->data_len; + /* Parse union data[version] */ + switch (obj->version) { + + case 0: + /* Skip to end of union */ + ptr += remaining; remaining = 0; + break; + + case 1: + + /* Parse u8 data_v1_digest[TRUNNEL_SENDME_V1_DIGEST_LEN] */ + CHECK_REMAINING(TRUNNEL_SENDME_V1_DIGEST_LEN, fail); + memcpy(obj->data_v1_digest, ptr, TRUNNEL_SENDME_V1_DIGEST_LEN); + remaining -= TRUNNEL_SENDME_V1_DIGEST_LEN; ptr += TRUNNEL_SENDME_V1_DIGEST_LEN; + break; + + default: + goto fail; + break; + } + if (remaining != 0) + goto fail; + remaining = remaining_after; + } trunnel_assert(ptr + remaining == input + len_in); return len_in - remaining; truncated: return -2; - trunnel_alloc_failed: - return -1; fail: result = -1; return result; @@ -313,180 +345,3 @@ sendme_cell_parse(sendme_cell_t **output, const uint8_t *input, const size_t len } return result; } -sendme_data_v1_t * -sendme_data_v1_new(void) -{ - sendme_data_v1_t *val = trunnel_calloc(1, sizeof(sendme_data_v1_t)); - if (NULL == val) - return NULL; - return val; -} - -/** Release all storage held inside 'obj', but do not free 'obj'. - */ -static void -sendme_data_v1_clear(sendme_data_v1_t *obj) -{ - (void) obj; -} - -void -sendme_data_v1_free(sendme_data_v1_t *obj) -{ - if (obj == NULL) - return; - sendme_data_v1_clear(obj); - trunnel_memwipe(obj, sizeof(sendme_data_v1_t)); - trunnel_free_(obj); -} - -size_t -sendme_data_v1_getlen_digest(const sendme_data_v1_t *inp) -{ - (void)inp; return 4; -} - -uint8_t -sendme_data_v1_get_digest(sendme_data_v1_t *inp, size_t idx) -{ - trunnel_assert(idx < 4); - return inp->digest[idx]; -} - -uint8_t -sendme_data_v1_getconst_digest(const sendme_data_v1_t *inp, size_t idx) -{ - return sendme_data_v1_get_digest((sendme_data_v1_t*)inp, idx); -} -int -sendme_data_v1_set_digest(sendme_data_v1_t *inp, size_t idx, uint8_t elt) -{ - trunnel_assert(idx < 4); - inp->digest[idx] = elt; - return 0; -} - -uint8_t * -sendme_data_v1_getarray_digest(sendme_data_v1_t *inp) -{ - return inp->digest; -} -const uint8_t * -sendme_data_v1_getconstarray_digest(const sendme_data_v1_t *inp) -{ - return (const uint8_t *)sendme_data_v1_getarray_digest((sendme_data_v1_t*)inp); -} -const char * -sendme_data_v1_check(const sendme_data_v1_t *obj) -{ - if (obj == NULL) - return "Object was NULL"; - if (obj->trunnel_error_code_) - return "A set function failed on this object"; - return NULL; -} - -ssize_t -sendme_data_v1_encoded_len(const sendme_data_v1_t *obj) -{ - ssize_t result = 0; - - if (NULL != sendme_data_v1_check(obj)) - return -1; - - - /* Length of u8 digest[4] */ - result += 4; - return result; -} -int -sendme_data_v1_clear_errors(sendme_data_v1_t *obj) -{ - int r = obj->trunnel_error_code_; - obj->trunnel_error_code_ = 0; - return r; -} -ssize_t -sendme_data_v1_encode(uint8_t *output, const size_t avail, const sendme_data_v1_t *obj) -{ - ssize_t result = 0; - size_t written = 0; - uint8_t *ptr = output; - const char *msg; -#ifdef TRUNNEL_CHECK_ENCODED_LEN - const ssize_t encoded_len = sendme_data_v1_encoded_len(obj); -#endif - - if (NULL != (msg = sendme_data_v1_check(obj))) - goto check_failed; - -#ifdef TRUNNEL_CHECK_ENCODED_LEN - trunnel_assert(encoded_len >= 0); -#endif - - /* Encode u8 digest[4] */ - trunnel_assert(written <= avail); - if (avail - written < 4) - goto truncated; - memcpy(ptr, obj->digest, 4); - written += 4; ptr += 4; - - - trunnel_assert(ptr == output + written); -#ifdef TRUNNEL_CHECK_ENCODED_LEN - { - trunnel_assert(encoded_len >= 0); - trunnel_assert((size_t)encoded_len == written); - } - -#endif - - return written; - - truncated: - result = -2; - goto fail; - check_failed: - (void)msg; - result = -1; - goto fail; - fail: - trunnel_assert(result < 0); - return result; -} - -/** As sendme_data_v1_parse(), but do not allocate the output object. - */ -static ssize_t -sendme_data_v1_parse_into(sendme_data_v1_t *obj, const uint8_t *input, const size_t len_in) -{ - const uint8_t *ptr = input; - size_t remaining = len_in; - ssize_t result = 0; - (void)result; - - /* Parse u8 digest[4] */ - CHECK_REMAINING(4, truncated); - memcpy(obj->digest, ptr, 4); - remaining -= 4; ptr += 4; - trunnel_assert(ptr + remaining == input + len_in); - return len_in - remaining; - - truncated: - return -2; -} - -ssize_t -sendme_data_v1_parse(sendme_data_v1_t **output, const uint8_t *input, const size_t len_in) -{ - ssize_t result; - *output = sendme_data_v1_new(); - if (NULL == *output) - return -1; - result = sendme_data_v1_parse_into(*output, input, len_in); - if (result < 0) { - sendme_data_v1_free(*output); - *output = NULL; - } - return result; -} diff --git a/src/trunnel/sendme.h b/src/trunnel/sendme.h index 8207cb56f7..f3c3dd78c4 100644 --- a/src/trunnel/sendme.h +++ b/src/trunnel/sendme.h @@ -8,22 +8,16 @@ #include #include "trunnel.h" +#define TRUNNEL_SENDME_V1_DIGEST_LEN 20 #if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_SENDME_CELL) struct sendme_cell_st { uint8_t version; uint16_t data_len; - TRUNNEL_DYNARRAY_HEAD(, uint8_t) data; + uint8_t data_v1_digest[TRUNNEL_SENDME_V1_DIGEST_LEN]; uint8_t trunnel_error_code_; }; #endif typedef struct sendme_cell_st sendme_cell_t; -#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_SENDME_DATA_V1) -struct sendme_data_v1_st { - uint8_t digest[4]; - uint8_t trunnel_error_code_; -}; -#endif -typedef struct sendme_data_v1_st sendme_data_v1_t; /** Return a newly allocated sendme_cell with all elements set to * zero. */ @@ -77,94 +71,31 @@ uint16_t sendme_cell_get_data_len(const sendme_cell_t *inp); * 'inp' on failure. */ int sendme_cell_set_data_len(sendme_cell_t *inp, uint16_t val); -/** Return the length of the dynamic array holding the data field of - * the sendme_cell_t in 'inp'. +/** Return the (constant) length of the array holding the + * data_v1_digest field of the sendme_cell_t in 'inp'. */ -size_t sendme_cell_getlen_data(const sendme_cell_t *inp); -/** Return the element at position 'idx' of the dynamic array field - * data of the sendme_cell_t in 'inp'. - */ -uint8_t sendme_cell_get_data(sendme_cell_t *inp, size_t idx); -/** As sendme_cell_get_data, but take and return a const pointer - */ -uint8_t sendme_cell_getconst_data(const sendme_cell_t *inp, size_t idx); -/** Change the element at position 'idx' of the dynamic array field - * data of the sendme_cell_t in 'inp', so that it will hold the value - * 'elt'. - */ -int sendme_cell_set_data(sendme_cell_t *inp, size_t idx, uint8_t elt); -/** Append a new element 'elt' to the dynamic array field data of the - * sendme_cell_t in 'inp'. - */ -int sendme_cell_add_data(sendme_cell_t *inp, uint8_t elt); -/** Return a pointer to the variable-length array field data of 'inp'. - */ -uint8_t * sendme_cell_getarray_data(sendme_cell_t *inp); -/** As sendme_cell_get_data, but take and return a const pointer - */ -const uint8_t * sendme_cell_getconstarray_data(const sendme_cell_t *inp); -/** Change the length of the variable-length array field data of 'inp' - * to 'newlen'.Fill extra elements with 0. Return 0 on success; return - * -1 and set the error code on 'inp' on failure. - */ -int sendme_cell_setlen_data(sendme_cell_t *inp, size_t newlen); -/** Return a newly allocated sendme_data_v1 with all elements set to - * zero. - */ -sendme_data_v1_t *sendme_data_v1_new(void); -/** Release all storage held by the sendme_data_v1 in 'victim'. (Do - * nothing if 'victim' is NULL.) - */ -void sendme_data_v1_free(sendme_data_v1_t *victim); -/** Try to parse a sendme_data_v1 from the buffer in 'input', using up - * to 'len_in' bytes from the input buffer. On success, return the - * number of bytes consumed and set *output to the newly allocated - * sendme_data_v1_t. On failure, return -2 if the input appears - * truncated, and -1 if the input is otherwise invalid. - */ -ssize_t sendme_data_v1_parse(sendme_data_v1_t **output, const uint8_t *input, const size_t len_in); -/** Return the number of bytes we expect to need to encode the - * sendme_data_v1 in 'obj'. On failure, return a negative value. Note - * that this value may be an overestimate, and can even be an - * underestimate for certain unencodeable objects. - */ -ssize_t sendme_data_v1_encoded_len(const sendme_data_v1_t *obj); -/** Try to encode the sendme_data_v1 from 'input' into the buffer at - * 'output', using up to 'avail' bytes of the output buffer. On - * success, return the number of bytes used. On failure, return -2 if - * the buffer was not long enough, and -1 if the input was invalid. - */ -ssize_t sendme_data_v1_encode(uint8_t *output, size_t avail, const sendme_data_v1_t *input); -/** Check whether the internal state of the sendme_data_v1 in 'obj' is - * consistent. Return NULL if it is, and a short message if it is not. - */ -const char *sendme_data_v1_check(const sendme_data_v1_t *obj); -/** Clear any errors that were set on the object 'obj' by its setter - * functions. Return true iff errors were cleared. - */ -int sendme_data_v1_clear_errors(sendme_data_v1_t *obj); -/** Return the (constant) length of the array holding the digest field - * of the sendme_data_v1_t in 'inp'. - */ -size_t sendme_data_v1_getlen_digest(const sendme_data_v1_t *inp); +size_t sendme_cell_getlen_data_v1_digest(const sendme_cell_t *inp); /** Return the element at position 'idx' of the fixed array field - * digest of the sendme_data_v1_t in 'inp'. + * data_v1_digest of the sendme_cell_t in 'inp'. */ -uint8_t sendme_data_v1_get_digest(sendme_data_v1_t *inp, size_t idx); -/** As sendme_data_v1_get_digest, but take and return a const pointer +uint8_t sendme_cell_get_data_v1_digest(sendme_cell_t *inp, size_t idx); +/** As sendme_cell_get_data_v1_digest, but take and return a const + * pointer */ -uint8_t sendme_data_v1_getconst_digest(const sendme_data_v1_t *inp, size_t idx); +uint8_t sendme_cell_getconst_data_v1_digest(const sendme_cell_t *inp, size_t idx); /** Change the element at position 'idx' of the fixed array field - * digest of the sendme_data_v1_t in 'inp', so that it will hold the - * value 'elt'. + * data_v1_digest of the sendme_cell_t in 'inp', so that it will hold + * the value 'elt'. */ -int sendme_data_v1_set_digest(sendme_data_v1_t *inp, size_t idx, uint8_t elt); -/** Return a pointer to the 4-element array field digest of 'inp'. +int sendme_cell_set_data_v1_digest(sendme_cell_t *inp, size_t idx, uint8_t elt); +/** Return a pointer to the TRUNNEL_SENDME_V1_DIGEST_LEN-element array + * field data_v1_digest of 'inp'. */ -uint8_t * sendme_data_v1_getarray_digest(sendme_data_v1_t *inp); -/** As sendme_data_v1_get_digest, but take and return a const pointer +uint8_t * sendme_cell_getarray_data_v1_digest(sendme_cell_t *inp); +/** As sendme_cell_get_data_v1_digest, but take and return a const + * pointer */ -const uint8_t * sendme_data_v1_getconstarray_digest(const sendme_data_v1_t *inp); +const uint8_t * sendme_cell_getconstarray_data_v1_digest(const sendme_cell_t *inp); #endif diff --git a/src/trunnel/sendme.trunnel b/src/trunnel/sendme.trunnel index 7294a09b47..300963e679 100644 --- a/src/trunnel/sendme.trunnel +++ b/src/trunnel/sendme.trunnel @@ -1,18 +1,19 @@ /* This file contains the SENDME cell definition. */ +/* v1 digest length in bytes. */ +const TRUNNEL_SENDME_V1_DIGEST_LEN = 20; + +/* SENDME cell declaration. */ struct sendme_cell { /* Version field. */ u8 version IN [0x00, 0x01]; - /* The data content depends on the version. */ + /* Length of data contained in this cell. */ u16 data_len; - u8 data[data_len]; -} -/* SENDME version 0. No data. */ - -/* SENDME version 1. Authenticated with digest. */ -struct sendme_data_v1 { - /* A 4 bytes digest. */ - u8 digest[4]; + /* The data content depends on the version. */ + union data[version] with length data_len { + 0x00: ignore; + 0x01: u8 v1_digest[TRUNNEL_SENDME_V1_DIGEST_LEN]; + }; } From 2ec25e847eb2d9af0a1a1c552ffa8dbd87cf6023 Mon Sep 17 00:00:00 2001 From: David Goulet Date: Thu, 7 Mar 2019 11:35:52 -0500 Subject: [PATCH 0880/2557] prop289: Move SENDME cell processing in a separate function No behavior change. Only moving code and fixing part of it in order to use the parameters passed as pointers. Part of #26288 Signed-off-by: David Goulet --- src/core/or/relay.c | 137 ++++++++++++++++++++++++-------------------- 1 file changed, 76 insertions(+), 61 deletions(-) diff --git a/src/core/or/relay.c b/src/core/or/relay.c index 63c406d8af..6bf7ac1a7a 100644 --- a/src/core/or/relay.c +++ b/src/core/or/relay.c @@ -1444,6 +1444,81 @@ connection_edge_process_relay_cell_not_open( // return -1; } +/** Process a SENDME cell that arrived on circ. If it is a stream level + * cell, it is destined for the given conn. If it is a circuit level + * cell, it is destined for the layer_hint. The domain is the + * logging domain that should be used. + * + * Return 0 if everything went well or a negative value representing a circuit + * end reason on error for which the caller is responsible for closing it. */ +static int +process_sendme_cell(const relay_header_t *rh, const cell_t *cell, + circuit_t *circ, edge_connection_t *conn, + crypt_path_t *layer_hint, int domain) +{ + int ret; + + tor_assert(rh); + + if (!rh->stream_id) { + /* Circuit level SENDME cell. */ + ret = sendme_process_circuit_level(layer_hint, circ, + cell->payload + RELAY_HEADER_SIZE, + rh->length); + if (ret < 0) { + return ret; + } + /* Resume reading on any streams now that we've processed a valid + * SENDME cell that updated our package window. */ + circuit_resume_edge_reading(circ, layer_hint); + /* We are done, the rest of the code is for the stream level. */ + return 0; + } + + /* No connection, might be half edge state. We are done if so. */ + if (!conn) { + if (CIRCUIT_IS_ORIGIN(circ)) { + origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ); + if (connection_half_edge_is_valid_sendme(ocirc->half_streams, + rh->stream_id)) { + circuit_read_valid_data(ocirc, rh->length); + log_info(domain, "Sendme cell on circ %u valid on half-closed " + "stream id %d", + ocirc->global_identifier, rh->stream_id); + } + } + + log_info(domain, "SENDME cell dropped, unknown stream (streamid %d).", + rh->stream_id); + return 0; + } + + /* Stream level SENDME cell. */ + ret = sendme_process_stream_level(conn, circ, rh->length); + if (ret < 0) { + /* Means we need to close the circuit with reason ret. */ + return ret; + } + + /* We've now processed properly a SENDME cell, all windows have been + * properly updated, we'll read on the edge connection to see if we can + * get data out towards the end point (Exit or client) since we are now + * allowed to deliver more cells. */ + + if (circuit_queue_streams_are_blocked(circ)) { + /* Still waiting for queue to flush; don't touch conn */ + return 0; + } + connection_start_reading(TO_CONN(conn)); + /* handle whatever might still be on the inbuf */ + if (connection_edge_package_raw_inbuf(conn, 1, NULL) < 0) { + /* (We already sent an end cell if possible) */ + connection_mark_for_close(TO_CONN(conn)); + return 0; + } + return 0; +} + /** An incoming relay cell has arrived on circuit circ. If * conn is NULL this is a control cell, else cell is * destined for conn. @@ -1825,67 +1900,7 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, (unsigned)circ->n_circ_id, rh.stream_id); return 0; case RELAY_COMMAND_SENDME: - { - int ret; - - if (!rh.stream_id) { - /* Circuit level SENDME cell. */ - ret = sendme_process_circuit_level(layer_hint, circ, - cell->payload + RELAY_HEADER_SIZE, - rh.length); - if (ret < 0) { - return ret; - } - /* Resume reading on any streams now that we've processed a valid - * SENDME cell that updated our package window. */ - circuit_resume_edge_reading(circ, layer_hint); - /* We are done, the rest of the code is for the stream level. */ - return 0; - } - - /* No connection, might be half edge state. We are done if so. */ - if (!conn) { - if (CIRCUIT_IS_ORIGIN(circ)) { - origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ); - if (connection_half_edge_is_valid_sendme(ocirc->half_streams, - rh.stream_id)) { - circuit_read_valid_data(ocirc, rh.length); - log_info(domain, "Sendme cell on circ %u valid on half-closed " - "stream id %d", - ocirc->global_identifier, rh.stream_id); - } - } - - log_info(domain, "SENDME cell dropped, unknown stream (streamid %d).", - rh.stream_id); - return 0; - } - - /* Stream level SENDME cell. */ - ret = sendme_process_stream_level(conn, circ, rh.length); - if (ret < 0) { - /* Means we need to close the circuit with reason ret. */ - return ret; - } - - /* We've now processed properly a SENDME cell, all windows have been - * properly updated, we'll read on the edge connection to see if we can - * get data out towards the end point (Exit or client) since we are now - * allowed to deliver more cells. */ - - if (circuit_queue_streams_are_blocked(circ)) { - /* Still waiting for queue to flush; don't touch conn */ - return 0; - } - connection_start_reading(TO_CONN(conn)); - /* handle whatever might still be on the inbuf */ - if (connection_edge_package_raw_inbuf(conn, 1, NULL) < 0) { - /* (We already sent an end cell if possible) */ - connection_mark_for_close(TO_CONN(conn)); - return 0; - } - return 0; - } + return process_sendme_cell(&rh, cell, circ, conn, layer_hint, domain); case RELAY_COMMAND_RESOLVE: if (layer_hint) { log_fn(LOG_PROTOCOL_WARN, LD_APP, From 217b55319336227f9e397db526cea551dbd796e4 Mon Sep 17 00:00:00 2001 From: David Goulet Date: Thu, 7 Mar 2019 11:45:38 -0500 Subject: [PATCH 0881/2557] prop289: Rename packaged functions with better name The circuit and stream level functions that update the package window have been renamed to have a "_note_" in them to make their purpose more clear. Part of #26288 Signed-off-by: David Goulet --- src/core/or/relay.c | 4 ++-- src/core/or/sendme.c | 4 ++-- src/core/or/sendme.h | 5 +++-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/core/or/relay.c b/src/core/or/relay.c index 6bf7ac1a7a..fa008120b3 100644 --- a/src/core/or/relay.c +++ b/src/core/or/relay.c @@ -2092,7 +2092,7 @@ connection_edge_package_raw_inbuf(edge_connection_t *conn, int package_partial, } /* Handle the circuit-level SENDME package window. */ - if (sendme_circuit_data_packaged(circ, cpath_layer) < 0) { + if (sendme_note_circuit_data_packaged(circ, cpath_layer) < 0) { /* Package window has gone under 0. Protocol issue. */ log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, "Circuit package window is below 0. Closing circuit."); @@ -2101,7 +2101,7 @@ connection_edge_package_raw_inbuf(edge_connection_t *conn, int package_partial, } /* Handle the stream-level SENDME package window. */ - if (sendme_stream_data_packaged(conn) < 0) { + if (sendme_note_stream_data_packaged(conn) < 0) { connection_stop_reading(TO_CONN(conn)); log_debug(domain,"conn->package_window reached 0."); circuit_consider_stop_edge_reading(circ, cpath_layer); diff --git a/src/core/or/sendme.c b/src/core/or/sendme.c index a333b02b60..16ff5bcb8f 100644 --- a/src/core/or/sendme.c +++ b/src/core/or/sendme.c @@ -491,7 +491,7 @@ sendme_stream_data_received(edge_connection_t *conn) * layer_hint is NULL, this means we are the Exit end point else we are the * Client. Update the package window and return its new value. */ int -sendme_circuit_data_packaged(circuit_t *circ, crypt_path_t *layer_hint) +sendme_note_circuit_data_packaged(circuit_t *circ, crypt_path_t *layer_hint) { int package_window, domain; @@ -518,7 +518,7 @@ sendme_circuit_data_packaged(circuit_t *circ, crypt_path_t *layer_hint) /* Called when a relay DATA cell is packaged for the given edge connection * conn. Update the package window and return its new value. */ int -sendme_stream_data_packaged(edge_connection_t *conn) +sendme_note_stream_data_packaged(edge_connection_t *conn) { tor_assert(conn); return --conn->package_window; diff --git a/src/core/or/sendme.h b/src/core/or/sendme.h index c2e2518da8..2154a29f4a 100644 --- a/src/core/or/sendme.h +++ b/src/core/or/sendme.h @@ -30,8 +30,9 @@ int sendme_stream_data_received(edge_connection_t *conn); int sendme_circuit_data_received(circuit_t *circ, crypt_path_t *layer_hint); /* Update package window functions. */ -int sendme_circuit_data_packaged(circuit_t *circ, crypt_path_t *layer_hint); -int sendme_stream_data_packaged(edge_connection_t *conn); +int sendme_note_circuit_data_packaged(circuit_t *circ, + crypt_path_t *layer_hint); +int sendme_note_stream_data_packaged(edge_connection_t *conn); /* Track cell digest. */ void sendme_note_cell_digest(circuit_t *circ); From 4efe9d653aa1d375d77d6dca83ca63787d6599d7 Mon Sep 17 00:00:00 2001 From: David Goulet Date: Thu, 7 Mar 2019 12:01:58 -0500 Subject: [PATCH 0882/2557] prop289: Move digest matching in its own function No behavior change but code had to be refactored a bit. Also, the tor_memcmp() was changed to tor_memneq(). Part of #26288 Signed-off-by: David Goulet --- src/core/or/sendme.c | 80 +++++++++++++++++++++++++------------------- 1 file changed, 45 insertions(+), 35 deletions(-) diff --git a/src/core/or/sendme.c b/src/core/or/sendme.c index 16ff5bcb8f..0a7b1cbc02 100644 --- a/src/core/or/sendme.c +++ b/src/core/or/sendme.c @@ -59,6 +59,49 @@ get_accept_min_version(void) SENDME_ACCEPT_MIN_VERSION_MAX); } +/* Return true iff the given cell digest matches the first digest in the + * circuit sendme list. */ +static bool +v1_digest_matches(const circuit_t *circ, const uint8_t *cell_digest) +{ + bool ret = false; + uint8_t *circ_digest = NULL; + + tor_assert(circ); + tor_assert(cell_digest); + + /* We shouldn't have received a SENDME if we have no digests. Log at + * protocol warning because it can be tricked by sending many SENDMEs + * without prior data cell. */ + if (circ->sendme_last_digests == NULL || + smartlist_len(circ->sendme_last_digests) == 0) { + log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, + "We received a SENDME but we have no cell digests to match. " + "Closing circuit."); + goto no_match; + } + + /* Pop the first element that was added (FIFO) and compare it. */ + circ_digest = smartlist_get(circ->sendme_last_digests, 0); + smartlist_del_keeporder(circ->sendme_last_digests, 0); + + /* Compare the digest with the one in the SENDME. This cell is invalid + * without a perfect match. */ + if (tor_memneq(circ_digest, cell_digest, TRUNNEL_SENDME_V1_DIGEST_LEN)) { + log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, + "SENDME v1 cell digest do not match."); + goto no_match; + } + /* Digests matches! */ + ret = true; + + no_match: + /* This digest was popped from the circuit list. Regardless of what happens, + * we have no more use for it. */ + tor_free(circ_digest); + return ret; +} + /* Return true iff the given decoded SENDME version 1 cell is valid and * matches the expected digest on the circuit. * @@ -68,44 +111,11 @@ get_accept_min_version(void) static bool cell_v1_is_valid(const sendme_cell_t *cell, const circuit_t *circ) { - const uint8_t *cell_digest = NULL; - tor_assert(cell); tor_assert(circ); - cell_digest = sendme_cell_getconstarray_data_v1_digest(cell); - - /* We shouldn't have received this SENDME if we have no digests. Log at - * protocol warning because it can be tricked by sending many SENDMEs - * without prior data cell. */ - if (circ->sendme_last_digests == NULL || - smartlist_len(circ->sendme_last_digests) == 0) { - log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, - "We received a SENDME but we have no cell digests to match. " - "Closing circuit."); - goto invalid; - } - - /* Pop the first element that was added (FIFO) and compare it. */ - { - uint8_t *digest = smartlist_get(circ->sendme_last_digests, 0); - smartlist_del_keeporder(circ->sendme_last_digests, 0); - - /* Compare the digest with the one in the SENDME. This cell is invalid - * without a perfect match. */ - if (tor_memcmp(digest, cell_digest, TRUNNEL_SENDME_V1_DIGEST_LEN)) { - tor_free(digest); - log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, - "SENDME v1 cell digest do not match."); - goto invalid; - } - tor_free(digest); - } - - /* Validated SENDME v1 cell. */ - return 1; - invalid: - return 0; + const uint8_t *cell_digest = sendme_cell_getconstarray_data_v1_digest(cell); + return v1_digest_matches(circ, cell_digest); } /* Return true iff the given cell version can be handled or if the minimum From 77d560af64226eaa0fde157d7a6607791975a7a9 Mon Sep 17 00:00:00 2001 From: David Goulet Date: Thu, 7 Mar 2019 12:30:13 -0500 Subject: [PATCH 0883/2557] prop289: Keep the digest bytes, not the object The digest object is as large as the entire internal digest object's state, which is often much larger than the actual set of bytes you're transmitting. This commit makes it that we keep the digest itself which is 20 bytes. Part of #26288 Signed-off-by: David Goulet --- src/core/crypto/relay_crypto.c | 17 ++++++----------- src/core/or/relay_crypto_st.h | 2 +- src/core/or/sendme.c | 29 +++++++++++------------------ src/core/or/sendme.h | 2 +- src/test/test_relaycell.c | 1 - src/test/test_sendme.c | 16 +++++++--------- 6 files changed, 26 insertions(+), 41 deletions(-) diff --git a/src/core/crypto/relay_crypto.c b/src/core/crypto/relay_crypto.c index d4116d47ab..94e9060651 100644 --- a/src/core/crypto/relay_crypto.c +++ b/src/core/crypto/relay_crypto.c @@ -143,12 +143,9 @@ relay_decrypt_cell(circuit_t *circ, cell_t *cell, *recognized = 1; *layer_hint = thishop; /* Keep current digest of this cell for the possible SENDME. */ - if (thishop->crypto.sendme_digest) { - crypto_digest_free(thishop->crypto.sendme_digest); - } - thishop->crypto.sendme_digest = - crypto_digest_dup(thishop->crypto.b_digest); - + crypto_digest_get_digest(thishop->crypto.b_digest, + (char *) thishop->crypto.sendme_digest, + sizeof(thishop->crypto.sendme_digest)); return 0; } } @@ -220,10 +217,9 @@ relay_encrypt_cell_inbound(cell_t *cell, { relay_set_digest(or_circ->crypto.b_digest, cell); /* Keep a record of this cell, we might use it for validating the SENDME. */ - if (or_circ->crypto.sendme_digest) { - crypto_digest_free(or_circ->crypto.sendme_digest); - } - or_circ->crypto.sendme_digest = crypto_digest_dup(or_circ->crypto.b_digest); + crypto_digest_get_digest(or_circ->crypto.b_digest, + (char *) or_circ->crypto.sendme_digest, + sizeof(or_circ->crypto.sendme_digest)); /* encrypt one layer */ relay_crypt_one_payload(or_circ->crypto.b_crypto, cell->payload); } @@ -241,7 +237,6 @@ relay_crypto_clear(relay_crypto_t *crypto) crypto_cipher_free(crypto->b_crypto); crypto_digest_free(crypto->f_digest); crypto_digest_free(crypto->b_digest); - crypto_digest_free(crypto->sendme_digest); } /** Initialize crypto from the key material in key_data. diff --git a/src/core/or/relay_crypto_st.h b/src/core/or/relay_crypto_st.h index dbdf1599dc..1f243ccdc8 100644 --- a/src/core/or/relay_crypto_st.h +++ b/src/core/or/relay_crypto_st.h @@ -26,7 +26,7 @@ struct relay_crypto_t { struct crypto_digest_t *b_digest; /** Digest used for the next SENDME cell if any. */ - struct crypto_digest_t *sendme_digest; + uint8_t sendme_digest[DIGEST_LEN]; }; #undef crypto_cipher_t diff --git a/src/core/or/sendme.c b/src/core/or/sendme.c index 0a7b1cbc02..3dcd9df08e 100644 --- a/src/core/or/sendme.c +++ b/src/core/or/sendme.c @@ -215,7 +215,7 @@ sendme_is_valid(const circuit_t *circ, const uint8_t *cell_payload, * Return the size in bytes of the encoded cell in payload. A negative value * is returned on encoding failure. */ STATIC ssize_t -build_cell_payload_v1(crypto_digest_t *cell_digest, uint8_t *payload) +build_cell_payload_v1(const uint8_t *cell_digest, uint8_t *payload) { ssize_t len = -1; sendme_cell_t *cell = NULL; @@ -231,9 +231,8 @@ build_cell_payload_v1(crypto_digest_t *cell_digest, uint8_t *payload) sendme_cell_set_data_len(cell, TRUNNEL_SENDME_V1_DIGEST_LEN); /* Copy the digest into the data payload. */ - crypto_digest_get_digest(cell_digest, - (char *) sendme_cell_getarray_data_v1_digest(cell), - sendme_cell_get_data_len(cell)); + memcpy(sendme_cell_getarray_data_v1_digest(cell), cell_digest, + sendme_cell_get_data_len(cell)); /* Finally, encode the cell into the payload. */ len = sendme_cell_encode(payload, RELAY_PAYLOAD_SIZE, cell); @@ -249,7 +248,7 @@ build_cell_payload_v1(crypto_digest_t *cell_digest, uint8_t *payload) * because we failed to send the cell on it. */ static int send_circuit_level_sendme(circuit_t *circ, crypt_path_t *layer_hint, - crypto_digest_t *cell_digest) + const uint8_t *cell_digest) { uint8_t emit_version; uint8_t payload[RELAY_PAYLOAD_SIZE]; @@ -340,7 +339,7 @@ sendme_connection_edge_consider_sending(edge_connection_t *conn) void sendme_circuit_consider_sending(circuit_t *circ, crypt_path_t *layer_hint) { - crypto_digest_t *digest; + const uint8_t *digest; while ((layer_hint ? layer_hint->deliver_window : circ->deliver_window) <= CIRCWINDOW_START - CIRCWINDOW_INCREMENT) { @@ -539,7 +538,7 @@ sendme_note_stream_data_packaged(edge_connection_t *conn) void sendme_note_cell_digest(circuit_t *circ) { - uint8_t *digest; + const uint8_t *digest; tor_assert(circ); @@ -555,16 +554,10 @@ sendme_note_cell_digest(circuit_t *circ) return; } - /* Only note the digest if we actually have the digest of the previous cell - * recorded. It should never happen in theory as we always record the last - * digest for the v1 SENDME. */ - if (TO_OR_CIRCUIT(circ)->crypto.sendme_digest) { - digest = tor_malloc_zero(TRUNNEL_SENDME_V1_DIGEST_LEN); - crypto_digest_get_digest(TO_OR_CIRCUIT(circ)->crypto.sendme_digest, - (char *) digest, TRUNNEL_SENDME_V1_DIGEST_LEN); - if (circ->sendme_last_digests == NULL) { - circ->sendme_last_digests = smartlist_new(); - } - smartlist_add(circ->sendme_last_digests, digest); + /* Add the digest to the last seen list in the circuit. */ + digest = TO_OR_CIRCUIT(circ)->crypto.sendme_digest; + if (circ->sendme_last_digests == NULL) { + circ->sendme_last_digests = smartlist_new(); } + smartlist_add(circ->sendme_last_digests, tor_memdup(digest, DIGEST_LEN)); } diff --git a/src/core/or/sendme.h b/src/core/or/sendme.h index 2154a29f4a..71df9b6f07 100644 --- a/src/core/or/sendme.h +++ b/src/core/or/sendme.h @@ -50,7 +50,7 @@ STATIC int get_accept_min_version(void); STATIC bool cell_version_is_valid(uint8_t cell_version); -STATIC ssize_t build_cell_payload_v1(crypto_digest_t *cell_digest, +STATIC ssize_t build_cell_payload_v1(const uint8_t *cell_digest, uint8_t *payload); STATIC bool sendme_is_valid(const circuit_t *circ, const uint8_t *cell_payload, diff --git a/src/test/test_relaycell.c b/src/test/test_relaycell.c index c4ed215c7c..0623583511 100644 --- a/src/test/test_relaycell.c +++ b/src/test/test_relaycell.c @@ -705,7 +705,6 @@ test_circbw_relay(void *arg) circ = helper_create_origin_circuit(CIRCUIT_PURPOSE_C_GENERAL, 0); circ->cpath->state = CPATH_STATE_AWAITING_KEYS; circ->cpath->deliver_window = CIRCWINDOW_START; - circ->cpath->crypto.sendme_digest = crypto_digest_new(); entryconn1 = fake_entry_conn(circ, 1); edgeconn = ENTRY_TO_EDGE_CONN(entryconn1); diff --git a/src/test/test_sendme.c b/src/test/test_sendme.c index 92f478df0e..d6410a7488 100644 --- a/src/test/test_sendme.c +++ b/src/test/test_sendme.c @@ -16,6 +16,8 @@ #include "feature/nodelist/networkstatus.h" #include "feature/nodelist/networkstatus_st.h" +#include "lib/crypt_ops/crypto_digest.h" + #include "test/test.h" #include "test/log_test_helpers.h" @@ -64,7 +66,6 @@ test_v1_note_digest(void *arg) circuit_free_(circ); /* Points it to the OR circuit now. */ circ = TO_CIRCUIT(or_circ); - or_circ->crypto.sendme_digest = crypto_digest_new(); /* The package window has to be a multiple of CIRCWINDOW_INCREMENT minus 1 * in order to catched the CIRCWINDOW_INCREMENT-nth cell. Try something that @@ -144,7 +145,7 @@ test_v1_consensus_params(void *arg) static void test_v1_build_cell(void *arg) { - uint8_t payload[RELAY_PAYLOAD_SIZE]; + uint8_t payload[RELAY_PAYLOAD_SIZE], digest[DIGEST_LEN]; ssize_t ret; crypto_digest_t *cell_digest = NULL; or_circuit_t *or_circ = NULL; @@ -156,11 +157,12 @@ test_v1_build_cell(void *arg) circ = TO_CIRCUIT(or_circ); cell_digest = crypto_digest_new(); - crypto_digest_add_bytes(cell_digest, "AAAAAAAAAAAAAAAAAAAA", 20); tt_assert(cell_digest); + crypto_digest_add_bytes(cell_digest, "AAAAAAAAAAAAAAAAAAAA", 20); + crypto_digest_get_digest(cell_digest, (char *) digest, sizeof(digest)); /* SENDME v1 payload is 3 bytes + 20 bytes digest. See spec. */ - ret = build_cell_payload_v1(cell_digest, payload); + ret = build_cell_payload_v1(digest, payload); tt_int_op(ret, OP_EQ, 23); /* Validation. */ @@ -183,7 +185,6 @@ test_v1_build_cell(void *arg) teardown_capture_of_logs(); /* Note the wrong digest in the circuit, cell should fail validation. */ - or_circ->crypto.sendme_digest = crypto_digest_new(); circ->package_window = CIRCWINDOW_INCREMENT + 1; sendme_note_cell_digest(circ); tt_int_op(smartlist_len(circ->sendme_last_digests), OP_EQ, 1); @@ -194,11 +195,8 @@ test_v1_build_cell(void *arg) expect_log_msg_containing("SENDME v1 cell digest do not match."); teardown_capture_of_logs(); - /* Cleanup */ - crypto_digest_free(or_circ->crypto.sendme_digest); - /* Record the cell digest into the circuit, cell should validate. */ - or_circ->crypto.sendme_digest = crypto_digest_dup(cell_digest); + memcpy(or_circ->crypto.sendme_digest, digest, sizeof(digest)); circ->package_window = CIRCWINDOW_INCREMENT + 1; sendme_note_cell_digest(circ); tt_int_op(smartlist_len(circ->sendme_last_digests), OP_EQ, 1); From 44750b0de60fe80ea81f7fc83f8713f814caf5a6 Mon Sep 17 00:00:00 2001 From: David Goulet Date: Thu, 7 Mar 2019 12:45:16 -0500 Subject: [PATCH 0884/2557] prop289: Skip the first 4 unused bytes in a cell When adding random to a cell, skip the first 4 bytes and leave them zeroed. It has been very useful in the past for us to keep bytes like this. Some code trickery was added to make sure we have enough room for this 4 bytes offset when adding random. Part of #26288 Signed-off-by: David Goulet --- src/core/or/relay.c | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/src/core/or/relay.c b/src/core/or/relay.c index fa008120b3..504f391d9a 100644 --- a/src/core/or/relay.c +++ b/src/core/or/relay.c @@ -547,6 +547,8 @@ relay_send_command_from_edge_,(streamid_t stream_id, circuit_t *circ, cell_t cell; relay_header_t rh; cell_direction_t cell_direction; + int random_bytes_len; + size_t random_bytes_offset = 0; /* XXXX NM Split this function into a separate versions per circuit type? */ tor_assert(circ); @@ -574,11 +576,20 @@ relay_send_command_from_edge_,(streamid_t stream_id, circuit_t *circ, /* Add random bytes to the unused portion of the payload, to foil attacks * where the other side can predict all of the bytes in the payload and thus - * compute authenticated sendme cells without seeing the traffic. See - * proposal 289. */ + * compute authenticated sendme cells without seeing the traffic. See + * proposal 289. + * + * We'll skip the first 4 bytes of unused data because having some unused + * zero bytes has saved us a lot of times in the past. */ + random_bytes_len = RELAY_PAYLOAD_SIZE - + (RELAY_HEADER_SIZE + payload_len + 4); + if (random_bytes_len < 0) { + random_bytes_len = 0; + } + random_bytes_offset = RELAY_PAYLOAD_SIZE - random_bytes_len; crypto_fast_rng_getbytes(get_thread_fast_rng(), - cell.payload + RELAY_HEADER_SIZE + payload_len, - RELAY_PAYLOAD_SIZE - payload_len); + cell.payload + random_bytes_offset, + random_bytes_len); log_debug(LD_OR,"delivering %d cell %s.", relay_command, cell_direction == CELL_DIRECTION_OUT ? "forward" : "backward"); From aef7095c3e52e2f98850e72c68b00f54a39608a6 Mon Sep 17 00:00:00 2001 From: David Goulet Date: Thu, 7 Mar 2019 12:57:15 -0500 Subject: [PATCH 0885/2557] prop289: Add documentation for the circuit FIFO list Part of #26288 Signed-off-by: David Goulet --- src/core/or/circuit_st.h | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/src/core/or/circuit_st.h b/src/core/or/circuit_st.h index 5adb158935..a68547ecb1 100644 --- a/src/core/or/circuit_st.h +++ b/src/core/or/circuit_st.h @@ -106,9 +106,23 @@ struct circuit_t { int deliver_window; /** FIFO containing the digest of the cells that are just before a SENDME is * sent by the client. It is done at the last cell before our package_window - * goes down to 0 which is when we expect a SENDME. The protocol doesn't - * allow more than 10 outstanding SENDMEs worth of data meaning this list - * should only contain at most 10 digests of 4 bytes each. */ + * goes down to 0 which is when we expect a SENDME. + * + * Our current circuit package window is capped to 1000 + * (CIRCWINDOW_START_MAX) which is also the start value. The increment is + * set to 100 (CIRCWINDOW_INCREMENT) which means we don't allow more than + * 1000/100 = 10 outstanding SENDME cells worth of data. Meaning that this + * list can not contain more than 10 digests of DIGEST_LEN bytes (20). + * + * At position i in the list, the digest corresponds to the + * ((CIRCWINDOW_INCREMENT * i) - 1)-nth cell received since we expect the + * (CIRCWINDOW_INCREMENT * i)-nth cell to be the SENDME and thus containing + * the previous cell digest. + * + * For example, position 2 (starting at 0) means that we've received 299 + * cells and the 299th cell digest is kept at index 2. + * + * At maximum, this list contains 200 bytes plus the smartlist overhead. */ smartlist_t *sendme_last_digests; /** Temporary field used during circuits_handle_oom. */ From 7c8e519b3452ce3eb3d3c854d80be5b7e49164b4 Mon Sep 17 00:00:00 2001 From: David Goulet Date: Wed, 24 Apr 2019 10:25:29 -0400 Subject: [PATCH 0886/2557] sendme: Helper to know if next cell is a SENDME We'll use it this in order to know when to hash the cell for the SENDME instead of doing it at every cell. Part of #26288 Signed-off-by: David Goulet --- src/core/or/sendme.c | 25 ++++++++++++++++++++++++- src/core/or/sendme.h | 3 +++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/src/core/or/sendme.c b/src/core/or/sendme.c index 3dcd9df08e..c66e947bc4 100644 --- a/src/core/or/sendme.c +++ b/src/core/or/sendme.c @@ -286,6 +286,29 @@ send_circuit_level_sendme(circuit_t *circ, crypt_path_t *layer_hint, return 0; } +/* + * Public API + */ + +/** Return true iff the next cell for the given cell window is expected to be + * a SENDME. + * + * We are able to know that because the package or deliver window value minus + * one cell (the possible SENDME cell) should be a multiple of the increment + * window value. */ +bool +sendme_circuit_is_next_cell(int window) +{ + /* Is this the last cell before a SENDME? The idea is that if the package or + * deliver window reaches a multiple of the increment, after this cell, we + * should expect a SENDME. */ + if (((window - 1) % CIRCWINDOW_INCREMENT) != 0) { + return false; + } + /* Next cell is expected to be a SENDME. */ + return true; +} + /** Called when we've just received a relay data cell, when we've just * finished flushing all bytes to stream conn, or when we've flushed * *some* bytes to the stream conn. @@ -550,7 +573,7 @@ sendme_note_cell_digest(circuit_t *circ) /* Is this the last cell before a SENDME? The idea is that if the * package_window reaches a multiple of the increment, after this cell, we * should expect a SENDME. */ - if (((circ->package_window - 1) % CIRCWINDOW_INCREMENT) != 0) { + if (!sendme_circuit_is_next_cell(circ->package_window)) { return; } diff --git a/src/core/or/sendme.h b/src/core/or/sendme.h index 71df9b6f07..0965b5b22a 100644 --- a/src/core/or/sendme.h +++ b/src/core/or/sendme.h @@ -37,6 +37,9 @@ int sendme_note_stream_data_packaged(edge_connection_t *conn); /* Track cell digest. */ void sendme_note_cell_digest(circuit_t *circ); +/* Circuit level information. */ +bool sendme_circuit_is_next_cell(int window); + /* Private section starts. */ #ifdef SENDME_PRIVATE From 805c81efed9bc2c474d3f10675846ee445a908d5 Mon Sep 17 00:00:00 2001 From: David Goulet Date: Wed, 24 Apr 2019 11:57:20 -0400 Subject: [PATCH 0887/2557] sendme: Add helper to note the cell digest Signed-off-by: David Goulet --- src/core/or/sendme.c | 27 +++++++++++++++++++++++++++ src/core/or/sendme.h | 2 ++ 2 files changed, 29 insertions(+) diff --git a/src/core/or/sendme.c b/src/core/or/sendme.c index c66e947bc4..b384a19164 100644 --- a/src/core/or/sendme.c +++ b/src/core/or/sendme.c @@ -286,10 +286,37 @@ send_circuit_level_sendme(circuit_t *circ, crypt_path_t *layer_hint, return 0; } +/* Put the crypto.b_digest in the sendme_digest. */ +static void +note_cell_digest(const relay_crypto_t *crypto) +{ + tor_assert(crypto); + crypto_digest_get_digest(crypto->b_digest, (char *) crypto->sendme_digest, + sizeof(crypto->sendme_digest)); +} + /* * Public API */ +/** Keep the current inbound cell digest for the next SENDME digest. This part + * is only done by the client as the circuit came back from the Exit. */ +void +sendme_circuit_note_outbound_cell(or_circuit_t *or_circ) +{ + tor_assert(or_circ); + note_cell_digest(&or_circ->crypto); +} + +/** Keep the current inbound cell digest for the next SENDME digest. This part + * is only done by the client as the circuit came back from the Exit. */ +void +sendme_circuit_note_inbound_cell(crypt_path_t *cpath) +{ + tor_assert(cpath); + note_cell_digest(&cpath->crypto); +} + /** Return true iff the next cell for the given cell window is expected to be * a SENDME. * diff --git a/src/core/or/sendme.h b/src/core/or/sendme.h index 0965b5b22a..5889b41e9c 100644 --- a/src/core/or/sendme.h +++ b/src/core/or/sendme.h @@ -36,6 +36,8 @@ int sendme_note_stream_data_packaged(edge_connection_t *conn); /* Track cell digest. */ void sendme_note_cell_digest(circuit_t *circ); +void sendme_circuit_note_inbound_cell(crypt_path_t *cpath); +void sendme_circuit_note_outbound_cell(or_circuit_t *or_circ); /* Circuit level information. */ bool sendme_circuit_is_next_cell(int window); From c7385b5b14b30774c1768798c4495465da4d995d Mon Sep 17 00:00:00 2001 From: David Goulet Date: Wed, 24 Apr 2019 13:38:47 -0400 Subject: [PATCH 0888/2557] sendme: Keep cell digest only if a SENDME is next This way, we reduce the load by only hashing when we absolutely must. Part of #26288 Signed-off-by: David Goulet --- src/core/crypto/relay_crypto.c | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/src/core/crypto/relay_crypto.c b/src/core/crypto/relay_crypto.c index 94e9060651..eddc4298e2 100644 --- a/src/core/crypto/relay_crypto.c +++ b/src/core/crypto/relay_crypto.c @@ -12,6 +12,7 @@ #include "core/crypto/hs_ntor.h" // for HS_NTOR_KEY_EXPANSION_KDF_OUT_LEN #include "core/or/relay.h" #include "core/crypto/relay_crypto.h" +#include "core/or/sendme.h" #include "core/or/cell_st.h" #include "core/or/or_circuit_st.h" @@ -142,10 +143,11 @@ relay_decrypt_cell(circuit_t *circ, cell_t *cell, if (relay_digest_matches(thishop->crypto.b_digest, cell)) { *recognized = 1; *layer_hint = thishop; - /* Keep current digest of this cell for the possible SENDME. */ - crypto_digest_get_digest(thishop->crypto.b_digest, - (char *) thishop->crypto.sendme_digest, - sizeof(thishop->crypto.sendme_digest)); + /* This cell is for us. Keep a record of this cell because we will + * use it in the next SENDME cell. */ + if (sendme_circuit_is_next_cell(thishop->deliver_window)) { + sendme_circuit_note_inbound_cell(thishop); + } return 0; } } @@ -216,10 +218,13 @@ relay_encrypt_cell_inbound(cell_t *cell, or_circuit_t *or_circ) { relay_set_digest(or_circ->crypto.b_digest, cell); - /* Keep a record of this cell, we might use it for validating the SENDME. */ - crypto_digest_get_digest(or_circ->crypto.b_digest, - (char *) or_circ->crypto.sendme_digest, - sizeof(or_circ->crypto.sendme_digest)); + + /* We are about to send this cell outbound on the circuit. Keep a record of + * this cell if we are expecting that the next cell is a SENDME. */ + if (sendme_circuit_is_next_cell(TO_CIRCUIT(or_circ)->package_window)) { + sendme_circuit_note_outbound_cell(or_circ); + } + /* encrypt one layer */ relay_crypt_one_payload(or_circ->crypto.b_crypto, cell->payload); } From d084f9115d7d46ad5e029b9c75cea716fa7d65a5 Mon Sep 17 00:00:00 2001 From: David Goulet Date: Wed, 24 Apr 2019 15:39:10 -0400 Subject: [PATCH 0889/2557] sendme: Better handle the random padding We add random padding to every cell if there is room. This commit not only fixes how we compute that random padding length/offset but also improves its safety with helper functions and a unit test. Part of #26288 Signed-off-by: David Goulet --- src/core/or/relay.c | 74 ++++++++++++++++++++++++++++++++---------- src/core/or/relay.h | 1 + src/test/test_sendme.c | 46 ++++++++++++++++++++++++++ 3 files changed, 103 insertions(+), 18 deletions(-) diff --git a/src/core/or/relay.c b/src/core/or/relay.c index 504f391d9a..d273facd55 100644 --- a/src/core/or/relay.c +++ b/src/core/or/relay.c @@ -529,6 +529,60 @@ relay_command_to_string(uint8_t command) } } +/** Return the offset where the padding should start. The data_len is + * the relay payload length expected to be put in the cell. It can not be + * bigger than RELAY_PAYLOAD_SIZE else this function assert(). + * + * Value will always be smaller than CELL_PAYLOAD_SIZE because this offset is + * for the entire cell length not just the data payload length. Zero is + * returned if there is no room for padding. + * + * This function always skips the first 4 bytes after the payload because + * having some unused zero bytes has saved us a lot of times in the past. */ + +STATIC size_t +get_pad_cell_offset(size_t data_len) +{ + /* This is never suppose to happen but in case it does, stop right away + * because if tor is tricked somehow into not adding random bytes to the + * payload with this function returning 0 for a bad data_len, the entire + * authenticated SENDME design can be bypassed leading to bad denial of + * service attacks. */ + tor_assert(data_len <= RELAY_PAYLOAD_SIZE); + + /* If the offset is larger than the cell payload size, we return an offset + * of zero indicating that no padding needs to be added. */ + size_t offset = RELAY_HEADER_SIZE + data_len + 4; + if (offset >= CELL_PAYLOAD_SIZE) { + return 0; + } + return offset; +} + +/* Add random bytes to the unused portion of the payload, to foil attacks + * where the other side can predict all of the bytes in the payload and thus + * compute the authenticated SENDME cells without seeing the traffic. See + * proposal 289. */ +static void +pad_cell_payload(uint8_t *cell_payload, size_t data_len) +{ + size_t pad_offset, pad_len; + + tor_assert(cell_payload); + + pad_offset = get_pad_cell_offset(data_len); + if (pad_offset == 0) { + /* We can't add padding so we are done. */ + return; + } + + /* Remember here that the cell_payload is the length of the header and + * payload size so we offset it using the full lenght of the cell. */ + pad_len = CELL_PAYLOAD_SIZE - pad_offset; + crypto_fast_rng_getbytes(get_thread_fast_rng(), + cell_payload + pad_offset, pad_len); +} + /** Make a relay cell out of relay_command and payload, and send * it onto the open circuit circ. stream_id is the ID on * circ for the stream that's sending the relay cell, or 0 if it's a @@ -547,8 +601,6 @@ relay_send_command_from_edge_,(streamid_t stream_id, circuit_t *circ, cell_t cell; relay_header_t rh; cell_direction_t cell_direction; - int random_bytes_len; - size_t random_bytes_offset = 0; /* XXXX NM Split this function into a separate versions per circuit type? */ tor_assert(circ); @@ -574,22 +626,8 @@ relay_send_command_from_edge_,(streamid_t stream_id, circuit_t *circ, if (payload_len) memcpy(cell.payload+RELAY_HEADER_SIZE, payload, payload_len); - /* Add random bytes to the unused portion of the payload, to foil attacks - * where the other side can predict all of the bytes in the payload and thus - * compute authenticated sendme cells without seeing the traffic. See - * proposal 289. - * - * We'll skip the first 4 bytes of unused data because having some unused - * zero bytes has saved us a lot of times in the past. */ - random_bytes_len = RELAY_PAYLOAD_SIZE - - (RELAY_HEADER_SIZE + payload_len + 4); - if (random_bytes_len < 0) { - random_bytes_len = 0; - } - random_bytes_offset = RELAY_PAYLOAD_SIZE - random_bytes_len; - crypto_fast_rng_getbytes(get_thread_fast_rng(), - cell.payload + random_bytes_offset, - random_bytes_len); + /* Add random padding to the cell if we can. */ + pad_cell_payload(cell.payload, payload_len); log_debug(LD_OR,"delivering %d cell %s.", relay_command, cell_direction == CELL_DIRECTION_OUT ? "forward" : "backward"); diff --git a/src/core/or/relay.h b/src/core/or/relay.h index ea1b358ffb..2248cdf381 100644 --- a/src/core/or/relay.h +++ b/src/core/or/relay.h @@ -120,6 +120,7 @@ STATIC int cell_queues_check_size(void); STATIC int connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, edge_connection_t *conn, crypt_path_t *layer_hint); +STATIC size_t get_pad_cell_offset(size_t payload_len); #endif /* defined(RELAY_PRIVATE) */ diff --git a/src/test/test_sendme.c b/src/test/test_sendme.c index d6410a7488..50943e5f46 100644 --- a/src/test/test_sendme.c +++ b/src/test/test_sendme.c @@ -6,11 +6,13 @@ #define CIRCUITLIST_PRIVATE #define NETWORKSTATUS_PRIVATE #define SENDME_PRIVATE +#define RELAY_PRIVATE #include "core/or/circuit_st.h" #include "core/or/or_circuit_st.h" #include "core/or/origin_circuit_st.h" #include "core/or/circuitlist.h" +#include "core/or/relay.h" #include "core/or/sendme.h" #include "feature/nodelist/networkstatus.h" @@ -209,6 +211,48 @@ test_v1_build_cell(void *arg) circuit_free_(circ); } +static void +test_cell_payload_pad(void *arg) +{ + size_t pad_offset, payload_len, expected_offset; + + (void) arg; + + /* Offset should be 0, not enough room for padding. */ + payload_len = RELAY_PAYLOAD_SIZE; + pad_offset = get_pad_cell_offset(payload_len); + tt_int_op(pad_offset, OP_EQ, 0); + tt_int_op(CELL_PAYLOAD_SIZE - pad_offset, OP_LE, CELL_PAYLOAD_SIZE); + + /* Still no room because we keep 4 extra bytes. */ + pad_offset = get_pad_cell_offset(payload_len - 4); + tt_int_op(pad_offset, OP_EQ, 0); + tt_int_op(CELL_PAYLOAD_SIZE - pad_offset, OP_LE, CELL_PAYLOAD_SIZE); + + /* We should have 1 byte of padding. Meaning, the offset should be the + * CELL_PAYLOAD_SIZE minus 1 byte. */ + expected_offset = CELL_PAYLOAD_SIZE - 1; + pad_offset = get_pad_cell_offset(payload_len - 5); + tt_int_op(pad_offset, OP_EQ, expected_offset); + tt_int_op(CELL_PAYLOAD_SIZE - pad_offset, OP_LE, CELL_PAYLOAD_SIZE); + + /* Now some arbitrary small payload length. The cell size is header + 10 + + * extra 4 bytes we keep so the offset should be there. */ + expected_offset = RELAY_HEADER_SIZE + 10 + 4; + pad_offset = get_pad_cell_offset(10); + tt_int_op(pad_offset, OP_EQ, expected_offset); + tt_int_op(CELL_PAYLOAD_SIZE - pad_offset, OP_LE, CELL_PAYLOAD_SIZE); + + /* Data length of 0. */ + expected_offset = RELAY_HEADER_SIZE + 4; + pad_offset = get_pad_cell_offset(0); + tt_int_op(pad_offset, OP_EQ, expected_offset); + tt_int_op(CELL_PAYLOAD_SIZE - pad_offset, OP_LE, CELL_PAYLOAD_SIZE); + + done: + ; +} + struct testcase_t sendme_tests[] = { { "v1_note_digest", test_v1_note_digest, TT_FORK, NULL, NULL }, @@ -216,6 +260,8 @@ struct testcase_t sendme_tests[] = { NULL, NULL }, { "v1_build_cell", test_v1_build_cell, TT_FORK, NULL, NULL }, + { "cell_payload_pad", test_cell_payload_pad, TT_FORK, + NULL, NULL }, END_OF_TESTCASES }; From 67c22541830cf1dfc1c02843a1d4dd81c2df16ff Mon Sep 17 00:00:00 2001 From: David Goulet Date: Mon, 29 Apr 2019 11:29:05 -0400 Subject: [PATCH 0890/2557] sendme: Move note_cell_digest() to relay_crypto module Because this function is poking within the relay_crypto_t object, move the function to the module so we can keep it opaque as much as possible. Part of #26288 Signed-off-by: David Goulet --- src/core/crypto/relay_crypto.c | 9 +++++++++ src/core/crypto/relay_crypto.h | 2 ++ src/core/or/sendme.c | 14 +++----------- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/src/core/crypto/relay_crypto.c b/src/core/crypto/relay_crypto.c index eddc4298e2..3a6dad4b1d 100644 --- a/src/core/crypto/relay_crypto.c +++ b/src/core/crypto/relay_crypto.c @@ -91,6 +91,15 @@ relay_crypt_one_payload(crypto_cipher_t *cipher, uint8_t *in) crypto_cipher_crypt_inplace(cipher, (char*) in, CELL_PAYLOAD_SIZE); } +/** Record the b_digest from crypto and put it in the sendme_digest. */ +void +relay_crypto_record_sendme_digest(relay_crypto_t *crypto) +{ + tor_assert(crypto); + crypto_digest_get_digest(crypto->b_digest, (char *) crypto->sendme_digest, + sizeof(crypto->sendme_digest)); +} + /** Do the appropriate en/decryptions for cell arriving on * circ in direction cell_direction. * diff --git a/src/core/crypto/relay_crypto.h b/src/core/crypto/relay_crypto.h index 45a21d14ab..1009f1841b 100644 --- a/src/core/crypto/relay_crypto.h +++ b/src/core/crypto/relay_crypto.h @@ -27,5 +27,7 @@ void relay_crypto_clear(relay_crypto_t *crypto); void relay_crypto_assert_ok(const relay_crypto_t *crypto); +void relay_crypto_record_sendme_digest(relay_crypto_t *crypto); + #endif /* !defined(TOR_RELAY_CRYPTO_H) */ diff --git a/src/core/or/sendme.c b/src/core/or/sendme.c index b384a19164..b65f30ba97 100644 --- a/src/core/or/sendme.c +++ b/src/core/or/sendme.c @@ -12,6 +12,7 @@ #include "core/or/or.h" #include "app/config/config.h" +#include "core/crypto/relay_crypto.h" #include "core/mainloop/connection.h" #include "core/or/cell_st.h" #include "core/or/circuitlist.h" @@ -286,15 +287,6 @@ send_circuit_level_sendme(circuit_t *circ, crypt_path_t *layer_hint, return 0; } -/* Put the crypto.b_digest in the sendme_digest. */ -static void -note_cell_digest(const relay_crypto_t *crypto) -{ - tor_assert(crypto); - crypto_digest_get_digest(crypto->b_digest, (char *) crypto->sendme_digest, - sizeof(crypto->sendme_digest)); -} - /* * Public API */ @@ -305,7 +297,7 @@ void sendme_circuit_note_outbound_cell(or_circuit_t *or_circ) { tor_assert(or_circ); - note_cell_digest(&or_circ->crypto); + relay_crypto_record_sendme_digest(&or_circ->crypto); } /** Keep the current inbound cell digest for the next SENDME digest. This part @@ -314,7 +306,7 @@ void sendme_circuit_note_inbound_cell(crypt_path_t *cpath) { tor_assert(cpath); - note_cell_digest(&cpath->crypto); + relay_crypto_record_sendme_digest(&cpath->crypto); } /** Return true iff the next cell for the given cell window is expected to be From 0d8b9b56c5332b8f0205f460d0b23bb7f5620eff Mon Sep 17 00:00:00 2001 From: David Goulet Date: Mon, 29 Apr 2019 11:38:11 -0400 Subject: [PATCH 0891/2557] sendme: Better function names From nickm's review, improve the names of some functions. Part of #26288 Signed-off-by: David Goulet --- src/core/crypto/relay_crypto.c | 8 ++++---- src/core/or/relay.c | 2 +- src/core/or/sendme.c | 10 +++++----- src/core/or/sendme.h | 8 ++++---- src/test/test_sendme.c | 18 +++++++++--------- 5 files changed, 23 insertions(+), 23 deletions(-) diff --git a/src/core/crypto/relay_crypto.c b/src/core/crypto/relay_crypto.c index 3a6dad4b1d..5be45d6c7a 100644 --- a/src/core/crypto/relay_crypto.c +++ b/src/core/crypto/relay_crypto.c @@ -154,8 +154,8 @@ relay_decrypt_cell(circuit_t *circ, cell_t *cell, *layer_hint = thishop; /* This cell is for us. Keep a record of this cell because we will * use it in the next SENDME cell. */ - if (sendme_circuit_is_next_cell(thishop->deliver_window)) { - sendme_circuit_note_inbound_cell(thishop); + if (sendme_circuit_cell_is_next(thishop->deliver_window)) { + sendme_circuit_record_inbound_cell(thishop); } return 0; } @@ -230,8 +230,8 @@ relay_encrypt_cell_inbound(cell_t *cell, /* We are about to send this cell outbound on the circuit. Keep a record of * this cell if we are expecting that the next cell is a SENDME. */ - if (sendme_circuit_is_next_cell(TO_CIRCUIT(or_circ)->package_window)) { - sendme_circuit_note_outbound_cell(or_circ); + if (sendme_circuit_cell_is_next(TO_CIRCUIT(or_circ)->package_window)) { + sendme_circuit_record_outbound_cell(or_circ); } /* encrypt one layer */ diff --git a/src/core/or/relay.c b/src/core/or/relay.c index d273facd55..1b2aafb866 100644 --- a/src/core/or/relay.c +++ b/src/core/or/relay.c @@ -701,7 +701,7 @@ relay_send_command_from_edge_,(streamid_t stream_id, circuit_t *circ, * we need to. This call needs to be after the circuit_package_relay_cell() * because the cell digest is set within that function. */ if (relay_command == RELAY_COMMAND_DATA) { - sendme_note_cell_digest(circ); + sendme_record_cell_digest(circ); } return 0; diff --git a/src/core/or/sendme.c b/src/core/or/sendme.c index b65f30ba97..ff58c1489d 100644 --- a/src/core/or/sendme.c +++ b/src/core/or/sendme.c @@ -294,7 +294,7 @@ send_circuit_level_sendme(circuit_t *circ, crypt_path_t *layer_hint, /** Keep the current inbound cell digest for the next SENDME digest. This part * is only done by the client as the circuit came back from the Exit. */ void -sendme_circuit_note_outbound_cell(or_circuit_t *or_circ) +sendme_circuit_record_outbound_cell(or_circuit_t *or_circ) { tor_assert(or_circ); relay_crypto_record_sendme_digest(&or_circ->crypto); @@ -303,7 +303,7 @@ sendme_circuit_note_outbound_cell(or_circuit_t *or_circ) /** Keep the current inbound cell digest for the next SENDME digest. This part * is only done by the client as the circuit came back from the Exit. */ void -sendme_circuit_note_inbound_cell(crypt_path_t *cpath) +sendme_circuit_record_inbound_cell(crypt_path_t *cpath) { tor_assert(cpath); relay_crypto_record_sendme_digest(&cpath->crypto); @@ -316,7 +316,7 @@ sendme_circuit_note_inbound_cell(crypt_path_t *cpath) * one cell (the possible SENDME cell) should be a multiple of the increment * window value. */ bool -sendme_circuit_is_next_cell(int window) +sendme_circuit_cell_is_next(int window) { /* Is this the last cell before a SENDME? The idea is that if the package or * deliver window reaches a multiple of the increment, after this cell, we @@ -578,7 +578,7 @@ sendme_note_stream_data_packaged(edge_connection_t *conn) /* Note the cell digest in the circuit sendme last digests FIFO if applicable. * It is safe to pass a circuit that isn't meant to track those digests. */ void -sendme_note_cell_digest(circuit_t *circ) +sendme_record_cell_digest(circuit_t *circ) { const uint8_t *digest; @@ -592,7 +592,7 @@ sendme_note_cell_digest(circuit_t *circ) /* Is this the last cell before a SENDME? The idea is that if the * package_window reaches a multiple of the increment, after this cell, we * should expect a SENDME. */ - if (!sendme_circuit_is_next_cell(circ->package_window)) { + if (!sendme_circuit_cell_is_next(circ->package_window)) { return; } diff --git a/src/core/or/sendme.h b/src/core/or/sendme.h index 5889b41e9c..78273eb9a8 100644 --- a/src/core/or/sendme.h +++ b/src/core/or/sendme.h @@ -35,12 +35,12 @@ int sendme_note_circuit_data_packaged(circuit_t *circ, int sendme_note_stream_data_packaged(edge_connection_t *conn); /* Track cell digest. */ -void sendme_note_cell_digest(circuit_t *circ); -void sendme_circuit_note_inbound_cell(crypt_path_t *cpath); -void sendme_circuit_note_outbound_cell(or_circuit_t *or_circ); +void sendme_record_cell_digest(circuit_t *circ); +void sendme_circuit_record_inbound_cell(crypt_path_t *cpath); +void sendme_circuit_record_outbound_cell(or_circuit_t *or_circ); /* Circuit level information. */ -bool sendme_circuit_is_next_cell(int window); +bool sendme_circuit_cell_is_next(int window); /* Private section starts. */ #ifdef SENDME_PRIVATE diff --git a/src/test/test_sendme.c b/src/test/test_sendme.c index 50943e5f46..d40fbaf862 100644 --- a/src/test/test_sendme.c +++ b/src/test/test_sendme.c @@ -43,7 +43,7 @@ free_mock_consensus(void) } static void -test_v1_note_digest(void *arg) +test_v1_record_digest(void *arg) { or_circuit_t *or_circ = NULL; origin_circuit_t *orig_circ = NULL; @@ -61,7 +61,7 @@ test_v1_note_digest(void *arg) circ->purpose = CIRCUIT_PURPOSE_S_REND_JOINED; /* We should never note SENDME digest on origin circuit. */ - sendme_note_cell_digest(circ); + sendme_record_cell_digest(circ); tt_assert(!circ->sendme_last_digests); /* We do not need the origin circuit for now. */ orig_circ = NULL; @@ -73,23 +73,23 @@ test_v1_note_digest(void *arg) * in order to catched the CIRCWINDOW_INCREMENT-nth cell. Try something that * shouldn't be noted. */ circ->package_window = CIRCWINDOW_INCREMENT; - sendme_note_cell_digest(circ); + sendme_record_cell_digest(circ); tt_assert(!circ->sendme_last_digests); /* This should work now. Package window at CIRCWINDOW_INCREMENT + 1. */ circ->package_window++; - sendme_note_cell_digest(circ); + sendme_record_cell_digest(circ); tt_assert(circ->sendme_last_digests); tt_int_op(smartlist_len(circ->sendme_last_digests), OP_EQ, 1); /* Next cell in the package window shouldn't do anything. */ circ->package_window++; - sendme_note_cell_digest(circ); + sendme_record_cell_digest(circ); tt_int_op(smartlist_len(circ->sendme_last_digests), OP_EQ, 1); /* The next CIRCWINDOW_INCREMENT should add one more digest. */ circ->package_window = (CIRCWINDOW_INCREMENT * 2) + 1; - sendme_note_cell_digest(circ); + sendme_record_cell_digest(circ); tt_int_op(smartlist_len(circ->sendme_last_digests), OP_EQ, 2); done: @@ -188,7 +188,7 @@ test_v1_build_cell(void *arg) /* Note the wrong digest in the circuit, cell should fail validation. */ circ->package_window = CIRCWINDOW_INCREMENT + 1; - sendme_note_cell_digest(circ); + sendme_record_cell_digest(circ); tt_int_op(smartlist_len(circ->sendme_last_digests), OP_EQ, 1); setup_full_capture_of_logs(LOG_INFO); tt_int_op(sendme_is_valid(circ, payload, sizeof(payload)), OP_EQ, false); @@ -200,7 +200,7 @@ test_v1_build_cell(void *arg) /* Record the cell digest into the circuit, cell should validate. */ memcpy(or_circ->crypto.sendme_digest, digest, sizeof(digest)); circ->package_window = CIRCWINDOW_INCREMENT + 1; - sendme_note_cell_digest(circ); + sendme_record_cell_digest(circ); tt_int_op(smartlist_len(circ->sendme_last_digests), OP_EQ, 1); tt_int_op(sendme_is_valid(circ, payload, sizeof(payload)), OP_EQ, true); /* After a validation, the last digests is always popped out. */ @@ -254,7 +254,7 @@ test_cell_payload_pad(void *arg) } struct testcase_t sendme_tests[] = { - { "v1_note_digest", test_v1_note_digest, TT_FORK, + { "v1_record_digest", test_v1_record_digest, TT_FORK, NULL, NULL }, { "v1_consensus_params", test_v1_consensus_params, TT_FORK, NULL, NULL }, From 0f2ff267c5b509d697882989341d91b9fb4c249d Mon Sep 17 00:00:00 2001 From: David Goulet Date: Mon, 29 Apr 2019 12:11:57 -0400 Subject: [PATCH 0892/2557] sendme: Do not poke at crypto.sendme_digest directly As per review from nickm, keep as much as we can the relay_crypto_t object opaque. Part of #26288 Signed-off-by: David Goulet --- src/core/crypto/relay_crypto.c | 8 ++++++++ src/core/crypto/relay_crypto.h | 1 + src/core/or/sendme.c | 6 +++--- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/src/core/crypto/relay_crypto.c b/src/core/crypto/relay_crypto.c index 5be45d6c7a..8931163161 100644 --- a/src/core/crypto/relay_crypto.c +++ b/src/core/crypto/relay_crypto.c @@ -91,6 +91,14 @@ relay_crypt_one_payload(crypto_cipher_t *cipher, uint8_t *in) crypto_cipher_crypt_inplace(cipher, (char*) in, CELL_PAYLOAD_SIZE); } +/** Return the sendme_digest within the crypto object. */ +uint8_t * +relay_crypto_get_sendme_digest(relay_crypto_t *crypto) +{ + tor_assert(crypto); + return crypto->sendme_digest; +} + /** Record the b_digest from crypto and put it in the sendme_digest. */ void relay_crypto_record_sendme_digest(relay_crypto_t *crypto) diff --git a/src/core/crypto/relay_crypto.h b/src/core/crypto/relay_crypto.h index 1009f1841b..bcc1531838 100644 --- a/src/core/crypto/relay_crypto.h +++ b/src/core/crypto/relay_crypto.h @@ -27,6 +27,7 @@ void relay_crypto_clear(relay_crypto_t *crypto); void relay_crypto_assert_ok(const relay_crypto_t *crypto); +uint8_t *relay_crypto_get_sendme_digest(relay_crypto_t *crypto); void relay_crypto_record_sendme_digest(relay_crypto_t *crypto); #endif /* !defined(TOR_RELAY_CRYPTO_H) */ diff --git a/src/core/or/sendme.c b/src/core/or/sendme.c index ff58c1489d..6f451d38e6 100644 --- a/src/core/or/sendme.c +++ b/src/core/or/sendme.c @@ -388,10 +388,10 @@ sendme_circuit_consider_sending(circuit_t *circ, crypt_path_t *layer_hint) log_debug(LD_CIRC,"Queuing circuit sendme."); if (layer_hint) { layer_hint->deliver_window += CIRCWINDOW_INCREMENT; - digest = layer_hint->crypto.sendme_digest; + digest = relay_crypto_get_sendme_digest(&layer_hint->crypto); } else { circ->deliver_window += CIRCWINDOW_INCREMENT; - digest = TO_OR_CIRCUIT(circ)->crypto.sendme_digest; + digest = relay_crypto_get_sendme_digest(&TO_OR_CIRCUIT(circ)->crypto); } if (send_circuit_level_sendme(circ, layer_hint, digest) < 0) { return; /* The circuit's closed, don't continue */ @@ -597,7 +597,7 @@ sendme_record_cell_digest(circuit_t *circ) } /* Add the digest to the last seen list in the circuit. */ - digest = TO_OR_CIRCUIT(circ)->crypto.sendme_digest; + digest = relay_crypto_get_sendme_digest(&TO_OR_CIRCUIT(circ)->crypto); if (circ->sendme_last_digests == NULL) { circ->sendme_last_digests = smartlist_new(); } From 535ba0d7c58c681af8251d0133b33dc5f787fb2f Mon Sep 17 00:00:00 2001 From: David Goulet Date: Mon, 29 Apr 2019 12:27:53 -0400 Subject: [PATCH 0893/2557] practracker: Update exceptions for #26288 Signed-off-by: David Goulet --- scripts/maint/practracker/exceptions.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/maint/practracker/exceptions.txt b/scripts/maint/practracker/exceptions.txt index 37f11bb440..f8bb2bd379 100644 --- a/scripts/maint/practracker/exceptions.txt +++ b/scripts/maint/practracker/exceptions.txt @@ -85,7 +85,7 @@ problem function-size /src/core/or/circuitbuild.c:circuit_extend() 147 problem function-size /src/core/or/circuitbuild.c:choose_good_exit_server_general() 206 problem include-count /src/core/or/circuitlist.c 54 problem function-size /src/core/or/circuitlist.c:HT_PROTOTYPE() 128 -problem function-size /src/core/or/circuitlist.c:circuit_free_() 137 +problem function-size /src/core/or/circuitlist.c:circuit_free_() 143 problem function-size /src/core/or/circuitlist.c:circuit_find_to_cannibalize() 102 problem function-size /src/core/or/circuitlist.c:circuit_about_to_free() 120 problem function-size /src/core/or/circuitlist.c:circuits_handle_oom() 117 @@ -102,8 +102,8 @@ problem function-size /src/core/or/circuituse.c:circuit_get_open_circ_or_launch( problem function-size /src/core/or/circuituse.c:connection_ap_handshake_attach_circuit() 244 problem function-size /src/core/or/command.c:command_process_create_cell() 156 problem function-size /src/core/or/command.c:command_process_relay_cell() 132 -problem file-size /src/core/or/connection_edge.c 4575 -problem include-count /src/core/or/connection_edge.c 64 +problem file-size /src/core/or/connection_edge.c 4595 +problem include-count /src/core/or/connection_edge.c 65 problem function-size /src/core/or/connection_edge.c:connection_ap_expire_beginning() 117 problem function-size /src/core/or/connection_edge.c:connection_ap_handshake_rewrite() 192 problem function-size /src/core/or/connection_edge.c:connection_ap_handle_onion() 188 @@ -122,11 +122,11 @@ problem function-size /src/core/or/policies.c:policy_summarize() 107 problem function-size /src/core/or/protover.c:protover_all_supported() 116 problem file-size /src/core/or/relay.c 3173 problem function-size /src/core/or/relay.c:circuit_receive_relay_cell() 123 -problem function-size /src/core/or/relay.c:relay_send_command_from_edge_() 101 +problem function-size /src/core/or/relay.c:relay_send_command_from_edge_() 112 problem function-size /src/core/or/relay.c:connection_ap_process_end_not_open() 194 problem function-size /src/core/or/relay.c:connection_edge_process_relay_cell_not_open() 139 problem function-size /src/core/or/relay.c:connection_edge_process_relay_cell() 520 -problem function-size /src/core/or/relay.c:connection_edge_package_raw_inbuf() 130 +problem function-size /src/core/or/relay.c:connection_edge_package_raw_inbuf() 132 problem function-size /src/core/or/relay.c:circuit_resume_edge_reading_helper() 148 problem function-size /src/core/or/scheduler_kist.c:kist_scheduler_run() 171 problem function-size /src/core/or/scheduler_vanilla.c:vanilla_scheduler_run() 109 From 10c71105e6403025f380dee7c3d7e1ce40e8f2b6 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Wed, 24 Apr 2019 13:29:23 +0300 Subject: [PATCH 0894/2557] Travis: remove sudo configuration See: https://blog.travis-ci.com/2018-11-19-required-linux-infrastructure-migration --- .travis.yml | 22 +--------------------- 1 file changed, 1 insertion(+), 21 deletions(-) diff --git a/.travis.yml b/.travis.yml index ab7db2a928..7f20875cf8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -39,7 +39,7 @@ env: - RUST_OPTIONS="--enable-rust" TOR_RUST_DEPENDENCIES=true matrix: - ## include creates builds with gcc, linux, sudo: false + ## include creates builds with gcc, linux include: ## We include a single coverage build with the best options for coverage - env: COVERAGE_OPTIONS="--enable-coverage" HARDENING_OPTIONS="" @@ -71,12 +71,6 @@ matrix: ## allow failures by env: ## https://docs.travis-ci.com/user/customizing-the-build#matching-jobs-with-allow_failures exclude: - ## Clang doesn't work in containerized builds, see below. - - compiler: clang - sudo: false - ## Non-containerized gcc are slow and redundant. - - compiler: gcc - sudo: required ## gcc on OSX is less useful, because the default compiler is clang. - compiler: gcc os: osx @@ -92,20 +86,6 @@ matrix: ## TOR_RUST_DEPENDENCIES is spelt RUST_DEPENDENCIES in 0.3.2 env: RUST_OPTIONS="--enable-rust" TOR_RUST_DEPENDENCIES=true HARDENING_OPTIONS="" -## We don't need sudo. (The "apt:" stanza after this allows us to not need -## sudo; otherwise, we would need it for getting dependencies.) -## -## But we use "sudo: required" to force non-containerized builds, working -## around a Travis CI environment issue: clang LeakAnalyzer fails -## because it requires ptrace and the containerized environment no -## longer allows ptrace. -## https://github.com/travis-ci/travis-ci/issues/9033 -## -## In the matrix above, we exclude redundant combinations. -sudo: - - false - - required - ## (Linux only) Use the latest Linux image (Ubuntu Trusty) dist: trusty From b05b165a75a9d9e912da64788fc873fb215f5d75 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Sun, 21 Apr 2019 21:06:57 +0300 Subject: [PATCH 0895/2557] Add changes file --- changes/ticket30213 | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 changes/ticket30213 diff --git a/changes/ticket30213 b/changes/ticket30213 new file mode 100644 index 0000000000..acb7614807 --- /dev/null +++ b/changes/ticket30213 @@ -0,0 +1,3 @@ + o Minor features (continuous integration): + - Remove sudo configuration lines from .travis.yml as they are no longer + needed with current Travis build environment. Resolves issue 30213. From 11eaed66bbc2105942c26a2dc9e502ce77e08b69 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 18 Mar 2019 09:37:13 -0400 Subject: [PATCH 0896/2557] Add support for deterministic override of crypto_rand() in tests We had this previously, but we did it differently in different places. This implementation is pulled from test_prob_distr.c --- src/test/include.am | 3 + src/test/rng_test_helpers.c | 121 ++++++++++++++++++++++++++++++++++++ src/test/rng_test_helpers.h | 21 +++++++ 3 files changed, 145 insertions(+) create mode 100644 src/test/rng_test_helpers.c create mode 100644 src/test/rng_test_helpers.h diff --git a/src/test/include.am b/src/test/include.am index 497aa320a4..022cdbe035 100644 --- a/src/test/include.am +++ b/src/test/include.am @@ -89,6 +89,7 @@ src_test_test_SOURCES += \ src/test/log_test_helpers.c \ src/test/hs_test_helpers.c \ src/test/rend_test_helpers.c \ + src/test/rng_test_helpers.c \ src/test/test.c \ src/test/test_accounting.c \ src/test/test_addr.c \ @@ -211,6 +212,7 @@ endif src_test_test_slow_SOURCES = if UNITTESTS_ENABLED src_test_test_slow_SOURCES += \ + src/test/rng_test_helpers.c \ src/test/test_slow.c \ src/test/test_crypto_slow.c \ src/test/test_process_slow.c \ @@ -319,6 +321,7 @@ noinst_HEADERS+= \ src/test/hs_test_helpers.h \ src/test/log_test_helpers.h \ src/test/rend_test_helpers.h \ + src/test/rng_test_helpers.h \ src/test/test.h \ src/test/ptr_helpers.h \ src/test/test_helpers.h \ diff --git a/src/test/rng_test_helpers.c b/src/test/rng_test_helpers.c new file mode 100644 index 0000000000..4384d2bb37 --- /dev/null +++ b/src/test/rng_test_helpers.c @@ -0,0 +1,121 @@ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file rng_test_helpers.c + * \brief Helpers for overriding PRNGs during unit tests. + * + * We define two PRNG overrides: a "reproducible PRNG" where the seed is + * chosen randomly but the stream can be replayed later on in case a bug is + * found, and a "deterministic PRNG" where the seed is fixed in the unit + * tests. + * + * Obviously, this code is testing-only. + */ + +#include "orconfig.h" +#include "core/or/or.h" + +#include "lib/crypt_ops/crypto_rand.h" + +#include "test/rng_test_helpers.h" + +#ifndef TOR_UNIT_TESTS +#error "No. Never link this code into Tor proper." +#endif + +/** + * Mutex to protect deterministic prng. + * + * Note that if you actually _use_ the prng from two threads at the same time, + * the results will probably be nondeterministic anyway. + */ +static tor_mutex_t *rng_mutex = NULL; + +/* This is the seed of the deterministic randomness. */ +static uint8_t rng_seed[16]; +static crypto_xof_t *rng_xof = NULL; + +/** + * Print the seed for our PRNG to stdout. We use this when we're + **/ +void +testing_dump_reproducible_rng_seed(void) +{ + printf("\n" + "Seed: %s\n", + hex_str((const char*)rng_seed, sizeof(rng_seed))); +} + +/** Produce deterministic randomness for the stochastic tests using the global + * rng_xof output. + * + * This function produces deterministic data over multiple calls iff it's + * called in the same call order with the same 'n' parameter. + * If not, outputs will deviate. */ +static void +crypto_rand_deterministic(char *out, size_t n) +{ + tor_assert(rng_xof); + tor_mutex_acquire(rng_mutex); + crypto_xof_squeeze_bytes(rng_xof, (uint8_t*)out, n); + tor_mutex_release(rng_mutex); +} + +/** + * Implementation helper: override our crypto_rand() PRNG with a given seed of + * length seed_len. Overlong seeds are truncated; short ones are + * padded. + **/ +static void +enable_deterministic_rng_impl(const uint8_t *seed, size_t seed_len) +{ + memset(rng_seed, 0, sizeof(rng_seed)); + memcpy(rng_seed, seed, MIN(seed_len, sizeof(rng_seed))); + + rng_mutex = tor_mutex_new(); + + crypto_xof_free(rng_xof); + rng_xof = crypto_xof_new(); + crypto_xof_add_bytes(rng_xof, rng_seed, sizeof(rng_seed)); + MOCK(crypto_rand, crypto_rand_deterministic); +} + +/** + * Replace our crypto_rand() prng with a variant that generates all of its + * output deterministically from a randomly chosen seed. In the event of an + * error, you can log the seed later on with + * testing_dump_reproducible_rng_seed. + **/ +void +testing_enable_reproducible_rng(void) +{ + uint8_t seed[16]; + crypto_rand((char*)seed, sizeof(seed)); + enable_deterministic_rng_impl(seed, sizeof(seed)); +} + +/** + * Replace our crypto_rand() prng with a variant that generates all of its + * output deterministically from a fixed seed. This variant is mainly useful + * for cases when we don't want coverage to change between runs. + **/ +void +testing_enable_deterministic_rng(void) +{ + static const uint8_t quotation[] = + "What will it be? A tree? A weed? " + "Each one is started from a seed."; // -- Mary Ann Hoberman + enable_deterministic_rng_impl(quotation, sizeof(quotation)); +} + +/** + * Undo the overrides for our PRNG. To be used at the end of testing. + **/ +void +testing_disable_rng_override(void) +{ + crypto_xof_free(rng_xof); + UNMOCK(crypto_rand); + tor_mutex_free(rng_mutex); +} diff --git a/src/test/rng_test_helpers.h b/src/test/rng_test_helpers.h new file mode 100644 index 0000000000..be95ec9d1d --- /dev/null +++ b/src/test/rng_test_helpers.h @@ -0,0 +1,21 @@ +/* Copyright (c) 2017-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef TOR_RNG_TEST_HELPERS_H +#define TOR_RNG_TEST_HELPERS_H + +#include "core/or/or.h" + +void testing_enable_deterministic_rng(void); +void testing_enable_reproducible_rng(void); + +void testing_disable_rng_override(void); + +#define testing_disable_reproducible_rng() \ + testing_disable_rng_override() +#define testing_disable_deterministic_rng() \ + testing_disable_rng_override() + +void testing_dump_reproducible_rng_seed(void); + +#endif /* !defined(TOR_RNG_TEST_HELPERS_H) */ From d3526d3f2c0e3e56dd9fa0e91b78d2787c2d8701 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 18 Mar 2019 09:38:04 -0400 Subject: [PATCH 0897/2557] Update test_prob_distr to use new reproducible RNG override code --- src/test/test_prob_distr.c | 75 ++++++++------------------------------ 1 file changed, 15 insertions(+), 60 deletions(-) diff --git a/src/test/test_prob_distr.c b/src/test/test_prob_distr.c index 37cfdae7d9..747c3d98e6 100644 --- a/src/test/test_prob_distr.c +++ b/src/test/test_prob_distr.c @@ -33,6 +33,7 @@ #include "lib/math/prob_distr.h" #include "lib/math/fp.h" #include "lib/crypt_ops/crypto_rand.h" +#include "test/rng_test_helpers.h" #include #include @@ -1117,49 +1118,14 @@ test_psi_dist_sample(const struct dist *dist) } } -/* This is the seed of the deterministic randomness */ -static uint8_t rng_seed[16]; -static crypto_xof_t *rng_xof = NULL; - -/** Initialize the seed of the deterministic randomness. */ -static void -init_deterministic_rand(void) -{ - crypto_rand((char*)rng_seed, sizeof(rng_seed)); - crypto_xof_free(rng_xof); - rng_xof = crypto_xof_new(); - crypto_xof_add_bytes(rng_xof, rng_seed, sizeof(rng_seed)); -} - -static void -teardown_deterministic_rand(void) -{ - crypto_xof_free(rng_xof); -} - static void dump_seed(void) { printf("\n" "NOTE: This is a stochastic test, and we expect it to fail from\n" "time to time, with some low probability. If you see it fail more\n" - "than one trial in 100, though, please tell us.\n\n" - "Seed: %s\n", - hex_str((const char*)rng_seed, sizeof(rng_seed))); -} - -/** Produce deterministic randomness for the stochastic tests using the global - * deterministic_rand_counter seed - * - * This function produces deterministic data over multiple calls iff it's - * called in the same call order with the same 'n' parameter (which is the - * case for the psi test). If not, outputs will deviate. */ -static void -crypto_rand_deterministic(char *out, size_t n) -{ - /* Use a XOF to squeeze bytes out of that silly counter */ - tor_assert(rng_xof); - crypto_xof_squeeze_bytes(rng_xof, (uint8_t*)out, n); + "than one trial in 100, though, please tell us.\n\n"); + testing_dump_reproducible_rng_seed(); } static void @@ -1199,8 +1165,7 @@ test_stochastic_uniform(void *arg) }; bool ok = true, tests_failed = true; - init_deterministic_rand(); - MOCK(crypto_rand, crypto_rand_deterministic); + testing_enable_reproducible_rng(); ok &= test_psi_dist_sample(&uniform01.base); ok &= test_psi_dist_sample(&uniform_pos.base); @@ -1217,8 +1182,7 @@ test_stochastic_uniform(void *arg) if (tests_failed) { dump_seed(); } - teardown_deterministic_rand(); - UNMOCK(crypto_rand); + testing_disable_reproducible_rng(); } static bool @@ -1288,8 +1252,7 @@ test_stochastic_genpareto(void *arg) bool tests_failed = true; (void) arg; - init_deterministic_rand(); - MOCK(crypto_rand, crypto_rand_deterministic); + testing_enable_reproducible_rng(); ok = test_stochastic_genpareto_impl(0, 1, -0.25); tt_assert(ok); @@ -1312,8 +1275,7 @@ test_stochastic_genpareto(void *arg) if (tests_failed) { dump_seed(); } - teardown_deterministic_rand(); - UNMOCK(crypto_rand); + testing_disable_reproducible_rng(); } static void @@ -1324,8 +1286,7 @@ test_stochastic_geometric(void *arg) (void) arg; - init_deterministic_rand(); - MOCK(crypto_rand, crypto_rand_deterministic); + testing_enable_reproducible_rng(); ok = test_stochastic_geometric_impl(0.1); tt_assert(ok); @@ -1342,8 +1303,7 @@ test_stochastic_geometric(void *arg) if (tests_failed) { dump_seed(); } - teardown_deterministic_rand(); - UNMOCK(crypto_rand); + testing_disable_reproducible_rng(); } static void @@ -1353,8 +1313,7 @@ test_stochastic_logistic(void *arg) bool tests_failed = true; (void) arg; - init_deterministic_rand(); - MOCK(crypto_rand, crypto_rand_deterministic); + testing_enable_reproducible_rng(); ok = test_stochastic_logistic_impl(0, 1); tt_assert(ok); @@ -1371,8 +1330,7 @@ test_stochastic_logistic(void *arg) if (tests_failed) { dump_seed(); } - teardown_deterministic_rand(); - UNMOCK(crypto_rand); + testing_disable_reproducible_rng(); } static void @@ -1382,8 +1340,7 @@ test_stochastic_log_logistic(void *arg) bool tests_failed = true; (void) arg; - init_deterministic_rand(); - MOCK(crypto_rand, crypto_rand_deterministic); + testing_enable_reproducible_rng(); ok = test_stochastic_log_logistic_impl(1, 1); tt_assert(ok); @@ -1400,8 +1357,7 @@ test_stochastic_log_logistic(void *arg) if (tests_failed) { dump_seed(); } - teardown_deterministic_rand(); - UNMOCK(crypto_rand); + testing_disable_reproducible_rng(); } static void @@ -1411,8 +1367,7 @@ test_stochastic_weibull(void *arg) bool tests_failed = true; (void) arg; - init_deterministic_rand(); - MOCK(crypto_rand, crypto_rand_deterministic); + testing_enable_reproducible_rng(); ok = test_stochastic_weibull_impl(1, 0.5); tt_assert(ok); @@ -1431,7 +1386,7 @@ test_stochastic_weibull(void *arg) if (tests_failed) { dump_seed(); } - teardown_deterministic_rand(); + testing_disable_reproducible_rng(); UNMOCK(crypto_rand); } From 64d5ed0415d69c226b56e518abc8e003780368f0 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 18 Mar 2019 09:41:59 -0400 Subject: [PATCH 0898/2557] Update circuit_timeout test to use deterministic prng --- src/test/test.c | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/src/test/test.c b/src/test/test.c index fbc30fb64e..be5cb12b1e 100644 --- a/src/test/test.c +++ b/src/test/test.c @@ -12,6 +12,7 @@ #include "lib/crypt_ops/crypto_dh.h" #include "lib/crypt_ops/crypto_rand.h" #include "app/config/or_state_st.h" +#include "test/rng_test_helpers.h" #include #ifdef HAVE_FCNTL_H @@ -354,18 +355,6 @@ test_onion_queues(void *arg) tor_free(onionskin); } -static crypto_cipher_t *crypto_rand_aes_cipher = NULL; - -// Mock replacement for crypto_rand: Generates bytes from a provided AES_CTR -// cipher in crypto_rand_aes_cipher. -static void -crypto_rand_deterministic_aes(char *out, size_t n) -{ - tor_assert(crypto_rand_aes_cipher); - memset(out, 0, n); - crypto_cipher_crypt_inplace(crypto_rand_aes_cipher, out, n); -} - static void test_circuit_timeout(void *arg) { @@ -397,8 +386,7 @@ test_circuit_timeout(void *arg) // Use a deterministic RNG here, or else we'll get nondeterministic // coverage in some of the circuitstats functions. - MOCK(crypto_rand, crypto_rand_deterministic_aes); - crypto_rand_aes_cipher = crypto_cipher_new("xyzzyplughplover"); + testing_enable_deterministic_rng(); circuitbuild_running_unit_tests(); #define timeout0 (build_time_t)(30*1000.0) @@ -534,8 +522,8 @@ test_circuit_timeout(void *arg) circuit_build_times_free_timeouts(&final); or_state_free(state); teardown_periodic_events(); - UNMOCK(crypto_rand); - crypto_cipher_free(crypto_rand_aes_cipher); + + testing_disable_deterministic_rng(); } /** Test encoding and parsing of rendezvous service descriptors. */ From fe173ce0bce9b54271d32e0b46a9f6d891b16970 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 18 Mar 2019 09:59:18 -0400 Subject: [PATCH 0899/2557] Add a testing PRNG replacement that returns canned data. --- src/test/rng_test_helpers.c | 58 +++++++++++++++++++++++++++++++++++++ src/test/rng_test_helpers.h | 5 ++++ 2 files changed, 63 insertions(+) diff --git a/src/test/rng_test_helpers.c b/src/test/rng_test_helpers.c index 4384d2bb37..424aedd19c 100644 --- a/src/test/rng_test_helpers.c +++ b/src/test/rng_test_helpers.c @@ -99,6 +99,10 @@ testing_enable_reproducible_rng(void) * Replace our crypto_rand() prng with a variant that generates all of its * output deterministically from a fixed seed. This variant is mainly useful * for cases when we don't want coverage to change between runs. + * + * USAGE NOTE: Test correctness SHOULD NOT depend on the specific output of + * this "rng". If you need a specific output, use + * testing_enable_prefilled_rng() instead. **/ void testing_enable_deterministic_rng(void) @@ -109,6 +113,59 @@ testing_enable_deterministic_rng(void) enable_deterministic_rng_impl(quotation, sizeof(quotation)); } +static uint8_t *prefilled_rng_buffer = NULL; +static size_t prefilled_rng_buflen; +static size_t prefilled_rng_idx; + +/** + * crypto_rand() replacement that returns canned data. + **/ +static void +crypto_rand_prefilled(char *out, size_t n) +{ + tor_mutex_acquire(rng_mutex); + while (n) { + size_t n_to_copy = MIN(prefilled_rng_buflen - prefilled_rng_idx, n); + memcpy(out, prefilled_rng_buffer + prefilled_rng_idx, n_to_copy); + out += n_to_copy; + n -= n_to_copy; + prefilled_rng_idx += n_to_copy; + + if (prefilled_rng_idx == prefilled_rng_buflen) { + prefilled_rng_idx = 0; + } + } + tor_mutex_release(rng_mutex); +} + +/** + * Replace our crypto_rand() prng with a variant that yields output + * from a buffer. If it reaches the end of the buffer, it starts over. + **/ +void +testing_enable_prefilled_rng(const void *buffer, size_t buflen) +{ + tor_assert(buflen > 0); + rng_mutex = tor_mutex_new(); + + prefilled_rng_buffer = tor_memdup(buffer, buflen); + prefilled_rng_buflen = buflen; + prefilled_rng_idx = 0; + + MOCK(crypto_rand, crypto_rand_prefilled); +} + +/** + * Reset the position in the prefilled RNG buffer to the start. + */ +void +testing_prefilled_rng_reset(void) +{ + tor_mutex_acquire(rng_mutex); + prefilled_rng_idx = 0; + tor_mutex_release(rng_mutex); +} + /** * Undo the overrides for our PRNG. To be used at the end of testing. **/ @@ -116,6 +173,7 @@ void testing_disable_rng_override(void) { crypto_xof_free(rng_xof); + tor_free(prefilled_rng_buffer); UNMOCK(crypto_rand); tor_mutex_free(rng_mutex); } diff --git a/src/test/rng_test_helpers.h b/src/test/rng_test_helpers.h index be95ec9d1d..907099450d 100644 --- a/src/test/rng_test_helpers.h +++ b/src/test/rng_test_helpers.h @@ -8,6 +8,9 @@ void testing_enable_deterministic_rng(void); void testing_enable_reproducible_rng(void); +void testing_enable_prefilled_rng(const void *buffer, size_t buflen); + +void testing_prefilled_rng_reset(void); void testing_disable_rng_override(void); @@ -15,6 +18,8 @@ void testing_disable_rng_override(void); testing_disable_rng_override() #define testing_disable_deterministic_rng() \ testing_disable_rng_override() +#define testing_disable_prefilled_rng() \ + testing_disable_rng_override() void testing_dump_reproducible_rng_seed(void); From 7bd34698afc19ef88940c6a1739ce23fc6ad2dd5 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 18 Mar 2019 10:05:50 -0400 Subject: [PATCH 0900/2557] Use prefilled_rng in test_addr.c in place of existing code. --- src/test/test_addr.c | 41 ++++++++++++++--------------------------- 1 file changed, 14 insertions(+), 27 deletions(-) diff --git a/src/test/test_addr.c b/src/test/test_addr.c index fb8df5f0fb..3a1a7b6997 100644 --- a/src/test/test_addr.c +++ b/src/test/test_addr.c @@ -11,6 +11,7 @@ #include "feature/client/addressmap.h" #include "test/log_test_helpers.h" #include "lib/net/resolve.h" +#include "test/rng_test_helpers.h" #ifdef HAVE_SYS_UN_H #include @@ -945,27 +946,6 @@ test_virtaddrmap(void *data) ; } -static const char *canned_data = NULL; -static size_t canned_data_len = 0; - -/* Mock replacement for crypto_rand() that returns canned data from - * canned_data above. */ -static void -crypto_canned(char *ptr, size_t n) -{ - if (canned_data_len) { - size_t to_copy = MIN(n, canned_data_len); - memcpy(ptr, canned_data, to_copy); - canned_data += to_copy; - canned_data_len -= to_copy; - n -= to_copy; - ptr += to_copy; - } - if (n) { - crypto_rand_unmocked(ptr, n); - } -} - static void test_virtaddrmap_persist(void *data) { @@ -973,6 +953,8 @@ test_virtaddrmap_persist(void *data) const char *a, *b, *c; tor_addr_t addr; char *ones = NULL; + const char *canned_data; + size_t canned_data_len; addressmap_init(); @@ -991,7 +973,7 @@ test_virtaddrmap_persist(void *data) "1234567890" // the second call returns this. "abcdefghij"; // the third call returns this. canned_data_len = 30; - MOCK(crypto_rand, crypto_canned); + testing_enable_prefilled_rng(canned_data, canned_data_len); a = addressmap_register_virtual_address(RESOLVED_TYPE_HOSTNAME, tor_strdup("quuxit.baz")); @@ -1001,9 +983,9 @@ test_virtaddrmap_persist(void *data) tt_assert(b); tt_str_op(a, OP_EQ, "gezdgnbvgy3tqojq.virtual"); tt_str_op(b, OP_EQ, "mfrggzdfmztwq2lk.virtual"); + testing_disable_prefilled_rng(); // Now try something to get us an ipv4 address - UNMOCK(crypto_rand); tt_int_op(0,OP_EQ, parse_virtual_addr_network("192.168.0.0/16", AF_INET, 0, NULL)); a = addressmap_register_virtual_address(RESOLVED_TYPE_IPV4, @@ -1020,22 +1002,23 @@ test_virtaddrmap_persist(void *data) // Try some canned entropy and verify all the we discard duplicates, // addresses that end with 0, and addresses that end with 255. - MOCK(crypto_rand, crypto_canned); canned_data = "\x01\x02\x03\x04" // okay "\x01\x02\x03\x04" // duplicate "\x03\x04\x00\x00" // bad ending 1 "\x05\x05\x00\xff" // bad ending 2 "\x05\x06\x07\xf0"; // okay canned_data_len = 20; + testing_enable_prefilled_rng(canned_data, canned_data_len); + a = addressmap_register_virtual_address(RESOLVED_TYPE_IPV4, tor_strdup("wumble.onion")); b = addressmap_register_virtual_address(RESOLVED_TYPE_IPV4, tor_strdup("wumpus.onion")); tt_str_op(a, OP_EQ, "192.168.3.4"); tt_str_op(b, OP_EQ, "192.168.7.240"); + testing_disable_prefilled_rng(); // Now try IPv6! - UNMOCK(crypto_rand); tt_int_op(0,OP_EQ, parse_virtual_addr_network("1010:F000::/20", AF_INET6, 0, NULL)); a = addressmap_register_virtual_address(RESOLVED_TYPE_IPV6, @@ -1051,7 +1034,7 @@ test_virtaddrmap_persist(void *data) tt_assert(!strcmpstart(b, "[1010:f")); // Try IPv6 with canned entropy, to make sure we detect duplicates. - MOCK(crypto_rand, crypto_canned); + canned_data = "acanthopterygian" // okay "cinematographist" // okay "acanthopterygian" // duplicate @@ -1060,6 +1043,8 @@ test_virtaddrmap_persist(void *data) "cinematographist" // duplicate "coadministration"; // okay canned_data_len = 16 * 7; + testing_enable_prefilled_rng(canned_data, canned_data_len); + a = addressmap_register_virtual_address(RESOLVED_TYPE_IPV6, tor_strdup("wuffle.baz")); b = addressmap_register_virtual_address(RESOLVED_TYPE_IPV6, @@ -1072,9 +1057,11 @@ test_virtaddrmap_persist(void *data) // Try address exhaustion: make sure we can actually fail if we // get too many already-existing addresses. + testing_disable_prefilled_rng(); canned_data_len = 128*1024; canned_data = ones = tor_malloc(canned_data_len); memset(ones, 1, canned_data_len); + testing_enable_prefilled_rng(canned_data, canned_data_len); // There is some chance this one will fail if a previous random // allocation gave out the address already. a = addressmap_register_virtual_address(RESOLVED_TYPE_IPV4, @@ -1091,7 +1078,7 @@ test_virtaddrmap_persist(void *data) expect_single_log_msg_containing("Ran out of virtual addresses!"); done: - UNMOCK(crypto_rand); + testing_disable_prefilled_rng(); tor_free(ones); addressmap_free_all(); teardown_capture_of_logs(); From 0a9fb6938d6047376ce78ee568e71a99c309fe5f Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 18 Mar 2019 10:10:30 -0400 Subject: [PATCH 0901/2557] Use prefilled PRNG replacement in test_extorport This is the last remaining place where our tests had mocked crypto_rand. --- src/test/test_extorport.c | 25 ++++++++----------------- 1 file changed, 8 insertions(+), 17 deletions(-) diff --git a/src/test/test_extorport.c b/src/test/test_extorport.c index f5d16af921..cfdd11d161 100644 --- a/src/test/test_extorport.c +++ b/src/test/test_extorport.c @@ -18,6 +18,7 @@ #include "test/test.h" #include "test/test_helpers.h" +#include "test/rng_test_helpers.h" #ifdef HAVE_SYS_STAT_H #include @@ -302,16 +303,6 @@ test_ext_or_cookie_auth(void *arg) tor_free(client_hash2); } -static void -crypto_rand_return_tse_str(char *to, size_t n) -{ - if (n != 32) { - TT_FAIL(("Asked for %d bytes, not 32", (int)n)); - return; - } - memcpy(to, "te road There is always another ", 32); -} - static void test_ext_or_cookie_auth_testvec(void *arg) { @@ -326,7 +317,7 @@ test_ext_or_cookie_auth_testvec(void *arg) memcpy(ext_or_auth_cookie, "Gliding wrapt in a brown mantle," , 32); ext_or_auth_cookie_is_set = 1; - MOCK(crypto_rand, crypto_rand_return_tse_str); + testing_enable_prefilled_rng("te road There is always another ", 32); tt_int_op(0, OP_EQ, handle_client_auth_nonce(client_nonce, 32, &client_hash, &reply, @@ -351,7 +342,7 @@ test_ext_or_cookie_auth_testvec(void *arg) "33b3cd77ff79bd80c2074bbf438119a2"); done: - UNMOCK(crypto_rand); + testing_disable_prefilled_rng(); tor_free(reply); tor_free(client_hash); tor_free(mem_op_hex_tmp); @@ -414,9 +405,9 @@ do_ext_or_handshake(or_connection_t *conn) CONTAINS("\x01\x00", 2); WRITE("\x01", 1); WRITE("But when I look ahead up the whi", 32); - MOCK(crypto_rand, crypto_rand_return_tse_str); + testing_enable_prefilled_rng("te road There is always another ", 32); tt_int_op(0, OP_EQ, connection_ext_or_process_inbuf(conn)); - UNMOCK(crypto_rand); + testing_disable_prefilled_rng(); tt_int_op(TO_CONN(conn)->state, OP_EQ, EXT_OR_CONN_STATE_AUTH_WAIT_CLIENT_HASH); CONTAINS("\xec\x80\xed\x6e\x54\x6d\x3b\x36\xfd\xfc\x22\xfe\x13\x15\x41\x6b" @@ -481,9 +472,9 @@ test_ext_or_handshake(void *arg) tt_int_op(0, OP_EQ, connection_ext_or_process_inbuf(conn)); /* send the rest of the nonce. */ WRITE("ahead up the whi", 16); - MOCK(crypto_rand, crypto_rand_return_tse_str); + testing_enable_prefilled_rng("te road There is always another ", 32); tt_int_op(0, OP_EQ, connection_ext_or_process_inbuf(conn)); - UNMOCK(crypto_rand); + testing_disable_prefilled_rng(); /* We should get the right reply from the server. */ CONTAINS("\xec\x80\xed\x6e\x54\x6d\x3b\x36\xfd\xfc\x22\xfe\x13\x15\x41\x6b" "\x02\x9f\x1a\xde\x76\x10\xd9\x10\x87\x8b\x62\xee\xb7\x40\x38\x21" @@ -582,7 +573,7 @@ test_ext_or_handshake(void *arg) done: UNMOCK(connection_write_to_buf_impl_); - UNMOCK(crypto_rand); + testing_disable_prefilled_rng(); if (conn) connection_free_minimal(TO_CONN(conn)); #undef CONTAINS From 7086a9f90e65b8f88063ca478e6f4d41866b2878 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 18 Mar 2019 11:44:07 -0400 Subject: [PATCH 0902/2557] Make rng mock code also cover strongest_rand. --- src/test/rng_test_helpers.c | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/src/test/rng_test_helpers.c b/src/test/rng_test_helpers.c index 424aedd19c..aaf42ca05a 100644 --- a/src/test/rng_test_helpers.c +++ b/src/test/rng_test_helpers.c @@ -32,6 +32,13 @@ */ static tor_mutex_t *rng_mutex = NULL; +/** replacement for crypto_strongest_rand that delegates to crypto_rand. */ +static void +mock_crypto_strongest_rand(uint8_t *out, size_t len) +{ + crypto_rand((char *)out, len); +} + /* This is the seed of the deterministic randomness. */ static uint8_t rng_seed[16]; static crypto_xof_t *rng_xof = NULL; @@ -79,12 +86,13 @@ enable_deterministic_rng_impl(const uint8_t *seed, size_t seed_len) rng_xof = crypto_xof_new(); crypto_xof_add_bytes(rng_xof, rng_seed, sizeof(rng_seed)); MOCK(crypto_rand, crypto_rand_deterministic); + MOCK(crypto_strongest_rand_, mock_crypto_strongest_rand); } /** - * Replace our crypto_rand() prng with a variant that generates all of its - * output deterministically from a randomly chosen seed. In the event of an - * error, you can log the seed later on with + * Replace our crypto_rand() and crypto_strongest_rand() prngs with a variant + * that generates all of its output deterministically from a randomly chosen + * seed. In the event of an error, you can log the seed later on with * testing_dump_reproducible_rng_seed. **/ void @@ -96,9 +104,10 @@ testing_enable_reproducible_rng(void) } /** - * Replace our crypto_rand() prng with a variant that generates all of its - * output deterministically from a fixed seed. This variant is mainly useful - * for cases when we don't want coverage to change between runs. + * Replace our crypto_rand() and crypto_strongest_rand() prngs with a variant + * that generates all of its output deterministically from a fixed seed. This + * variant is mainly useful for cases when we don't want coverage to change + * between runs. * * USAGE NOTE: Test correctness SHOULD NOT depend on the specific output of * this "rng". If you need a specific output, use @@ -139,8 +148,9 @@ crypto_rand_prefilled(char *out, size_t n) } /** - * Replace our crypto_rand() prng with a variant that yields output - * from a buffer. If it reaches the end of the buffer, it starts over. + * Replace our crypto_rand() and crypto_strongest_rand() prngs with a variant + * that yields output from a buffer. If it reaches the end of the buffer, it + * starts over. **/ void testing_enable_prefilled_rng(const void *buffer, size_t buflen) @@ -153,6 +163,7 @@ testing_enable_prefilled_rng(const void *buffer, size_t buflen) prefilled_rng_idx = 0; MOCK(crypto_rand, crypto_rand_prefilled); + MOCK(crypto_strongest_rand_, mock_crypto_strongest_rand); } /** @@ -175,5 +186,6 @@ testing_disable_rng_override(void) crypto_xof_free(rng_xof); tor_free(prefilled_rng_buffer); UNMOCK(crypto_rand); + UNMOCK(crypto_strongest_rand_); tor_mutex_free(rng_mutex); } From c6a93beed83a6271eb029d4b045be9d92b34031d Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 18 Mar 2019 11:44:11 -0400 Subject: [PATCH 0903/2557] Use preloaded-rng code in test_hs_descriptor.c --- src/test/test_hs_descriptor.c | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/src/test/test_hs_descriptor.c b/src/test/test_hs_descriptor.c index 09c6c3e700..2563a0bdf9 100644 --- a/src/test/test_hs_descriptor.c +++ b/src/test/test_hs_descriptor.c @@ -21,6 +21,7 @@ #include "test/hs_test_helpers.h" #include "test/test_helpers.h" #include "test/log_test_helpers.h" +#include "test/rng_test_helpers.h" #ifdef HAVE_CFLAG_WOVERLENGTH_STRINGS DISABLE_GCC_WARNING(overlength-strings) @@ -30,13 +31,6 @@ DISABLE_GCC_WARNING(overlength-strings) #include "test_hs_descriptor.inc" ENABLE_GCC_WARNING(overlength-strings) -/* Mock function to fill all bytes with 1 */ -static void -mock_crypto_strongest_rand(uint8_t *out, size_t out_len) -{ - memset(out, 1, out_len); -} - /* Test certificate encoding put in a descriptor. */ static void test_cert_encoding(void *arg) @@ -800,7 +794,7 @@ test_build_authorized_client(void *arg) client_pubkey_b16, strlen(client_pubkey_b16)); - MOCK(crypto_strongest_rand_, mock_crypto_strongest_rand); + testing_enable_prefilled_rng("\x01", 1); hs_desc_build_authorized_client(subcredential, &client_auth_pk, &auth_ephemeral_sk, @@ -816,7 +810,7 @@ test_build_authorized_client(void *arg) done: tor_free(desc_client); tor_free(mem_op_hex_tmp); - UNMOCK(crypto_strongest_rand_); + testing_disable_prefilled_rng(); } struct testcase_t hs_descriptor[] = { From e66b5153bd5feeb16bb18b735745d37310ae63fa Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 18 Mar 2019 11:54:37 -0400 Subject: [PATCH 0904/2557] Extract add-entropy code from crypto_fast_rng to a new function --- src/lib/crypt_ops/crypto_rand_fast.c | 36 +++++++++++++++++----------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/src/lib/crypt_ops/crypto_rand_fast.c b/src/lib/crypt_ops/crypto_rand_fast.c index 01817c618f..dd9bf051c8 100644 --- a/src/lib/crypt_ops/crypto_rand_fast.c +++ b/src/lib/crypt_ops/crypto_rand_fast.c @@ -192,6 +192,26 @@ cipher_from_seed(const uint8_t *seed) return crypto_cipher_new_with_iv_and_bits(seed, seed+KEY_LEN, KEY_BITS); } +/** + * Helper: mix additional entropy into rng by using our XOF to mix the + * old value for the seed with some additional bytes from + * crypto_strongest_rand(). + **/ +static void +crypto_fast_rng_add_entopy(crypto_fast_rng_t *rng) +{ + crypto_xof_t *xof = crypto_xof_new(); + crypto_xof_add_bytes(xof, rng->buf.seed, SEED_LEN); + { + uint8_t seedbuf[SEED_LEN]; + crypto_strongest_rand(seedbuf, SEED_LEN); + crypto_xof_add_bytes(xof, seedbuf, SEED_LEN); + memwipe(seedbuf, 0, SEED_LEN); + } + crypto_xof_squeeze_bytes(xof, rng->buf.seed, SEED_LEN); + crypto_xof_free(xof); +} + /** * Helper: refill the seed bytes and output buffer of rng, using * the input seed bytes as input (key and IV) for the stream cipher. @@ -203,20 +223,8 @@ static void crypto_fast_rng_refill(crypto_fast_rng_t *rng) { if (rng->n_till_reseed-- == 0) { - /* It's time to reseed the RNG. We'll do this by using our XOF to mix the - * old value for the seed with some additional bytes from - * crypto_strongest_rand(). */ - crypto_xof_t *xof = crypto_xof_new(); - crypto_xof_add_bytes(xof, rng->buf.seed, SEED_LEN); - { - uint8_t seedbuf[SEED_LEN]; - crypto_strongest_rand(seedbuf, SEED_LEN); - crypto_xof_add_bytes(xof, seedbuf, SEED_LEN); - memwipe(seedbuf, 0, SEED_LEN); - } - crypto_xof_squeeze_bytes(xof, rng->buf.seed, SEED_LEN); - crypto_xof_free(xof); - + /* It's time to reseed the RNG. */ + crypto_fast_rng_add_entopy(rng); rng->n_till_reseed = RESEED_AFTER; } /* Now fill rng->buf with output from our stream cipher, initialized from From 587a525cc5aedaee51fff33a78f004f103a4e0c4 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 18 Mar 2019 12:03:48 -0400 Subject: [PATCH 0905/2557] Add improved debugging support to crypto_rand_fast code. --- src/lib/crypt_ops/crypto_rand.h | 4 +++ src/lib/crypt_ops/crypto_rand_fast.c | 46 ++++++++++++++++++++++++++-- 2 files changed, 47 insertions(+), 3 deletions(-) diff --git a/src/lib/crypt_ops/crypto_rand.h b/src/lib/crypt_ops/crypto_rand.h index c51d6a4480..528f238fa5 100644 --- a/src/lib/crypt_ops/crypto_rand.h +++ b/src/lib/crypt_ops/crypto_rand.h @@ -92,6 +92,10 @@ void crypto_rand_fast_shutdown(void); #if defined(TOR_UNIT_TESTS) /* Used for white-box testing */ size_t crypto_fast_rng_get_bytes_used_per_stream(void); +/* For deterministic prng implementations */ +void crypto_fast_rng_disable_reseed(crypto_fast_rng_t *rng); +/* To override the prng for testing. */ +crypto_fast_rng_t *crypto_replace_thread_fast_rng(crypto_fast_rng_t *rng); #endif #ifdef CRYPTO_RAND_PRIVATE diff --git a/src/lib/crypt_ops/crypto_rand_fast.c b/src/lib/crypt_ops/crypto_rand_fast.c index dd9bf051c8..b71ade81bd 100644 --- a/src/lib/crypt_ops/crypto_rand_fast.c +++ b/src/lib/crypt_ops/crypto_rand_fast.c @@ -95,8 +95,13 @@ CTASSERT(KEY_BITS == 128 || KEY_BITS == 192 || KEY_BITS == 256); struct crypto_fast_rng_t { /** How many more fills does this buffer have before we should mix - * in the output of crypto_rand()? */ - uint16_t n_till_reseed; + * in the output of crypto_strongest_rand()? + * + * This value may be negative if unit tests are enabled. If so, it + * indicates that we should never mix in extra data from + * crypto_strongest_rand(). + */ + int16_t n_till_reseed; /** How many bytes are remaining in cbuf.bytes? */ uint16_t bytes_left; #ifdef CHECK_PID @@ -181,6 +186,18 @@ crypto_fast_rng_new_from_seed(const uint8_t *seed) return result; } +#ifdef TOR_UNIT_TESTS +/** + * Unit tests only: prevent a crypto_fast_rng_t from ever mixing in more + * entropy. + */ +void +crypto_fast_rng_disable_reseed(crypto_fast_rng_t *rng) +{ + rng->n_till_reseed = -1; +} +#endif + /** * Helper: create a crypto_cipher_t object from SEED_LEN bytes of * input. The first KEY_LEN bytes are used as the stream cipher's key, @@ -222,10 +239,19 @@ crypto_fast_rng_add_entopy(crypto_fast_rng_t *rng) static void crypto_fast_rng_refill(crypto_fast_rng_t *rng) { - if (rng->n_till_reseed-- == 0) { + rng->n_till_reseed--; + if (rng->n_till_reseed == 0) { /* It's time to reseed the RNG. */ crypto_fast_rng_add_entopy(rng); rng->n_till_reseed = RESEED_AFTER; + } else if (rng->n_till_reseed < 0) { +#ifdef TOR_UNIT_TESTS + /* Reseeding is disabled for testing; never do it on this prng. */ + rng->n_till_reseed = -1; +#else + /* If testing is disabled, this shouldn't be able to become negative. */ + tor_assert_unreached(); +#endif } /* Now fill rng->buf with output from our stream cipher, initialized from * that seed value. */ @@ -371,6 +397,20 @@ destroy_thread_fast_rng(void) tor_threadlocal_set(&thread_rng, NULL); } +#ifdef TOR_UNIT_TESTS +/** + * Replace the current thread's rng with rng. For use by the + * unit tests only. Returns the previous thread rng. + **/ +crypto_fast_rng_t * +crypto_replace_thread_fast_rng(crypto_fast_rng_t *rng) +{ + crypto_fast_rng_t *old_rng = tor_threadlocal_get(&thread_rng); + tor_threadlocal_set(&thread_rng, rng); + return old_rng; +} +#endif + /** * Initialize the global thread-local key that will be used to keep track * of per-thread fast RNG instances. Called from the crypto subsystem's From 604e849d36232def4fa186c4eee4c1caa551e894 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 18 Mar 2019 12:15:46 -0400 Subject: [PATCH 0906/2557] Make the deterministic and reproducible rng test code handle fast_rng --- src/test/rng_test_helpers.c | 34 +++++++++++++++++++++++++++------- 1 file changed, 27 insertions(+), 7 deletions(-) diff --git a/src/test/rng_test_helpers.c b/src/test/rng_test_helpers.c index aaf42ca05a..c11709ed88 100644 --- a/src/test/rng_test_helpers.c +++ b/src/test/rng_test_helpers.c @@ -32,6 +32,11 @@ */ static tor_mutex_t *rng_mutex = NULL; +/** + * Cached old value for the thread prng. + **/ +static crypto_fast_rng_t *stored_fast_rng = NULL; + /** replacement for crypto_strongest_rand that delegates to crypto_rand. */ static void mock_crypto_strongest_rand(uint8_t *out, size_t len) @@ -87,12 +92,21 @@ enable_deterministic_rng_impl(const uint8_t *seed, size_t seed_len) crypto_xof_add_bytes(rng_xof, rng_seed, sizeof(rng_seed)); MOCK(crypto_rand, crypto_rand_deterministic); MOCK(crypto_strongest_rand_, mock_crypto_strongest_rand); + + uint8_t fast_rng_seed[CRYPTO_FAST_RNG_SEED_LEN]; + memset(fast_rng_seed, 0xff, sizeof(fast_rng_seed)); + memcpy(fast_rng_seed, rng_seed, MIN(sizeof(rng_seed), + sizeof(fast_rng_seed))); + crypto_fast_rng_t *fast_rng = crypto_fast_rng_new_from_seed(fast_rng_seed); + crypto_fast_rng_disable_reseed(fast_rng); + stored_fast_rng = crypto_replace_thread_fast_rng(fast_rng); } /** - * Replace our crypto_rand() and crypto_strongest_rand() prngs with a variant - * that generates all of its output deterministically from a randomly chosen - * seed. In the event of an error, you can log the seed later on with + * Replace our get_thread_fast_rng(), crypto_rand() and + * crypto_strongest_rand() prngs with a variant that generates all of its + * output deterministically from a randomly chosen seed. In the event of an + * error, you can log the seed later on with * testing_dump_reproducible_rng_seed. **/ void @@ -104,10 +118,10 @@ testing_enable_reproducible_rng(void) } /** - * Replace our crypto_rand() and crypto_strongest_rand() prngs with a variant - * that generates all of its output deterministically from a fixed seed. This - * variant is mainly useful for cases when we don't want coverage to change - * between runs. + * Replace our get_thread_fast_rng(), crypto_rand() and + * crypto_strongest_rand() prngs with a variant that generates all of its + * output deterministically from a fixed seed. This variant is mainly useful + * for cases when we don't want coverage to change between runs. * * USAGE NOTE: Test correctness SHOULD NOT depend on the specific output of * this "rng". If you need a specific output, use @@ -151,6 +165,9 @@ crypto_rand_prefilled(char *out, size_t n) * Replace our crypto_rand() and crypto_strongest_rand() prngs with a variant * that yields output from a buffer. If it reaches the end of the buffer, it * starts over. + * + * Note: the get_thread_fast_rng() prng is not replaced by this; we'll need + * more code to support that. **/ void testing_enable_prefilled_rng(const void *buffer, size_t buflen) @@ -188,4 +205,7 @@ testing_disable_rng_override(void) UNMOCK(crypto_rand); UNMOCK(crypto_strongest_rand_); tor_mutex_free(rng_mutex); + + crypto_fast_rng_t *rng = crypto_replace_thread_fast_rng(stored_fast_rng); + crypto_fast_rng_free(rng); } From 730dddc3806f904b4328f4e9281d370c65522c30 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 18 Mar 2019 12:50:11 -0400 Subject: [PATCH 0907/2557] Make sure that the rng is not replaced if it is already replaced. --- src/test/rng_test_helpers.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/test/rng_test_helpers.c b/src/test/rng_test_helpers.c index c11709ed88..262d380bda 100644 --- a/src/test/rng_test_helpers.c +++ b/src/test/rng_test_helpers.c @@ -24,6 +24,11 @@ #error "No. Never link this code into Tor proper." #endif +/** + * True iff the RNG is currently replaced. Prevents double-replacement. + **/ +static bool rng_is_replaced = false; + /** * Mutex to protect deterministic prng. * @@ -82,6 +87,9 @@ crypto_rand_deterministic(char *out, size_t n) static void enable_deterministic_rng_impl(const uint8_t *seed, size_t seed_len) { + tor_assert(!rng_is_replaced); + tor_assert(crypto_rand == crypto_rand__real); + memset(rng_seed, 0, sizeof(rng_seed)); memcpy(rng_seed, seed, MIN(seed_len, sizeof(rng_seed))); @@ -100,6 +108,8 @@ enable_deterministic_rng_impl(const uint8_t *seed, size_t seed_len) crypto_fast_rng_t *fast_rng = crypto_fast_rng_new_from_seed(fast_rng_seed); crypto_fast_rng_disable_reseed(fast_rng); stored_fast_rng = crypto_replace_thread_fast_rng(fast_rng); + + rng_is_replaced = true; } /** @@ -196,6 +206,9 @@ testing_prefilled_rng_reset(void) /** * Undo the overrides for our PRNG. To be used at the end of testing. + * + * Note that this function should be safe to call even if the rng has not + * yet been replaced. **/ void testing_disable_rng_override(void) @@ -208,4 +221,6 @@ testing_disable_rng_override(void) crypto_fast_rng_t *rng = crypto_replace_thread_fast_rng(stored_fast_rng); crypto_fast_rng_free(rng); + + rng_is_replaced = false; } From 48e1ab17204944e6197002dfa3d3d0c8a08184dc Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 18 Mar 2019 12:50:35 -0400 Subject: [PATCH 0908/2557] Changes file for 29732. --- changes/ticket29732 | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 changes/ticket29732 diff --git a/changes/ticket29732 b/changes/ticket29732 new file mode 100644 index 0000000000..bb72361c48 --- /dev/null +++ b/changes/ticket29732 @@ -0,0 +1,5 @@ + o Minor features (testing): + - Tor's unit test code now contains a standard set of functions to + replace the PRNG with a deterministic or reproducible version for + testing. Previously, various tests implemented this in various ways. + Implements ticket 29732. From 6eb1b8da0ab2a58157c4eb7db0a8471cb9dfefda Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 25 Apr 2019 14:20:41 -0400 Subject: [PATCH 0909/2557] Turn 'mainloop' into a subsystem. We need a little refactoring for this to work, since the initialization code for the periodic events assumes that libevent is already initialized, which it can't be until it's configured. This change, combined with the previous ones, lets other subsystems declare their own periodic events, without mainloop.c having to know about them. Implements ticket 30293. --- src/app/main/shutdown.c | 2 -- src/app/main/subsystem_list.c | 3 +++ src/core/include.am | 2 ++ src/core/mainloop/mainloop.c | 16 ++++++++-------- src/core/mainloop/mainloop.h | 2 +- src/core/mainloop/mainloop_sys.c | 32 ++++++++++++++++++++++++++++++++ src/core/mainloop/mainloop_sys.h | 12 ++++++++++++ src/test/test_periodic_event.c | 3 +++ 8 files changed, 61 insertions(+), 11 deletions(-) create mode 100644 src/core/mainloop/mainloop_sys.c create mode 100644 src/core/mainloop/mainloop_sys.h diff --git a/src/app/main/shutdown.c b/src/app/main/shutdown.c index 314e33f228..2dc18e19a7 100644 --- a/src/app/main/shutdown.c +++ b/src/app/main/shutdown.c @@ -18,7 +18,6 @@ #include "app/main/shutdown.h" #include "app/main/subsysmgr.h" #include "core/mainloop/connection.h" -#include "core/mainloop/mainloop.h" #include "core/mainloop/mainloop_pubsub.h" #include "core/or/channeltls.h" #include "core/or/circuitlist.h" @@ -176,7 +175,6 @@ tor_free_all(int postfork) /* stuff in main.c */ tor_mainloop_disconnect_pubsub(); - tor_mainloop_free_all(); if (!postfork) { release_lockfile(); diff --git a/src/app/main/subsystem_list.c b/src/app/main/subsystem_list.c index 3834176182..7ffdcc8053 100644 --- a/src/app/main/subsystem_list.c +++ b/src/app/main/subsystem_list.c @@ -8,6 +8,7 @@ #include "lib/cc/compat_compiler.h" #include "lib/cc/torint.h" +#include "core/mainloop/mainloop_sys.h" #include "core/or/ocirc_event_sys.h" #include "core/or/orconn_event_sys.h" #include "feature/control/btrack_sys.h" @@ -44,6 +45,8 @@ const subsys_fns_t *tor_subsystems[] = { &sys_orconn_event, /* -33 */ &sys_ocirc_event, /* -32 */ &sys_btrack, /* -30 */ + + &sys_mainloop, /* 5 */ }; const unsigned n_tor_subsystems = ARRAY_LENGTH(tor_subsystems); diff --git a/src/core/include.am b/src/core/include.am index 9824601725..4645bca76f 100644 --- a/src/core/include.am +++ b/src/core/include.am @@ -24,6 +24,7 @@ LIBTOR_APP_A_SOURCES = \ src/core/mainloop/cpuworker.c \ src/core/mainloop/mainloop.c \ src/core/mainloop/mainloop_pubsub.c \ + src/core/mainloop/mainloop_sys.c \ src/core/mainloop/netstatus.c \ src/core/mainloop/periodic.c \ src/core/or/address_set.c \ @@ -222,6 +223,7 @@ noinst_HEADERS += \ src/core/mainloop/cpuworker.h \ src/core/mainloop/mainloop.h \ src/core/mainloop/mainloop_pubsub.h \ + src/core/mainloop/mainloop_sys.h \ src/core/mainloop/netstatus.h \ src/core/mainloop/periodic.h \ src/core/or/addr_policy_st.h \ diff --git a/src/core/mainloop/mainloop.c b/src/core/mainloop/mainloop.c index ddcc3bcd76..8c90103a79 100644 --- a/src/core/mainloop/mainloop.c +++ b/src/core/mainloop/mainloop.c @@ -1550,7 +1550,7 @@ initialize_periodic_events_cb(evutil_socket_t fd, short events, void *data) /** Set up all the members of mainloop_periodic_events[], and configure them * all to be launched from a callback. */ -STATIC void +void initialize_periodic_events(void) { if (periodic_events_initialized) @@ -1563,7 +1563,6 @@ initialize_periodic_events(void) } /* Set up all periodic events. We'll launch them by roles. */ - periodic_events_setup_all(); #define NAMED_CALLBACK(name) \ STMT_BEGIN name ## _event = periodic_events_find( #name ); STMT_END @@ -1575,12 +1574,6 @@ initialize_periodic_events(void) NAMED_CALLBACK(launch_descriptor_fetches); NAMED_CALLBACK(check_dns_honesty); NAMED_CALLBACK(save_state); - - struct timeval one_second = { 1, 0 }; - initialize_periodic_events_event = tor_evtimer_new( - tor_libevent_get_base(), - initialize_periodic_events_cb, NULL); - event_add(initialize_periodic_events_event, &one_second); } STATIC void @@ -2778,6 +2771,13 @@ do_main_loop(void) */ initialize_periodic_events(); initialize_mainloop_events(); + periodic_events_setup_all(); + + struct timeval one_second = { 1, 0 }; + initialize_periodic_events_event = tor_evtimer_new( + tor_libevent_get_base(), + initialize_periodic_events_cb, NULL); + event_add(initialize_periodic_events_event, &one_second); #ifdef HAVE_SYSTEMD_209 uint64_t watchdog_delay; diff --git a/src/core/mainloop/mainloop.h b/src/core/mainloop/mainloop.h index 850918c35e..2dccdf0e1a 100644 --- a/src/core/mainloop/mainloop.h +++ b/src/core/mainloop/mainloop.h @@ -90,6 +90,7 @@ void mainloop_schedule_shutdown(int delay_sec); void tor_init_connection_lists(void); void initialize_mainloop_events(void); +void initialize_periodic_events(void); void tor_mainloop_free_all(void); struct token_bucket_rw_t; @@ -102,7 +103,6 @@ extern struct token_bucket_rw_t global_relayed_bucket; #ifdef MAINLOOP_PRIVATE STATIC int run_main_loop_until_done(void); STATIC void close_closeable_connections(void); -STATIC void initialize_periodic_events(void); STATIC void teardown_periodic_events(void); STATIC int get_my_roles(const or_options_t *); STATIC int check_network_participation_callback(time_t now, diff --git a/src/core/mainloop/mainloop_sys.c b/src/core/mainloop/mainloop_sys.c new file mode 100644 index 0000000000..fbd5a40327 --- /dev/null +++ b/src/core/mainloop/mainloop_sys.c @@ -0,0 +1,32 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "core/or/or.h" +#include "core/mainloop/mainloop_sys.h" +#include "core/mainloop/mainloop.h" + +#include "lib/subsys/subsys.h" + +static int +subsys_mainloop_initialize(void) +{ + initialize_periodic_events(); + return 0; +} + +static void +subsys_mainloop_shutdown(void) +{ + tor_mainloop_free_all(); +} + +const struct subsys_fns_t sys_mainloop = { + .name = "mainloop", + .supported = true, + .level = 5, + .initialize = subsys_mainloop_initialize, + .shutdown = subsys_mainloop_shutdown, +}; diff --git a/src/core/mainloop/mainloop_sys.h b/src/core/mainloop/mainloop_sys.h new file mode 100644 index 0000000000..14c567278c --- /dev/null +++ b/src/core/mainloop/mainloop_sys.h @@ -0,0 +1,12 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef MAINLOOP_SYS_H +#define MAINLOOP_SYS_H + +extern const struct subsys_fns_t sys_mainloop; + +#endif diff --git a/src/test/test_periodic_event.c b/src/test/test_periodic_event.c index 645274d371..19325ed3b7 100644 --- a/src/test/test_periodic_event.c +++ b/src/test/test_periodic_event.c @@ -51,6 +51,7 @@ test_pe_initialize(void *arg) * need to run the main loop and then wait for a second delaying the unit * tests. Instead, we'll test the callback work indepedently elsewhere. */ initialize_periodic_events(); + periodic_events_setup_all(); set_network_participation(false); rescan_periodic_events(get_options()); @@ -110,6 +111,7 @@ test_pe_launch(void *arg) #endif initialize_periodic_events(); + periodic_events_setup_all(); /* Now that we've initialized, rescan the list to launch. */ periodic_events_on_new_options(options); @@ -300,6 +302,7 @@ test_pe_hs_service(void *arg) consider_hibernation(time(NULL)); /* Initialize the events so we can enable them */ initialize_periodic_events(); + periodic_events_setup_all(); /* Hack: We'll set a dumb fn() of each events so they don't get called when * dispatching them. We just want to test the state of the callbacks, not From b5a62b1ef5cfbe3e56caa110d9024f7fa0ac6ae6 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 25 Apr 2019 15:09:24 -0400 Subject: [PATCH 0910/2557] Move dirauth periodic events into dirauth module. Closes ticket 30294. --- scripts/maint/practracker/exceptions.txt | 6 +- src/app/config/config.c | 1 + src/app/main/shutdown.c | 1 - src/app/main/subsystem_list.c | 6 + src/core/include.am | 4 + src/core/mainloop/mainloop.c | 140 ---------------------- src/core/mainloop/mainloop.h | 1 - src/core/mainloop/periodic.c | 27 +++++ src/core/mainloop/periodic.h | 2 + src/feature/dirauth/dirauth_periodic.c | 142 +++++++++++++++++++++++ src/feature/dirauth/dirauth_periodic.h | 25 ++++ src/feature/dirauth/dirauth_sys.c | 33 ++++++ src/feature/dirauth/dirauth_sys.h | 12 ++ src/feature/nodelist/networkstatus.c | 1 + 14 files changed, 256 insertions(+), 145 deletions(-) create mode 100644 src/feature/dirauth/dirauth_periodic.c create mode 100644 src/feature/dirauth/dirauth_periodic.h create mode 100644 src/feature/dirauth/dirauth_sys.c create mode 100644 src/feature/dirauth/dirauth_sys.h diff --git a/scripts/maint/practracker/exceptions.txt b/scripts/maint/practracker/exceptions.txt index 37f11bb440..a0358ae834 100644 --- a/scripts/maint/practracker/exceptions.txt +++ b/scripts/maint/practracker/exceptions.txt @@ -29,8 +29,8 @@ # # Remember: It is better to fix the problem than to add a new exception! -problem file-size /src/app/config/config.c 8491 -problem include-count /src/app/config/config.c 86 +problem file-size /src/app/config/config.c 8492 +problem include-count /src/app/config/config.c 87 problem function-size /src/app/config/config.c:options_act_reversible() 296 problem function-size /src/app/config/config.c:options_act() 588 problem function-size /src/app/config/config.c:resolve_my_address() 192 @@ -214,7 +214,7 @@ problem function-size /src/feature/nodelist/authcert.c:trusted_dirs_load_certs_f problem function-size /src/feature/nodelist/authcert.c:authority_certs_fetch_missing() 296 problem function-size /src/feature/nodelist/fmt_routerstatus.c:routerstatus_format_entry() 166 problem function-size /src/feature/nodelist/microdesc.c:microdesc_cache_rebuild() 134 -problem include-count /src/feature/nodelist/networkstatus.c 61 +problem include-count /src/feature/nodelist/networkstatus.c 62 problem function-size /src/feature/nodelist/networkstatus.c:networkstatus_check_consensus_signature() 176 problem function-size /src/feature/nodelist/networkstatus.c:networkstatus_set_current_consensus() 293 problem function-size /src/feature/nodelist/node_select.c:router_pick_directory_server_impl() 123 diff --git a/src/app/config/config.c b/src/app/config/config.c index 46dc15b069..81a83e2c5f 100644 --- a/src/app/config/config.c +++ b/src/app/config/config.c @@ -156,6 +156,7 @@ #include "lib/evloop/procmon.h" #include "feature/dirauth/dirvote.h" +#include "feature/dirauth/dirauth_periodic.h" #include "feature/dirauth/recommend_pkg.h" #include "feature/dirauth/authmode.h" diff --git a/src/app/main/shutdown.c b/src/app/main/shutdown.c index 2dc18e19a7..9239a0cf0f 100644 --- a/src/app/main/shutdown.c +++ b/src/app/main/shutdown.c @@ -125,7 +125,6 @@ tor_free_all(int postfork) } geoip_free_all(); geoip_stats_free_all(); - dirvote_free_all(); routerlist_free_all(); networkstatus_free_all(); addressmap_free_all(); diff --git a/src/app/main/subsystem_list.c b/src/app/main/subsystem_list.c index 7ffdcc8053..00effe01aa 100644 --- a/src/app/main/subsystem_list.c +++ b/src/app/main/subsystem_list.c @@ -24,6 +24,8 @@ #include "lib/wallclock/wallclock_sys.h" #include "lib/process/process_sys.h" +#include "feature/dirauth/dirauth_sys.h" + #include /** @@ -47,6 +49,10 @@ const subsys_fns_t *tor_subsystems[] = { &sys_btrack, /* -30 */ &sys_mainloop, /* 5 */ + +#ifdef HAVE_MODULE_DIRAUTH + &sys_dirauth, /* 70 */ +#endif }; const unsigned n_tor_subsystems = ARRAY_LENGTH(tor_subsystems); diff --git a/src/core/include.am b/src/core/include.am index 4645bca76f..dc1ff2407b 100644 --- a/src/core/include.am +++ b/src/core/include.am @@ -176,6 +176,8 @@ LIBTOR_APP_TESTING_A_SOURCES = $(LIBTOR_APP_A_SOURCES) # The Directory Authority module. MODULE_DIRAUTH_SOURCES = \ src/feature/dirauth/authmode.c \ + src/feature/dirauth/dirauth_periodic.c \ + src/feature/dirauth/dirauth_sys.c \ src/feature/dirauth/dircollate.c \ src/feature/dirauth/dirvote.c \ src/feature/dirauth/shared_random.c \ @@ -308,6 +310,8 @@ noinst_HEADERS += \ src/feature/control/getinfo_geoip.h \ src/feature/dirauth/authmode.h \ src/feature/dirauth/bwauth.h \ + src/feature/dirauth/dirauth_periodic.h \ + src/feature/dirauth/dirauth_sys.h \ src/feature/dirauth/dircollate.h \ src/feature/dirauth/dirvote.h \ src/feature/dirauth/dsigs_parse.h \ diff --git a/src/core/mainloop/mainloop.c b/src/core/mainloop/mainloop.c index 8c90103a79..8e61bd7e6c 100644 --- a/src/core/mainloop/mainloop.c +++ b/src/core/mainloop/mainloop.c @@ -75,7 +75,6 @@ #include "feature/control/control.h" #include "feature/control/control_events.h" #include "feature/dirauth/authmode.h" -#include "feature/dirauth/reachability.h" #include "feature/dircache/consdiffmgr.h" #include "feature/dircache/dirserv.h" #include "feature/dircommon/directory.h" @@ -106,9 +105,6 @@ #include -#include "feature/dirauth/dirvote.h" -#include "feature/dirauth/authmode.h" - #include "core/or/cell_st.h" #include "core/or/entry_connection_st.h" #include "feature/nodelist/networkstatus_st.h" @@ -1346,7 +1342,6 @@ static int periodic_events_initialized = 0; #define CALLBACK(name) \ static int name ## _callback(time_t, const or_options_t *) CALLBACK(add_entropy); -CALLBACK(check_authority_cert); CALLBACK(check_canonical_channels); CALLBACK(check_descriptor); CALLBACK(check_dns_honesty); @@ -1356,14 +1351,11 @@ CALLBACK(check_for_reachability_bw); CALLBACK(check_onion_keys_expiry_time); CALLBACK(clean_caches); CALLBACK(clean_consdiffmgr); -CALLBACK(dirvote); -CALLBACK(downrate_stability); CALLBACK(expire_old_ciruits_serverside); CALLBACK(fetch_networkstatus); CALLBACK(heartbeat); CALLBACK(hs_service); CALLBACK(launch_descriptor_fetches); -CALLBACK(launch_reachability_tests); CALLBACK(prune_old_routers); CALLBACK(reachability_warnings); CALLBACK(record_bridge_stats); @@ -1373,7 +1365,6 @@ CALLBACK(retry_dns); CALLBACK(retry_listeners); CALLBACK(rotate_onion_key); CALLBACK(rotate_x509_certificate); -CALLBACK(save_stability); CALLBACK(save_state); CALLBACK(write_bridge_ns); CALLBACK(write_stats_file); @@ -1428,15 +1419,6 @@ STATIC periodic_event_item_t mainloop_periodic_events[] = { CALLBACK(retry_dns, ROUTER, 0), CALLBACK(rotate_onion_key, ROUTER, 0), - /* Authorities (bridge and directory) only. */ - CALLBACK(downrate_stability, AUTHORITIES, 0), - CALLBACK(launch_reachability_tests, AUTHORITIES, FL(NEED_NET)), - CALLBACK(save_stability, AUTHORITIES, 0), - - /* Directory authority only. */ - CALLBACK(check_authority_cert, DIRAUTH, 0), - CALLBACK(dirvote, DIRAUTH, FL(NEED_NET)), - /* Relay only. */ CALLBACK(check_canonical_channels, RELAY, FL(NEED_NET)), CALLBACK(check_dns_honesty, RELAY, FL(NEED_NET)), @@ -1470,7 +1452,6 @@ STATIC periodic_event_item_t mainloop_periodic_events[] = { * can access them by name. We also keep them inside periodic_events[] * so that we can implement "reset all timers" in a reasonable way. */ static periodic_event_item_t *check_descriptor_event=NULL; -static periodic_event_item_t *dirvote_event=NULL; static periodic_event_item_t *fetch_networkstatus_event=NULL; static periodic_event_item_t *launch_descriptor_fetches_event=NULL; static periodic_event_item_t *check_dns_honesty_event=NULL; @@ -1569,7 +1550,6 @@ initialize_periodic_events(void) NAMED_CALLBACK(check_descriptor); NAMED_CALLBACK(prune_old_routers); - NAMED_CALLBACK(dirvote); NAMED_CALLBACK(fetch_networkstatus); NAMED_CALLBACK(launch_descriptor_fetches); NAMED_CALLBACK(check_dns_honesty); @@ -1581,7 +1561,6 @@ teardown_periodic_events(void) { periodic_events_destroy_all(); check_descriptor_event = NULL; - dirvote_event = NULL; fetch_networkstatus_event = NULL; launch_descriptor_fetches_event = NULL; check_dns_honesty_event = NULL; @@ -1712,29 +1691,6 @@ mainloop_schedule_shutdown(int delay_sec) mainloop_event_schedule(scheduled_shutdown_ev, &delay_tv); } -#define LONGEST_TIMER_PERIOD (30 * 86400) -/** Helper: Return the number of seconds between now and next, - * clipped to the range [1 second, LONGEST_TIMER_PERIOD]. */ -static inline int -safe_timer_diff(time_t now, time_t next) -{ - if (next > now) { - /* There were no computers at signed TIME_MIN (1902 on 32-bit systems), - * and nothing that could run Tor. It's a bug if 'next' is around then. - * On 64-bit systems with signed TIME_MIN, TIME_MIN is before the Big - * Bang. We cannot extrapolate past a singularity, but there was probably - * nothing that could run Tor then, either. - **/ - tor_assert(next > TIME_MIN + LONGEST_TIMER_PERIOD); - - if (next - LONGEST_TIMER_PERIOD > now) - return LONGEST_TIMER_PERIOD; - return (int)(next - now); - } else { - return 1; - } -} - /** Perform regular maintenance tasks. This function gets run once per * second. */ @@ -2003,102 +1959,6 @@ check_network_participation_callback(time_t now, const or_options_t *options) return CHECK_PARTICIPATION_INTERVAL; } -/** - * Periodic callback: if we're an authority, make sure we test - * the routers on the network for reachability. - */ -static int -launch_reachability_tests_callback(time_t now, const or_options_t *options) -{ - if (authdir_mode_tests_reachability(options) && - !net_is_disabled()) { - /* try to determine reachability of the other Tor relays */ - dirserv_test_reachability(now); - } - return REACHABILITY_TEST_INTERVAL; -} - -/** - * Periodic callback: if we're an authority, discount the stability - * information (and other rephist information) that's older. - */ -static int -downrate_stability_callback(time_t now, const or_options_t *options) -{ - (void)options; - /* 1d. Periodically, we discount older stability information so that new - * stability info counts more, and save the stability information to disk as - * appropriate. */ - time_t next = rep_hist_downrate_old_runs(now); - return safe_timer_diff(now, next); -} - -/** - * Periodic callback: if we're an authority, record our measured stability - * information from rephist in an mtbf file. - */ -static int -save_stability_callback(time_t now, const or_options_t *options) -{ - if (authdir_mode_tests_reachability(options)) { - if (rep_hist_record_mtbf_data(now, 1)<0) { - log_warn(LD_GENERAL, "Couldn't store mtbf data."); - } - } -#define SAVE_STABILITY_INTERVAL (30*60) - return SAVE_STABILITY_INTERVAL; -} - -/** - * Periodic callback: if we're an authority, check on our authority - * certificate (the one that authenticates our authority signing key). - */ -static int -check_authority_cert_callback(time_t now, const or_options_t *options) -{ - (void)now; - (void)options; - /* 1e. Periodically, if we're a v3 authority, we check whether our cert is - * close to expiring and warn the admin if it is. */ - v3_authority_check_key_expiry(); -#define CHECK_V3_CERTIFICATE_INTERVAL (5*60) - return CHECK_V3_CERTIFICATE_INTERVAL; -} - -/** - * Scheduled callback: Run directory-authority voting functionality. - * - * The schedule is a bit complicated here, so dirvote_act() manages the - * schedule itself. - **/ -static int -dirvote_callback(time_t now, const or_options_t *options) -{ - if (!authdir_mode_v3(options)) { - tor_assert_nonfatal_unreached(); - return 3600; - } - - time_t next = dirvote_act(options, now); - if (BUG(next == TIME_MAX)) { - /* This shouldn't be returned unless we called dirvote_act() without - * being an authority. If it happens, maybe our configuration will - * fix itself in an hour or so? */ - return 3600; - } - return safe_timer_diff(now, next); -} - -/** Reschedule the directory-authority voting event. Run this whenever the - * schedule has changed. */ -void -reschedule_dirvote(const or_options_t *options) -{ - if (authdir_mode_v3(options) && dirvote_event) { - periodic_event_reschedule(dirvote_event); - } -} - /** * Periodic callback: If our consensus is too old, recalculate whether * we can actually use it. diff --git a/src/core/mainloop/mainloop.h b/src/core/mainloop/mainloop.h index 2dccdf0e1a..3a611f81aa 100644 --- a/src/core/mainloop/mainloop.h +++ b/src/core/mainloop/mainloop.h @@ -62,7 +62,6 @@ void reset_all_main_loop_timers(void); void reschedule_descriptor_update_check(void); void reschedule_directory_downloads(void); void reschedule_or_state_save(void); -void reschedule_dirvote(const or_options_t *options); void mainloop_schedule_postloop_cleanup(void); void rescan_periodic_events(const or_options_t *options); MOCK_DECL(void, schedule_rescan_periodic_events,(void)); diff --git a/src/core/mainloop/periodic.c b/src/core/mainloop/periodic.c index a4a03ed297..ceef740992 100644 --- a/src/core/mainloop/periodic.c +++ b/src/core/mainloop/periodic.c @@ -323,3 +323,30 @@ periodic_events_destroy_all(void) smartlist_free(the_periodic_events); } + +#define LONGEST_TIMER_PERIOD (30 * 86400) +/** Helper: Return the number of seconds between now and next, + * clipped to the range [1 second, LONGEST_TIMER_PERIOD]. + * + * We use this to answer the question, "how many seconds is it from now until + * next" in periodic timer callbacks. Don't use it for other purposes + **/ +int +safe_timer_diff(time_t now, time_t next) +{ + if (next > now) { + /* There were no computers at signed TIME_MIN (1902 on 32-bit systems), + * and nothing that could run Tor. It's a bug if 'next' is around then. + * On 64-bit systems with signed TIME_MIN, TIME_MIN is before the Big + * Bang. We cannot extrapolate past a singularity, but there was probably + * nothing that could run Tor then, either. + **/ + tor_assert(next > TIME_MIN + LONGEST_TIMER_PERIOD); + + if (next - LONGEST_TIMER_PERIOD > now) + return LONGEST_TIMER_PERIOD; + return (int)(next - now); + } else { + return 1; + } +} diff --git a/src/core/mainloop/periodic.h b/src/core/mainloop/periodic.h index a021a141db..4e4e8b2c96 100644 --- a/src/core/mainloop/periodic.h +++ b/src/core/mainloop/periodic.h @@ -97,4 +97,6 @@ periodic_event_item_t *periodic_events_find(const char *name); void periodic_events_rescan_by_roles(int roles, bool net_disabled); void periodic_events_destroy_all(void); +int safe_timer_diff(time_t now, time_t next); + #endif /* !defined(TOR_PERIODIC_H) */ diff --git a/src/feature/dirauth/dirauth_periodic.c b/src/feature/dirauth/dirauth_periodic.c new file mode 100644 index 0000000000..1fa0ca9cfa --- /dev/null +++ b/src/feature/dirauth/dirauth_periodic.c @@ -0,0 +1,142 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "core/or/or.h" + +#include "app/config/or_options_st.h" +#include "core/mainloop/netstatus.h" +#include "feature/dirauth/reachability.h" +#include "feature/stats/rephist.h" + +#include "feature/dirauth/dirvote.h" +#include "feature/dirauth/dirauth_periodic.h" +#include "feature/dirauth/authmode.h" + +#include "core/mainloop/periodic.h" + +#define DECLARE_EVENT(name, roles, flags) \ + static periodic_event_item_t name ## _event = \ + PERIODIC_EVENT(name, \ + PERIODIC_EVENT_ROLE_##roles, \ + flags) + +#define FL(name) (PERIODIC_EVENT_FLAG_##name) + +/** + * Periodic callback: if we're an authority, check on our authority + * certificate (the one that authenticates our authority signing key). + */ +static int +check_authority_cert_callback(time_t now, const or_options_t *options) +{ + (void)now; + (void)options; + /* 1e. Periodically, if we're a v3 authority, we check whether our cert is + * close to expiring and warn the admin if it is. */ + v3_authority_check_key_expiry(); +#define CHECK_V3_CERTIFICATE_INTERVAL (5*60) + return CHECK_V3_CERTIFICATE_INTERVAL; +} + +DECLARE_EVENT(check_authority_cert, DIRAUTH, 0); + +/** + * Scheduled callback: Run directory-authority voting functionality. + * + * The schedule is a bit complicated here, so dirvote_act() manages the + * schedule itself. + **/ +static int +dirvote_callback(time_t now, const or_options_t *options) +{ + if (!authdir_mode_v3(options)) { + tor_assert_nonfatal_unreached(); + return 3600; + } + + time_t next = dirvote_act(options, now); + if (BUG(next == TIME_MAX)) { + /* This shouldn't be returned unless we called dirvote_act() without + * being an authority. If it happens, maybe our configuration will + * fix itself in an hour or so? */ + return 3600; + } + return safe_timer_diff(now, next); +} + +DECLARE_EVENT(dirvote, DIRAUTH, FL(NEED_NET)); + +/** Reschedule the directory-authority voting event. Run this whenever the + * schedule has changed. */ +void +reschedule_dirvote(const or_options_t *options) +{ + if (authdir_mode_v3(options)) { + periodic_event_reschedule(&dirvote_event); + } +} + +/** + * Periodic callback: if we're an authority, record our measured stability + * information from rephist in an mtbf file. + */ +static int +save_stability_callback(time_t now, const or_options_t *options) +{ + if (authdir_mode_tests_reachability(options)) { + if (rep_hist_record_mtbf_data(now, 1)<0) { + log_warn(LD_GENERAL, "Couldn't store mtbf data."); + } + } +#define SAVE_STABILITY_INTERVAL (30*60) + return SAVE_STABILITY_INTERVAL; +} + +DECLARE_EVENT(save_stability, AUTHORITIES, 0); + +/** + * Periodic callback: if we're an authority, make sure we test + * the routers on the network for reachability. + */ +static int +launch_reachability_tests_callback(time_t now, const or_options_t *options) +{ + if (authdir_mode_tests_reachability(options) && + !net_is_disabled()) { + /* try to determine reachability of the other Tor relays */ + dirserv_test_reachability(now); + } + return REACHABILITY_TEST_INTERVAL; +} + +DECLARE_EVENT(launch_reachability_tests, AUTHORITIES, FL(NEED_NET)); + +/** + * Periodic callback: if we're an authority, discount the stability + * information (and other rephist information) that's older. + */ +static int +downrate_stability_callback(time_t now, const or_options_t *options) +{ + (void)options; + /* 1d. Periodically, we discount older stability information so that new + * stability info counts more, and save the stability information to disk as + * appropriate. */ + time_t next = rep_hist_downrate_old_runs(now); + return safe_timer_diff(now, next); +} + +DECLARE_EVENT(downrate_stability, AUTHORITIES, 0); + +void +dirauth_add_periodic_events(void) +{ + periodic_events_add(&downrate_stability_event); + periodic_events_add(&launch_reachability_tests_event); + periodic_events_add(&save_stability_event); + periodic_events_add(&check_authority_cert_event); + periodic_events_add(&dirvote_event); +} diff --git a/src/feature/dirauth/dirauth_periodic.h b/src/feature/dirauth/dirauth_periodic.h new file mode 100644 index 0000000000..680c0c4bed --- /dev/null +++ b/src/feature/dirauth/dirauth_periodic.h @@ -0,0 +1,25 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef DIRVOTE_PERIODIC_H +#define DIRVOTE_PERIODIC_H + +#ifdef HAVE_MODULE_DIRAUTH + +void dirauth_add_periodic_events(void); +void reschedule_dirvote(const or_options_t *options); + +#else + +static inline void +reschedule_dirvote(const or_options_t *options) +{ + (void)options; +} + +#endif + +#endif diff --git a/src/feature/dirauth/dirauth_sys.c b/src/feature/dirauth/dirauth_sys.c new file mode 100644 index 0000000000..6845e62c27 --- /dev/null +++ b/src/feature/dirauth/dirauth_sys.c @@ -0,0 +1,33 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "core/or/or.h" + +#include "feature/dirauth/dirauth_sys.h" +#include "feature/dirauth/dirvote.h" +#include "feature/dirauth/dirauth_periodic.h" +#include "lib/subsys/subsys.h" + +static int +subsys_dirauth_initialize(void) +{ + dirauth_add_periodic_events(); + return 0; +} + +static void +subsys_dirauth_shutdown(void) +{ + dirvote_free_all(); +} + +const struct subsys_fns_t sys_dirauth = { + .name = "dirauth", + .supported = true, + .level = 70, + .initialize = subsys_dirauth_initialize, + .shutdown = subsys_dirauth_shutdown, +}; diff --git a/src/feature/dirauth/dirauth_sys.h b/src/feature/dirauth/dirauth_sys.h new file mode 100644 index 0000000000..e10f4c9589 --- /dev/null +++ b/src/feature/dirauth/dirauth_sys.h @@ -0,0 +1,12 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef DIRAUTH_SYS_H +#define DIRAUTH_SYS_H + +extern const struct subsys_fns_t sys_dirauth; + +#endif diff --git a/src/feature/nodelist/networkstatus.c b/src/feature/nodelist/networkstatus.c index ee22e16323..8b0fc102c8 100644 --- a/src/feature/nodelist/networkstatus.c +++ b/src/feature/nodelist/networkstatus.c @@ -82,6 +82,7 @@ #include "lib/crypt_ops/crypto_rand.h" #include "lib/crypt_ops/crypto_util.h" +#include "feature/dirauth/dirauth_periodic.h" #include "feature/dirauth/dirvote.h" #include "feature/dirauth/authmode.h" #include "feature/dirauth/shared_random.h" From 7e03500eefacdb8a987b16d4a9b256443216a27e Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 25 Apr 2019 15:12:10 -0400 Subject: [PATCH 0911/2557] Changes file for periodic event movement --- changes/ticket30293 | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 changes/ticket30293 diff --git a/changes/ticket30293 b/changes/ticket30293 new file mode 100644 index 0000000000..c74b6cd346 --- /dev/null +++ b/changes/ticket30293 @@ -0,0 +1,5 @@ + o Code simplification and refactoring: + - Start move responsibility for knowing about periodic events to the + appropriate subsystems, so that the mainloop doesn't need to know all + the periodic events in the rest of the codebase. Implements tickets + 30293 and 30294. From 9a62a820fb2b0319c27d5f5131c2227a66133ee0 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 26 Apr 2019 12:46:14 -0400 Subject: [PATCH 0912/2557] Remove now-extraneous calls to initialize_periodic_events(). This is now the responsibility of the mainloop's subsystem initializer. --- src/core/mainloop/mainloop.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/core/mainloop/mainloop.c b/src/core/mainloop/mainloop.c index 8e61bd7e6c..0c825bb1be 100644 --- a/src/core/mainloop/mainloop.c +++ b/src/core/mainloop/mainloop.c @@ -2610,8 +2610,6 @@ dns_servers_relaunch_checks(void) void initialize_mainloop_events(void) { - initialize_periodic_events(); - if (!schedule_active_linked_connections_event) { schedule_active_linked_connections_event = mainloop_event_postloop_new(schedule_active_linked_connections_cb, NULL); @@ -2629,8 +2627,9 @@ do_main_loop(void) /* initialize the periodic events first, so that code that depends on the * events being present does not assert. */ - initialize_periodic_events(); + tor_assert(periodic_events_initialized); initialize_mainloop_events(); + periodic_events_setup_all(); struct timeval one_second = { 1, 0 }; From b7cc631d2391d55c078e0693303f6a0d4d9e963a Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 26 Apr 2019 13:17:35 -0400 Subject: [PATCH 0913/2557] Rename and clarify some functions for periodic events When we tell the periodic event manager about an event, we are "registering" that event. The event sits around without being usable, however, until we "connect" the event to libevent. In the end, we "disconnect" the event and remove its libevent parts. Previously, we called these operations "add", "setup", and "destroy", which led to confusion. --- src/core/mainloop/mainloop.c | 6 ++-- src/core/mainloop/periodic.c | 46 ++++++++++++++++---------- src/core/mainloop/periodic.h | 10 +++--- src/feature/dirauth/dirauth_periodic.c | 12 +++---- src/feature/dirauth/dirauth_periodic.h | 2 +- src/feature/dirauth/dirauth_sys.c | 2 +- src/test/test_periodic_event.c | 6 ++-- 7 files changed, 48 insertions(+), 36 deletions(-) diff --git a/src/core/mainloop/mainloop.c b/src/core/mainloop/mainloop.c index 0c825bb1be..30dad956ae 100644 --- a/src/core/mainloop/mainloop.c +++ b/src/core/mainloop/mainloop.c @@ -1540,7 +1540,7 @@ initialize_periodic_events(void) periodic_events_initialized = 1; for (int i = 0; mainloop_periodic_events[i].name; ++i) { - periodic_events_add(&mainloop_periodic_events[i]); + periodic_events_register(&mainloop_periodic_events[i]); } /* Set up all periodic events. We'll launch them by roles. */ @@ -1559,7 +1559,7 @@ initialize_periodic_events(void) STATIC void teardown_periodic_events(void) { - periodic_events_destroy_all(); + periodic_events_disconnect_all(); check_descriptor_event = NULL; fetch_networkstatus_event = NULL; launch_descriptor_fetches_event = NULL; @@ -2630,7 +2630,7 @@ do_main_loop(void) tor_assert(periodic_events_initialized); initialize_mainloop_events(); - periodic_events_setup_all(); + periodic_events_connect_all(); struct timeval one_second = { 1, 0 }; initialize_periodic_events_event = tor_evtimer_new( diff --git a/src/core/mainloop/periodic.c b/src/core/mainloop/periodic.c index ceef740992..dbc4553a73 100644 --- a/src/core/mainloop/periodic.c +++ b/src/core/mainloop/periodic.c @@ -12,7 +12,12 @@ * * This module manages a global list of periodic_event_item_t objects, * each corresponding to a single event. To register an event, pass it to - * periodic_events_add() when initializing your subsystem. + * periodic_events_register() when initializing your subsystem. + * + * Registering an event makes the periodic event subsystem know about it, but + * doesn't cause the event to get created immediately. Before the event can + * be started, periodic_event_connect_all() must be called by mainloop.c to + * connect all the events to Libevent. * * We expect that periodic_event_item_t objects will be statically allocated; * we set them up and tear them down here, but we don't take ownership of @@ -33,7 +38,8 @@ static const int MAX_INTERVAL = 10 * 365 * 86400; /** - * + * Global list of periodic events that have been registered with + * periodic_event_register. **/ static smartlist_t *the_periodic_events = NULL; @@ -106,9 +112,10 @@ periodic_event_reschedule(periodic_event_item_t *event) } } -/** Initializes the libevent backend for a periodic event. */ +/** Connects a periodic event to the Libevent backend. Does not launch the + * event immediately. */ void -periodic_event_setup(periodic_event_item_t *event) +periodic_event_connect(periodic_event_item_t *event) { if (event->ev) { /* Already setup? This is a bug */ log_err(LD_BUG, "Initial dispatch should only be done once."); @@ -126,7 +133,7 @@ void periodic_event_launch(periodic_event_item_t *event) { if (! event->ev) { /* Not setup? This is a bug */ - log_err(LD_BUG, "periodic_event_launch without periodic_event_setup"); + log_err(LD_BUG, "periodic_event_launch without periodic_event_connect"); tor_assert(0); } /* Event already enabled? This is a bug */ @@ -140,9 +147,9 @@ periodic_event_launch(periodic_event_item_t *event) periodic_event_dispatch(event->ev, event); } -/** Release all storage associated with event */ -void -periodic_event_destroy(periodic_event_item_t *event) +/** Disconnect and unregister the periodic event in event */ +static void +periodic_event_disconnect(periodic_event_item_t *event) { if (!event) return; @@ -205,7 +212,7 @@ periodic_event_schedule_and_disable(periodic_event_item_t *event) * take ownership of it. **/ void -periodic_events_add(periodic_event_item_t *item) +periodic_events_register(periodic_event_item_t *item) { if (!the_periodic_events) the_periodic_events = smartlist_new(); @@ -216,9 +223,11 @@ periodic_events_add(periodic_event_item_t *item) smartlist_add(the_periodic_events, item); } -/** Set up all not-previously setup periodic events. */ +/** + * Make all registered periodic events connect to the libevent backend. + */ void -periodic_events_setup_all(void) +periodic_events_connect_all(void) { if (! the_periodic_events) return; @@ -226,11 +235,12 @@ periodic_events_setup_all(void) SMARTLIST_FOREACH_BEGIN(the_periodic_events, periodic_event_item_t *, item) { if (item->ev) continue; - periodic_event_setup(item); + periodic_event_connect(item); } SMARTLIST_FOREACH_END(item); } -/** Reset all the registered periodic events so we'll do all our actions again +/** + * Reset all the registered periodic events so we'll do all our actions again * as if we just started up. * * Useful if our clock just moved back a long time from the future, @@ -307,18 +317,20 @@ periodic_events_rescan_by_roles(int roles, bool net_disabled) } SMARTLIST_FOREACH_END(item); } -/** Invoked at shutdown: free resources used in this module. +/** + * Invoked at shutdown: disconnect and unregister all periodic events. * * Does not free the periodic_event_item_t object themselves, because we do - * not own them. */ + * not own them. + */ void -periodic_events_destroy_all(void) +periodic_events_disconnect_all(void) { if (! the_periodic_events) return; SMARTLIST_FOREACH_BEGIN(the_periodic_events, periodic_event_item_t *, item) { - periodic_event_destroy(item); + periodic_event_disconnect(item); } SMARTLIST_FOREACH_END(item); smartlist_free(the_periodic_events); diff --git a/src/core/mainloop/periodic.h b/src/core/mainloop/periodic.h index 4e4e8b2c96..a9aa461969 100644 --- a/src/core/mainloop/periodic.h +++ b/src/core/mainloop/periodic.h @@ -83,19 +83,19 @@ periodic_event_is_enabled(const periodic_event_item_t *item) } void periodic_event_launch(periodic_event_item_t *event); -void periodic_event_setup(periodic_event_item_t *event); -void periodic_event_destroy(periodic_event_item_t *event); +void periodic_event_connect(periodic_event_item_t *event); +//void periodic_event_disconnect(periodic_event_item_t *event); void periodic_event_reschedule(periodic_event_item_t *event); void periodic_event_enable(periodic_event_item_t *event); void periodic_event_disable(periodic_event_item_t *event); void periodic_event_schedule_and_disable(periodic_event_item_t *event); -void periodic_events_add(periodic_event_item_t *item); -void periodic_events_setup_all(void); +void periodic_events_register(periodic_event_item_t *item); +void periodic_events_connect_all(void); void periodic_events_reset_all(void); periodic_event_item_t *periodic_events_find(const char *name); void periodic_events_rescan_by_roles(int roles, bool net_disabled); -void periodic_events_destroy_all(void); +void periodic_events_disconnect_all(void); int safe_timer_diff(time_t now, time_t next); diff --git a/src/feature/dirauth/dirauth_periodic.c b/src/feature/dirauth/dirauth_periodic.c index 1fa0ca9cfa..cfbb156b9f 100644 --- a/src/feature/dirauth/dirauth_periodic.c +++ b/src/feature/dirauth/dirauth_periodic.c @@ -132,11 +132,11 @@ downrate_stability_callback(time_t now, const or_options_t *options) DECLARE_EVENT(downrate_stability, AUTHORITIES, 0); void -dirauth_add_periodic_events(void) +dirauth_register_periodic_events(void) { - periodic_events_add(&downrate_stability_event); - periodic_events_add(&launch_reachability_tests_event); - periodic_events_add(&save_stability_event); - periodic_events_add(&check_authority_cert_event); - periodic_events_add(&dirvote_event); + periodic_events_register(&downrate_stability_event); + periodic_events_register(&launch_reachability_tests_event); + periodic_events_register(&save_stability_event); + periodic_events_register(&check_authority_cert_event); + periodic_events_register(&dirvote_event); } diff --git a/src/feature/dirauth/dirauth_periodic.h b/src/feature/dirauth/dirauth_periodic.h index 680c0c4bed..de14cbb3c8 100644 --- a/src/feature/dirauth/dirauth_periodic.h +++ b/src/feature/dirauth/dirauth_periodic.h @@ -9,7 +9,7 @@ #ifdef HAVE_MODULE_DIRAUTH -void dirauth_add_periodic_events(void); +void dirauth_register_periodic_events(void); void reschedule_dirvote(const or_options_t *options); #else diff --git a/src/feature/dirauth/dirauth_sys.c b/src/feature/dirauth/dirauth_sys.c index 6845e62c27..bb482f2685 100644 --- a/src/feature/dirauth/dirauth_sys.c +++ b/src/feature/dirauth/dirauth_sys.c @@ -14,7 +14,7 @@ static int subsys_dirauth_initialize(void) { - dirauth_add_periodic_events(); + dirauth_register_periodic_events(); return 0; } diff --git a/src/test/test_periodic_event.c b/src/test/test_periodic_event.c index 19325ed3b7..961a8be698 100644 --- a/src/test/test_periodic_event.c +++ b/src/test/test_periodic_event.c @@ -51,7 +51,7 @@ test_pe_initialize(void *arg) * need to run the main loop and then wait for a second delaying the unit * tests. Instead, we'll test the callback work indepedently elsewhere. */ initialize_periodic_events(); - periodic_events_setup_all(); + periodic_events_connect_all(); set_network_participation(false); rescan_periodic_events(get_options()); @@ -111,7 +111,7 @@ test_pe_launch(void *arg) #endif initialize_periodic_events(); - periodic_events_setup_all(); + periodic_events_connect_all(); /* Now that we've initialized, rescan the list to launch. */ periodic_events_on_new_options(options); @@ -302,7 +302,7 @@ test_pe_hs_service(void *arg) consider_hibernation(time(NULL)); /* Initialize the events so we can enable them */ initialize_periodic_events(); - periodic_events_setup_all(); + periodic_events_connect_all(); /* Hack: We'll set a dumb fn() of each events so they don't get called when * dispatching them. We just want to test the state of the callbacks, not From 965c2064da9ae731873110a1f6da66e2f6fa020e Mon Sep 17 00:00:00 2001 From: Taylor Yu Date: Fri, 5 Apr 2019 17:27:15 -0500 Subject: [PATCH 0914/2557] Correct file name in doxygen comment --- src/feature/control/control_fmt.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/feature/control/control_fmt.c b/src/feature/control/control_fmt.c index b2ab4f10bb..dac3f87795 100644 --- a/src/feature/control/control_fmt.c +++ b/src/feature/control/control_fmt.c @@ -3,7 +3,7 @@ /* See LICENSE for licensing information */ /** - * \file control.c + * \file control_fmt.c * \brief Formatting functions for controller data. */ From 8e7316bae43e666a9fa3b99448561af90cbc21b8 Mon Sep 17 00:00:00 2001 From: Taylor Yu Date: Mon, 8 Apr 2019 11:34:12 -0500 Subject: [PATCH 0915/2557] Split reply formatting out of control_fmt.c Split the core reply formatting code out of control_fmt.c into control_proto.c. The remaining code in control_format.c deals with specific subsystems and will eventually move to join those subsystems. --- scripts/maint/practracker/exceptions.txt | 2 +- src/core/include.am | 2 + src/feature/control/control.c | 2 +- src/feature/control/control_auth.c | 2 +- src/feature/control/control_cmd.c | 2 +- src/feature/control/control_events.c | 1 + src/feature/control/control_fmt.c | 142 +------------------ src/feature/control/control_fmt.h | 9 -- src/feature/control/control_getinfo.c | 1 + src/feature/control/control_proto.c | 165 +++++++++++++++++++++++ src/feature/control/control_proto.h | 24 ++++ src/test/test_util.c | 2 +- 12 files changed, 199 insertions(+), 155 deletions(-) create mode 100644 src/feature/control/control_proto.c create mode 100644 src/feature/control/control_proto.h diff --git a/scripts/maint/practracker/exceptions.txt b/scripts/maint/practracker/exceptions.txt index d90ed1f4bd..ea774e2e83 100644 --- a/scripts/maint/practracker/exceptions.txt +++ b/scripts/maint/practracker/exceptions.txt @@ -152,7 +152,7 @@ problem function-size /src/feature/control/control_cmd.c:handle_control_add_onio problem function-size /src/feature/control/control_cmd.c:add_onion_helper_keyarg() 125 problem function-size /src/feature/control/control_cmd.c:handle_control_command() 104 problem function-size /src/feature/control/control_events.c:control_event_stream_status() 119 -problem include-count /src/feature/control/control_getinfo.c 53 +problem include-count /src/feature/control/control_getinfo.c 54 problem function-size /src/feature/control/control_getinfo.c:getinfo_helper_misc() 109 problem function-size /src/feature/control/control_getinfo.c:getinfo_helper_dir() 304 problem function-size /src/feature/control/control_getinfo.c:getinfo_helper_events() 236 diff --git a/src/core/include.am b/src/core/include.am index 4ec42182a6..2533a4990f 100644 --- a/src/core/include.am +++ b/src/core/include.am @@ -79,6 +79,7 @@ LIBTOR_APP_A_SOURCES = \ src/feature/control/control_events.c \ src/feature/control/control_fmt.c \ src/feature/control/control_getinfo.c \ + src/feature/control/control_proto.c \ src/feature/control/fmt_serverstatus.c \ src/feature/control/getinfo_geoip.c \ src/feature/dirauth/keypin.c \ @@ -307,6 +308,7 @@ noinst_HEADERS += \ src/feature/control/control_events.h \ src/feature/control/control_fmt.h \ src/feature/control/control_getinfo.h \ + src/feature/control/control_proto.h \ src/feature/control/fmt_serverstatus.h \ src/feature/control/getinfo_geoip.h \ src/feature/dirauth/authmode.h \ diff --git a/src/feature/control/control.c b/src/feature/control/control.c index 23ef83ef95..1f85bd3fc0 100644 --- a/src/feature/control/control.c +++ b/src/feature/control/control.c @@ -47,7 +47,7 @@ #include "feature/control/control_auth.h" #include "feature/control/control_cmd.h" #include "feature/control/control_events.h" -#include "feature/control/control_fmt.h" +#include "feature/control/control_proto.h" #include "feature/rend/rendcommon.h" #include "feature/rend/rendservice.h" #include "lib/evloop/procmon.h" diff --git a/src/feature/control/control_auth.c b/src/feature/control/control_auth.c index a86442c21f..bdabdc9896 100644 --- a/src/feature/control/control_auth.c +++ b/src/feature/control/control_auth.c @@ -15,7 +15,7 @@ #include "feature/control/control_auth.h" #include "feature/control/control_cmd_args_st.h" #include "feature/control/control_connection_st.h" -#include "feature/control/control_fmt.h" +#include "feature/control/control_proto.h" #include "lib/crypt_ops/crypto_rand.h" #include "lib/crypt_ops/crypto_util.h" #include "lib/encoding/confline.h" diff --git a/src/feature/control/control_cmd.c b/src/feature/control/control_cmd.c index 9afa734d86..d69177f4cd 100644 --- a/src/feature/control/control_cmd.c +++ b/src/feature/control/control_cmd.c @@ -27,8 +27,8 @@ #include "feature/control/control_auth.h" #include "feature/control/control_cmd.h" #include "feature/control/control_events.h" -#include "feature/control/control_fmt.h" #include "feature/control/control_getinfo.h" +#include "feature/control/control_proto.h" #include "feature/hs/hs_control.h" #include "feature/nodelist/nodelist.h" #include "feature/nodelist/routerinfo.h" diff --git a/src/feature/control/control_events.c b/src/feature/control/control_events.c index 129776f49f..e596a8aee2 100644 --- a/src/feature/control/control_events.c +++ b/src/feature/control/control_events.c @@ -24,6 +24,7 @@ #include "feature/control/control.h" #include "feature/control/control_events.h" #include "feature/control/control_fmt.h" +#include "feature/control/control_proto.h" #include "feature/dircommon/directory.h" #include "feature/nodelist/networkstatus.h" #include "feature/nodelist/nodelist.h" diff --git a/src/feature/control/control_fmt.c b/src/feature/control/control_fmt.c index dac3f87795..e0e77eb2d0 100644 --- a/src/feature/control/control_fmt.c +++ b/src/feature/control/control_fmt.c @@ -14,6 +14,7 @@ #include "core/or/circuitlist.h" #include "core/or/connection_edge.h" #include "feature/control/control_fmt.h" +#include "feature/control/control_proto.h" #include "feature/nodelist/nodelist.h" #include "core/or/cpath_build_state_st.h" @@ -23,39 +24,6 @@ #include "core/or/socks_request_st.h" #include "feature/control/control_connection_st.h" -/** Append a NUL-terminated string s to the end of - * conn-\>outbuf. - */ -void -connection_write_str_to_buf(const char *s, control_connection_t *conn) -{ - size_t len = strlen(s); - connection_buf_add(s, len, TO_CONN(conn)); -} - -/** Acts like sprintf, but writes its formatted string to the end of - * conn-\>outbuf. */ -void -connection_printf_to_buf(control_connection_t *conn, const char *format, ...) -{ - va_list ap; - char *buf = NULL; - int len; - - va_start(ap,format); - len = tor_vasprintf(&buf, format, ap); - va_end(ap); - - if (len < 0) { - log_err(LD_BUG, "Unable to format string for controller."); - tor_assert(0); - } - - connection_buf_add(buf, (size_t)len, TO_CONN(conn)); - - tor_free(buf); -} - /** Given an AP connection conn and a len-character buffer * buf, determine the address:port combination requested on * conn, and write it to buf. Return 0 on success, -1 on @@ -197,114 +165,6 @@ circuit_describe_status_for_controller(origin_circuit_t *circ) return rv; } -/** Given a len-character string in data, made of lines - * terminated by CRLF, allocate a new string in *out, and copy the - * contents of data into *out, adding a period before any period - * that appears at the start of a line, and adding a period-CRLF line at - * the end. Replace all LF characters sequences with CRLF. Return the number - * of bytes in *out. - */ -size_t -write_escaped_data(const char *data, size_t len, char **out) -{ - tor_assert(len < SIZE_MAX - 9); - size_t sz_out = len+8+1; - char *outp; - const char *start = data, *end; - size_t i; - int start_of_line; - for (i=0; i < len; ++i) { - if (data[i] == '\n') { - sz_out += 2; /* Maybe add a CR; maybe add a dot. */ - if (sz_out >= SIZE_T_CEILING) { - log_warn(LD_BUG, "Input to write_escaped_data was too long"); - *out = tor_strdup(".\r\n"); - return 3; - } - } - } - *out = outp = tor_malloc(sz_out); - end = data+len; - start_of_line = 1; - while (data < end) { - if (*data == '\n') { - if (data > start && data[-1] != '\r') - *outp++ = '\r'; - start_of_line = 1; - } else if (*data == '.') { - if (start_of_line) { - start_of_line = 0; - *outp++ = '.'; - } - } else { - start_of_line = 0; - } - *outp++ = *data++; - } - if (outp < *out+2 || fast_memcmp(outp-2, "\r\n", 2)) { - *outp++ = '\r'; - *outp++ = '\n'; - } - *outp++ = '.'; - *outp++ = '\r'; - *outp++ = '\n'; - *outp = '\0'; /* NUL-terminate just in case. */ - tor_assert(outp >= *out); - tor_assert((size_t)(outp - *out) <= sz_out); - return outp - *out; -} - -/** Given a len-character string in data, made of lines - * terminated by CRLF, allocate a new string in *out, and copy - * the contents of data into *out, removing any period - * that appears at the start of a line, and replacing all CRLF sequences - * with LF. Return the number of - * bytes in *out. */ -size_t -read_escaped_data(const char *data, size_t len, char **out) -{ - char *outp; - const char *next; - const char *end; - - *out = outp = tor_malloc(len+1); - - end = data+len; - - while (data < end) { - /* we're at the start of a line. */ - if (*data == '.') - ++data; - next = memchr(data, '\n', end-data); - if (next) { - size_t n_to_copy = next-data; - /* Don't copy a CR that precedes this LF. */ - if (n_to_copy && *(next-1) == '\r') - --n_to_copy; - memcpy(outp, data, n_to_copy); - outp += n_to_copy; - data = next+1; /* This will point at the start of the next line, - * or the end of the string, or a period. */ - } else { - memcpy(outp, data, end-data); - outp += (end-data); - *outp = '\0'; - return outp - *out; - } - *outp++ = '\n'; - } - - *outp = '\0'; - return outp - *out; -} - -/** Send a "DONE" message down the control connection conn. */ -void -send_control_done(control_connection_t *conn) -{ - connection_write_str_to_buf("250 OK\r\n", conn); -} - /** Return a longname the node whose identity is id_digest. If * node_get_by_id() returns NULL, base 16 encoding of id_digest is * returned instead. diff --git a/src/feature/control/control_fmt.h b/src/feature/control/control_fmt.h index 8bbbaa95d0..6446e37079 100644 --- a/src/feature/control/control_fmt.h +++ b/src/feature/control/control_fmt.h @@ -12,21 +12,12 @@ #ifndef TOR_CONTROL_FMT_H #define TOR_CONTROL_FMT_H -void connection_write_str_to_buf(const char *s, control_connection_t *conn); -void connection_printf_to_buf(control_connection_t *conn, - const char *format, ...) - CHECK_PRINTF(2,3); - int write_stream_target_to_buf(entry_connection_t *conn, char *buf, size_t len); void orconn_target_get_name(char *buf, size_t len, or_connection_t *conn); char *circuit_describe_status_for_controller(origin_circuit_t *circ); -size_t write_escaped_data(const char *data, size_t len, char **out); -size_t read_escaped_data(const char *data, size_t len, char **out); -void send_control_done(control_connection_t *conn); - MOCK_DECL(const char *, node_describe_longname_by_id,(const char *id_digest)); #endif /* !defined(TOR_CONTROL_FMT_H) */ diff --git a/src/feature/control/control_getinfo.c b/src/feature/control/control_getinfo.c index 5c6a0d4aa2..6f5e69dc0b 100644 --- a/src/feature/control/control_getinfo.c +++ b/src/feature/control/control_getinfo.c @@ -28,6 +28,7 @@ #include "feature/control/control_events.h" #include "feature/control/control_fmt.h" #include "feature/control/control_getinfo.h" +#include "feature/control/control_proto.h" #include "feature/control/fmt_serverstatus.h" #include "feature/control/getinfo_geoip.h" #include "feature/dircache/dirserv.h" diff --git a/src/feature/control/control_proto.c b/src/feature/control/control_proto.c new file mode 100644 index 0000000000..daf5b1189e --- /dev/null +++ b/src/feature/control/control_proto.c @@ -0,0 +1,165 @@ +/* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file control_proto.c + * \brief Formatting functions for controller data. + */ + +#include "core/or/or.h" + +#include "core/mainloop/connection.h" +#include "core/or/circuitbuild.h" +#include "core/or/circuitlist.h" +#include "core/or/connection_edge.h" +#include "feature/control/control_proto.h" +#include "feature/nodelist/nodelist.h" + +#include "core/or/cpath_build_state_st.h" +#include "core/or/entry_connection_st.h" +#include "core/or/or_connection_st.h" +#include "core/or/origin_circuit_st.h" +#include "core/or/socks_request_st.h" +#include "feature/control/control_connection_st.h" + +/** Append a NUL-terminated string s to the end of + * conn-\>outbuf. + */ +void +connection_write_str_to_buf(const char *s, control_connection_t *conn) +{ + size_t len = strlen(s); + connection_buf_add(s, len, TO_CONN(conn)); +} + +/** Acts like sprintf, but writes its formatted string to the end of + * conn-\>outbuf. */ +void +connection_printf_to_buf(control_connection_t *conn, const char *format, ...) +{ + va_list ap; + char *buf = NULL; + int len; + + va_start(ap,format); + len = tor_vasprintf(&buf, format, ap); + va_end(ap); + + if (len < 0) { + log_err(LD_BUG, "Unable to format string for controller."); + tor_assert(0); + } + + connection_buf_add(buf, (size_t)len, TO_CONN(conn)); + + tor_free(buf); +} + +/** Given a len-character string in data, made of lines + * terminated by CRLF, allocate a new string in *out, and copy the + * contents of data into *out, adding a period before any period + * that appears at the start of a line, and adding a period-CRLF line at + * the end. Replace all LF characters sequences with CRLF. Return the number + * of bytes in *out. + */ +size_t +write_escaped_data(const char *data, size_t len, char **out) +{ + tor_assert(len < SIZE_MAX - 9); + size_t sz_out = len+8+1; + char *outp; + const char *start = data, *end; + size_t i; + int start_of_line; + for (i=0; i < len; ++i) { + if (data[i] == '\n') { + sz_out += 2; /* Maybe add a CR; maybe add a dot. */ + if (sz_out >= SIZE_T_CEILING) { + log_warn(LD_BUG, "Input to write_escaped_data was too long"); + *out = tor_strdup(".\r\n"); + return 3; + } + } + } + *out = outp = tor_malloc(sz_out); + end = data+len; + start_of_line = 1; + while (data < end) { + if (*data == '\n') { + if (data > start && data[-1] != '\r') + *outp++ = '\r'; + start_of_line = 1; + } else if (*data == '.') { + if (start_of_line) { + start_of_line = 0; + *outp++ = '.'; + } + } else { + start_of_line = 0; + } + *outp++ = *data++; + } + if (outp < *out+2 || fast_memcmp(outp-2, "\r\n", 2)) { + *outp++ = '\r'; + *outp++ = '\n'; + } + *outp++ = '.'; + *outp++ = '\r'; + *outp++ = '\n'; + *outp = '\0'; /* NUL-terminate just in case. */ + tor_assert(outp >= *out); + tor_assert((size_t)(outp - *out) <= sz_out); + return outp - *out; +} + +/** Given a len-character string in data, made of lines + * terminated by CRLF, allocate a new string in *out, and copy + * the contents of data into *out, removing any period + * that appears at the start of a line, and replacing all CRLF sequences + * with LF. Return the number of + * bytes in *out. */ +size_t +read_escaped_data(const char *data, size_t len, char **out) +{ + char *outp; + const char *next; + const char *end; + + *out = outp = tor_malloc(len+1); + + end = data+len; + + while (data < end) { + /* we're at the start of a line. */ + if (*data == '.') + ++data; + next = memchr(data, '\n', end-data); + if (next) { + size_t n_to_copy = next-data; + /* Don't copy a CR that precedes this LF. */ + if (n_to_copy && *(next-1) == '\r') + --n_to_copy; + memcpy(outp, data, n_to_copy); + outp += n_to_copy; + data = next+1; /* This will point at the start of the next line, + * or the end of the string, or a period. */ + } else { + memcpy(outp, data, end-data); + outp += (end-data); + *outp = '\0'; + return outp - *out; + } + *outp++ = '\n'; + } + + *outp = '\0'; + return outp - *out; +} + +/** Send a "DONE" message down the control connection conn. */ +void +send_control_done(control_connection_t *conn) +{ + connection_write_str_to_buf("250 OK\r\n", conn); +} diff --git a/src/feature/control/control_proto.h b/src/feature/control/control_proto.h new file mode 100644 index 0000000000..5720b22601 --- /dev/null +++ b/src/feature/control/control_proto.h @@ -0,0 +1,24 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file control_proto.h + * \brief Header file for control_proto.c. + **/ + +#ifndef TOR_CONTROL_PROTO_H +#define TOR_CONTROL_PROTO_H + +void connection_write_str_to_buf(const char *s, control_connection_t *conn); +void connection_printf_to_buf(control_connection_t *conn, + const char *format, ...) + CHECK_PRINTF(2,3); + +size_t write_escaped_data(const char *data, size_t len, char **out); +size_t read_escaped_data(const char *data, size_t len, char **out); +void send_control_done(control_connection_t *conn); + +#endif /* !defined(TOR_CONTROL_PROTO_H) */ diff --git a/src/test/test_util.c b/src/test/test_util.c index 61e41e9a9b..88eda847a2 100644 --- a/src/test/test_util.c +++ b/src/test/test_util.c @@ -15,7 +15,7 @@ #include "lib/buf/buffers.h" #include "app/config/config.h" #include "feature/control/control.h" -#include "feature/control/control_fmt.h" +#include "feature/control/control_proto.h" #include "feature/client/transports.h" #include "lib/crypt_ops/crypto_format.h" #include "lib/crypt_ops/crypto_rand.h" From 482437754a7a30b651f096d4432a42aa61bd3b17 Mon Sep 17 00:00:00 2001 From: Taylor Yu Date: Mon, 8 Apr 2019 13:39:04 -0500 Subject: [PATCH 0916/2557] Add clarifying comments to control_proto.c Refer to control-spec.txt grammar productions in comments in control_proto.c for clarity. --- src/feature/control/control_proto.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/feature/control/control_proto.c b/src/feature/control/control_proto.c index daf5b1189e..4756a02632 100644 --- a/src/feature/control/control_proto.c +++ b/src/feature/control/control_proto.c @@ -62,6 +62,8 @@ connection_printf_to_buf(control_connection_t *conn, const char *format, ...) * that appears at the start of a line, and adding a period-CRLF line at * the end. Replace all LF characters sequences with CRLF. Return the number * of bytes in *out. + * + * This corresponds to CmdData in control-spec.txt. */ size_t write_escaped_data(const char *data, size_t len, char **out) @@ -118,7 +120,10 @@ write_escaped_data(const char *data, size_t len, char **out) * the contents of data into *out, removing any period * that appears at the start of a line, and replacing all CRLF sequences * with LF. Return the number of - * bytes in *out. */ + * bytes in *out. + * + * This corresponds to CmdData in control-spec.txt. + */ size_t read_escaped_data(const char *data, size_t len, char **out) { From 61976a4b1ca5dc9c7f9494ee3bf17a96c1182cf0 Mon Sep 17 00:00:00 2001 From: Taylor Yu Date: Tue, 9 Apr 2019 12:22:31 -0500 Subject: [PATCH 0917/2557] Factor out control reply output Create a set of abstractions for controller commands and events to output replies to the control channel. The control protocol has a relatively consistent SMTP-like structure, so it's helpful when code that implements control commands and events doesn't explicitly format everything on its own. --- src/feature/control/control_proto.c | 106 ++++++++++++++++++++++++++++ src/feature/control/control_proto.h | 24 +++++++ 2 files changed, 130 insertions(+) diff --git a/src/feature/control/control_proto.c b/src/feature/control/control_proto.c index 4756a02632..7bd237de94 100644 --- a/src/feature/control/control_proto.c +++ b/src/feature/control/control_proto.c @@ -168,3 +168,109 @@ send_control_done(control_connection_t *conn) { connection_write_str_to_buf("250 OK\r\n", conn); } + +/** Write a reply to the control channel. + * + * @param conn control connection + * @param code numeric result code + * @param c separator character, usually ' ', '-', or '+' + * @param s string + */ +void +control_write_reply(control_connection_t *conn, int code, int c, const char *s) +{ + connection_printf_to_buf(conn, "%03d%c%s\r\n", code, c, s); +} + +/** Write a formatted reply to the control channel. + * + * @param conn control connection + * @param code numeric result code + * @param c separator character, usually ' ', '-', or '+' + * @param fmt format string + * @param ap va_list from caller + */ +void +control_vprintf_reply(control_connection_t *conn, int code, int c, + const char *fmt, va_list ap) +{ + char *buf = NULL; + int len; + + len = tor_vasprintf(&buf, fmt, ap); + if (len < 0) { + log_err(LD_BUG, "Unable to format string for controller."); + tor_assert(0); + } + control_write_reply(conn, code, c, buf); + tor_free(buf); +} + +/** Write an EndReplyLine */ +void +control_write_endreply(control_connection_t *conn, int code, const char *s) +{ + control_write_reply(conn, code, ' ', s); +} + +/** Write a formatted EndReplyLine */ +void +control_printf_endreply(control_connection_t *conn, int code, + const char *fmt, ...) +{ + va_list ap; + + va_start(ap, fmt); + control_vprintf_reply(conn, code, ' ', fmt, ap); + va_end(ap); +} + +/** Write a MidReplyLine */ +void +control_write_midreply(control_connection_t *conn, int code, const char *s) +{ + control_write_reply(conn, code, '-', s); +} + +/** Write a formatted MidReplyLine */ +void +control_printf_midreply(control_connection_t *conn, int code, const char *fmt, + ...) +{ + va_list ap; + + va_start(ap, fmt); + control_vprintf_reply(conn, code, '-', fmt, ap); + va_end(ap); +} + +/** Write a DataReplyLine */ +void +control_write_datareply(control_connection_t *conn, int code, const char *s) +{ + control_write_reply(conn, code, '+', s); +} + +/** Write a formatted DataReplyLine */ +void +control_printf_datareply(control_connection_t *conn, int code, const char *fmt, + ...) +{ + va_list ap; + + va_start(ap, fmt); + control_vprintf_reply(conn, code, '+', fmt, ap); + va_end(ap); +} + +/** Write a CmdData */ +void +control_write_data(control_connection_t *conn, const char *data) +{ + char *esc = NULL; + size_t esc_len; + + esc_len = write_escaped_data(data, strlen(data), &esc); + connection_buf_add(esc, esc_len, TO_CONN(conn)); + tor_free(esc); +} diff --git a/src/feature/control/control_proto.h b/src/feature/control/control_proto.h index 5720b22601..101b808d88 100644 --- a/src/feature/control/control_proto.h +++ b/src/feature/control/control_proto.h @@ -21,4 +21,28 @@ size_t write_escaped_data(const char *data, size_t len, char **out); size_t read_escaped_data(const char *data, size_t len, char **out); void send_control_done(control_connection_t *conn); +void control_write_reply(control_connection_t *conn, int code, int c, + const char *s); +void control_vprintf_reply(control_connection_t *conn, int code, int c, + const char *fmt, va_list ap) + CHECK_PRINTF(4, 0); +void control_write_endreply(control_connection_t *conn, int code, + const char *s); +void control_printf_endreply(control_connection_t *conn, int code, + const char *fmt, ...) + CHECK_PRINTF(3, 4); +void control_write_midreply(control_connection_t *conn, int code, + const char *s); +void control_printf_midreply(control_connection_t *conn, int code, + const char *fmt, + ...) + CHECK_PRINTF(3, 4); +void control_write_datareply(control_connection_t *conn, int code, + const char *s); +void control_printf_datareply(control_connection_t *conn, int code, + const char *fmt, + ...) + CHECK_PRINTF(3, 4); +void control_write_data(control_connection_t *conn, const char *data); + #endif /* !defined(TOR_CONTROL_PROTO_H) */ From 769eb07a7a3bd93091ca1677564d3de62e3c5c2c Mon Sep 17 00:00:00 2001 From: Taylor Yu Date: Wed, 10 Apr 2019 15:11:36 -0500 Subject: [PATCH 0918/2557] Manually fix some control replies Manually fix up some reply-generating code that the Coccinelle scripts won't match. Some more complicated ones remain -- these are mostly ones that accumulate data to send, and then call connection_buf_add() or connection_write_str_to_buf() directly. --- src/feature/control/control_cmd.c | 53 +++++++++++++-------------- src/feature/control/control_getinfo.c | 12 ++---- 2 files changed, 28 insertions(+), 37 deletions(-) diff --git a/src/feature/control/control_cmd.c b/src/feature/control/control_cmd.c index d69177f4cd..5aa828ca53 100644 --- a/src/feature/control/control_cmd.c +++ b/src/feature/control/control_cmd.c @@ -355,7 +355,6 @@ handle_control_loadconf(control_connection_t *conn, { setopt_err_t retval; char *errstring = NULL; - const char *msg = NULL; retval = options_init_from_string(NULL, args->cmddata, CMD_RUN_TOR, NULL, &errstring); @@ -365,31 +364,29 @@ handle_control_loadconf(control_connection_t *conn, "Controller gave us config file that didn't validate: %s", errstring); +#define SEND_ERRMSG(code, msg) \ + control_printf_endreply(conn, code, msg "%s%s", \ + errstring ? ": " : "", \ + errstring ? errstring : "") switch (retval) { case SETOPT_ERR_PARSE: - msg = "552 Invalid config file"; + SEND_ERRMSG(552, "Invalid config file"); break; case SETOPT_ERR_TRANSITION: - msg = "553 Transition not allowed"; + SEND_ERRMSG(553, "Transition not allowed"); break; case SETOPT_ERR_SETTING: - msg = "553 Unable to set option"; + SEND_ERRMSG(553, "Unable to set option"); break; case SETOPT_ERR_MISC: default: - msg = "550 Unable to load config"; + SEND_ERRMSG(550, "Unable to load config"); break; case SETOPT_OK: + send_control_done(conn); break; } - if (msg) { - if (errstring) - connection_printf_to_buf(conn, "%s: %s\r\n", msg, errstring); - else - connection_printf_to_buf(conn, "%s\r\n", msg); - } else { - send_control_done(conn); - } +#undef SEND_ERRMSG tor_free(errstring); return 0; } @@ -600,30 +597,32 @@ control_setconf_helper(control_connection_t *conn, opt_err = options_trial_assign(lines, flags, &errstring); { - const char *msg; +#define SEND_ERRMSG(code, msg) \ + control_printf_endreply(conn, code, msg ": %s", errstring); + switch (opt_err) { case SETOPT_ERR_MISC: - msg = "552 Unrecognized option"; + SEND_ERRMSG(552, "Unrecognized option"); break; case SETOPT_ERR_PARSE: - msg = "513 Unacceptable option value"; + SEND_ERRMSG(513, "Unacceptable option value"); break; case SETOPT_ERR_TRANSITION: - msg = "553 Transition not allowed"; + SEND_ERRMSG(553, "Transition not allowed"); break; case SETOPT_ERR_SETTING: default: - msg = "553 Unable to set option"; + SEND_ERRMSG(553, "Unable to set option"); break; case SETOPT_OK: config_free_lines(lines); send_control_done(conn); return 0; } +#undef SEND_ERRMSG log_warn(LD_CONTROL, "Controller gave us config lines that didn't validate: %s", errstring); - connection_printf_to_buf(conn, "%s: %s\r\n", msg, errstring); config_free_lines(lines); tor_free(errstring); return 0; @@ -1309,15 +1308,13 @@ handle_control_protocolinfo(control_connection_t *conn, smartlist_free(mlist); } - connection_printf_to_buf(conn, - "250-PROTOCOLINFO 1\r\n" - "250-AUTH METHODS=%s%s%s\r\n" - "250-VERSION Tor=%s\r\n" - "250 OK\r\n", - methods, - cookies?" COOKIEFILE=":"", - cookies?esc_cfile:"", - escaped(VERSION)); + control_write_midreply(conn, 250, "PROTOCOLINFO 1"); + control_printf_midreply(conn, 250, "AUTH METHODS=%s%s%s", methods, + cookies?" COOKIEFILE=":"", + cookies?esc_cfile:""); + control_printf_midreply(conn, 250, "VERSION Tor=%s", escaped(VERSION)); + send_control_done(conn); + tor_free(methods); tor_free(cfile); tor_free(abs_cfile); diff --git a/src/feature/control/control_getinfo.c b/src/feature/control/control_getinfo.c index 6f5e69dc0b..c10229bfbb 100644 --- a/src/feature/control/control_getinfo.c +++ b/src/feature/control/control_getinfo.c @@ -1639,16 +1639,10 @@ handle_control_getinfo(control_connection_t *conn, char *k = smartlist_get(answers, i); char *v = smartlist_get(answers, i+1); if (!strchr(v, '\n') && !strchr(v, '\r')) { - connection_printf_to_buf(conn, "250-%s=", k); - connection_write_str_to_buf(v, conn); - connection_write_str_to_buf("\r\n", conn); + control_printf_midreply(conn, 250, "%s=%s", k, v); } else { - char *esc = NULL; - size_t esc_len; - esc_len = write_escaped_data(v, strlen(v), &esc); - connection_printf_to_buf(conn, "250+%s=\r\n", k); - connection_buf_add(esc, esc_len, TO_CONN(conn)); - tor_free(esc); + control_printf_datareply(conn, 250, "%s=", k); + control_write_data(conn, v); } } connection_write_str_to_buf("250 OK\r\n", conn); From 58ec88e8060508a253bbe8ba5c19b020a2629ea0 Mon Sep 17 00:00:00 2001 From: Taylor Yu Date: Wed, 10 Apr 2019 12:27:50 -0500 Subject: [PATCH 0919/2557] Coccinelle scripts for control.c refactor --- scripts/coccinelle/ctrl-reply-cleanup.cocci | 43 ++++++++++ scripts/coccinelle/ctrl-reply.cocci | 87 +++++++++++++++++++++ scripts/coccinelle/tor-coccinelle.h | 3 + 3 files changed, 133 insertions(+) create mode 100644 scripts/coccinelle/ctrl-reply-cleanup.cocci create mode 100644 scripts/coccinelle/ctrl-reply.cocci create mode 100644 scripts/coccinelle/tor-coccinelle.h diff --git a/scripts/coccinelle/ctrl-reply-cleanup.cocci b/scripts/coccinelle/ctrl-reply-cleanup.cocci new file mode 100644 index 0000000000..f085cd4684 --- /dev/null +++ b/scripts/coccinelle/ctrl-reply-cleanup.cocci @@ -0,0 +1,43 @@ +// Script to clean up after ctrl-reply.cocci -- run as a separate step +// because cleanup_write2 (even when disabled) somehow prevents the +// match rule in ctrl-reply.cocci from matching. + +// If it doesn't have to be a printf, turn it into a write + +@ cleanup_write @ +expression E; +constant code, s; +@@ +-control_printf_endreply(E, code, s) ++control_write_endreply(E, code, s) + +// Use send_control_done() instead of explicitly writing it out +@ cleanup_send_done @ +type T; +identifier f != send_control_done; +expression E; +@@ + T f(...) { +<... +-control_write_endreply(E, 250, "OK") ++send_control_done(E) + ...> + } + +// Clean up more printfs that could be writes +// +// For some reason, including this rule, even disabled, causes the +// match rule in ctrl-reply.cocci to fail to match some code that has +// %s in its format strings + +@ cleanup_write2 @ +expression E1, E2; +constant code; +@@ +( +-control_printf_endreply(E1, code, "%s", E2) ++control_write_endreply(E1, code, E2) +| +-control_printf_midreply(E1, code, "%s", E2) ++control_write_midreply(E1, code, E2) +) diff --git a/scripts/coccinelle/ctrl-reply.cocci b/scripts/coccinelle/ctrl-reply.cocci new file mode 100644 index 0000000000..d6e9aeedd7 --- /dev/null +++ b/scripts/coccinelle/ctrl-reply.cocci @@ -0,0 +1,87 @@ +// Script to edit control_*.c for refactored control reply output functions + +@ initialize:python @ +@@ +import re +from coccilib.report import * + +# reply strings "NNN-foo", "NNN+foo", "NNN foo", etc. +r = re.compile(r'^"(\d+)([ +-])(.*)\\r\\n"$') + +# Generate name of function to call based on which separator character +# comes between the numeric code and the text +def idname(sep, base): + if sep == '+': + return base + "datareply" + elif sep == '-': + return base + "midreply" + else: + return base + "endreply" + +# Generate the actual replacements used by the rules +def gen(s, base, p): + pos = p[0] + print_report(pos, "%s %s" % (base, s)) + m = r.match(s) + if m is None: + # String not correct format, so fail match + cocci.include_match(False) + print_report(pos, "BAD STRING %s" % s) + return + + code, sep, s1 = m.groups() + + if r'\r\n' in s1: + # Extra CRLF in string, so fail match + cocci.include_match(False) + print_report(pos, "extra CRLF in string %s" % s) + return + + coccinelle.code = code + # Need a string that is a single C token, because Coccinelle only allows + # "identifiers" to be output from Python scripts? + coccinelle.body = '"%s"' % s1 + coccinelle.id = idname(sep, base) + return + +@ match @ +identifier f; +position p; +expression E; +constant s; +@@ +( + connection_printf_to_buf@f@p(E, s, ...) +| + connection_write_str_to_buf@f@p(s, E) +) + +@ script:python sc1 @ +s << match.s; +p << match.p; +f << match.f; +id; +body; +code; +@@ +if f == 'connection_printf_to_buf': + gen(s, 'control_printf_', p) +elif f == 'connection_write_str_to_buf': + gen(s, 'control_write_', p) +else: + raise(ValueError("%s: %s" % (f, s))) + +@ replace @ +constant match.s; +expression match.E; +identifier match.f; +identifier sc1.body, sc1.id, sc1.code; +@@ +( +-connection_write_str_to_buf@f(s, E) ++id(E, code, body) +| +-connection_printf_to_buf@f(E, s ++id(E, code, body + , ...) +) diff --git a/scripts/coccinelle/tor-coccinelle.h b/scripts/coccinelle/tor-coccinelle.h new file mode 100644 index 0000000000..8f625dcee4 --- /dev/null +++ b/scripts/coccinelle/tor-coccinelle.h @@ -0,0 +1,3 @@ +#define MOCK_IMPL(a, b, c) a b c +#define CHECK_PRINTF(a, b) +#define STATIC static From 983452e2215d27033ed91012a53c1b9ad3dd4a34 Mon Sep 17 00:00:00 2001 From: Taylor Yu Date: Wed, 10 Apr 2019 17:04:09 -0500 Subject: [PATCH 0920/2557] Run Coccinelle for control.c refactor --- src/feature/control/control.c | 8 +- src/feature/control/control_auth.c | 38 +++--- src/feature/control/control_cmd.c | 166 +++++++++++++------------- src/feature/control/control_getinfo.c | 13 +- src/feature/control/control_proto.c | 2 +- 5 files changed, 106 insertions(+), 121 deletions(-) diff --git a/src/feature/control/control.c b/src/feature/control/control.c index 1f85bd3fc0..436bf423cf 100644 --- a/src/feature/control/control.c +++ b/src/feature/control/control.c @@ -401,7 +401,7 @@ connection_control_process_inbuf(control_connection_t *conn) return 0; else if (r == -1) { if (data_len + conn->incoming_cmd_cur_len > MAX_COMMAND_LINE_LENGTH) { - connection_write_str_to_buf("500 Line too long.\r\n", conn); + control_write_endreply(conn, 500, "Line too long."); connection_stop_reading(TO_CONN(conn)); connection_mark_and_flush(TO_CONN(conn)); } @@ -455,20 +455,20 @@ connection_control_process_inbuf(control_connection_t *conn) /* Otherwise, Quit is always valid. */ if (!strcasecmp(conn->current_cmd, "QUIT")) { - connection_write_str_to_buf("250 closing connection\r\n", conn); + control_write_endreply(conn, 250, "closing connection"); connection_mark_and_flush(TO_CONN(conn)); return 0; } if (conn->base_.state == CONTROL_CONN_STATE_NEEDAUTH && !is_valid_initial_command(conn, conn->current_cmd)) { - connection_write_str_to_buf("514 Authentication required.\r\n", conn); + control_write_endreply(conn, 514, "Authentication required."); connection_mark_for_close(TO_CONN(conn)); return 0; } if (data_len >= UINT32_MAX) { - connection_write_str_to_buf("500 A 4GB command? Nice try.\r\n", conn); + control_write_endreply(conn, 500, "A 4GB command? Nice try."); connection_mark_for_close(TO_CONN(conn)); return 0; } diff --git a/src/feature/control/control_auth.c b/src/feature/control/control_auth.c index bdabdc9896..51c8da1b99 100644 --- a/src/feature/control/control_auth.c +++ b/src/feature/control/control_auth.c @@ -141,27 +141,26 @@ handle_control_authchallenge(control_connection_t *conn, char server_nonce_encoded[(2*SAFECOOKIE_SERVER_NONCE_LEN) + 1]; if (strcasecmp(smartlist_get(args->args, 0), "SAFECOOKIE")) { - connection_write_str_to_buf("513 AUTHCHALLENGE only supports SAFECOOKIE " - "authentication\r\n", conn); + control_write_endreply(conn, 513, + "AUTHCHALLENGE only supports SAFECOOKIE " "authentication"); goto fail; } if (!authentication_cookie_is_set) { - connection_write_str_to_buf("515 Cookie authentication is disabled\r\n", - conn); + control_write_endreply(conn, 515, "Cookie authentication is disabled"); goto fail; } if (args->kwargs == NULL || args->kwargs->next != NULL) { /* connection_write_str_to_buf("512 AUTHCHALLENGE requires exactly " "2 arguments.\r\n", conn); */ - connection_printf_to_buf(conn, - "512 AUTHCHALLENGE dislikes argument list %s\r\n", + control_printf_endreply(conn, 512, + "AUTHCHALLENGE dislikes argument list %s", escaped(args->raw_body)); goto fail; } if (strcmp(args->kwargs->key, "")) { - connection_write_str_to_buf("512 AUTHCHALLENGE does not accept keyword " - "arguments.\r\n", conn); + control_write_endreply(conn, 512, + "AUTHCHALLENGE does not accept keyword " "arguments."); goto fail; } @@ -177,8 +176,7 @@ handle_control_authchallenge(control_connection_t *conn, client_nonce = tor_malloc(client_nonce_len); if (base16_decode(client_nonce, client_nonce_len, hex_nonce, strlen(hex_nonce)) != (int)client_nonce_len) { - connection_write_str_to_buf("513 Invalid base16 client nonce\r\n", - conn); + control_write_endreply(conn, 513, "Invalid base16 client nonce"); tor_free(client_nonce); goto fail; } @@ -223,9 +221,8 @@ handle_control_authchallenge(control_connection_t *conn, base16_encode(server_nonce_encoded, sizeof(server_nonce_encoded), server_nonce, sizeof(server_nonce)); - connection_printf_to_buf(conn, - "250 AUTHCHALLENGE SERVERHASH=%s " - "SERVERNONCE=%s\r\n", + control_printf_endreply(conn, 250, + "AUTHCHALLENGE SERVERHASH=%s " "SERVERNONCE=%s", server_hash_encoded, server_nonce_encoded); @@ -263,13 +260,12 @@ handle_control_authenticate(control_connection_t *conn, password = tor_strdup(""); password_len = 0; } else if (args->kwargs->next) { - connection_write_str_to_buf( - "512 Too many arguments to AUTHENTICATE.\r\n", conn); + control_write_endreply(conn, 512, "Too many arguments to AUTHENTICATE."); connection_mark_for_close(TO_CONN(conn)); return 0; } else if (strcmp(args->kwargs->key, "")) { - connection_write_str_to_buf( - "512 AUTHENTICATE does not accept keyword arguments.\r\n", conn); + control_write_endreply(conn, 512, + "AUTHENTICATE does not accept keyword arguments."); connection_mark_for_close(TO_CONN(conn)); return 0; } else if (strchr(args->raw_body, '\"')) { @@ -282,10 +278,8 @@ handle_control_authenticate(control_connection_t *conn, password = tor_malloc(password_len+1); if (base16_decode(password, password_len+1, hex_passwd, strlen(hex_passwd)) != (int) password_len) { - connection_write_str_to_buf( - "551 Invalid hexadecimal encoding. Maybe you tried a plain text " - "password? If so, the standard requires that you put it in " - "double quotes.\r\n", conn); + control_write_endreply(conn, 551, + "Invalid hexadecimal encoding. Maybe you tried a plain text " "password? If so, the standard requires that you put it in " "double quotes."); connection_mark_for_close(TO_CONN(conn)); tor_free(password); return 0; @@ -418,7 +412,7 @@ handle_control_authenticate(control_connection_t *conn, err: tor_free(password); - connection_printf_to_buf(conn, "515 Authentication failed: %s\r\n", errstr); + control_printf_endreply(conn, 515, "Authentication failed: %s", errstr); connection_mark_for_close(TO_CONN(conn)); if (sl) { /* clean up */ SMARTLIST_FOREACH(sl, char *, str, tor_free(str)); diff --git a/src/feature/control/control_cmd.c b/src/feature/control/control_cmd.c index 5aa828ca53..ae4d6584cc 100644 --- a/src/feature/control/control_cmd.c +++ b/src/feature/control/control_cmd.c @@ -319,11 +319,11 @@ handle_control_getconf(control_connection_t *conn, if ((len = smartlist_len(unrecognized))) { for (i=0; i < len-1; ++i) - connection_printf_to_buf(conn, - "552-Unrecognized configuration key \"%s\"\r\n", + control_printf_midreply(conn, 552, + "Unrecognized configuration key \"%s\"", (char*)smartlist_get(unrecognized, i)); - connection_printf_to_buf(conn, - "552 Unrecognized configuration key \"%s\"\r\n", + control_printf_endreply(conn, 552, + "Unrecognized configuration key \"%s\"", (char*)smartlist_get(unrecognized, len-1)); } else if ((len = smartlist_len(answers))) { char *tmp = smartlist_get(answers, len-1); @@ -332,7 +332,7 @@ handle_control_getconf(control_connection_t *conn, msg = smartlist_join_strings(answers, "", 0, &msg_len); connection_buf_add(msg, msg_len, TO_CONN(conn)); } else { - connection_write_str_to_buf("250 OK\r\n", conn); + send_control_done(conn); } SMARTLIST_FOREACH(answers, char *, cp, tor_free(cp)); @@ -424,7 +424,7 @@ handle_control_setevents(control_connection_t *conn, } if (event_code == -1) { - connection_printf_to_buf(conn, "552 Unrecognized event \"%s\"\r\n", + control_printf_endreply(conn, 552, "Unrecognized event \"%s\"", ev); return 0; } @@ -455,8 +455,8 @@ handle_control_saveconf(control_connection_t *conn, bool force = config_lines_contain_flag(args->kwargs, "FORCE"); const or_options_t *options = get_options(); if ((!force && options->IncludeUsed) || options_save_current() < 0) { - connection_write_str_to_buf( - "551 Unable to write configuration to disk.\r\n", conn); + control_write_endreply(conn, 551, + "Unable to write configuration to disk."); } else { send_control_done(conn); } @@ -489,7 +489,7 @@ handle_control_signal(control_connection_t *conn, } if (sig < 0) - connection_printf_to_buf(conn, "552 Unrecognized signal code \"%s\"\r\n", + control_printf_endreply(conn, 552, "Unrecognized signal code \"%s\"", s); if (sig < 0) return 0; @@ -776,7 +776,7 @@ handle_control_extendcircuit(control_connection_t *conn, if (purpose_line) { intended_purpose = circuit_purpose_from_string(purpose_line->value); if (intended_purpose == CIRCUIT_PURPOSE_UNKNOWN) { - connection_printf_to_buf(conn, "552 Unknown purpose \"%s\"\r\n", + control_printf_endreply(conn, 552, "Unknown purpose \"%s\"", purpose_line->value); goto done; } @@ -787,22 +787,22 @@ handle_control_extendcircuit(control_connection_t *conn, // "EXTENDCIRCUIT 0" with no path. circ = circuit_launch(intended_purpose, CIRCLAUNCH_NEED_CAPACITY); if (!circ) { - connection_write_str_to_buf("551 Couldn't start circuit\r\n", conn); + control_write_endreply(conn, 551, "Couldn't start circuit"); } else { - connection_printf_to_buf(conn, "250 EXTENDED %lu\r\n", - (unsigned long)circ->global_identifier); + control_printf_endreply(conn, 250, "EXTENDED %lu", + (unsigned long)circ->global_identifier); } goto done; } } if (!zero_circ && !(circ = get_circ(circ_id))) { - connection_printf_to_buf(conn, "552 Unknown circuit \"%s\"\r\n", circ_id); + control_printf_endreply(conn, 552, "Unknown circuit \"%s\"", circ_id); goto done; } if (!path_str) { - connection_printf_to_buf(conn, "512 syntax error: path required.\r\n"); + control_write_endreply(conn, 512, "syntax error: path required."); goto done; } @@ -813,11 +813,11 @@ handle_control_extendcircuit(control_connection_t *conn, SMARTLIST_FOREACH_BEGIN(router_nicknames, const char *, n) { const node_t *node = node_get_by_nickname(n, 0); if (!node) { - connection_printf_to_buf(conn, "552 No such router \"%s\"\r\n", n); + control_printf_endreply(conn, 552, "No such router \"%s\"", n); goto done; } if (!node_has_preferred_descriptor(node, first_node)) { - connection_printf_to_buf(conn, "552 No descriptor for \"%s\"\r\n", n); + control_printf_endreply(conn, 552, "No descriptor for \"%s\"", n); goto done; } smartlist_add(nodes, (void*)node); @@ -825,7 +825,7 @@ handle_control_extendcircuit(control_connection_t *conn, } SMARTLIST_FOREACH_END(n); if (!smartlist_len(nodes)) { - connection_write_str_to_buf("512 No router names provided\r\n", conn); + control_write_endreply(conn, 512, "No router names provided"); goto done; } @@ -863,7 +863,7 @@ handle_control_extendcircuit(control_connection_t *conn, int err_reason = 0; if ((err_reason = circuit_handle_first_hop(circ)) < 0) { circuit_mark_for_close(TO_CIRCUIT(circ), -err_reason); - connection_write_str_to_buf("551 Couldn't start circuit\r\n", conn); + control_write_endreply(conn, 551, "Couldn't start circuit"); goto done; } } else { @@ -875,13 +875,13 @@ handle_control_extendcircuit(control_connection_t *conn, log_info(LD_CONTROL, "send_next_onion_skin failed; circuit marked for closing."); circuit_mark_for_close(TO_CIRCUIT(circ), -err_reason); - connection_write_str_to_buf("551 Couldn't send onion skin\r\n", conn); + control_write_endreply(conn, 551, "Couldn't send onion skin"); goto done; } } } - connection_printf_to_buf(conn, "250 EXTENDED %lu\r\n", + control_printf_endreply(conn, 250, "EXTENDED %lu", (unsigned long)circ->global_identifier); if (zero_circ) /* send a 'launched' event, for completeness */ circuit_event_status(circ, CIRC_EVENT_LAUNCHED, 0); @@ -909,26 +909,26 @@ handle_control_setcircuitpurpose(control_connection_t *conn, const char *circ_id = smartlist_get(args->args,0); if (!(circ = get_circ(circ_id))) { - connection_printf_to_buf(conn, "552 Unknown circuit \"%s\"\r\n", circ_id); + control_printf_endreply(conn, 552, "Unknown circuit \"%s\"", circ_id); goto done; } { const config_line_t *purp = config_line_find_case(args->kwargs, "PURPOSE"); if (!purp) { - connection_write_str_to_buf("552 No purpose given\r\n", conn); + control_write_endreply(conn, 552, "No purpose given"); goto done; } new_purpose = circuit_purpose_from_string(purp->value); if (new_purpose == CIRCUIT_PURPOSE_UNKNOWN) { - connection_printf_to_buf(conn, "552 Unknown purpose \"%s\"\r\n", + control_printf_endreply(conn, 552, "Unknown purpose \"%s\"", purp->value); goto done; } } circuit_change_purpose(TO_CIRCUIT(circ), new_purpose); - connection_write_str_to_buf("250 OK\r\n", conn); + send_control_done(conn); done: return 0; @@ -959,17 +959,17 @@ handle_control_attachstream(control_connection_t *conn, const config_line_t *hoparg = config_line_find_case(args->kwargs, "HOP"); if (!(ap_conn = get_stream(stream_id))) { - connection_printf_to_buf(conn, "552 Unknown stream \"%s\"\r\n", stream_id); + control_printf_endreply(conn, 552, "Unknown stream \"%s\"", stream_id); return 0; } else if (!zero_circ && !(circ = get_circ(circ_id))) { - connection_printf_to_buf(conn, "552 Unknown circuit \"%s\"\r\n", circ_id); + control_printf_endreply(conn, 552, "Unknown circuit \"%s\"", circ_id); return 0; } else if (circ) { if (hoparg) { hop = (int) tor_parse_ulong(hoparg->value, 10, 0, INT_MAX, &hop_line_ok, NULL); if (!hop_line_ok) { /* broken hop line */ - connection_printf_to_buf(conn, "552 Bad value hop=%s\r\n", + control_printf_endreply(conn, 552, "Bad value hop=%s", hoparg->value); return 0; } @@ -979,9 +979,8 @@ handle_control_attachstream(control_connection_t *conn, if (ENTRY_TO_CONN(ap_conn)->state != AP_CONN_STATE_CONTROLLER_WAIT && ENTRY_TO_CONN(ap_conn)->state != AP_CONN_STATE_CONNECT_WAIT && ENTRY_TO_CONN(ap_conn)->state != AP_CONN_STATE_RESOLVE_WAIT) { - connection_write_str_to_buf( - "555 Connection is not managed by controller.\r\n", - conn); + control_write_endreply(conn, 555, + "Connection is not managed by controller."); return 0; } @@ -1000,15 +999,14 @@ handle_control_attachstream(control_connection_t *conn, } if (circ && (circ->base_.state != CIRCUIT_STATE_OPEN)) { - connection_write_str_to_buf( - "551 Can't attach stream to non-open origin circuit\r\n", - conn); + control_write_endreply(conn, 551, + "Can't attach stream to non-open origin circuit"); return 0; } /* Is this a single hop circuit? */ if (circ && (circuit_get_cpath_len(circ)<2 || hop==1)) { - connection_write_str_to_buf( - "551 Can't attach stream to this one-hop circuit.\r\n", conn); + control_write_endreply(conn, 551, + "Can't attach stream to this one-hop circuit."); return 0; } @@ -1016,13 +1014,12 @@ handle_control_attachstream(control_connection_t *conn, /* find this hop in the circuit, and set cpath */ cpath = circuit_get_cpath_hop(circ, hop); if (!cpath) { - connection_printf_to_buf(conn, - "551 Circuit doesn't have %d hops.\r\n", hop); + control_printf_endreply(conn, 551, "Circuit doesn't have %d hops.", hop); return 0; } } if (connection_ap_handshake_rewrite_and_attach(ap_conn, circ, cpath) < 0) { - connection_write_str_to_buf("551 Unable to attach stream\r\n", conn); + control_write_endreply(conn, 551, "Unable to attach stream"); return 0; } send_control_done(conn); @@ -1054,7 +1051,7 @@ handle_control_postdescriptor(control_connection_t *conn, line = config_line_find_case(args->kwargs, "purpose"); if (line) { purpose = router_purpose_from_string(line->value); - connection_printf_to_buf(conn, "552 Unknown purpose \"%s\"\r\n", + control_printf_endreply(conn, 552, "Unknown purpose \"%s\"", line->value); goto done; } @@ -1065,7 +1062,7 @@ handle_control_postdescriptor(control_connection_t *conn, else if (!strcasecmp(line->value, "yes")) cache = 1; else { - connection_printf_to_buf(conn, "552 Unknown cache request \"%s\"\r\n", + control_printf_endreply(conn, 552, "Unknown cache request \"%s\"", line->value); goto done; } @@ -1074,11 +1071,11 @@ handle_control_postdescriptor(control_connection_t *conn, switch (router_load_single_router(args->cmddata, purpose, cache, &msg)) { case -1: if (!msg) msg = "Could not parse descriptor"; - connection_printf_to_buf(conn, "554 %s\r\n", msg); + control_write_endreply(conn, 554, msg); break; case 0: if (!msg) msg = "Descriptor not added"; - connection_printf_to_buf(conn, "251 %s\r\n",msg); + control_write_endreply(conn, 251, msg); break; case 1: send_control_done(conn); @@ -1107,7 +1104,7 @@ handle_control_redirectstream(control_connection_t *conn, if (!(ap_conn = get_stream(smartlist_get(args, 0))) || !ap_conn->socks_request) { - connection_printf_to_buf(conn, "552 Unknown stream \"%s\"\r\n", + control_printf_endreply(conn, 552, "Unknown stream \"%s\"", (char*)smartlist_get(args, 0)); } else { int ok = 1; @@ -1116,7 +1113,7 @@ handle_control_redirectstream(control_connection_t *conn, 10, 1, 65535, &ok, NULL); } if (!ok) { - connection_printf_to_buf(conn, "512 Cannot parse port \"%s\"\r\n", + control_printf_endreply(conn, 512, "Cannot parse port \"%s\"", (char*)smartlist_get(args, 2)); } else { new_addr = tor_strdup(smartlist_get(args, 1)); @@ -1155,13 +1152,13 @@ handle_control_closestream(control_connection_t *conn, tor_assert(smartlist_len(args) >= 2); if (!(ap_conn = get_stream(smartlist_get(args, 0)))) - connection_printf_to_buf(conn, "552 Unknown stream \"%s\"\r\n", + control_printf_endreply(conn, 552, "Unknown stream \"%s\"", (char*)smartlist_get(args, 0)); else { reason = (uint8_t) tor_parse_ulong(smartlist_get(args,1), 10, 0, 255, &ok, NULL); if (!ok) { - connection_printf_to_buf(conn, "552 Unrecognized reason \"%s\"\r\n", + control_printf_endreply(conn, 552, "Unrecognized reason \"%s\"", (char*)smartlist_get(args, 1)); ap_conn = NULL; } @@ -1192,7 +1189,7 @@ handle_control_closecircuit(control_connection_t *conn, origin_circuit_t *circ = NULL; if (!(circ=get_circ(circ_id))) { - connection_printf_to_buf(conn, "552 Unknown circuit \"%s\"\r\n", circ_id); + control_printf_endreply(conn, 552, "Unknown circuit \"%s\"", circ_id); return 0; } @@ -1277,7 +1274,7 @@ handle_control_protocolinfo(control_connection_t *conn, } }); if (bad_arg) { - connection_printf_to_buf(conn, "513 No such version %s\r\n", + control_printf_endreply(conn, 513, "No such version %s", escaped(bad_arg)); /* Don't tolerate bad arguments when not authenticated. */ if (!STATE_IS_OPEN(TO_CONN(conn)->state)) @@ -1342,7 +1339,7 @@ handle_control_usefeature(control_connection_t *conn, else if (!strcasecmp(arg, "EXTENDED_EVENTS")) ; else { - connection_printf_to_buf(conn, "552 Unrecognized feature \"%s\"\r\n", + control_printf_endreply(conn, 552, "Unrecognized feature \"%s\"", arg); bad = 1; break; @@ -1426,7 +1423,7 @@ handle_control_hsfetch(control_connection_t *conn, version = HS_VERSION_THREE; hs_parse_address(hsaddress, &v3_pk, NULL, NULL); } else { - connection_printf_to_buf(conn, "513 Invalid argument \"%s\"\r\n", + control_printf_endreply(conn, 513, "Invalid argument \"%s\"", arg1); goto done; } @@ -1437,7 +1434,7 @@ handle_control_hsfetch(control_connection_t *conn, const node_t *node = node_get_by_hex_id(server, 0); if (!node) { - connection_printf_to_buf(conn, "552 Server \"%s\" not found\r\n", + control_printf_endreply(conn, 552, "Server \"%s\" not found", server); goto done; } @@ -1456,7 +1453,7 @@ handle_control_hsfetch(control_connection_t *conn, rend_query = rend_data_client_create(hsaddress, desc_id, NULL, REND_NO_AUTH); if (rend_query == NULL) { - connection_printf_to_buf(conn, "551 Error creating the HS query\r\n"); + control_write_endreply(conn, 551, "Error creating the HS query"); goto done; } } @@ -1464,7 +1461,7 @@ handle_control_hsfetch(control_connection_t *conn, /* Using a descriptor ID, we force the user to provide at least one * hsdir server using the SERVER= option. */ if (desc_id && (!hsdirs || !smartlist_len(hsdirs))) { - connection_printf_to_buf(conn, "512 SERVER option is required\r\n"); + control_write_endreply(conn, 512, "SERVER option is required"); goto done; } @@ -1516,7 +1513,7 @@ handle_control_hspost(control_connection_t *conn, const node_t *node = node_get_by_hex_id(server, 0); if (!node || !node->rs) { - connection_printf_to_buf(conn, "552 Server \"%s\" not found\r\n", + control_printf_endreply(conn, 552, "Server \"%s\" not found", server); goto done; } @@ -1527,7 +1524,7 @@ handle_control_hspost(control_connection_t *conn, } else if (!strcasecmpstart(line->key, "HSADDRESS")) { const char *address = line->value; if (!hs_address_is_valid(address)) { - connection_printf_to_buf(conn, "512 Malformed onion address\r\n"); + control_write_endreply(conn, 512, "Malformed onion address"); goto done; } onion_address = address; @@ -1539,7 +1536,7 @@ handle_control_hspost(control_connection_t *conn, /* Handle the v3 case. */ if (onion_address) { if (hs_control_hspost_command(encoded_desc, onion_address, hs_dirs) < 0) { - connection_printf_to_buf(conn, "554 Invalid descriptor\r\n"); + control_write_endreply(conn, 554, "Invalid descriptor"); } else { send_control_done(conn); } @@ -1579,7 +1576,7 @@ handle_control_hspost(control_connection_t *conn, rend_service_descriptor_free(parsed); } else { - connection_printf_to_buf(conn, "554 Invalid descriptor\r\n"); + control_write_endreply(conn, 554, "Invalid descriptor"); } tor_free(intro_content); @@ -1685,7 +1682,7 @@ handle_control_add_onion(control_connection_t *conn, rend_service_port_config_t *cfg = rend_service_parse_port_config(arg->value, ",", NULL); if (!cfg) { - connection_printf_to_buf(conn, "512 Invalid VIRTPORT/TARGET\r\n"); + control_write_endreply(conn, 512, "Invalid VIRTPORT/TARGET"); goto out; } smartlist_add(port_cfgs, cfg); @@ -1694,7 +1691,7 @@ handle_control_add_onion(control_connection_t *conn, int ok = 0; max_streams = (int)tor_parse_long(arg->value, 10, 0, 65535, &ok, NULL); if (!ok) { - connection_printf_to_buf(conn, "512 Invalid MaxStreams\r\n"); + control_write_endreply(conn, 512, "Invalid MaxStreams"); goto out; } } else if (!strcasecmp(arg->key, "Flags")) { @@ -1722,7 +1719,7 @@ handle_control_add_onion(control_connection_t *conn, smartlist_split_string(flags, arg->value, ",", SPLIT_IGNORE_BLANK, 0); if (smartlist_len(flags) < 1) { - connection_printf_to_buf(conn, "512 Invalid 'Flags' argument\r\n"); + control_write_endreply(conn, 512, "Invalid 'Flags' argument"); bad = 1; } SMARTLIST_FOREACH_BEGIN(flags, const char *, flag) @@ -1738,8 +1735,7 @@ handle_control_add_onion(control_connection_t *conn, } else if (!strcasecmp(flag, non_anonymous_flag)) { non_anonymous = 1; } else { - connection_printf_to_buf(conn, - "512 Invalid 'Flags' argument: %s\r\n", + control_printf_endreply(conn, 512, "Invalid 'Flags' argument: %s", escaped(flag)); bad = 1; break; @@ -1773,8 +1769,7 @@ handle_control_add_onion(control_connection_t *conn, } } SMARTLIST_FOREACH_END(ac); if (bad) { - connection_printf_to_buf(conn, - "512 Duplicate name in ClientAuth\r\n"); + control_write_endreply(conn, 512, "Duplicate name in ClientAuth"); rend_authorized_client_free(client); goto out; } @@ -1792,19 +1787,19 @@ handle_control_add_onion(control_connection_t *conn, } } if (smartlist_len(port_cfgs) == 0) { - connection_printf_to_buf(conn, "512 Missing 'Port' argument\r\n"); + control_write_endreply(conn, 512, "Missing 'Port' argument"); goto out; } else if (auth_type == REND_NO_AUTH && auth_clients != NULL) { - connection_printf_to_buf(conn, "512 No auth type specified\r\n"); + control_write_endreply(conn, 512, "No auth type specified"); goto out; } else if (auth_type != REND_NO_AUTH && auth_clients == NULL) { - connection_printf_to_buf(conn, "512 No auth clients specified\r\n"); + control_write_endreply(conn, 512, "No auth clients specified"); goto out; } else if ((auth_type == REND_BASIC_AUTH && smartlist_len(auth_clients) > 512) || (auth_type == REND_STEALTH_AUTH && smartlist_len(auth_clients) > 16)) { - connection_printf_to_buf(conn, "512 Too many auth clients\r\n"); + control_write_endreply(conn, 512, "Too many auth clients"); goto out; } else if (non_anonymous != rend_service_non_anonymous_mode_enabled( get_options())) { @@ -1815,8 +1810,8 @@ handle_control_add_onion(control_connection_t *conn, * 512 Tor is in non-anonymous hidden service mode * (I've deliberately written them out in full here to aid searchability.) */ - connection_printf_to_buf(conn, "512 Tor is in %sanonymous hidden service " - "mode\r\n", + control_printf_endreply(conn, 512, + "Tor is in %sanonymous hidden service " "mode", non_anonymous ? "" : "non-"); goto out; } @@ -1843,7 +1838,7 @@ handle_control_add_onion(control_connection_t *conn, /* Hidden service version 3 don't have client authentication support so if * ClientAuth was given, send back an error. */ if (hs_version == HS_VERSION_THREE && auth_clients) { - connection_printf_to_buf(conn, "513 ClientAuth not supported\r\n"); + control_write_endreply(conn, 513, "ClientAuth not supported"); goto out; } @@ -1873,10 +1868,10 @@ handle_control_add_onion(control_connection_t *conn, } tor_assert(service_id); - connection_printf_to_buf(conn, "250-ServiceID=%s\r\n", service_id); + control_printf_midreply(conn, 250, "ServiceID=%s", service_id); if (key_new_alg) { tor_assert(key_new_blob); - connection_printf_to_buf(conn, "250-PrivateKey=%s:%s\r\n", + control_printf_midreply(conn, 250, "PrivateKey=%s:%s", key_new_alg, key_new_blob); } if (auth_created_clients) { @@ -1891,24 +1886,24 @@ handle_control_add_onion(control_connection_t *conn, }); } - connection_printf_to_buf(conn, "250 OK\r\n"); + send_control_done(conn); break; } case RSAE_BADPRIVKEY: - connection_printf_to_buf(conn, "551 Failed to generate onion address\r\n"); + control_write_endreply(conn, 551, "Failed to generate onion address"); break; case RSAE_ADDREXISTS: - connection_printf_to_buf(conn, "550 Onion address collision\r\n"); + control_write_endreply(conn, 550, "Onion address collision"); break; case RSAE_BADVIRTPORT: - connection_printf_to_buf(conn, "512 Invalid VIRTPORT/TARGET\r\n"); + control_write_endreply(conn, 512, "Invalid VIRTPORT/TARGET"); break; case RSAE_BADAUTH: - connection_printf_to_buf(conn, "512 Invalid client authorization\r\n"); + control_write_endreply(conn, 512, "Invalid client authorization"); break; case RSAE_INTERNAL: /* FALLSTHROUGH */ default: - connection_printf_to_buf(conn, "551 Failed to add Onion Service\r\n"); + control_write_endreply(conn, 551, "Failed to add Onion Service"); } if (key_new_blob) { memwipe(key_new_blob, 0, strlen(key_new_blob)); @@ -2150,7 +2145,7 @@ handle_control_del_onion(control_connection_t *conn, } else if (hs_address_is_valid(service_id)) { hs_version = HS_VERSION_THREE; } else { - connection_printf_to_buf(conn, "512 Malformed Onion Service id\r\n"); + control_write_endreply(conn, 512, "Malformed Onion Service id"); goto out; } @@ -2174,7 +2169,7 @@ handle_control_del_onion(control_connection_t *conn, } } if (onion_services == NULL) { - connection_printf_to_buf(conn, "552 Unknown Onion Service id\r\n"); + control_write_endreply(conn, 552, "Unknown Onion Service id"); } else { int ret = -1; switch (hs_version) { @@ -2226,7 +2221,7 @@ handle_control_obsolete(control_connection_t *conn, (void)args; char *command = tor_strdup(conn->current_cmd); tor_strupper(command); - connection_printf_to_buf(conn, "511 %s is obsolete.\r\n", command); + control_printf_endreply(conn, 511, "%s is obsolete.", command); tor_free(command); return 0; } @@ -2359,8 +2354,7 @@ handle_single_control_command(const control_cmd_def_t *def, cmd_data_len, args, &err); if (!parsed_args) { - connection_printf_to_buf(conn, - "512 Bad arguments to %s: %s\r\n", + control_printf_endreply(conn, 512, "Bad arguments to %s: %s", conn->current_cmd, err?err:""); tor_free(err); } else { @@ -2401,7 +2395,7 @@ handle_control_command(control_connection_t *conn, } } - connection_printf_to_buf(conn, "510 Unrecognized command \"%s\"\r\n", + control_printf_endreply(conn, 510, "Unrecognized command \"%s\"", conn->current_cmd); return 0; diff --git a/src/feature/control/control_getinfo.c b/src/feature/control/control_getinfo.c index c10229bfbb..3e31bb9e8f 100644 --- a/src/feature/control/control_getinfo.c +++ b/src/feature/control/control_getinfo.c @@ -1608,7 +1608,7 @@ handle_control_getinfo(control_connection_t *conn, if (handle_getinfo_helper(conn, q, &ans, &errmsg) < 0) { if (!errmsg) errmsg = "Internal error"; - connection_printf_to_buf(conn, "551 %s\r\n", errmsg); + control_write_endreply(conn, 551, errmsg); goto done; } if (!ans) { @@ -1625,13 +1625,10 @@ handle_control_getinfo(control_connection_t *conn, if (smartlist_len(unrecognized)) { /* control-spec section 2.3, mid-reply '-' or end of reply ' ' */ for (i=0; i < smartlist_len(unrecognized)-1; ++i) - connection_printf_to_buf(conn, - "552-%s\r\n", - (char *)smartlist_get(unrecognized, i)); - - connection_printf_to_buf(conn, - "552 %s\r\n", + control_write_midreply(conn, 552, (char *)smartlist_get(unrecognized, i)); + + control_write_endreply(conn, 552, (char *)smartlist_get(unrecognized, i)); goto done; } @@ -1645,7 +1642,7 @@ handle_control_getinfo(control_connection_t *conn, control_write_data(conn, v); } } - connection_write_str_to_buf("250 OK\r\n", conn); + send_control_done(conn); done: SMARTLIST_FOREACH(answers, char *, cp, tor_free(cp)); diff --git a/src/feature/control/control_proto.c b/src/feature/control/control_proto.c index 7bd237de94..1dd62da2be 100644 --- a/src/feature/control/control_proto.c +++ b/src/feature/control/control_proto.c @@ -166,7 +166,7 @@ read_escaped_data(const char *data, size_t len, char **out) void send_control_done(control_connection_t *conn) { - connection_write_str_to_buf("250 OK\r\n", conn); + control_write_endreply(conn, 250, "OK"); } /** Write a reply to the control channel. From 68caca58a88633c5da1860eec9e17155beaf9417 Mon Sep 17 00:00:00 2001 From: Taylor Yu Date: Wed, 10 Apr 2019 17:22:36 -0500 Subject: [PATCH 0921/2557] Clean up formatting after Coccinelle Clean up some minor formatting quirks after the Coccinelle run. --- src/feature/control/control_auth.c | 20 ++++++----- src/feature/control/control_cmd.c | 58 ++++++++++++++---------------- 2 files changed, 39 insertions(+), 39 deletions(-) diff --git a/src/feature/control/control_auth.c b/src/feature/control/control_auth.c index 51c8da1b99..49d4d415c6 100644 --- a/src/feature/control/control_auth.c +++ b/src/feature/control/control_auth.c @@ -142,7 +142,8 @@ handle_control_authchallenge(control_connection_t *conn, if (strcasecmp(smartlist_get(args->args, 0), "SAFECOOKIE")) { control_write_endreply(conn, 513, - "AUTHCHALLENGE only supports SAFECOOKIE " "authentication"); + "AUTHCHALLENGE only supports SAFECOOKIE " + "authentication"); goto fail; } if (!authentication_cookie_is_set) { @@ -154,13 +155,14 @@ handle_control_authchallenge(control_connection_t *conn, "2 arguments.\r\n", conn); */ control_printf_endreply(conn, 512, - "AUTHCHALLENGE dislikes argument list %s", - escaped(args->raw_body)); + "AUTHCHALLENGE dislikes argument list %s", + escaped(args->raw_body)); goto fail; } if (strcmp(args->kwargs->key, "")) { control_write_endreply(conn, 512, - "AUTHCHALLENGE does not accept keyword " "arguments."); + "AUTHCHALLENGE does not accept keyword " + "arguments."); goto fail; } @@ -222,9 +224,9 @@ handle_control_authchallenge(control_connection_t *conn, server_nonce, sizeof(server_nonce)); control_printf_endreply(conn, 250, - "AUTHCHALLENGE SERVERHASH=%s " "SERVERNONCE=%s", - server_hash_encoded, - server_nonce_encoded); + "AUTHCHALLENGE SERVERHASH=%s SERVERNONCE=%s", + server_hash_encoded, + server_nonce_encoded); tor_free(client_nonce); return 0; @@ -279,7 +281,9 @@ handle_control_authenticate(control_connection_t *conn, if (base16_decode(password, password_len+1, hex_passwd, strlen(hex_passwd)) != (int) password_len) { control_write_endreply(conn, 551, - "Invalid hexadecimal encoding. Maybe you tried a plain text " "password? If so, the standard requires that you put it in " "double quotes."); + "Invalid hexadecimal encoding. Maybe you tried a plain text " + "password? If so, the standard requires that you put it in " + "double quotes."); connection_mark_for_close(TO_CONN(conn)); tor_free(password); return 0; diff --git a/src/feature/control/control_cmd.c b/src/feature/control/control_cmd.c index ae4d6584cc..5555a2c5c4 100644 --- a/src/feature/control/control_cmd.c +++ b/src/feature/control/control_cmd.c @@ -320,11 +320,11 @@ handle_control_getconf(control_connection_t *conn, if ((len = smartlist_len(unrecognized))) { for (i=0; i < len-1; ++i) control_printf_midreply(conn, 552, - "Unrecognized configuration key \"%s\"", - (char*)smartlist_get(unrecognized, i)); + "Unrecognized configuration key \"%s\"", + (char*)smartlist_get(unrecognized, i)); control_printf_endreply(conn, 552, - "Unrecognized configuration key \"%s\"", - (char*)smartlist_get(unrecognized, len-1)); + "Unrecognized configuration key \"%s\"", + (char*)smartlist_get(unrecognized, len-1)); } else if ((len = smartlist_len(answers))) { char *tmp = smartlist_get(answers, len-1); tor_assert(strlen(tmp)>4); @@ -424,8 +424,7 @@ handle_control_setevents(control_connection_t *conn, } if (event_code == -1) { - control_printf_endreply(conn, 552, "Unrecognized event \"%s\"", - ev); + control_printf_endreply(conn, 552, "Unrecognized event \"%s\"", ev); return 0; } } @@ -489,8 +488,7 @@ handle_control_signal(control_connection_t *conn, } if (sig < 0) - control_printf_endreply(conn, 552, "Unrecognized signal code \"%s\"", - s); + control_printf_endreply(conn, 552, "Unrecognized signal code \"%s\"", s); if (sig < 0) return 0; @@ -777,7 +775,7 @@ handle_control_extendcircuit(control_connection_t *conn, intended_purpose = circuit_purpose_from_string(purpose_line->value); if (intended_purpose == CIRCUIT_PURPOSE_UNKNOWN) { control_printf_endreply(conn, 552, "Unknown purpose \"%s\"", - purpose_line->value); + purpose_line->value); goto done; } } @@ -882,7 +880,7 @@ handle_control_extendcircuit(control_connection_t *conn, } control_printf_endreply(conn, 250, "EXTENDED %lu", - (unsigned long)circ->global_identifier); + (unsigned long)circ->global_identifier); if (zero_circ) /* send a 'launched' event, for completeness */ circuit_event_status(circ, CIRC_EVENT_LAUNCHED, 0); done: @@ -922,7 +920,7 @@ handle_control_setcircuitpurpose(control_connection_t *conn, new_purpose = circuit_purpose_from_string(purp->value); if (new_purpose == CIRCUIT_PURPOSE_UNKNOWN) { control_printf_endreply(conn, 552, "Unknown purpose \"%s\"", - purp->value); + purp->value); goto done; } } @@ -970,7 +968,7 @@ handle_control_attachstream(control_connection_t *conn, &hop_line_ok, NULL); if (!hop_line_ok) { /* broken hop line */ control_printf_endreply(conn, 552, "Bad value hop=%s", - hoparg->value); + hoparg->value); return 0; } } @@ -1052,7 +1050,7 @@ handle_control_postdescriptor(control_connection_t *conn, if (line) { purpose = router_purpose_from_string(line->value); control_printf_endreply(conn, 552, "Unknown purpose \"%s\"", - line->value); + line->value); goto done; } line = config_line_find_case(args->kwargs, "cache"); @@ -1063,7 +1061,7 @@ handle_control_postdescriptor(control_connection_t *conn, cache = 1; else { control_printf_endreply(conn, 552, "Unknown cache request \"%s\"", - line->value); + line->value); goto done; } } @@ -1105,7 +1103,7 @@ handle_control_redirectstream(control_connection_t *conn, if (!(ap_conn = get_stream(smartlist_get(args, 0))) || !ap_conn->socks_request) { control_printf_endreply(conn, 552, "Unknown stream \"%s\"", - (char*)smartlist_get(args, 0)); + (char*)smartlist_get(args, 0)); } else { int ok = 1; if (smartlist_len(args) > 2) { /* they included a port too */ @@ -1114,7 +1112,7 @@ handle_control_redirectstream(control_connection_t *conn, } if (!ok) { control_printf_endreply(conn, 512, "Cannot parse port \"%s\"", - (char*)smartlist_get(args, 2)); + (char*)smartlist_get(args, 2)); } else { new_addr = tor_strdup(smartlist_get(args, 1)); } @@ -1153,13 +1151,13 @@ handle_control_closestream(control_connection_t *conn, if (!(ap_conn = get_stream(smartlist_get(args, 0)))) control_printf_endreply(conn, 552, "Unknown stream \"%s\"", - (char*)smartlist_get(args, 0)); + (char*)smartlist_get(args, 0)); else { reason = (uint8_t) tor_parse_ulong(smartlist_get(args,1), 10, 0, 255, &ok, NULL); if (!ok) { control_printf_endreply(conn, 552, "Unrecognized reason \"%s\"", - (char*)smartlist_get(args, 1)); + (char*)smartlist_get(args, 1)); ap_conn = NULL; } } @@ -1275,7 +1273,7 @@ handle_control_protocolinfo(control_connection_t *conn, }); if (bad_arg) { control_printf_endreply(conn, 513, "No such version %s", - escaped(bad_arg)); + escaped(bad_arg)); /* Don't tolerate bad arguments when not authenticated. */ if (!STATE_IS_OPEN(TO_CONN(conn)->state)) connection_mark_for_close(TO_CONN(conn)); @@ -1340,7 +1338,7 @@ handle_control_usefeature(control_connection_t *conn, ; else { control_printf_endreply(conn, 552, "Unrecognized feature \"%s\"", - arg); + arg); bad = 1; break; } @@ -1423,8 +1421,7 @@ handle_control_hsfetch(control_connection_t *conn, version = HS_VERSION_THREE; hs_parse_address(hsaddress, &v3_pk, NULL, NULL); } else { - control_printf_endreply(conn, 513, "Invalid argument \"%s\"", - arg1); + control_printf_endreply(conn, 513, "Invalid argument \"%s\"", arg1); goto done; } @@ -1434,8 +1431,7 @@ handle_control_hsfetch(control_connection_t *conn, const node_t *node = node_get_by_hex_id(server, 0); if (!node) { - control_printf_endreply(conn, 552, "Server \"%s\" not found", - server); + control_printf_endreply(conn, 552, "Server \"%s\" not found", server); goto done; } if (!hsdirs) { @@ -1514,7 +1510,7 @@ handle_control_hspost(control_connection_t *conn, if (!node || !node->rs) { control_printf_endreply(conn, 552, "Server \"%s\" not found", - server); + server); goto done; } /* Valid server, add it to our local list. */ @@ -1736,7 +1732,7 @@ handle_control_add_onion(control_connection_t *conn, non_anonymous = 1; } else { control_printf_endreply(conn, 512, "Invalid 'Flags' argument: %s", - escaped(flag)); + escaped(flag)); bad = 1; break; } @@ -1811,8 +1807,8 @@ handle_control_add_onion(control_connection_t *conn, * (I've deliberately written them out in full here to aid searchability.) */ control_printf_endreply(conn, 512, - "Tor is in %sanonymous hidden service " "mode", - non_anonymous ? "" : "non-"); + "Tor is in %sanonymous hidden service " "mode", + non_anonymous ? "" : "non-"); goto out; } @@ -1872,7 +1868,7 @@ handle_control_add_onion(control_connection_t *conn, if (key_new_alg) { tor_assert(key_new_blob); control_printf_midreply(conn, 250, "PrivateKey=%s:%s", - key_new_alg, key_new_blob); + key_new_alg, key_new_blob); } if (auth_created_clients) { SMARTLIST_FOREACH(auth_created_clients, rend_authorized_client_t *, ac, { @@ -2355,7 +2351,7 @@ handle_single_control_command(const control_cmd_def_t *def, &err); if (!parsed_args) { control_printf_endreply(conn, 512, "Bad arguments to %s: %s", - conn->current_cmd, err?err:""); + conn->current_cmd, err?err:""); tor_free(err); } else { if (BUG(err)) @@ -2396,7 +2392,7 @@ handle_control_command(control_connection_t *conn, } control_printf_endreply(conn, 510, "Unrecognized command \"%s\"", - conn->current_cmd); + conn->current_cmd); return 0; } From 09003679962e5d0e0c15cb278b83e3ef79affdda Mon Sep 17 00:00:00 2001 From: Taylor Yu Date: Fri, 26 Apr 2019 13:25:12 -0500 Subject: [PATCH 0922/2557] Changes file for ticket30007 --- changes/ticket30007 | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 changes/ticket30007 diff --git a/changes/ticket30007 b/changes/ticket30007 new file mode 100644 index 0000000000..e87f6b956f --- /dev/null +++ b/changes/ticket30007 @@ -0,0 +1,3 @@ + o Code simplification and refactoring: + - Abstract out the low-level formatting of replies on the control + port. Implements ticket 30007. From 309467c64e007ea6841c07fdee35eaff0146d541 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 30 Apr 2019 14:43:35 -0400 Subject: [PATCH 0923/2557] Rename tor_mem_is_zero to fast_mem_is_zero() For memeq and friends, "tor_" indicates constant-time and "fast_" indicates optimized. I'm fine with leaving the constant-time "safe_mem_is_zero" with its current name, but the "tor_" prefix on the current optimized version is misleading. Also, make the tor_digest*_is_zero() uniformly constant-time, and add a fast_digest*_is_zero() version to use as needed. A later commit in this branch will fix all the users of tor_mem_is_zero(). Closes ticket 30309. --- changes/bug30309 | 3 +++ src/lib/string/util_string.c | 9 +++------ src/lib/string/util_string.h | 8 +++++++- 3 files changed, 13 insertions(+), 7 deletions(-) create mode 100644 changes/bug30309 diff --git a/changes/bug30309 b/changes/bug30309 new file mode 100644 index 0000000000..6cbbe8d156 --- /dev/null +++ b/changes/bug30309 @@ -0,0 +1,3 @@ + o Code simplification and refactoring: + - Rename tor_mem_is_zero() to fast_mem_is_zero(), to emphasize that + it is not a constant-time function. Closes ticket 30309. diff --git a/src/lib/string/util_string.c b/src/lib/string/util_string.c index 0c4e399008..f5061a11d2 100644 --- a/src/lib/string/util_string.c +++ b/src/lib/string/util_string.c @@ -71,7 +71,7 @@ tor_memstr(const void *haystack, size_t hlen, const char *needle) /** Return true iff the 'len' bytes at 'mem' are all zero. */ int -tor_mem_is_zero(const char *mem, size_t len) +fast_mem_is_zero(const char *mem, size_t len) { static const char ZERO[] = { 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, @@ -95,17 +95,14 @@ tor_mem_is_zero(const char *mem, size_t len) int tor_digest_is_zero(const char *digest) { - static const uint8_t ZERO_DIGEST[] = { - 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0 - }; - return tor_memeq(digest, ZERO_DIGEST, DIGEST_LEN); + return safe_mem_is_zero(digest, DIGEST_LEN); } /** Return true iff the DIGEST256_LEN bytes in digest are all zero. */ int tor_digest256_is_zero(const char *digest) { - return tor_mem_is_zero(digest, DIGEST256_LEN); + return safe_mem_is_zero(digest, DIGEST256_LEN); } /** Remove from the string s every character which appears in diff --git a/src/lib/string/util_string.h b/src/lib/string/util_string.h index da4fab159c..7e8af0578c 100644 --- a/src/lib/string/util_string.h +++ b/src/lib/string/util_string.h @@ -20,7 +20,13 @@ const void *tor_memmem(const void *haystack, size_t hlen, const void *needle, size_t nlen); const void *tor_memstr(const void *haystack, size_t hlen, const char *needle); -int tor_mem_is_zero(const char *mem, size_t len); +int fast_mem_is_zero(const char *mem, size_t len); +#define fast_digest_is_zero(d) fast_mem_is_zero((d), DIGEST_LEN) +#define fast_digetst256_is_zero(d) fast_mem_is_zero((d), DIGEST256_LEN) + +// XXXX remove this after we replace all users. +#define tor_mem_is_zero fast_mem_is_zero + int tor_digest_is_zero(const char *digest); int tor_digest256_is_zero(const char *digest); From 0034f1095680e2b05c19ec13368ddc936a53058a Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 30 Apr 2019 14:45:58 -0400 Subject: [PATCH 0924/2557] Use safe_mem_is_zero in a few more places. I don't believe any of these represent a real timing vulnerability (remote timing against memcmp() on a modern CPU is not easy), but these are the ones where I believe we should be more careful. --- src/feature/relay/routerkeys.c | 2 +- src/feature/rend/rendcache.c | 6 ++++-- src/lib/crypt_ops/crypto_ed25519.c | 2 +- src/lib/crypt_ops/crypto_rand.c | 3 ++- 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/feature/relay/routerkeys.c b/src/feature/relay/routerkeys.c index 5db7ed7268..a9190b2e13 100644 --- a/src/feature/relay/routerkeys.c +++ b/src/feature/relay/routerkeys.c @@ -226,7 +226,7 @@ load_ed_keys(const or_options_t *options, time_t now) tor_free(fname); } } - if (tor_mem_is_zero((char*)id->seckey.seckey, sizeof(id->seckey))) + if (safe_mem_is_zero((char*)id->seckey.seckey, sizeof(id->seckey))) sign_signing_key_with_id = NULL; else sign_signing_key_with_id = id; diff --git a/src/feature/rend/rendcache.c b/src/feature/rend/rendcache.c index abeb150685..c3f86d8c82 100644 --- a/src/feature/rend/rendcache.c +++ b/src/feature/rend/rendcache.c @@ -19,6 +19,8 @@ #include "feature/rend/rend_intro_point_st.h" #include "feature/rend/rend_service_descriptor_st.h" +#include "lib/ctime/di_ops.h" + /** Map from service id (as generated by rend_get_service_id) to * rend_cache_entry_t. */ STATIC strmap_t *rend_cache = NULL; @@ -889,8 +891,8 @@ rend_cache_store_v2_desc_as_client(const char *desc, if (intro_content && intro_size > 0) { int n_intro_points; if (rend_data->auth_type != REND_NO_AUTH && - !tor_mem_is_zero(rend_data->descriptor_cookie, - sizeof(rend_data->descriptor_cookie))) { + !safe_mem_is_zero(rend_data->descriptor_cookie, + sizeof(rend_data->descriptor_cookie))) { char *ipos_decrypted = NULL; size_t ipos_decrypted_size; if (rend_decrypt_introduction_points(&ipos_decrypted, diff --git a/src/lib/crypt_ops/crypto_ed25519.c b/src/lib/crypt_ops/crypto_ed25519.c index 400f963898..0581529125 100644 --- a/src/lib/crypt_ops/crypto_ed25519.c +++ b/src/lib/crypt_ops/crypto_ed25519.c @@ -226,7 +226,7 @@ ed25519_keypair_generate(ed25519_keypair_t *keypair_out, int extra_strong) int ed25519_public_key_is_zero(const ed25519_public_key_t *pubkey) { - return tor_mem_is_zero((char*)pubkey->pubkey, ED25519_PUBKEY_LEN); + return safe_mem_is_zero((char*)pubkey->pubkey, ED25519_PUBKEY_LEN); } /* Return a heap-allocated array that contains msg prefixed by the diff --git a/src/lib/crypt_ops/crypto_rand.c b/src/lib/crypt_ops/crypto_rand.c index 0b1cb96c1b..79c8ed1eed 100644 --- a/src/lib/crypt_ops/crypto_rand.c +++ b/src/lib/crypt_ops/crypto_rand.c @@ -36,6 +36,7 @@ #include "lib/defs/digest_sizes.h" #include "lib/crypt_ops/crypto_digest.h" +#include "lib/ctime/di_ops.h" #ifdef ENABLE_NSS #include "lib/crypt_ops/crypto_nss_mgt.h" @@ -314,7 +315,7 @@ crypto_strongest_rand_raw(uint8_t *out, size_t out_len) } } - if ((out_len < sanity_min_size) || !tor_mem_is_zero((char*)out, out_len)) + if ((out_len < sanity_min_size) || !safe_mem_is_zero((char*)out, out_len)) return 0; } From 295feeb09377c4d78f9ee43ec3197b908d7cb960 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 30 Apr 2019 14:49:05 -0400 Subject: [PATCH 0925/2557] Replace all remaining tor_mem_is_zero() with fast_mem_is_zero() --- src/core/mainloop/connection.c | 2 +- src/core/or/channeltls.c | 2 +- src/core/or/circuitbuild.c | 2 +- src/feature/client/bridges.c | 2 +- src/feature/dirauth/dirvote.c | 4 ++-- src/feature/dirauth/dsigs_parse.c | 2 +- src/feature/dirauth/shared_random.c | 4 ++-- src/feature/dirauth/shared_random_state.c | 2 +- src/feature/dirclient/dirclient.c | 2 +- src/feature/dircommon/voting_schedule.c | 2 +- src/feature/dirparse/ns_parse.c | 2 +- src/feature/hs/hs_common.c | 12 ++++++------ src/feature/hs/hs_descriptor.c | 20 ++++++++++---------- src/feature/hs/hs_intropoint.c | 4 ++-- src/feature/hs/hs_service.c | 8 ++++---- src/feature/nodelist/fmt_routerstatus.c | 2 +- src/feature/nodelist/microdesc.c | 2 +- src/feature/nodelist/nodelist.c | 2 +- src/feature/nodelist/routerlist.c | 4 ++-- src/feature/nodelist/torcert.c | 2 +- src/feature/relay/router.c | 8 ++++---- src/lib/crypt_ops/crypto_format.c | 2 +- src/lib/string/util_string.h | 3 --- src/test/test.c | 2 +- src/test/test_addr.c | 4 ++-- src/test/test_crypto.c | 4 ++-- src/test/test_crypto_slow.c | 2 +- src/test/test_extorport.c | 4 ++-- src/test/test_hs.c | 2 +- src/test/test_hs_cache.c | 2 +- src/test/test_hs_client.c | 4 ++-- src/test/test_hs_descriptor.c | 6 +++--- src/test/test_hs_service.c | 16 ++++++++-------- src/test/test_link_handshake.c | 4 ++-- src/test/test_routerkeys.c | 4 ++-- src/test/test_shared_random.c | 4 ++-- src/test/test_util.c | 14 +++++++------- src/test/test_util_format.c | 2 +- 38 files changed, 83 insertions(+), 86 deletions(-) diff --git a/src/core/mainloop/connection.c b/src/core/mainloop/connection.c index 30504e4edb..b9d1b95c2a 100644 --- a/src/core/mainloop/connection.c +++ b/src/core/mainloop/connection.c @@ -1654,7 +1654,7 @@ check_sockaddr(const struct sockaddr *sa, int len, int level) len,(int)sizeof(struct sockaddr_in6)); ok = 0; } - if (tor_mem_is_zero((void*)sin6->sin6_addr.s6_addr, 16) || + if (fast_mem_is_zero((void*)sin6->sin6_addr.s6_addr, 16) || sin6->sin6_port == 0) { log_fn(level, LD_NET, "Address for new connection has address/port equal to zero."); diff --git a/src/core/or/channeltls.c b/src/core/or/channeltls.c index f552b20770..772f583f2e 100644 --- a/src/core/or/channeltls.c +++ b/src/core/or/channeltls.c @@ -1722,7 +1722,7 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan) tor_assert(tor_digest_is_zero( (const char*)(chan->conn->handshake_state-> authenticated_rsa_peer_id))); - tor_assert(tor_mem_is_zero( + tor_assert(fast_mem_is_zero( (const char*)(chan->conn->handshake_state-> authenticated_ed25519_peer_id.pubkey), 32)); /* If the client never authenticated, it's a tor client or bridge diff --git a/src/core/or/circuitbuild.c b/src/core/or/circuitbuild.c index cfe0a97bcf..ececd83778 100644 --- a/src/core/or/circuitbuild.c +++ b/src/core/or/circuitbuild.c @@ -2989,7 +2989,7 @@ extend_info_supports_ntor(const extend_info_t* ei) { tor_assert(ei); /* Valid ntor keys have at least one non-zero byte */ - return !tor_mem_is_zero( + return !fast_mem_is_zero( (const char*)ei->curve25519_onion_key.public_key, CURVE25519_PUBKEY_LEN); } diff --git a/src/feature/client/bridges.c b/src/feature/client/bridges.c index ea1aff9519..f517876609 100644 --- a/src/feature/client/bridges.c +++ b/src/feature/client/bridges.c @@ -348,7 +348,7 @@ int node_is_a_configured_bridge(const node_t *node) { /* First, let's try searching for a bridge with matching identity. */ - if (BUG(tor_mem_is_zero(node->identity, DIGEST_LEN))) + if (BUG(fast_mem_is_zero(node->identity, DIGEST_LEN))) return 0; if (find_bridge_by_digest(node->identity) != NULL) diff --git a/src/feature/dirauth/dirvote.c b/src/feature/dirauth/dirvote.c index 1f861d2417..a02933b298 100644 --- a/src/feature/dirauth/dirvote.c +++ b/src/feature/dirauth/dirvote.c @@ -2599,7 +2599,7 @@ networkstatus_add_detached_signatures(networkstatus_t *target, return -1; } for (alg = DIGEST_SHA1; alg < N_COMMON_DIGEST_ALGORITHMS; ++alg) { - if (!tor_mem_is_zero(digests->d[alg], DIGEST256_LEN)) { + if (!fast_mem_is_zero(digests->d[alg], DIGEST256_LEN)) { if (fast_memeq(target->digests.d[alg], digests->d[alg], DIGEST256_LEN)) { ++n_matches; @@ -2795,7 +2795,7 @@ networkstatus_get_detached_signatures(smartlist_t *consensuses) char d[HEX_DIGEST256_LEN+1]; const char *alg_name = crypto_digest_algorithm_get_name(alg); - if (tor_mem_is_zero(ns->digests.d[alg], DIGEST256_LEN)) + if (fast_mem_is_zero(ns->digests.d[alg], DIGEST256_LEN)) continue; base16_encode(d, sizeof(d), ns->digests.d[alg], DIGEST256_LEN); smartlist_add_asprintf(elements, "additional-digest %s %s %s\n", diff --git a/src/feature/dirauth/dsigs_parse.c b/src/feature/dirauth/dsigs_parse.c index d88176fee9..c5c8e18866 100644 --- a/src/feature/dirauth/dsigs_parse.c +++ b/src/feature/dirauth/dsigs_parse.c @@ -127,7 +127,7 @@ networkstatus_parse_detached_signatures(const char *s, const char *eos) } digests = detached_get_digests(sigs, flavor); tor_assert(digests); - if (!tor_mem_is_zero(digests->d[alg], digest_length)) { + if (!fast_mem_is_zero(digests->d[alg], digest_length)) { log_warn(LD_DIR, "Multiple digests for %s with %s on detached " "signatures document", flavor, algname); continue; diff --git a/src/feature/dirauth/shared_random.c b/src/feature/dirauth/shared_random.c index 137c49800f..5ccf1a95e5 100644 --- a/src/feature/dirauth/shared_random.c +++ b/src/feature/dirauth/shared_random.c @@ -224,7 +224,7 @@ verify_commit_and_reveal(const sr_commit_t *commit) STATIC int commit_has_reveal_value(const sr_commit_t *commit) { - return !tor_mem_is_zero(commit->encoded_reveal, + return !fast_mem_is_zero(commit->encoded_reveal, sizeof(commit->encoded_reveal)); } @@ -486,7 +486,7 @@ get_vote_line_from_commit(const sr_commit_t *commit, sr_phase_t phase) { /* Send a reveal value for this commit if we have one. */ const char *reveal_str = commit->encoded_reveal; - if (tor_mem_is_zero(commit->encoded_reveal, + if (fast_mem_is_zero(commit->encoded_reveal, sizeof(commit->encoded_reveal))) { reveal_str = ""; } diff --git a/src/feature/dirauth/shared_random_state.c b/src/feature/dirauth/shared_random_state.c index a7b7480edd..9a045b283d 100644 --- a/src/feature/dirauth/shared_random_state.c +++ b/src/feature/dirauth/shared_random_state.c @@ -538,7 +538,7 @@ disk_state_put_commit_line(const sr_commit_t *commit, config_line_t *line) tor_assert(commit); tor_assert(line); - if (!tor_mem_is_zero(commit->encoded_reveal, + if (!fast_mem_is_zero(commit->encoded_reveal, sizeof(commit->encoded_reveal))) { /* Add extra whitespace so we can format the line correctly. */ tor_asprintf(&reveal_str, " %s", commit->encoded_reveal); diff --git a/src/feature/dirclient/dirclient.c b/src/feature/dirclient/dirclient.c index 0b79b07799..1ea50fd350 100644 --- a/src/feature/dirclient/dirclient.c +++ b/src/feature/dirclient/dirclient.c @@ -2531,7 +2531,7 @@ handle_response_fetch_microdesc(dir_connection_t *conn, conn->base_.port); tor_assert(conn->requested_resource && !strcmpstart(conn->requested_resource, "d/")); - tor_assert_nonfatal(!tor_mem_is_zero(conn->identity_digest, DIGEST_LEN)); + tor_assert_nonfatal(!fast_mem_is_zero(conn->identity_digest, DIGEST_LEN)); which = smartlist_new(); dir_split_resource_into_fingerprints(conn->requested_resource+2, which, NULL, diff --git a/src/feature/dircommon/voting_schedule.c b/src/feature/dircommon/voting_schedule.c index 0a7476eda7..5576ec69f7 100644 --- a/src/feature/dircommon/voting_schedule.c +++ b/src/feature/dircommon/voting_schedule.c @@ -150,7 +150,7 @@ voting_schedule_get_next_valid_after_time(void) /* This is a safe guard in order to make sure that the voting schedule * static object is at least initialized. Using this function with a zeroed * voting schedule can lead to bugs. */ - if (tor_mem_is_zero((const char *) &voting_schedule, + if (fast_mem_is_zero((const char *) &voting_schedule, sizeof(voting_schedule))) { need_to_recalculate_voting_schedule = true; goto done; /* no need for next check if we have to recalculate anyway */ diff --git a/src/feature/dirparse/ns_parse.c b/src/feature/dirparse/ns_parse.c index d653a59826..d5405e6464 100644 --- a/src/feature/dirparse/ns_parse.c +++ b/src/feature/dirparse/ns_parse.c @@ -1478,7 +1478,7 @@ networkstatus_parse_vote_from_string(const char *s, SMARTLIST_FOREACH_BEGIN(ns->routerstatus_list, vote_routerstatus_t *, vrs) { if (! vrs->has_ed25519_listing || - tor_mem_is_zero((const char *)vrs->ed25519_id, DIGEST256_LEN)) + fast_mem_is_zero((const char *)vrs->ed25519_id, DIGEST256_LEN)) continue; if (digest256map_get(ed_id_map, vrs->ed25519_id) != NULL) { log_warn(LD_DIR, "Vote networkstatus ed25519 identities were not " diff --git a/src/feature/hs/hs_common.c b/src/feature/hs/hs_common.c index d4736c2862..1e4f6d9166 100644 --- a/src/feature/hs/hs_common.c +++ b/src/feature/hs/hs_common.c @@ -1025,7 +1025,7 @@ hs_build_blinded_pubkey(const ed25519_public_key_t *pk, tor_assert(pk); tor_assert(blinded_pk_out); - tor_assert(!tor_mem_is_zero((char *) pk, ED25519_PUBKEY_LEN)); + tor_assert(!fast_mem_is_zero((char *) pk, ED25519_PUBKEY_LEN)); build_blinded_key_param(pk, secret, secret_len, time_period_num, get_time_period_length(), param); @@ -1050,8 +1050,8 @@ hs_build_blinded_keypair(const ed25519_keypair_t *kp, tor_assert(kp); tor_assert(blinded_kp_out); /* Extra safety. A zeroed key is bad. */ - tor_assert(!tor_mem_is_zero((char *) &kp->pubkey, ED25519_PUBKEY_LEN)); - tor_assert(!tor_mem_is_zero((char *) &kp->seckey, ED25519_SECKEY_LEN)); + tor_assert(!fast_mem_is_zero((char *) &kp->pubkey, ED25519_PUBKEY_LEN)); + tor_assert(!fast_mem_is_zero((char *) &kp->seckey, ED25519_SECKEY_LEN)); build_blinded_key_param(&kp->pubkey, secret, secret_len, time_period_num, get_time_period_length(), param); @@ -1283,15 +1283,15 @@ node_has_hsdir_index(const node_t *node) /* At this point, since the node has a desc, this node must also have an * hsdir index. If not, something went wrong, so BUG out. */ - if (BUG(tor_mem_is_zero((const char*)node->hsdir_index.fetch, + if (BUG(fast_mem_is_zero((const char*)node->hsdir_index.fetch, DIGEST256_LEN))) { return 0; } - if (BUG(tor_mem_is_zero((const char*)node->hsdir_index.store_first, + if (BUG(fast_mem_is_zero((const char*)node->hsdir_index.store_first, DIGEST256_LEN))) { return 0; } - if (BUG(tor_mem_is_zero((const char*)node->hsdir_index.store_second, + if (BUG(fast_mem_is_zero((const char*)node->hsdir_index.store_second, DIGEST256_LEN))) { return 0; } diff --git a/src/feature/hs/hs_descriptor.c b/src/feature/hs/hs_descriptor.c index b526da6661..52dc8690c3 100644 --- a/src/feature/hs/hs_descriptor.c +++ b/src/feature/hs/hs_descriptor.c @@ -678,7 +678,7 @@ get_auth_client_str(const hs_desc_authorized_client_t *client) char encrypted_cookie_b64[HS_DESC_ENCRYPED_COOKIE_LEN * 2]; #define ASSERT_AND_BASE64(field) STMT_BEGIN \ - tor_assert(!tor_mem_is_zero((char *) client->field, \ + tor_assert(!fast_mem_is_zero((char *) client->field, \ sizeof(client->field))); \ ret = base64_encode_nopad(field##_b64, sizeof(field##_b64), \ client->field, sizeof(client->field)); \ @@ -809,7 +809,7 @@ get_outer_encrypted_layer_plaintext(const hs_descriptor_t *desc, const curve25519_public_key_t *ephemeral_pubkey; ephemeral_pubkey = &desc->superencrypted_data.auth_ephemeral_pubkey; - tor_assert(!tor_mem_is_zero((char *) ephemeral_pubkey->public_key, + tor_assert(!fast_mem_is_zero((char *) ephemeral_pubkey->public_key, CURVE25519_PUBKEY_LEN)); curve25519_public_to_base64(ephemeral_key_base64, ephemeral_pubkey); @@ -1421,12 +1421,12 @@ decrypt_descriptor_cookie(const hs_descriptor_t *desc, tor_assert(desc); tor_assert(client); tor_assert(client_auth_sk); - tor_assert(!tor_mem_is_zero( + tor_assert(!fast_mem_is_zero( (char *) &desc->superencrypted_data.auth_ephemeral_pubkey, sizeof(desc->superencrypted_data.auth_ephemeral_pubkey))); - tor_assert(!tor_mem_is_zero((char *) client_auth_sk, + tor_assert(!fast_mem_is_zero((char *) client_auth_sk, sizeof(*client_auth_sk))); - tor_assert(!tor_mem_is_zero((char *) desc->subcredential, DIGEST256_LEN)); + tor_assert(!fast_mem_is_zero((char *) desc->subcredential, DIGEST256_LEN)); /* Get the KEYS component to derive the CLIENT-ID and COOKIE-KEY. */ keystream_length = @@ -2571,7 +2571,7 @@ hs_desc_decode_descriptor(const char *encoded, /* Subcredentials are not optional. */ if (BUG(!subcredential || - tor_mem_is_zero((char*)subcredential, DIGEST256_LEN))) { + fast_mem_is_zero((char*)subcredential, DIGEST256_LEN))) { log_warn(LD_GENERAL, "Tried to decrypt without subcred. Impossible!"); goto err; } @@ -2884,13 +2884,13 @@ hs_desc_build_authorized_client(const uint8_t *subcredential, tor_assert(descriptor_cookie); tor_assert(client_out); tor_assert(subcredential); - tor_assert(!tor_mem_is_zero((char *) auth_ephemeral_sk, + tor_assert(!fast_mem_is_zero((char *) auth_ephemeral_sk, sizeof(*auth_ephemeral_sk))); - tor_assert(!tor_mem_is_zero((char *) client_auth_pk, + tor_assert(!fast_mem_is_zero((char *) client_auth_pk, sizeof(*client_auth_pk))); - tor_assert(!tor_mem_is_zero((char *) descriptor_cookie, + tor_assert(!fast_mem_is_zero((char *) descriptor_cookie, HS_DESC_DESCRIPTOR_COOKIE_LEN)); - tor_assert(!tor_mem_is_zero((char *) subcredential, + tor_assert(!fast_mem_is_zero((char *) subcredential, DIGEST256_LEN)); /* Get the KEYS part so we can derive the CLIENT-ID and COOKIE-KEY. */ diff --git a/src/feature/hs/hs_intropoint.c b/src/feature/hs/hs_intropoint.c index c9cd3a0419..a568014a6d 100644 --- a/src/feature/hs/hs_intropoint.c +++ b/src/feature/hs/hs_intropoint.c @@ -392,7 +392,7 @@ validate_introduce1_parsed_cell(const trn_cell_introduce1_t *cell) * safety net here. The legacy ID must be zeroes in this case. */ legacy_key_id_len = trn_cell_introduce1_getlen_legacy_key_id(cell); legacy_key_id = trn_cell_introduce1_getconstarray_legacy_key_id(cell); - if (BUG(!tor_mem_is_zero((char *) legacy_key_id, legacy_key_id_len))) { + if (BUG(!fast_mem_is_zero((char *) legacy_key_id, legacy_key_id_len))) { goto invalid; } @@ -517,7 +517,7 @@ introduce1_cell_is_legacy(const uint8_t *request) /* If the first 20 bytes of the cell (DIGEST_LEN) are NOT zeroes, it * indicates a legacy cell (v2). */ - if (!tor_mem_is_zero((const char *) request, DIGEST_LEN)) { + if (!fast_mem_is_zero((const char *) request, DIGEST_LEN)) { /* Legacy cell. */ return 1; } diff --git a/src/feature/hs/hs_service.c b/src/feature/hs/hs_service.c index e3d0043460..0e145524a1 100644 --- a/src/feature/hs/hs_service.c +++ b/src/feature/hs/hs_service.c @@ -1746,7 +1746,7 @@ build_service_desc_superencrypted(const hs_service_t *service, sizeof(curve25519_public_key_t)); /* Test that subcred is not zero because we might use it below */ - if (BUG(tor_mem_is_zero((char*)desc->desc->subcredential, DIGEST256_LEN))) { + if (BUG(fast_mem_is_zero((char*)desc->desc->subcredential, DIGEST256_LEN))) { return -1; } @@ -1812,9 +1812,9 @@ build_service_desc_plaintext(const hs_service_t *service, tor_assert(service); tor_assert(desc); - tor_assert(!tor_mem_is_zero((char *) &desc->blinded_kp, + tor_assert(!fast_mem_is_zero((char *) &desc->blinded_kp, sizeof(desc->blinded_kp))); - tor_assert(!tor_mem_is_zero((char *) &desc->signing_kp, + tor_assert(!fast_mem_is_zero((char *) &desc->signing_kp, sizeof(desc->signing_kp))); /* Set the subcredential. */ @@ -1864,7 +1864,7 @@ build_service_desc_keys(const hs_service_t *service, ed25519_keypair_t kp; tor_assert(desc); - tor_assert(!tor_mem_is_zero((char *) &service->keys.identity_pk, + tor_assert(!fast_mem_is_zero((char *) &service->keys.identity_pk, ED25519_PUBKEY_LEN)); /* XXX: Support offline key feature (#18098). */ diff --git a/src/feature/nodelist/fmt_routerstatus.c b/src/feature/nodelist/fmt_routerstatus.c index 8c9212e05c..9a7d3082ef 100644 --- a/src/feature/nodelist/fmt_routerstatus.c +++ b/src/feature/nodelist/fmt_routerstatus.c @@ -233,7 +233,7 @@ routerstatus_format_entry(const routerstatus_t *rs, const char *version, } if (format == NS_V3_VOTE && vrs) { - if (tor_mem_is_zero((char*)vrs->ed25519_id, ED25519_PUBKEY_LEN)) { + if (fast_mem_is_zero((char*)vrs->ed25519_id, ED25519_PUBKEY_LEN)) { smartlist_add_strdup(chunks, "id ed25519 none\n"); } else { char ed_b64[BASE64_DIGEST256_LEN+1]; diff --git a/src/feature/nodelist/microdesc.c b/src/feature/nodelist/microdesc.c index 36922561a0..db2149754a 100644 --- a/src/feature/nodelist/microdesc.c +++ b/src/feature/nodelist/microdesc.c @@ -970,7 +970,7 @@ microdesc_list_missing_digest256(networkstatus_t *ns, microdesc_cache_t *cache, continue; if (skip && digest256map_get(skip, (const uint8_t*)rs->descriptor_digest)) continue; - if (tor_mem_is_zero(rs->descriptor_digest, DIGEST256_LEN)) + if (fast_mem_is_zero(rs->descriptor_digest, DIGEST256_LEN)) continue; /* XXXX Also skip if we're a noncache and wouldn't use this router. * XXXX NM Microdesc diff --git a/src/feature/nodelist/nodelist.c b/src/feature/nodelist/nodelist.c index 8aa4915107..21914c6c6d 100644 --- a/src/feature/nodelist/nodelist.c +++ b/src/feature/nodelist/nodelist.c @@ -1859,7 +1859,7 @@ microdesc_has_curve25519_onion_key(const microdesc_t *md) return 0; } - if (tor_mem_is_zero((const char*)md->onion_curve25519_pkey->public_key, + if (fast_mem_is_zero((const char*)md->onion_curve25519_pkey->public_key, CURVE25519_PUBKEY_LEN)) { return 0; } diff --git a/src/feature/nodelist/routerlist.c b/src/feature/nodelist/routerlist.c index 48f448ad1e..5fea1d1a8b 100644 --- a/src/feature/nodelist/routerlist.c +++ b/src/feature/nodelist/routerlist.c @@ -2969,7 +2969,7 @@ routerinfo_incompatible_with_extrainfo(const crypto_pk_t *identity_pkey, digest256_matches = tor_memeq(ei->digest256, sd->extra_info_digest256, DIGEST256_LEN); digest256_matches |= - tor_mem_is_zero(sd->extra_info_digest256, DIGEST256_LEN); + fast_mem_is_zero(sd->extra_info_digest256, DIGEST256_LEN); /* The identity must match exactly to have been generated at the same time * by the same router. */ @@ -3053,7 +3053,7 @@ routerinfo_has_curve25519_onion_key(const routerinfo_t *ri) return 0; } - if (tor_mem_is_zero((const char*)ri->onion_curve25519_pkey->public_key, + if (fast_mem_is_zero((const char*)ri->onion_curve25519_pkey->public_key, CURVE25519_PUBKEY_LEN)) { return 0; } diff --git a/src/feature/nodelist/torcert.c b/src/feature/nodelist/torcert.c index 56f1a8ac9f..270c14eb1c 100644 --- a/src/feature/nodelist/torcert.c +++ b/src/feature/nodelist/torcert.c @@ -74,7 +74,7 @@ tor_cert_sign_impl(const ed25519_keypair_t *signing_key, tor_assert(real_len == alloc_len); tor_assert(real_len > ED25519_SIG_LEN); uint8_t *sig = encoded + (real_len - ED25519_SIG_LEN); - tor_assert(tor_mem_is_zero((char*)sig, ED25519_SIG_LEN)); + tor_assert(fast_mem_is_zero((char*)sig, ED25519_SIG_LEN)); ed25519_signature_t signature; if (ed25519_sign(&signature, encoded, diff --git a/src/feature/relay/router.c b/src/feature/relay/router.c index ac4b3b7a02..82dad3191d 100644 --- a/src/feature/relay/router.c +++ b/src/feature/relay/router.c @@ -244,7 +244,7 @@ expire_old_onion_keys(void) lastonionkey = NULL; } - /* We zero out the keypair. See the tor_mem_is_zero() check made in + /* We zero out the keypair. See the fast_mem_is_zero() check made in * construct_ntor_key_map() below. */ memset(&last_curve25519_onion_key, 0, sizeof(last_curve25519_onion_key)); @@ -284,7 +284,7 @@ construct_ntor_key_map(void) { di_digest256_map_t *m = NULL; - if (!tor_mem_is_zero((const char*) + if (!fast_mem_is_zero((const char*) curve25519_onion_key.pubkey.public_key, CURVE25519_PUBKEY_LEN)) { dimap_add_entry(&m, @@ -292,7 +292,7 @@ construct_ntor_key_map(void) tor_memdup(&curve25519_onion_key, sizeof(curve25519_keypair_t))); } - if (!tor_mem_is_zero((const char*) + if (!fast_mem_is_zero((const char*) last_curve25519_onion_key.pubkey.public_key, CURVE25519_PUBKEY_LEN)) { dimap_add_entry(&m, @@ -1049,7 +1049,7 @@ init_keys(void) return -1; keydir = get_keydir_fname("secret_onion_key_ntor.old"); - if (tor_mem_is_zero((const char *) + if (fast_mem_is_zero((const char *) last_curve25519_onion_key.pubkey.public_key, CURVE25519_PUBKEY_LEN) && file_status(keydir) == FN_FILE) { diff --git a/src/lib/crypt_ops/crypto_format.c b/src/lib/crypt_ops/crypto_format.c index e11b391194..118cd79045 100644 --- a/src/lib/crypt_ops/crypto_format.c +++ b/src/lib/crypt_ops/crypto_format.c @@ -104,7 +104,7 @@ crypto_read_tagged_contents_from_file(const char *fname, prefix[32] = 0; /* Check type, extract tag. */ if (strcmpstart(prefix, "== ") || strcmpend(prefix, " ==") || - ! tor_mem_is_zero(prefix+strlen(prefix), 32-strlen(prefix))) { + ! fast_mem_is_zero(prefix+strlen(prefix), 32-strlen(prefix))) { saved_errno = EINVAL; goto end; } diff --git a/src/lib/string/util_string.h b/src/lib/string/util_string.h index 7e8af0578c..b3c6841d41 100644 --- a/src/lib/string/util_string.h +++ b/src/lib/string/util_string.h @@ -24,9 +24,6 @@ int fast_mem_is_zero(const char *mem, size_t len); #define fast_digest_is_zero(d) fast_mem_is_zero((d), DIGEST_LEN) #define fast_digetst256_is_zero(d) fast_mem_is_zero((d), DIGEST256_LEN) -// XXXX remove this after we replace all users. -#define tor_mem_is_zero fast_mem_is_zero - int tor_digest_is_zero(const char *digest); int tor_digest256_is_zero(const char *digest); diff --git a/src/test/test.c b/src/test/test.c index be5cb12b1e..77c58ee681 100644 --- a/src/test/test.c +++ b/src/test/test.c @@ -284,7 +284,7 @@ test_fast_handshake(void *arg) /* First, test an entire handshake. */ memset(client_handshake, 0, sizeof(client_handshake)); tt_int_op(0, OP_EQ, fast_onionskin_create(&state, client_handshake)); - tt_assert(! tor_mem_is_zero((char*)client_handshake, + tt_assert(! fast_mem_is_zero((char*)client_handshake, sizeof(client_handshake))); tt_int_op(0, OP_EQ, diff --git a/src/test/test_addr.c b/src/test/test_addr.c index 3a1a7b6997..05d8bf6c7b 100644 --- a/src/test/test_addr.c +++ b/src/test/test_addr.c @@ -240,7 +240,7 @@ test_addr_ip6_helpers(void *arg) tt_int_op(0,OP_EQ, tor_addr_lookup("9000::5", AF_UNSPEC, &t1)); tt_int_op(AF_INET6,OP_EQ, tor_addr_family(&t1)); tt_int_op(0x90,OP_EQ, tor_addr_to_in6_addr8(&t1)[0]); - tt_assert(tor_mem_is_zero((char*)tor_addr_to_in6_addr8(&t1)+1, 14)); + tt_assert(fast_mem_is_zero((char*)tor_addr_to_in6_addr8(&t1)+1, 14)); tt_int_op(0x05,OP_EQ, tor_addr_to_in6_addr8(&t1)[15]); /* === Test pton: valid af_inet6 */ @@ -697,7 +697,7 @@ test_addr_ip6_helpers(void *arg) &t1,&mask,&port1,&port2); tt_int_op(r,OP_EQ,AF_INET6); tt_int_op(tor_addr_family(&t1),OP_EQ,AF_INET6); - tt_assert(tor_mem_is_zero((const char*)tor_addr_to_in6_addr32(&t1), 16)); + tt_assert(fast_mem_is_zero((const char*)tor_addr_to_in6_addr32(&t1), 16)); tt_int_op(mask,OP_EQ,0); tt_int_op(port1,OP_EQ,1); tt_int_op(port2,OP_EQ,65535); diff --git a/src/test/test_crypto.c b/src/test/test_crypto.c index 08dfb6bcdd..872da3d2c5 100644 --- a/src/test/test_crypto.c +++ b/src/test/test_crypto.c @@ -389,7 +389,7 @@ test_crypto_aes128(void *arg) "\xff\xff\xff\xff\xff\xff\xff\xff" "\xff\xff\xff\xff\xff\xff\xff\xff"); crypto_cipher_crypt_inplace(env1, data2, 64); - tt_assert(tor_mem_is_zero(data2, 64)); + tt_assert(fast_mem_is_zero(data2, 64)); done: tor_free(mem_op_hex_tmp); @@ -2134,7 +2134,7 @@ test_crypto_curve25519_persist(void *arg) tt_u64_op((uint64_t)st.st_size, OP_EQ, 32+CURVE25519_PUBKEY_LEN+CURVE25519_SECKEY_LEN); tt_assert(fast_memeq(content, "== c25519v1: testing ==", taglen)); - tt_assert(tor_mem_is_zero(content+taglen, 32-taglen)); + tt_assert(fast_mem_is_zero(content+taglen, 32-taglen)); cp = content + 32; tt_mem_op(keypair.seckey.secret_key,OP_EQ, cp, diff --git a/src/test/test_crypto_slow.c b/src/test/test_crypto_slow.c index e24aee8930..3b20dfa587 100644 --- a/src/test/test_crypto_slow.c +++ b/src/test/test_crypto_slow.c @@ -109,7 +109,7 @@ run_s2k_tests(const unsigned flags, const unsigned type, secret_to_key_derivekey(buf3, sizeof(buf3), buf, speclen, pw1, strlen(pw1))); tt_mem_op(buf2, OP_EQ, buf3, sizeof(buf3)); - tt_assert(!tor_mem_is_zero((char*)buf2+keylen, sizeof(buf2)-keylen)); + tt_assert(!fast_mem_is_zero((char*)buf2+keylen, sizeof(buf2)-keylen)); done: ; diff --git a/src/test/test_extorport.c b/src/test/test_extorport.c index cfdd11d161..38aca90266 100644 --- a/src/test/test_extorport.c +++ b/src/test/test_extorport.c @@ -177,7 +177,7 @@ test_ext_or_init_auth(void *arg) /* Shouldn't be initialized already, or our tests will be a bit * meaningless */ ext_or_auth_cookie = tor_malloc_zero(32); - tt_assert(tor_mem_is_zero((char*)ext_or_auth_cookie, 32)); + tt_assert(fast_mem_is_zero((char*)ext_or_auth_cookie, 32)); /* Now make sure we use a temporary file */ fn = get_fname("ext_cookie_file"); @@ -202,7 +202,7 @@ test_ext_or_init_auth(void *arg) tt_mem_op(cp,OP_EQ, "! Extended ORPort Auth Cookie !\x0a", 32); tt_mem_op(cp+32,OP_EQ, ext_or_auth_cookie, 32); memcpy(cookie0, ext_or_auth_cookie, 32); - tt_assert(!tor_mem_is_zero((char*)ext_or_auth_cookie, 32)); + tt_assert(!fast_mem_is_zero((char*)ext_or_auth_cookie, 32)); /* Operation should be idempotent. */ tt_int_op(0, OP_EQ, init_ext_or_cookie_authentication(1)); diff --git a/src/test/test_hs.c b/src/test/test_hs.c index 5d3327c777..2b69aae547 100644 --- a/src/test/test_hs.c +++ b/src/test/test_hs.c @@ -448,7 +448,7 @@ test_hs_rend_data(void *arg) tt_int_op(client_v2->auth_type, OP_EQ, REND_BASIC_AUTH); tt_int_op(strlen(client_v2->onion_address), OP_EQ, 0); tt_mem_op(client_v2->desc_id_fetch, OP_EQ, desc_id, sizeof(desc_id)); - tt_int_op(tor_mem_is_zero(client_v2->descriptor_cookie, + tt_int_op(fast_mem_is_zero(client_v2->descriptor_cookie, sizeof(client_v2->descriptor_cookie)), OP_EQ, 1); tt_assert(client->hsdirs_fp); tt_int_op(smartlist_len(client->hsdirs_fp), OP_EQ, 0); diff --git a/src/test/test_hs_cache.c b/src/test/test_hs_cache.c index 2187c2be39..d71f8b6b18 100644 --- a/src/test/test_hs_cache.c +++ b/src/test/test_hs_cache.c @@ -486,7 +486,7 @@ test_client_cache(void *arg) NULL, &published_desc_str); tt_int_op(retval, OP_EQ, 0); memcpy(wanted_subcredential, published_desc->subcredential, DIGEST256_LEN); - tt_assert(!tor_mem_is_zero((char*)wanted_subcredential, DIGEST256_LEN)); + tt_assert(!fast_mem_is_zero((char*)wanted_subcredential, DIGEST256_LEN)); } /* Test handle_response_fetch_hsdesc_v3() */ diff --git a/src/test/test_hs_client.c b/src/test/test_hs_client.c index 8362b6cbda..6fd80051fd 100644 --- a/src/test/test_hs_client.c +++ b/src/test/test_hs_client.c @@ -395,7 +395,7 @@ test_client_pick_intro(void *arg) tt_assert(fetched_desc); tt_mem_op(fetched_desc->subcredential, OP_EQ, desc->subcredential, DIGEST256_LEN); - tt_assert(!tor_mem_is_zero((char*)fetched_desc->subcredential, + tt_assert(!fast_mem_is_zero((char*)fetched_desc->subcredential, DIGEST256_LEN)); tor_free(encoded); } @@ -433,7 +433,7 @@ test_client_pick_intro(void *arg) for (int i = 0; i < 64; ++i) { extend_info_t *ip = client_get_random_intro(&service_kp.pubkey); tor_assert(ip); - tt_assert(!tor_mem_is_zero((char*)ip->identity_digest, DIGEST_LEN)); + tt_assert(!fast_mem_is_zero((char*)ip->identity_digest, DIGEST_LEN)); tt_mem_op(ip->identity_digest, OP_EQ, chosen_intro_ei->identity_digest, DIGEST_LEN); extend_info_free(ip); diff --git a/src/test/test_hs_descriptor.c b/src/test/test_hs_descriptor.c index 5a3fd46dbe..6fe5573c0f 100644 --- a/src/test/test_hs_descriptor.c +++ b/src/test/test_hs_descriptor.c @@ -126,7 +126,7 @@ test_descriptor_padding(void *arg) tt_assert(padded_plaintext); tor_free(plaintext); /* Make sure our padding has been zeroed. */ - tt_int_op(tor_mem_is_zero((char *) padded_plaintext + plaintext_len, + tt_int_op(fast_mem_is_zero((char *) padded_plaintext + plaintext_len, padded_len - plaintext_len), OP_EQ, 1); tor_free(padded_plaintext); /* Never never have a padded length smaller than the plaintext. */ @@ -143,7 +143,7 @@ test_descriptor_padding(void *arg) tt_assert(padded_plaintext); tor_free(plaintext); /* Make sure our padding has been zeroed. */ - tt_int_op(tor_mem_is_zero((char *) padded_plaintext + plaintext_len, + tt_int_op(fast_mem_is_zero((char *) padded_plaintext + plaintext_len, padded_len - plaintext_len), OP_EQ, 1); tor_free(padded_plaintext); /* Never never have a padded length smaller than the plaintext. */ @@ -160,7 +160,7 @@ test_descriptor_padding(void *arg) tt_assert(padded_plaintext); tor_free(plaintext); /* Make sure our padding has been zeroed. */ - tt_int_op(tor_mem_is_zero((char *) padded_plaintext + plaintext_len, + tt_int_op(fast_mem_is_zero((char *) padded_plaintext + plaintext_len, padded_len - plaintext_len), OP_EQ, 1); tor_free(padded_plaintext); /* Never never have a padded length smaller than the plaintext. */ diff --git a/src/test/test_hs_service.c b/src/test/test_hs_service.c index 57132e6197..63f6069227 100644 --- a/src/test/test_hs_service.c +++ b/src/test/test_hs_service.c @@ -393,11 +393,11 @@ test_load_keys(void *arg) tt_assert(s); /* Ok we have the service object. Validate few things. */ - tt_assert(!tor_mem_is_zero(s->onion_address, sizeof(s->onion_address))); + tt_assert(!fast_mem_is_zero(s->onion_address, sizeof(s->onion_address))); tt_int_op(hs_address_is_valid(s->onion_address), OP_EQ, 1); - tt_assert(!tor_mem_is_zero((char *) s->keys.identity_sk.seckey, + tt_assert(!fast_mem_is_zero((char *) s->keys.identity_sk.seckey, ED25519_SECKEY_LEN)); - tt_assert(!tor_mem_is_zero((char *) s->keys.identity_pk.pubkey, + tt_assert(!fast_mem_is_zero((char *) s->keys.identity_pk.pubkey, ED25519_PUBKEY_LEN)); /* Check onion address from identity key. */ hs_build_address(&s->keys.identity_pk, s->config.version, addr); @@ -677,7 +677,7 @@ test_service_intro_point(void *arg) ip = helper_create_service_ip(); tt_assert(ip); /* Make sure the authentication keypair is not zeroes. */ - tt_int_op(tor_mem_is_zero((const char *) &ip->auth_key_kp, + tt_int_op(fast_mem_is_zero((const char *) &ip->auth_key_kp, sizeof(ed25519_keypair_t)), OP_EQ, 0); /* The introduce2_max MUST be in that range. */ tt_u64_op(ip->introduce2_max, OP_GE, @@ -1562,9 +1562,9 @@ test_build_update_descriptors(void *arg) tt_int_op(smartlist_len(ip_cur->base.link_specifiers), OP_EQ, 3); /* Make sure we have a valid encryption keypair generated when we pick an * intro point in the update process. */ - tt_assert(!tor_mem_is_zero((char *) ip_cur->enc_key_kp.seckey.secret_key, + tt_assert(!fast_mem_is_zero((char *) ip_cur->enc_key_kp.seckey.secret_key, CURVE25519_SECKEY_LEN)); - tt_assert(!tor_mem_is_zero((char *) ip_cur->enc_key_kp.pubkey.public_key, + tt_assert(!fast_mem_is_zero((char *) ip_cur->enc_key_kp.pubkey.public_key, CURVE25519_PUBKEY_LEN)); tt_u64_op(ip_cur->time_to_expire, OP_GE, now + INTRO_POINT_LIFETIME_MIN_SECONDS); @@ -1884,9 +1884,9 @@ test_rendezvous1_parsing(void *arg) } /* Send out the RENDEZVOUS1 and make sure that our mock func worked */ - tt_assert(tor_mem_is_zero(rend1_payload, 32)); + tt_assert(fast_mem_is_zero(rend1_payload, 32)); hs_circ_service_rp_has_opened(service, service_circ); - tt_assert(!tor_mem_is_zero(rend1_payload, 32)); + tt_assert(!fast_mem_is_zero(rend1_payload, 32)); tt_int_op(rend1_payload_len, OP_EQ, HS_LEGACY_RENDEZVOUS_CELL_SIZE); /******************************/ diff --git a/src/test/test_link_handshake.c b/src/test/test_link_handshake.c index 34f59f26cd..4e4c86aa0a 100644 --- a/src/test/test_link_handshake.c +++ b/src/test/test_link_handshake.c @@ -263,7 +263,7 @@ test_link_handshake_certs_ok(void *arg) tt_assert(c1->handshake_state->authenticated_rsa); tt_assert(! c1->handshake_state->authenticated_ed25519); } - tt_assert(! tor_mem_is_zero( + tt_assert(! fast_mem_is_zero( (char*)c1->handshake_state->authenticated_rsa_peer_id, 20)); chan2 = tor_malloc_zero(sizeof(*chan2)); @@ -290,7 +290,7 @@ test_link_handshake_certs_ok(void *arg) tt_ptr_op(c2->handshake_state->certs->ed_id_sign, OP_EQ, NULL); } tt_assert(c2->handshake_state->certs->id_cert); - tt_assert(tor_mem_is_zero( + tt_assert(fast_mem_is_zero( (char*)c2->handshake_state->authenticated_rsa_peer_id, 20)); /* no authentication has happened yet, since we haen't gotten an AUTH cell. */ diff --git a/src/test/test_routerkeys.c b/src/test/test_routerkeys.c index 102d9334a1..0c6b533698 100644 --- a/src/test/test_routerkeys.c +++ b/src/test/test_routerkeys.c @@ -399,7 +399,7 @@ test_routerkeys_ed_key_init_split(void *arg) tt_assert(kp2 != NULL); tt_assert(cert == NULL); tt_mem_op(&kp1->pubkey, OP_EQ, &kp2->pubkey, sizeof(kp2->pubkey)); - tt_assert(tor_mem_is_zero((char*)kp2->seckey.seckey, + tt_assert(fast_mem_is_zero((char*)kp2->seckey.seckey, sizeof(kp2->seckey.seckey))); ed25519_keypair_free(kp2); kp2 = NULL; @@ -409,7 +409,7 @@ test_routerkeys_ed_key_init_split(void *arg) tt_assert(kp2 != NULL); tt_assert(cert == NULL); tt_mem_op(&kp1->pubkey, OP_EQ, &kp2->pubkey, sizeof(kp2->pubkey)); - tt_assert(tor_mem_is_zero((char*)kp2->seckey.seckey, + tt_assert(fast_mem_is_zero((char*)kp2->seckey.seckey, sizeof(kp2->seckey.seckey))); ed25519_keypair_free(kp2); kp2 = NULL; diff --git a/src/test/test_shared_random.c b/src/test/test_shared_random.c index 480799383b..9fb88b9bee 100644 --- a/src/test/test_shared_random.c +++ b/src/test/test_shared_random.c @@ -449,12 +449,12 @@ test_sr_commit(void *arg) /* We should have a reveal value. */ tt_assert(commit_has_reveal_value(our_commit)); /* We should have a random value. */ - tt_assert(!tor_mem_is_zero((char *) our_commit->random_number, + tt_assert(!fast_mem_is_zero((char *) our_commit->random_number, sizeof(our_commit->random_number))); /* Commit and reveal timestamp should be the same. */ tt_u64_op(our_commit->commit_ts, OP_EQ, our_commit->reveal_ts); /* We should have a hashed reveal. */ - tt_assert(!tor_mem_is_zero(our_commit->hashed_reveal, + tt_assert(!fast_mem_is_zero(our_commit->hashed_reveal, sizeof(our_commit->hashed_reveal))); /* Do we have a valid encoded commit and reveal. Note the following only * tests if the generated values are correct. Their could be a bug in diff --git a/src/test/test_util.c b/src/test/test_util.c index 61e41e9a9b..1c6ac61eba 100644 --- a/src/test/test_util.c +++ b/src/test/test_util.c @@ -2087,14 +2087,14 @@ test_util_strmisc(void *arg) /* Test mem_is_zero */ memset(buf,0,128); buf[128] = 'x'; - tt_assert(tor_mem_is_zero(buf, 10)); - tt_assert(tor_mem_is_zero(buf, 20)); - tt_assert(tor_mem_is_zero(buf, 128)); - tt_assert(!tor_mem_is_zero(buf, 129)); + tt_assert(fast_mem_is_zero(buf, 10)); + tt_assert(fast_mem_is_zero(buf, 20)); + tt_assert(fast_mem_is_zero(buf, 128)); + tt_assert(!fast_mem_is_zero(buf, 129)); buf[60] = (char)255; - tt_assert(!tor_mem_is_zero(buf, 128)); + tt_assert(!fast_mem_is_zero(buf, 128)); buf[0] = (char)1; - tt_assert(!tor_mem_is_zero(buf, 10)); + tt_assert(!fast_mem_is_zero(buf, 10)); /* Test 'escaped' */ tt_ptr_op(escaped(NULL), OP_EQ, NULL); @@ -3789,7 +3789,7 @@ test_util_memarea(void *arg) tt_int_op(((uintptr_t)p3) % sizeof(void*),OP_EQ, 0); tt_assert(!memarea_owns_ptr(area, p3+8192)); tt_assert(!memarea_owns_ptr(area, p3+30)); - tt_assert(tor_mem_is_zero(p2, 52)); + tt_assert(fast_mem_is_zero(p2, 52)); /* Make sure we don't overalign. */ p1 = memarea_alloc(area, 1); p2 = memarea_alloc(area, 1); diff --git a/src/test/test_util_format.c b/src/test/test_util_format.c index c8945a707c..2859da66b2 100644 --- a/src/test/test_util_format.c +++ b/src/test/test_util_format.c @@ -367,7 +367,7 @@ test_util_format_base32_decode(void *arg) ret = base32_decode(dst, real_dstlen, "#abcde", 6); tt_int_op(ret, OP_EQ, -1); /* Make sure the destination buffer has been zeroed even on error. */ - tt_int_op(tor_mem_is_zero(dst, real_dstlen), OP_EQ, 1); + tt_int_op(fast_mem_is_zero(dst, real_dstlen), OP_EQ, 1); } done: From 853942b71e1a397cbe025dd9c101ccbab4bdaa11 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 30 Apr 2019 11:25:41 -0400 Subject: [PATCH 0926/2557] Make the recommend_pkg file dirauth-only. --- src/core/include.am | 2 +- src/feature/dirauth/recommend_pkg.h | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/core/include.am b/src/core/include.am index 4ec42182a6..dba957b6b5 100644 --- a/src/core/include.am +++ b/src/core/include.am @@ -157,7 +157,6 @@ LIBTOR_APP_A_SOURCES += \ src/feature/dirauth/dsigs_parse.c \ src/feature/dirauth/guardfraction.c \ src/feature/dirauth/reachability.c \ - src/feature/dirauth/recommend_pkg.c \ src/feature/dirauth/process_descs.c \ src/feature/dirauth/voteflags.c @@ -180,6 +179,7 @@ MODULE_DIRAUTH_SOURCES = \ src/feature/dirauth/dirauth_sys.c \ src/feature/dirauth/dircollate.c \ src/feature/dirauth/dirvote.c \ + src/feature/dirauth/recommend_pkg.c \ src/feature/dirauth/shared_random.c \ src/feature/dirauth/shared_random_state.c diff --git a/src/feature/dirauth/recommend_pkg.h b/src/feature/dirauth/recommend_pkg.h index 8200d78f72..1f97d50177 100644 --- a/src/feature/dirauth/recommend_pkg.h +++ b/src/feature/dirauth/recommend_pkg.h @@ -12,6 +12,18 @@ #ifndef TOR_RECOMMEND_PKG_H #define TOR_RECOMMEND_PKG_H +#ifdef HAVE_MODULE_DIRAUTH int validate_recommended_package_line(const char *line); +#else + +static inline int +validate_recommended_package_line(const char *line) +{ + (void) line; + return 0; +} + +#endif + #endif From aab02459243ef282849cd5015f3102a907ed1c17 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 30 Apr 2019 11:29:31 -0400 Subject: [PATCH 0927/2557] Make the dsigs_parse.c module dirauth-only. --- src/core/include.am | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/include.am b/src/core/include.am index dba957b6b5..f3ba10a3df 100644 --- a/src/core/include.am +++ b/src/core/include.am @@ -154,7 +154,6 @@ LIBTOR_APP_A_SOURCES = \ # the separation is only in the code location. LIBTOR_APP_A_SOURCES += \ src/feature/dirauth/bwauth.c \ - src/feature/dirauth/dsigs_parse.c \ src/feature/dirauth/guardfraction.c \ src/feature/dirauth/reachability.c \ src/feature/dirauth/process_descs.c \ @@ -179,6 +178,7 @@ MODULE_DIRAUTH_SOURCES = \ src/feature/dirauth/dirauth_sys.c \ src/feature/dirauth/dircollate.c \ src/feature/dirauth/dirvote.c \ + src/feature/dirauth/dsigs_parse.c \ src/feature/dirauth/recommend_pkg.c \ src/feature/dirauth/shared_random.c \ src/feature/dirauth/shared_random_state.c From 857bfc70330f9e808c99310f1efbc3fb5738c4fb Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 30 Apr 2019 11:31:03 -0400 Subject: [PATCH 0928/2557] Make the process_descs.c module dirauth-only. --- src/app/main/shutdown.c | 2 -- src/core/include.am | 2 +- src/feature/dirauth/dirauth_sys.c | 3 +++ 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/app/main/shutdown.c b/src/app/main/shutdown.c index 9239a0cf0f..fd9512ce4b 100644 --- a/src/app/main/shutdown.c +++ b/src/app/main/shutdown.c @@ -39,7 +39,6 @@ #include "feature/dirauth/bwauth.h" #include "feature/dirauth/dirvote.h" #include "feature/dirauth/keypin.h" -#include "feature/dirauth/process_descs.h" #include "feature/dirauth/shared_random.h" #include "feature/dircache/consdiffmgr.h" #include "feature/dircache/dirserv.h" @@ -128,7 +127,6 @@ tor_free_all(int postfork) routerlist_free_all(); networkstatus_free_all(); addressmap_free_all(); - dirserv_free_fingerprint_list(); dirserv_free_all(); dirserv_clear_measured_bw_cache(); rend_cache_free_all(); diff --git a/src/core/include.am b/src/core/include.am index f3ba10a3df..02b90ba180 100644 --- a/src/core/include.am +++ b/src/core/include.am @@ -156,7 +156,6 @@ LIBTOR_APP_A_SOURCES += \ src/feature/dirauth/bwauth.c \ src/feature/dirauth/guardfraction.c \ src/feature/dirauth/reachability.c \ - src/feature/dirauth/process_descs.c \ src/feature/dirauth/voteflags.c if BUILD_NT_SERVICES @@ -179,6 +178,7 @@ MODULE_DIRAUTH_SOURCES = \ src/feature/dirauth/dircollate.c \ src/feature/dirauth/dirvote.c \ src/feature/dirauth/dsigs_parse.c \ + src/feature/dirauth/process_descs.c \ src/feature/dirauth/recommend_pkg.c \ src/feature/dirauth/shared_random.c \ src/feature/dirauth/shared_random_state.c diff --git a/src/feature/dirauth/dirauth_sys.c b/src/feature/dirauth/dirauth_sys.c index bb482f2685..b87fa5dc29 100644 --- a/src/feature/dirauth/dirauth_sys.c +++ b/src/feature/dirauth/dirauth_sys.c @@ -9,6 +9,8 @@ #include "feature/dirauth/dirauth_sys.h" #include "feature/dirauth/dirvote.h" #include "feature/dirauth/dirauth_periodic.h" +#include "feature/dirauth/process_descs.h" + #include "lib/subsys/subsys.h" static int @@ -21,6 +23,7 @@ subsys_dirauth_initialize(void) static void subsys_dirauth_shutdown(void) { + dirserv_free_fingerprint_list(); dirvote_free_all(); } From 339ac4dc67af912e3c8608627e436fb866714538 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 30 Apr 2019 11:31:57 -0400 Subject: [PATCH 0929/2557] Make the guardfraction.c module dirauth-only. --- src/app/config/config.c | 2 ++ src/core/include.am | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/app/config/config.c b/src/app/config/config.c index 81a83e2c5f..3e0683eb56 100644 --- a/src/app/config/config.c +++ b/src/app/config/config.c @@ -3561,10 +3561,12 @@ options_validate(or_options_t *old_options, or_options_t *options, dirserv_read_measured_bandwidths(options->V3BandwidthsFile, NULL, NULL, NULL); } +#ifdef HAVE_MODULE_DIRAUTH /* same for guardfraction file */ if (options->GuardfractionFile && !old_options) { dirserv_read_guardfraction_file(options->GuardfractionFile, NULL); } +#endif } if (options->AuthoritativeDir && !options->DirPort_set) diff --git a/src/core/include.am b/src/core/include.am index 02b90ba180..8ddbdbc0b7 100644 --- a/src/core/include.am +++ b/src/core/include.am @@ -154,7 +154,6 @@ LIBTOR_APP_A_SOURCES = \ # the separation is only in the code location. LIBTOR_APP_A_SOURCES += \ src/feature/dirauth/bwauth.c \ - src/feature/dirauth/guardfraction.c \ src/feature/dirauth/reachability.c \ src/feature/dirauth/voteflags.c @@ -178,6 +177,7 @@ MODULE_DIRAUTH_SOURCES = \ src/feature/dirauth/dircollate.c \ src/feature/dirauth/dirvote.c \ src/feature/dirauth/dsigs_parse.c \ + src/feature/dirauth/guardfraction.c \ src/feature/dirauth/process_descs.c \ src/feature/dirauth/recommend_pkg.c \ src/feature/dirauth/shared_random.c \ From 996f7c75ba6abd9c4bc884f2c9a3b62889da9134 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 30 Apr 2019 11:33:00 -0400 Subject: [PATCH 0930/2557] Make the reachability.c module dirauth-only. --- src/core/include.am | 2 +- src/feature/nodelist/routerlist.c | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/core/include.am b/src/core/include.am index 8ddbdbc0b7..6367b01978 100644 --- a/src/core/include.am +++ b/src/core/include.am @@ -154,7 +154,6 @@ LIBTOR_APP_A_SOURCES = \ # the separation is only in the code location. LIBTOR_APP_A_SOURCES += \ src/feature/dirauth/bwauth.c \ - src/feature/dirauth/reachability.c \ src/feature/dirauth/voteflags.c if BUILD_NT_SERVICES @@ -179,6 +178,7 @@ MODULE_DIRAUTH_SOURCES = \ src/feature/dirauth/dsigs_parse.c \ src/feature/dirauth/guardfraction.c \ src/feature/dirauth/process_descs.c \ + src/feature/dirauth/reachability.c \ src/feature/dirauth/recommend_pkg.c \ src/feature/dirauth/shared_random.c \ src/feature/dirauth/shared_random_state.c diff --git a/src/feature/nodelist/routerlist.c b/src/feature/nodelist/routerlist.c index 48f448ad1e..88a5eeac6a 100644 --- a/src/feature/nodelist/routerlist.c +++ b/src/feature/nodelist/routerlist.c @@ -1926,6 +1926,8 @@ routerlist_remove_old_routers(void) void routerlist_descriptors_added(smartlist_t *sl, int from_cache) { + // XXXX use pubsub mechanism here. + tor_assert(sl); control_event_descriptors_changed(sl); SMARTLIST_FOREACH_BEGIN(sl, routerinfo_t *, ri) { @@ -1933,7 +1935,9 @@ routerlist_descriptors_added(smartlist_t *sl, int from_cache) learned_bridge_descriptor(ri, from_cache); if (ri->needs_retest_if_added) { ri->needs_retest_if_added = 0; +#ifdef HAVE_MODULE_DIRAUTH dirserv_single_reachability_test(approx_time(), ri); +#endif } } SMARTLIST_FOREACH_END(ri); } From cafb999810d89a1afc4e23561fb368d07650ccd5 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 30 Apr 2019 15:53:53 -0400 Subject: [PATCH 0931/2557] bump to 0.4.0.5 --- configure.ac | 4 ++-- contrib/win32build/tor-mingw.nsi.in | 2 +- src/win32/orconfig.h | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/configure.ac b/configure.ac index 062ff5168a..28bf5d1239 100644 --- a/configure.ac +++ b/configure.ac @@ -4,7 +4,7 @@ dnl Copyright (c) 2007-2019, The Tor Project, Inc. dnl See LICENSE for licensing information AC_PREREQ([2.63]) -AC_INIT([tor],[0.4.0.4-rc-dev]) +AC_INIT([tor],[0.4.0.5]) AC_CONFIG_SRCDIR([src/app/main/tor_main.c]) AC_CONFIG_MACRO_DIR([m4]) @@ -14,7 +14,7 @@ AC_CONFIG_MACRO_DIR([m4]) # version number changes. Tor uses it to make sure that it # only shuts down for missing "required protocols" when those protocols # are listed as required by a consensus after this date. -AC_DEFINE(APPROX_RELEASE_DATE, ["2019-04-11"], # for 0.4.0.4-rc-dev +AC_DEFINE(APPROX_RELEASE_DATE, ["2019-04-30"], # for 0.4.0.5 [Approximate date when this software was released. (Updated when the version changes.)]) # "foreign" means we don't follow GNU package layout standards diff --git a/contrib/win32build/tor-mingw.nsi.in b/contrib/win32build/tor-mingw.nsi.in index 38bbca4c66..3538b7caaa 100644 --- a/contrib/win32build/tor-mingw.nsi.in +++ b/contrib/win32build/tor-mingw.nsi.in @@ -8,7 +8,7 @@ !include "LogicLib.nsh" !include "FileFunc.nsh" !insertmacro GetParameters -!define VERSION "0.4.0.4-rc-dev" +!define VERSION "0.4.0.5" !define INSTALLER "tor-${VERSION}-win32.exe" !define WEBSITE "https://www.torproject.org/" !define LICENSE "LICENSE" diff --git a/src/win32/orconfig.h b/src/win32/orconfig.h index 3a7410b8d8..23722d6174 100644 --- a/src/win32/orconfig.h +++ b/src/win32/orconfig.h @@ -218,7 +218,7 @@ #define USING_TWOS_COMPLEMENT /* Version number of package */ -#define VERSION "0.4.0.4-rc-dev" +#define VERSION "0.4.0.5" From d91deeee45c3433cc7dcaf1e67a84692420db870 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Wed, 1 May 2019 12:41:49 +0300 Subject: [PATCH 0932/2557] pre-push.git-hook: Allow fixup and squash commits when pushing to non-upstream branches --- scripts/git/pre-push.git-hook | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/scripts/git/pre-push.git-hook b/scripts/git/pre-push.git-hook index 740180d6f6..238541d9f0 100755 --- a/scripts/git/pre-push.git-hook +++ b/scripts/git/pre-push.git-hook @@ -16,16 +16,6 @@ echo "Running pre-push hook" z40=0000000000000000000000000000000000000000 -remote="$1" - -ref_is_upstream_branch() { - if [ "$1" == "refs/heads/master" ] || - [[ "$1" == refs/heads/release-* ]] || - [[ "$1" == refs/heads/maint-* ]] - then - return 1 - fi -} workdir=$(git rev-parse --show-toplevel) if [ -x "$workdir/.git/hooks/pre-commit" ]; then @@ -40,6 +30,24 @@ if [ -e scripts/maint/practracker/practracker.py ]; then fi fi +remote="$1" +remote_loc="$2" + +if [[ "$remote_loc" != *github.com/torproject/tor.git ]] && + [[ "$remote_loc" != *torproject.org/tor.git ]]; then + echo "Not pushing to upstream - refraining from further checks" + exit 0 +fi + +ref_is_upstream_branch() { + if [ "$1" == "refs/heads/master" ] || + [[ "$1" == refs/heads/release-* ]] || + [[ "$1" == refs/heads/maint-* ]] + then + return 1 + fi +} + # shellcheck disable=SC2034 while read -r local_ref local_sha remote_ref remote_sha do From d0fb74c902e19bdec6e3a84a9ec5c70dccbf88d0 Mon Sep 17 00:00:00 2001 From: Mike Perry Date: Wed, 1 May 2019 21:00:26 +0000 Subject: [PATCH 0933/2557] Bug 29231: Report correct padding write totals and enabled totals. --- src/core/or/channel.c | 11 +++++++++++ src/core/or/channeltls.c | 4 ++-- src/core/or/circuitpadding.c | 2 +- src/core/or/connection_or.c | 5 ++++- src/core/or/relay.c | 2 +- src/core/or/relay.h | 1 + 6 files changed, 20 insertions(+), 5 deletions(-) diff --git a/src/core/or/channel.c b/src/core/or/channel.c index fd7bf62789..0e190809ba 100644 --- a/src/core/or/channel.c +++ b/src/core/or/channel.c @@ -1418,6 +1418,7 @@ write_packed_cell(channel_t *chan, packed_cell_t *cell) { int ret = -1; size_t cell_bytes; + uint8_t command = packed_cell_get_command(cell, chan->wide_circ_ids); tor_assert(chan); tor_assert(cell); @@ -1452,6 +1453,16 @@ write_packed_cell(channel_t *chan, packed_cell_t *cell) /* Successfully sent the cell. */ ret = 0; + /* Update padding statistics for the packed codepath.. */ + rep_hist_padding_count_write(PADDING_TYPE_TOTAL); + if (command == CELL_PADDING) + rep_hist_padding_count_write(PADDING_TYPE_CELL); + if (chan->padding_enabled) { + rep_hist_padding_count_write(PADDING_TYPE_ENABLED_TOTAL); + if (command == CELL_PADDING) + rep_hist_padding_count_write(PADDING_TYPE_ENABLED_CELL); + } + done: return ret; } diff --git a/src/core/or/channeltls.c b/src/core/or/channeltls.c index f552b20770..96d50f8a55 100644 --- a/src/core/or/channeltls.c +++ b/src/core/or/channeltls.c @@ -1094,13 +1094,13 @@ channel_tls_handle_cell(cell_t *cell, or_connection_t *conn) entry_guards_note_internet_connectivity(get_guard_selection_info()); rep_hist_padding_count_read(PADDING_TYPE_TOTAL); - if (TLS_CHAN_TO_BASE(chan)->currently_padding) + if (TLS_CHAN_TO_BASE(chan)->padding_enabled) rep_hist_padding_count_read(PADDING_TYPE_ENABLED_TOTAL); switch (cell->command) { case CELL_PADDING: rep_hist_padding_count_read(PADDING_TYPE_CELL); - if (TLS_CHAN_TO_BASE(chan)->currently_padding) + if (TLS_CHAN_TO_BASE(chan)->padding_enabled) rep_hist_padding_count_read(PADDING_TYPE_ENABLED_CELL); ++stats_n_padding_cells_processed; /* do nothing */ diff --git a/src/core/or/circuitpadding.c b/src/core/or/circuitpadding.c index d30edf9db8..9253c9e282 100644 --- a/src/core/or/circuitpadding.c +++ b/src/core/or/circuitpadding.c @@ -1020,6 +1020,7 @@ circpad_send_padding_cell_for_callback(circpad_machine_runtime_t *mi) "Callback: Sending padding to non-origin circuit."); relay_send_command_from_edge(0, mi->on_circ, RELAY_COMMAND_DROP, NULL, 0, NULL); + rep_hist_padding_count_write(PADDING_TYPE_DROP); } else { static ratelim_t cell_lim = RATELIM_INIT(600); log_fn_ratelim(&cell_lim,LOG_NOTICE,LD_CIRC, @@ -1028,7 +1029,6 @@ circpad_send_padding_cell_for_callback(circpad_machine_runtime_t *mi) } } - rep_hist_padding_count_write(PADDING_TYPE_DROP); /* This is a padding cell sent from the client or from the middle node, * (because it's invoked from circuitpadding.c) */ circpad_cell_event_padding_sent(circ); diff --git a/src/core/or/connection_or.c b/src/core/or/connection_or.c index e9b4b21955..830e09fd54 100644 --- a/src/core/or/connection_or.c +++ b/src/core/or/connection_or.c @@ -2308,6 +2308,8 @@ connection_or_write_cell_to_buf(const cell_t *cell, or_connection_t *conn) cell_pack(&networkcell, cell, conn->wide_circ_ids); + /* We need to count padding cells from this non-packed code path + * since they are sent via chan->write_cell() (which is not packed) */ rep_hist_padding_count_write(PADDING_TYPE_TOTAL); if (cell->command == CELL_PADDING) rep_hist_padding_count_write(PADDING_TYPE_CELL); @@ -2318,7 +2320,7 @@ connection_or_write_cell_to_buf(const cell_t *cell, or_connection_t *conn) if (conn->chan) { channel_timestamp_active(TLS_CHAN_TO_BASE(conn->chan)); - if (TLS_CHAN_TO_BASE(conn->chan)->currently_padding) { + if (TLS_CHAN_TO_BASE(conn->chan)->padding_enabled) { rep_hist_padding_count_write(PADDING_TYPE_ENABLED_TOTAL); if (cell->command == CELL_PADDING) rep_hist_padding_count_write(PADDING_TYPE_ENABLED_CELL); @@ -2348,6 +2350,7 @@ connection_or_write_var_cell_to_buf,(const var_cell_t *cell, if (conn->base_.state == OR_CONN_STATE_OR_HANDSHAKING_V3) or_handshake_state_record_var_cell(conn, conn->handshake_state, cell, 0); + rep_hist_padding_count_write(PADDING_TYPE_TOTAL); /* Touch the channel's active timestamp if there is one */ if (conn->chan) channel_timestamp_active(TLS_CHAN_TO_BASE(conn->chan)); diff --git a/src/core/or/relay.c b/src/core/or/relay.c index a166904a5f..59b580c141 100644 --- a/src/core/or/relay.c +++ b/src/core/or/relay.c @@ -2770,7 +2770,7 @@ set_streams_blocked_on_circ(circuit_t *circ, channel_t *chan, } /** Extract the command from a packed cell. */ -static uint8_t +uint8_t packed_cell_get_command(const packed_cell_t *cell, int wide_circ_ids) { if (wide_circ_ids) { diff --git a/src/core/or/relay.h b/src/core/or/relay.h index ea1b358ffb..126f3f868e 100644 --- a/src/core/or/relay.h +++ b/src/core/or/relay.h @@ -95,6 +95,7 @@ const uint8_t *decode_address_from_payload(tor_addr_t *addr_out, void circuit_clear_cell_queue(circuit_t *circ, channel_t *chan); circid_t packed_cell_get_circid(const packed_cell_t *cell, int wide_circ_ids); +uint8_t packed_cell_get_command(const packed_cell_t *cell, int wide_circ_ids); #ifdef RELAY_PRIVATE STATIC int connected_cell_parse(const relay_header_t *rh, const cell_t *cell, From 332617a81a874412bbadf331be74b7414545c197 Mon Sep 17 00:00:00 2001 From: Mike Perry Date: Wed, 1 May 2019 21:03:23 +0000 Subject: [PATCH 0934/2557] Changes file for bug29231. --- changes/bug29231 | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 changes/bug29231 diff --git a/changes/bug29231 b/changes/bug29231 new file mode 100644 index 0000000000..bcc19e1b48 --- /dev/null +++ b/changes/bug29231 @@ -0,0 +1,4 @@ + o Minor bugfixes (Channel padding statistics): + - Channel padding write totals and padding-enabled totals are now + counted properly in relay extrainfo descriptors. Fixes bug 29231; + bugfix on 0.3.1.1-alpha From e1771aeb51fd8f92db66fc1e9d0d8aad5ff9f169 Mon Sep 17 00:00:00 2001 From: Mike Perry Date: Wed, 1 May 2019 21:04:40 +0000 Subject: [PATCH 0935/2557] The practracker beatings will continue until our files get smaller. --- scripts/maint/practracker/exceptions.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/maint/practracker/exceptions.txt b/scripts/maint/practracker/exceptions.txt index d90ed1f4bd..49384029bb 100644 --- a/scripts/maint/practracker/exceptions.txt +++ b/scripts/maint/practracker/exceptions.txt @@ -72,7 +72,7 @@ problem include-count /src/core/mainloop/mainloop.c 66 problem function-size /src/core/mainloop/mainloop.c:conn_close_if_marked() 108 problem function-size /src/core/mainloop/mainloop.c:run_connection_housekeeping() 123 problem function-size /src/core/mainloop/mainloop.c:CALLBACK() 116 -problem file-size /src/core/or/channel.c 3476 +problem file-size /src/core/or/channel.c 3487 problem function-size /src/core/or/channeltls.c:channel_tls_handle_var_cell() 160 problem function-size /src/core/or/channeltls.c:channel_tls_process_versions_cell() 170 problem function-size /src/core/or/channeltls.c:channel_tls_process_netinfo_cell() 214 @@ -112,7 +112,7 @@ problem function-size /src/core/or/connection_edge.c:connection_ap_handshake_sen problem function-size /src/core/or/connection_edge.c:connection_ap_handshake_socks_resolved() 106 problem function-size /src/core/or/connection_edge.c:connection_exit_begin_conn() 184 problem function-size /src/core/or/connection_edge.c:connection_exit_connect() 102 -problem file-size /src/core/or/connection_or.c 3121 +problem file-size /src/core/or/connection_or.c 3124 problem include-count /src/core/or/connection_or.c 51 problem function-size /src/core/or/connection_or.c:connection_or_group_set_badness_() 105 problem function-size /src/core/or/connection_or.c:connection_or_client_learned_peer_id() 144 From 3d13841fa542889df741c8c5630ec2dd576f645d Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 2 May 2019 08:42:01 -0400 Subject: [PATCH 0936/2557] Remove changes files that are already in 0.4.0.5 or earlier --- changes/29241_diagnostic | 4 ---- changes/bug13221 | 5 ----- changes/bug27199 | 3 --- changes/bug28525 | 7 ------- changes/bug28614_better_logging | 6 ------ changes/bug28656 | 3 --- changes/bug28925 | 4 ---- changes/bug29017 | 4 ---- changes/bug29036 | 5 ----- changes/bug29144 | 5 ----- changes/bug29241 | 6 ------ changes/bug29500 | 3 --- changes/bug29527 | 5 ----- changes/bug29530_035 | 5 ----- changes/bug29562 | 4 ---- changes/bug29599 | 3 --- changes/bug29601 | 6 ------ changes/bug29665 | 7 ------- changes/bug29693 | 3 --- changes/bug29703 | 4 ---- changes/bug29706_minimal | 4 ---- changes/bug29706_refactor | 4 ---- changes/bug29874 | 4 ---- changes/bug29922 | 4 ---- changes/bug29930 | 4 ---- changes/bug29959-040 | 3 --- changes/bug30001 | 7 ------- changes/bug30011 | 4 ---- changes/bug30021 | 8 -------- changes/bug30040 | 9 --------- changes/bug30041 | 5 ----- changes/bug30263 | 3 --- changes/cid1444119 | 3 --- changes/diagnostic_28223_redux | 4 ---- changes/doc29121 | 3 --- changes/geoip-2019-03-04 | 4 ---- changes/geoip-2019-04-02 | 4 ---- changes/ticket21377 | 4 ---- changes/ticket29357 | 7 ------- changes/ticket29435 | 3 --- changes/ticket29631 | 4 ---- changes/ticket29806 | 7 ------- changes/ticket29897 | 3 --- changes/ticket29962 | 3 --- changes/ticket30117 | 4 ---- 45 files changed, 204 deletions(-) delete mode 100644 changes/29241_diagnostic delete mode 100644 changes/bug13221 delete mode 100644 changes/bug27199 delete mode 100644 changes/bug28525 delete mode 100644 changes/bug28614_better_logging delete mode 100644 changes/bug28656 delete mode 100644 changes/bug28925 delete mode 100644 changes/bug29017 delete mode 100644 changes/bug29036 delete mode 100644 changes/bug29144 delete mode 100644 changes/bug29241 delete mode 100644 changes/bug29500 delete mode 100644 changes/bug29527 delete mode 100644 changes/bug29530_035 delete mode 100644 changes/bug29562 delete mode 100644 changes/bug29599 delete mode 100644 changes/bug29601 delete mode 100644 changes/bug29665 delete mode 100644 changes/bug29693 delete mode 100644 changes/bug29703 delete mode 100644 changes/bug29706_minimal delete mode 100644 changes/bug29706_refactor delete mode 100644 changes/bug29874 delete mode 100644 changes/bug29922 delete mode 100644 changes/bug29930 delete mode 100644 changes/bug29959-040 delete mode 100644 changes/bug30001 delete mode 100644 changes/bug30011 delete mode 100644 changes/bug30021 delete mode 100644 changes/bug30040 delete mode 100644 changes/bug30041 delete mode 100644 changes/bug30263 delete mode 100644 changes/cid1444119 delete mode 100644 changes/diagnostic_28223_redux delete mode 100644 changes/doc29121 delete mode 100644 changes/geoip-2019-03-04 delete mode 100644 changes/geoip-2019-04-02 delete mode 100644 changes/ticket21377 delete mode 100644 changes/ticket29357 delete mode 100644 changes/ticket29435 delete mode 100644 changes/ticket29631 delete mode 100644 changes/ticket29806 delete mode 100644 changes/ticket29897 delete mode 100644 changes/ticket29962 delete mode 100644 changes/ticket30117 diff --git a/changes/29241_diagnostic b/changes/29241_diagnostic deleted file mode 100644 index 1e38654957..0000000000 --- a/changes/29241_diagnostic +++ /dev/null @@ -1,4 +0,0 @@ - o Minor features (NSS, diagnostic): - - Try to log an error from NSS (if there is any) and a more useful - description of our situation if we are using NSS and a call to - SSL_ExportKeyingMaterial() fails. Diagnostic for ticket 29241. diff --git a/changes/bug13221 b/changes/bug13221 deleted file mode 100644 index 13935a1921..0000000000 --- a/changes/bug13221 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor bugfixes (logging): - - Correct a misleading error message when IPv4Only or IPv6Only - is used but the resolved address can not be interpreted as an - address of the specified IP version. Fixes bug 13221; bugfix - on 0.2.3.9-alpha. Patch from Kris Katterjohn. diff --git a/changes/bug27199 b/changes/bug27199 deleted file mode 100644 index f9d2a422f9..0000000000 --- a/changes/bug27199 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor bugfixes (rust): - - Abort on panic in all build profiles, instead of potentially unwinding - into C code. Fixes bug 27199; bugfix on 0.3.3.1-alpha. diff --git a/changes/bug28525 b/changes/bug28525 deleted file mode 100644 index 988ffb2192..0000000000 --- a/changes/bug28525 +++ /dev/null @@ -1,7 +0,0 @@ - o Minor features (address selection): - - Make Tor aware of the RFC 6598 (Carrier Grade NAT) IP range, which is the - subnet 100.64.0.0/10. This is deployed by many ISPs as an alternative to - RFC 1918 that does not break existing internal networks. This patch fixes - security issues caused by RFC 6518 by blocking control ports on these - addresses and warns users if client ports or ExtORPorts are listening on - a RFC 6598 address. Closes ticket 28525. Patch by Neel Chauhan. diff --git a/changes/bug28614_better_logging b/changes/bug28614_better_logging deleted file mode 100644 index 26d19c3c11..0000000000 --- a/changes/bug28614_better_logging +++ /dev/null @@ -1,6 +0,0 @@ - o Minor bugfixes (logging): - - On Windows, when errors cause us to reload a consensus from disk, tell - the user that we are retrying at log level "notice". Previously we only - logged this information at "info", which was confusing because the - errors themselves were logged at "warning". Improves previous fix for - 28614. Fixes bug 30004; bugfix on 0.4.0.2-alpha. diff --git a/changes/bug28656 b/changes/bug28656 deleted file mode 100644 index d3a13d196c..0000000000 --- a/changes/bug28656 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor bugfixes (logging): - - Stop logging a BUG() warning when tor is waiting for exit descriptors. - Fixes bug 28656; bugfix on 0.3.5.1-alpha. diff --git a/changes/bug28925 b/changes/bug28925 deleted file mode 100644 index a867443885..0000000000 --- a/changes/bug28925 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (bootstrap reporting): - - During bootstrap reporting, correctly distinguish pluggable - transports from plain proxies. Fixes bug 28925; bugfix on - 0.4.0.1-alpha. diff --git a/changes/bug29017 b/changes/bug29017 deleted file mode 100644 index 5c4a53c43f..0000000000 --- a/changes/bug29017 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (stats): - - When ExtraInfoStatistics is 0, stop including PaddingStatistics in - relay and bridge extra-info documents. Fixes bug 29017; - bugfix on 0.3.1.1-alpha. diff --git a/changes/bug29036 b/changes/bug29036 deleted file mode 100644 index 8b96c5c8fa..0000000000 --- a/changes/bug29036 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor bugfix (continuous integration): - - Reset coverage state on disk after Travis CI has finished. This is being - done to prevent future gcda file merge errors which causes the test suite - for the process subsystem to fail. The process subsystem was introduced - in 0.4.0.1-alpha. Fixes bug 29036; bugfix on 0.2.9.15. diff --git a/changes/bug29144 b/changes/bug29144 deleted file mode 100644 index 5801224f14..0000000000 --- a/changes/bug29144 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor bugfixes (logging): - - Log the correct port number for listening sockets when "auto" is - used to let Tor pick the port number. Previously, port 0 was - logged instead of the actual port number. Fixes bug 29144; - bugfix on 0.3.5.1-alpha. Patch from Kris Katterjohn. diff --git a/changes/bug29241 b/changes/bug29241 deleted file mode 100644 index 7f25e154d1..0000000000 --- a/changes/bug29241 +++ /dev/null @@ -1,6 +0,0 @@ - o Major bugfixes (NSS, relay): - - When running with NSS, disable TLS 1.2 ciphersuites that use SHA384 - for their PRF. Due to an NSS bug, the TLS key exporters for these - ciphersuites don't work -- which caused relays to fail to handshake - with one another when these ciphersuites were enabled. - Fixes bug 29241; bugfix on 0.3.5.1-alpha. diff --git a/changes/bug29500 b/changes/bug29500 deleted file mode 100644 index 16550935b2..0000000000 --- a/changes/bug29500 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor bugfixes (circuitpadding testing): - - Minor tweaks to avoid very rare test failures related to timers and - monotime. Fixes bug 29500; bugfix on 0.4.0.1-alpha diff --git a/changes/bug29527 b/changes/bug29527 deleted file mode 100644 index 6f36a9e1a0..0000000000 --- a/changes/bug29527 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor features (circuit padding): - - Stop warning about undefined behavior in the probability distribution - tests. Float division by zero may technically be undefined behaviour in - C, but it's well-defined in IEEE 754. Partial backport of 29298. - Closes ticket 29527; bugfix on 0.4.0.1-alpha. diff --git a/changes/bug29530_035 b/changes/bug29530_035 deleted file mode 100644 index 6dfcd51e7b..0000000000 --- a/changes/bug29530_035 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor bugfixes (testing): - - Downgrade some LOG_ERR messages in the address/* tests to warnings. - The LOG_ERR messages were occurring when we had no configured network. - We were failing the unit tests, because we backported 28668 to 0.3.5.8, - but did not backport 29530. Fixes bug 29530; bugfix on 0.3.5.8. diff --git a/changes/bug29562 b/changes/bug29562 deleted file mode 100644 index 0621cd09a0..0000000000 --- a/changes/bug29562 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (pluggable transports): - - Fix an assertion failure crash bug when a pluggable transport process is - terminated during the bootstrap phase. Fixes bug 29562; bugfix on - 0.4.0.1-alpha. diff --git a/changes/bug29599 b/changes/bug29599 deleted file mode 100644 index 14e2f5d077..0000000000 --- a/changes/bug29599 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor bugfixes (memory management, testing): - - Stop leaking parts of the shared random state in the shared-random unit - tests. Fixes bug 29599; bugfix on 0.2.9.1-alpha. diff --git a/changes/bug29601 b/changes/bug29601 deleted file mode 100644 index c4ba5fbc8b..0000000000 --- a/changes/bug29601 +++ /dev/null @@ -1,6 +0,0 @@ - o Minor bugfixes (Windows, CI): - - Skip the Appveyor 32-bit Windows Server 2016 job, and 64-bit Windows - Server 2012 R2 job. The remaining 2 jobs still provide coverage of - 64/32-bit, and Windows Server 2016/2012 R2. Also set fast_finish, so - failed jobs terminate the build immediately. - Fixes bug 29601; bugfix on 0.3.5.4-alpha. diff --git a/changes/bug29665 b/changes/bug29665 deleted file mode 100644 index d89046faf5..0000000000 --- a/changes/bug29665 +++ /dev/null @@ -1,7 +0,0 @@ - o Minor bugfixes (single onion services): - - Allow connections to single onion services to remain idle without - being disconnected. Relays acting as rendezvous points for - single onion services were mistakenly closing idle established - rendezvous circuits after 60 seconds, thinking that they are unused - directory-fetching circuits that had served their purpose. Fixes - bug 29665; bugfix on 0.2.1.26. diff --git a/changes/bug29693 b/changes/bug29693 deleted file mode 100644 index 33ce051c40..0000000000 --- a/changes/bug29693 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor bugfixes (unit tests): - - Decrease the false positive rate of stochastic probability distribution - tests. Fixes bug 29693; bugfix on 0.4.0.1-alpha. \ No newline at end of file diff --git a/changes/bug29703 b/changes/bug29703 deleted file mode 100644 index 0e17ee45e6..0000000000 --- a/changes/bug29703 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (testing): - - Backport the 0.3.4 src/test/test-network.sh to 0.2.9. - We need a recent test-network.sh to use new chutney features in CI. - Fixes bug 29703; bugfix on 0.2.9.1-alpha. diff --git a/changes/bug29706_minimal b/changes/bug29706_minimal deleted file mode 100644 index 9d4a43326c..0000000000 --- a/changes/bug29706_minimal +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (memory management, testing): - - Stop leaking parts of the shared random state in the shared-random unit - tests. The previous fix in 29599 was incomplete. - Fixes bug 29706; bugfix on 0.2.9.1-alpha. diff --git a/changes/bug29706_refactor b/changes/bug29706_refactor deleted file mode 100644 index ba1d0c7edd..0000000000 --- a/changes/bug29706_refactor +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (memory management): - - Refactor the shared random state's memory management so that it actually - takes ownership of the shared random value pointers. - Fixes bug 29706; bugfix on 0.2.9.1-alpha. diff --git a/changes/bug29874 b/changes/bug29874 deleted file mode 100644 index 8534753b51..0000000000 --- a/changes/bug29874 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (pluggable transports): - - Restore old behaviour when it comes to discovering the path of a given - Pluggable Transport exe-file. Fixes bug 29874; bugfix on 0.4.0.1-alpha. - diff --git a/changes/bug29922 b/changes/bug29922 deleted file mode 100644 index dacb951097..0000000000 --- a/changes/bug29922 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (testing, windows): - - Fix a test failure caused by an unexpected bug warning in - our test for tor_gmtime_r(-1). Fixes bug 29922; - bugfix on 0.2.9.3-alpha. diff --git a/changes/bug29930 b/changes/bug29930 deleted file mode 100644 index a99b11430b..0000000000 --- a/changes/bug29930 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (UI): - - Lower log level of unlink() errors during bootstrap. Fixes bug 29930; - bugfix on 0.4.0.1-alpha. - diff --git a/changes/bug29959-040 b/changes/bug29959-040 deleted file mode 100644 index 3740e0169a..0000000000 --- a/changes/bug29959-040 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor bugfixes (directory authorities): - - Actually include the bandwidth-file-digest line in directory authority - votes. Fixes bug 29959; bugfix on 0.4.0.2-alpha. diff --git a/changes/bug30001 b/changes/bug30001 deleted file mode 100644 index 52e58872ef..0000000000 --- a/changes/bug30001 +++ /dev/null @@ -1,7 +0,0 @@ - o Minor features (testing): - - Use the approx_time() function when setting the "Expires" header - in directory replies, to make them more testable. Needed for - ticket 30001. - o Minor bug fixes (testing): - - Check the time in the "Expires" header with approx_time(). - Fixes bug 30001; bugfix on 0.4.0.4-rc. diff --git a/changes/bug30011 b/changes/bug30011 deleted file mode 100644 index 4c9069e291..0000000000 --- a/changes/bug30011 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (CI): - - Terminate test-stem if it takes more than 9.5 minutes to run. - (Travis terminates the job after 10 minutes of no output.) - Diagnostic for 29437. Fixes bug 30011; bugfix on 0.3.5.4-alpha. diff --git a/changes/bug30021 b/changes/bug30021 deleted file mode 100644 index 2a887f3cf2..0000000000 --- a/changes/bug30021 +++ /dev/null @@ -1,8 +0,0 @@ - o Minor bugfixes (TLS protocol, integration tests): - - When classifying a client's selection of TLS ciphers, if the client - ciphers are not yet available, do not cache the result. Previously, - we had cached the unavailability of the cipher list and never looked - again, which in turn led us to assume that the client only supported - the ancient V1 link protocol. This, in turn, was causing Stem - integration tests to stall in some cases. - Fixes bug 30021; bugfix on 0.2.4.8-alpha. diff --git a/changes/bug30040 b/changes/bug30040 deleted file mode 100644 index 7d80528a10..0000000000 --- a/changes/bug30040 +++ /dev/null @@ -1,9 +0,0 @@ - o Minor bugfixes (security): - - Fix a potential double free bug when reading huge bandwidth files. The - issue is not exploitable in the current Tor network because the - vulnerable code is only reached when directory authorities read bandwidth - files, but bandwidth files come from a trusted source (usually the - authorities themselves). Furthermore, the issue is only exploitable in - rare (non-POSIX) 32-bit architectures which are not used by any of the - current authorities. Fixes bug 30040; bugfix on 0.3.5.1-alpha. Bug found - and fixed by Tobias Stoeckmann. diff --git a/changes/bug30041 b/changes/bug30041 deleted file mode 100644 index 801c8f67ac..0000000000 --- a/changes/bug30041 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor bugfixes (hardening): - - Verify in more places that we are not about to create a buffer - with more than INT_MAX bytes, to avoid possible OOB access in the event - of bugs. Fixes bug 30041; bugfix on 0.2.0.16. Found and fixed by - Tobias Stoeckmann. diff --git a/changes/bug30263 b/changes/bug30263 deleted file mode 100644 index ba81c1b8a1..0000000000 --- a/changes/bug30263 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor bugfixes (shellcheck): - - Stop looking for scripts in the build directory during - "make shellcheck". Fixes bug 30263; bugfix on 0.4.0.1-alpha. diff --git a/changes/cid1444119 b/changes/cid1444119 deleted file mode 100644 index bb6854e66f..0000000000 --- a/changes/cid1444119 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor bugfixes (C correctness): - - Fix an unlikely memory leak in consensus_diff_apply(). Fixes bug 29824; - bugfix on 0.3.1.1-alpha. This is Coverity warning CID 1444119. diff --git a/changes/diagnostic_28223_redux b/changes/diagnostic_28223_redux deleted file mode 100644 index 0d7499832e..0000000000 --- a/changes/diagnostic_28223_redux +++ /dev/null @@ -1,4 +0,0 @@ - o Minor features (diagnostic): - - Add more diagnostic log messages in an attempt to solve - the issue of NUL bytes appearing in a microdescriptor cache. - Related to ticket 28223. diff --git a/changes/doc29121 b/changes/doc29121 deleted file mode 100644 index dd31cc9c70..0000000000 --- a/changes/doc29121 +++ /dev/null @@ -1,3 +0,0 @@ - o Documentation: - - Clarify that Tor performs stream isolation between *Port listeners by - default. Resolves issue 29121. diff --git a/changes/geoip-2019-03-04 b/changes/geoip-2019-03-04 deleted file mode 100644 index c8ce5dad5d..0000000000 --- a/changes/geoip-2019-03-04 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor features (geoip): - - Update geoip and geoip6 to the March 4 2019 Maxmind GeoLite2 - Country database. Closes ticket 29666. - diff --git a/changes/geoip-2019-04-02 b/changes/geoip-2019-04-02 deleted file mode 100644 index 7302d939f6..0000000000 --- a/changes/geoip-2019-04-02 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor features (geoip): - - Update geoip and geoip6 to the April 2 2019 Maxmind GeoLite2 - Country database. Closes ticket 29992. - diff --git a/changes/ticket21377 b/changes/ticket21377 deleted file mode 100644 index 2bf5149a0a..0000000000 --- a/changes/ticket21377 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor features (dircache): - - When a directory authority is using a bandwidth file to obtain the - bandwidth values that will be included in the next vote, serve this - bandwidth file at /tor/status-vote/next/bandwidth. Closes ticket 21377. \ No newline at end of file diff --git a/changes/ticket29357 b/changes/ticket29357 deleted file mode 100644 index 3aab930cd4..0000000000 --- a/changes/ticket29357 +++ /dev/null @@ -1,7 +0,0 @@ - o Minor features (dormant mode): - - Add a DormantCanceledByStartup option to tell Tor that it should - treat a startup event as cancelling any previous dormant state. - Integrators should use this option with caution: it should - only be used if Tor is being started because of something that the - user did, and not if Tor is being automatically started in the - background. Closes ticket 29357. diff --git a/changes/ticket29435 b/changes/ticket29435 deleted file mode 100644 index d48ae98e4b..0000000000 --- a/changes/ticket29435 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor bugfixes (testing): - - Fix our gcov wrapper script to look for object files at the - correct locations. Fixes bug 29435; bugfix on 0.3.5.1-alpha. diff --git a/changes/ticket29631 b/changes/ticket29631 deleted file mode 100644 index 9fc194ba96..0000000000 --- a/changes/ticket29631 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (Rust, protover): - - The Rust implementation of protover was missing the "Padding" value in - the translate function from C to Rust. Fixes bug 29631; bugfix on - 0.4.0.1-alpha. diff --git a/changes/ticket29806 b/changes/ticket29806 deleted file mode 100644 index 6afefd4c04..0000000000 --- a/changes/ticket29806 +++ /dev/null @@ -1,7 +0,0 @@ - o Minor features (bandwidth authority): - - Make bandwidth authorities to ignore relays that are reported in the - bandwidth file with the key-value "vote=0". - This change allows to report the relays that were not measured due - some failure and diagnose the reasons without the bandwidth being included in the - bandwidth authorities vote. - Closes ticket 29806. diff --git a/changes/ticket29897 b/changes/ticket29897 deleted file mode 100644 index 232a79fbce..0000000000 --- a/changes/ticket29897 +++ /dev/null @@ -1,3 +0,0 @@ - o Code simplification and refactoring: - - Refactor handle_get_next_bandwidth() to use connection_dir_buf_add(). - Implements ticket 29897. diff --git a/changes/ticket29962 b/changes/ticket29962 deleted file mode 100644 index e36cc0cf9a..0000000000 --- a/changes/ticket29962 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor features (continuous integration): - - On Travis Rust builds, cleanup Rust registry and refrain from caching - target/ directory to speed up builds. Resolves issue 29962. diff --git a/changes/ticket30117 b/changes/ticket30117 deleted file mode 100644 index 5b6e6dabf7..0000000000 --- a/changes/ticket30117 +++ /dev/null @@ -1,4 +0,0 @@ - o Testing (continuous integration): - - In Travis, tell timelimit to use stem's backtrace signals. And launch - python directly from timelimit, so python receives the signals from - timelimit, rather than make. Closes ticket 30117. From 77bd219808ac82c231aef37672e7fb212cd83d15 Mon Sep 17 00:00:00 2001 From: David Goulet Date: Thu, 2 May 2019 08:58:58 -0400 Subject: [PATCH 0937/2557] sendme: Improve logging messages Signed-off-by: David Goulet --- src/core/or/sendme.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/core/or/sendme.c b/src/core/or/sendme.c index 6f451d38e6..70ff3798ba 100644 --- a/src/core/or/sendme.c +++ b/src/core/or/sendme.c @@ -129,8 +129,8 @@ cell_version_is_valid(uint8_t cell_version) /* Can we handle this version? */ if (accept_version > SENDME_MAX_SUPPORTED_VERSION) { log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, - "Unable to handle SENDME version %u. We only support <= %d " - "(from consensus). Probably your tor is too old?", + "Unable to accept SENDME version %u (from consensus). " + "We only support <= %d. Probably your tor is too old?", accept_version, cell_version); goto invalid; } @@ -138,8 +138,7 @@ cell_version_is_valid(uint8_t cell_version) /* We only accept a SENDME cell from what the consensus tells us. */ if (cell_version < accept_version) { log_info(LD_PROTOCOL, "Unacceptable SENDME version %d. Only " - "accepting %u (taken from the consensus). " - "Closing circuit.", + "accepting %u (from consensus). Closing circuit.", cell_version, accept_version); goto invalid; } From 6f42efaa5926a2f1be89e3b591311f2130931db8 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 30 Apr 2019 12:42:33 -0400 Subject: [PATCH 0938/2557] Move voteflags.[ch] to become dirauth only. For various reasons, this was a nontrivial movement. There are several places in the code where we do something like "update the flags on this routerstatus or node if we're an authority", and at least one where we pretended to be an authority when we weren't. --- src/core/include.am | 9 ++- src/core/mainloop/mainloop.c | 21 +------ src/feature/control/fmt_serverstatus.c | 1 - src/feature/dirauth/bridgeauth.c | 55 ++++++++++++++++ src/feature/dirauth/bridgeauth.h | 12 ++++ src/feature/dirauth/dirauth_periodic.c | 19 ++++++ src/feature/dirauth/dirvote.c | 4 +- src/feature/dirauth/voteflags.c | 66 +++++++------------ src/feature/dirauth/voteflags.h | 12 ++-- src/feature/nodelist/networkstatus.c | 87 +++++++++++++------------- src/feature/nodelist/networkstatus.h | 5 +- src/test/test_voting_flags.c | 2 +- 12 files changed, 173 insertions(+), 120 deletions(-) create mode 100644 src/feature/dirauth/bridgeauth.c create mode 100644 src/feature/dirauth/bridgeauth.h diff --git a/src/core/include.am b/src/core/include.am index 6367b01978..63d4264210 100644 --- a/src/core/include.am +++ b/src/core/include.am @@ -153,8 +153,8 @@ LIBTOR_APP_A_SOURCES = \ # These should eventually move into module_dirauth_sources, but for now # the separation is only in the code location. LIBTOR_APP_A_SOURCES += \ - src/feature/dirauth/bwauth.c \ - src/feature/dirauth/voteflags.c + src/feature/dirauth/bwauth.c + if BUILD_NT_SERVICES LIBTOR_APP_A_SOURCES += src/app/main/ntmain.c @@ -171,6 +171,7 @@ LIBTOR_APP_TESTING_A_SOURCES = $(LIBTOR_APP_A_SOURCES) # The Directory Authority module. MODULE_DIRAUTH_SOURCES = \ src/feature/dirauth/authmode.c \ + src/feature/dirauth/bridgeauth.c \ src/feature/dirauth/dirauth_periodic.c \ src/feature/dirauth/dirauth_sys.c \ src/feature/dirauth/dircollate.c \ @@ -181,7 +182,8 @@ MODULE_DIRAUTH_SOURCES = \ src/feature/dirauth/reachability.c \ src/feature/dirauth/recommend_pkg.c \ src/feature/dirauth/shared_random.c \ - src/feature/dirauth/shared_random_state.c + src/feature/dirauth/shared_random_state.c \ + src/feature/dirauth/voteflags.c if BUILD_MODULE_DIRAUTH LIBTOR_APP_A_SOURCES += $(MODULE_DIRAUTH_SOURCES) @@ -310,6 +312,7 @@ noinst_HEADERS += \ src/feature/control/fmt_serverstatus.h \ src/feature/control/getinfo_geoip.h \ src/feature/dirauth/authmode.h \ + src/feature/dirauth/bridgeauth.h \ src/feature/dirauth/bwauth.h \ src/feature/dirauth/dirauth_periodic.h \ src/feature/dirauth/dirauth_sys.h \ diff --git a/src/core/mainloop/mainloop.c b/src/core/mainloop/mainloop.c index 30dad956ae..eb875df023 100644 --- a/src/core/mainloop/mainloop.c +++ b/src/core/mainloop/mainloop.c @@ -75,6 +75,7 @@ #include "feature/control/control.h" #include "feature/control/control_events.h" #include "feature/dirauth/authmode.h" +#include "feature/dirauth/bridgeauth.h" #include "feature/dircache/consdiffmgr.h" #include "feature/dircache/dirserv.h" #include "feature/dircommon/directory.h" @@ -1366,7 +1367,6 @@ CALLBACK(retry_listeners); CALLBACK(rotate_onion_key); CALLBACK(rotate_x509_certificate); CALLBACK(save_state); -CALLBACK(write_bridge_ns); CALLBACK(write_stats_file); CALLBACK(control_per_second_events); CALLBACK(second_elapsed); @@ -1433,9 +1433,6 @@ STATIC periodic_event_item_t mainloop_periodic_events[] = { /* XXXX this could be restricted to CLIENT+NET_PARTICIPANT */ CALLBACK(rend_cache_failure_clean, NET_PARTICIPANT, FL(RUN_ON_DISABLE)), - /* Bridge Authority only. */ - CALLBACK(write_bridge_ns, BRIDGEAUTH, 0), - /* Directory server only. */ CALLBACK(clean_consdiffmgr, DIRSERVER, 0), @@ -2369,22 +2366,6 @@ check_dns_honesty_callback(time_t now, const or_options_t *options) return 12*3600 + crypto_rand_int(12*3600); } -/** - * Periodic callback: if we're the bridge authority, write a networkstatus - * file to disk. - */ -static int -write_bridge_ns_callback(time_t now, const or_options_t *options) -{ - /* 10. write bridge networkstatus file to disk */ - if (options->BridgeAuthoritativeDir) { - networkstatus_dump_bridge_status_to_file(now); -#define BRIDGE_STATUSFILE_INTERVAL (30*60) - return BRIDGE_STATUSFILE_INTERVAL; - } - return PERIODIC_EVENT_NO_UPDATE; -} - static int heartbeat_callback_first_time = 1; /** diff --git a/src/feature/control/fmt_serverstatus.c b/src/feature/control/fmt_serverstatus.c index d224a1d234..a80bf50ad9 100644 --- a/src/feature/control/fmt_serverstatus.c +++ b/src/feature/control/fmt_serverstatus.c @@ -76,7 +76,6 @@ list_server_status_v1(smartlist_t *routers, char **router_status_out, SMARTLIST_FOREACH_BEGIN(routers, routerinfo_t *, ri) { const node_t *node = node_get_by_id(ri->cache_info.identity_digest); tor_assert(node); - if (for_controller) { char name_buf[MAX_VERBOSE_NICKNAME_LEN+2]; char *cp = name_buf; diff --git a/src/feature/dirauth/bridgeauth.c b/src/feature/dirauth/bridgeauth.c new file mode 100644 index 0000000000..4aaefc7a6d --- /dev/null +++ b/src/feature/dirauth/bridgeauth.c @@ -0,0 +1,55 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "core/or/or.h" +#include "feature/dirauth/bridgeauth.h" +#include "feature/dirauth/voteflags.h" +#include "feature/nodelist/networkstatus.h" +#include "feature/relay/router.h" +#include "app/config/config.h" + +#include "feature/nodelist/routerinfo_st.h" + +/** Write out router status entries for all our bridge descriptors. Here, we + * also mark routers as running. */ +void +bridgeauth_dump_bridge_status_to_file(time_t now) +{ + char *status; + char *fname = NULL; + char *thresholds = NULL; + char *published_thresholds_and_status = NULL; + char published[ISO_TIME_LEN+1]; + const routerinfo_t *me = router_get_my_routerinfo(); + char fingerprint[FINGERPRINT_LEN+1]; + char *fingerprint_line = NULL; + + dirserv_set_bridges_running(now); + status = networkstatus_getinfo_by_purpose("bridge", now); + + if (me && crypto_pk_get_fingerprint(me->identity_pkey, + fingerprint, 0) >= 0) { + tor_asprintf(&fingerprint_line, "fingerprint %s\n", fingerprint); + } else { + log_warn(LD_BUG, "Error computing fingerprint for bridge status."); + } + format_iso_time(published, now); + dirserv_compute_bridge_flag_thresholds(); + thresholds = dirserv_get_flag_thresholds_line(); + tor_asprintf(&published_thresholds_and_status, + "published %s\nflag-thresholds %s\n%s%s", + published, thresholds, fingerprint_line ? fingerprint_line : "", + status); + fname = get_datadir_fname("networkstatus-bridges"); + if (write_str_to_file(fname,published_thresholds_and_status,0)<0) { + log_warn(LD_DIRSERV, "Unable to write networkstatus-bridges file."); + } + tor_free(thresholds); + tor_free(published_thresholds_and_status); + tor_free(fname); + tor_free(status); + tor_free(fingerprint_line); +} diff --git a/src/feature/dirauth/bridgeauth.h b/src/feature/dirauth/bridgeauth.h new file mode 100644 index 0000000000..cc80fd6375 --- /dev/null +++ b/src/feature/dirauth/bridgeauth.h @@ -0,0 +1,12 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef TOR_DIRAUTH_BRIDGEAUTH_H +#define TOR_DIRAUTH_BRIDGEAUTH_H + +void bridgeauth_dump_bridge_status_to_file(time_t now); + +#endif diff --git a/src/feature/dirauth/dirauth_periodic.c b/src/feature/dirauth/dirauth_periodic.c index cfbb156b9f..02727d61b4 100644 --- a/src/feature/dirauth/dirauth_periodic.c +++ b/src/feature/dirauth/dirauth_periodic.c @@ -11,6 +11,7 @@ #include "feature/dirauth/reachability.h" #include "feature/stats/rephist.h" +#include "feature/dirauth/bridgeauth.h" #include "feature/dirauth/dirvote.h" #include "feature/dirauth/dirauth_periodic.h" #include "feature/dirauth/authmode.h" @@ -131,6 +132,23 @@ downrate_stability_callback(time_t now, const or_options_t *options) DECLARE_EVENT(downrate_stability, AUTHORITIES, 0); +/** + * Periodic callback: if we're the bridge authority, write a networkstatus + * file to disk. + */ +static int +write_bridge_ns_callback(time_t now, const or_options_t *options) +{ + if (options->BridgeAuthoritativeDir) { + bridgeauth_dump_bridge_status_to_file(now); +#define BRIDGE_STATUSFILE_INTERVAL (30*60) + return BRIDGE_STATUSFILE_INTERVAL; + } + return PERIODIC_EVENT_NO_UPDATE; +} + +DECLARE_EVENT(write_bridge_ns, BRIDGEAUTH, 0); + void dirauth_register_periodic_events(void) { @@ -139,4 +157,5 @@ dirauth_register_periodic_events(void) periodic_events_register(&save_stability_event); periodic_events_register(&check_authority_cert_event); periodic_events_register(&dirvote_event); + periodic_events_register(&write_bridge_ns_event); } diff --git a/src/feature/dirauth/dirvote.c b/src/feature/dirauth/dirvote.c index 1f861d2417..38b583c46a 100644 --- a/src/feature/dirauth/dirvote.c +++ b/src/feature/dirauth/dirvote.c @@ -4545,8 +4545,8 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key, vrs = tor_malloc_zero(sizeof(vote_routerstatus_t)); rs = &vrs->status; - set_routerstatus_from_routerinfo(rs, node, ri, now, - listbadexits); + dirauth_set_routerstatus_from_routerinfo(rs, node, ri, now, + listbadexits); if (ri->cache_info.signing_key_cert) { memcpy(vrs->ed25519_id, diff --git a/src/feature/dirauth/voteflags.c b/src/feature/dirauth/voteflags.c index 957ebe4a4f..f552af98c4 100644 --- a/src/feature/dirauth/voteflags.c +++ b/src/feature/dirauth/voteflags.c @@ -546,38 +546,31 @@ should_publish_node_ipv6(const node_t *node, const routerinfo_t *ri, router_is_me(ri)); } -/** Extract status information from ri and from other authority - * functions and store it in rs. rs is zeroed out before it is - * set. - * - * We assume that ri-\>is_running has already been set, e.g. by - * dirserv_set_router_is_running(ri, now); +/** + * Extract status information from ri and from other authority + * functions and store it in rs, as per + * set_routerstatus_from_routerinfo. Additionally, sets information + * in from the authority subsystem. */ void -set_routerstatus_from_routerinfo(routerstatus_t *rs, - node_t *node, - const routerinfo_t *ri, - time_t now, - int listbadexits) +dirauth_set_routerstatus_from_routerinfo(routerstatus_t *rs, + node_t *node, + const routerinfo_t *ri, + time_t now, + int listbadexits) { const or_options_t *options = get_options(); uint32_t routerbw_kb = dirserv_get_credible_bandwidth_kb(ri); - memset(rs, 0, sizeof(routerstatus_t)); + /* Set these flags so that set_routerstatus_from_routerinfo can copy them. + */ + node->is_stable = !dirserv_thinks_router_is_unreliable(now, ri, 1, 0); + node->is_fast = !dirserv_thinks_router_is_unreliable(now, ri, 0, 1); + node->is_hs_dir = dirserv_thinks_router_is_hs_dir(ri, node, now); - rs->is_authority = - router_digest_is_trusted_dir(ri->cache_info.identity_digest); - - /* Already set by compute_performance_thresholds. */ - rs->is_exit = node->is_exit; - rs->is_stable = node->is_stable = - !dirserv_thinks_router_is_unreliable(now, ri, 1, 0); - rs->is_fast = node->is_fast = - !dirserv_thinks_router_is_unreliable(now, ri, 0, 1); - rs->is_flagged_running = node->is_running; /* computed above */ - - rs->is_valid = node->is_valid; + set_routerstatus_from_routerinfo(rs, node, ri); + /* Override rs->is_possible_guard. */ if (node->is_fast && node->is_stable && ri->supports_tunnelled_dir_requests && ((options->AuthDirGuardBWGuarantee && @@ -593,31 +586,16 @@ set_routerstatus_from_routerinfo(routerstatus_t *rs, rs->is_possible_guard = 0; } + /* Override rs->is_bad_exit */ rs->is_bad_exit = listbadexits && node->is_bad_exit; - rs->is_hs_dir = node->is_hs_dir = - dirserv_thinks_router_is_hs_dir(ri, node, now); - - rs->is_named = rs->is_unnamed = 0; - - rs->published_on = ri->cache_info.published_on; - memcpy(rs->identity_digest, node->identity, DIGEST_LEN); - memcpy(rs->descriptor_digest, ri->cache_info.signed_descriptor_digest, - DIGEST_LEN); - rs->addr = ri->addr; - strlcpy(rs->nickname, ri->nickname, sizeof(rs->nickname)); - rs->or_port = ri->or_port; - rs->dir_port = ri->dir_port; - rs->is_v2_dir = ri->supports_tunnelled_dir_requests; + /* Set rs->is_staledesc. */ rs->is_staledesc = (ri->cache_info.published_on + DESC_IS_STALE_INTERVAL) < now; - if (should_publish_node_ipv6(node, ri, now)) { - /* We're configured as having IPv6 connectivity. There's an IPv6 - OR port and it's reachable so copy it to the routerstatus. */ - tor_addr_copy(&rs->ipv6_addr, &ri->ipv6_addr); - rs->ipv6_orport = ri->ipv6_orport; - } else { + if (! should_publish_node_ipv6(node, ri, now)) { + /* We're not configured as having IPv6 connectivity or the node isn't: + * zero its IPv6 information. */ tor_addr_make_null(&rs->ipv6_addr, AF_INET6); rs->ipv6_orport = 0; } diff --git a/src/feature/dirauth/voteflags.h b/src/feature/dirauth/voteflags.h index 18b29a5183..ee809a290d 100644 --- a/src/feature/dirauth/voteflags.h +++ b/src/feature/dirauth/voteflags.h @@ -12,18 +12,20 @@ #ifndef TOR_VOTEFLAGS_H #define TOR_VOTEFLAGS_H +#ifdef HAVE_MODULE_DIRAUTH void dirserv_set_router_is_running(routerinfo_t *router, time_t now); char *dirserv_get_flag_thresholds_line(void); void dirserv_compute_bridge_flag_thresholds(void); int running_long_enough_to_decide_unreachable(void); -void set_routerstatus_from_routerinfo(routerstatus_t *rs, - node_t *node, - const routerinfo_t *ri, - time_t now, - int listbadexits); +void dirauth_set_routerstatus_from_routerinfo(routerstatus_t *rs, + node_t *node, + const routerinfo_t *ri, + time_t now, + int listbadexits); void dirserv_compute_performance_thresholds(digestmap_t *omit_as_sybil); +#endif void dirserv_set_bridges_running(time_t now); diff --git a/src/feature/nodelist/networkstatus.c b/src/feature/nodelist/networkstatus.c index 22fef81085..c7e337309e 100644 --- a/src/feature/nodelist/networkstatus.c +++ b/src/feature/nodelist/networkstatus.c @@ -2366,6 +2366,49 @@ networkstatus_getinfo_helper_single(const routerstatus_t *rs) NULL); } +/** + * Extract status information from ri and from other authority + * functions and store it in rs. rs is zeroed out before it is + * set. + * + * We assume that node-\>is_running has already been set, e.g. by + * dirserv_set_router_is_running(ri, now); + */ +void +set_routerstatus_from_routerinfo(routerstatus_t *rs, + const node_t *node, + const routerinfo_t *ri) +{ + memset(rs, 0, sizeof(routerstatus_t)); + + rs->is_authority = + router_digest_is_trusted_dir(ri->cache_info.identity_digest); + + /* Set by compute_performance_thresholds or from consensus */ + rs->is_exit = node->is_exit; + rs->is_stable = node->is_stable; + rs->is_fast = node->is_fast; + rs->is_flagged_running = node->is_running; + rs->is_valid = node->is_valid; + rs->is_possible_guard = node->is_possible_guard; + rs->is_bad_exit = node->is_bad_exit; + rs->is_hs_dir = node->is_hs_dir; + rs->is_named = rs->is_unnamed = 0; + + rs->published_on = ri->cache_info.published_on; + memcpy(rs->identity_digest, node->identity, DIGEST_LEN); + memcpy(rs->descriptor_digest, ri->cache_info.signed_descriptor_digest, + DIGEST_LEN); + rs->addr = ri->addr; + strlcpy(rs->nickname, ri->nickname, sizeof(rs->nickname)); + rs->or_port = ri->or_port; + rs->dir_port = ri->dir_port; + rs->is_v2_dir = ri->supports_tunnelled_dir_requests; + + tor_addr_copy(&rs->ipv6_addr, &ri->ipv6_addr); + rs->ipv6_orport = ri->ipv6_orport; +} + /** Alloc and return a string describing routerstatuses for the most * recent info of each router we know about that is of purpose * purpose_string. Return NULL if unrecognized purpose. @@ -2398,8 +2441,7 @@ networkstatus_getinfo_by_purpose(const char *purpose_string, time_t now) continue; if (ri->purpose != purpose) continue; - /* then generate and write out status lines for each of them */ - set_routerstatus_from_routerinfo(&rs, node, ri, now, 0); + set_routerstatus_from_routerinfo(&rs, node, ri); smartlist_add(statuses, networkstatus_getinfo_helper_single(&rs)); } SMARTLIST_FOREACH_END(ri); @@ -2409,47 +2451,6 @@ networkstatus_getinfo_by_purpose(const char *purpose_string, time_t now) return answer; } -/** Write out router status entries for all our bridge descriptors. Here, we - * also mark routers as running. */ -void -networkstatus_dump_bridge_status_to_file(time_t now) -{ - char *status; - char *fname = NULL; - char *thresholds = NULL; - char *published_thresholds_and_status = NULL; - char published[ISO_TIME_LEN+1]; - const routerinfo_t *me = router_get_my_routerinfo(); - char fingerprint[FINGERPRINT_LEN+1]; - char *fingerprint_line = NULL; - - dirserv_set_bridges_running(now); - status = networkstatus_getinfo_by_purpose("bridge", now); - - if (me && crypto_pk_get_fingerprint(me->identity_pkey, - fingerprint, 0) >= 0) { - tor_asprintf(&fingerprint_line, "fingerprint %s\n", fingerprint); - } else { - log_warn(LD_BUG, "Error computing fingerprint for bridge status."); - } - format_iso_time(published, now); - dirserv_compute_bridge_flag_thresholds(); - thresholds = dirserv_get_flag_thresholds_line(); - tor_asprintf(&published_thresholds_and_status, - "published %s\nflag-thresholds %s\n%s%s", - published, thresholds, fingerprint_line ? fingerprint_line : "", - status); - fname = get_datadir_fname("networkstatus-bridges"); - if (write_str_to_file(fname,published_thresholds_and_status,0)<0) { - log_warn(LD_DIRSERV, "Unable to write networkstatus-bridges file."); - } - tor_free(thresholds); - tor_free(published_thresholds_and_status); - tor_free(fname); - tor_free(status); - tor_free(fingerprint_line); -} - /* DOCDOC get_net_param_from_list */ static int32_t get_net_param_from_list(smartlist_t *net_params, const char *param_name, diff --git a/src/feature/nodelist/networkstatus.h b/src/feature/nodelist/networkstatus.h index 8269fc6182..600fd7fbd5 100644 --- a/src/feature/nodelist/networkstatus.h +++ b/src/feature/nodelist/networkstatus.h @@ -122,7 +122,6 @@ void signed_descs_update_status_from_consensus_networkstatus( char *networkstatus_getinfo_helper_single(const routerstatus_t *rs); char *networkstatus_getinfo_by_purpose(const char *purpose_string, time_t now); -void networkstatus_dump_bridge_status_to_file(time_t now); MOCK_DECL(int32_t, networkstatus_get_param, (const networkstatus_t *ns, const char *param_name, int32_t default_val, int32_t min_val, int32_t max_val)); @@ -149,6 +148,10 @@ void vote_routerstatus_free_(vote_routerstatus_t *rs); #define vote_routerstatus_free(rs) \ FREE_AND_NULL(vote_routerstatus_t, vote_routerstatus_free_, (rs)) +void set_routerstatus_from_routerinfo(routerstatus_t *rs, + const node_t *node, + const routerinfo_t *ri); + #ifdef NETWORKSTATUS_PRIVATE #ifdef TOR_UNIT_TESTS STATIC int networkstatus_set_current_consensus_from_ns(networkstatus_t *c, diff --git a/src/test/test_voting_flags.c b/src/test/test_voting_flags.c index 5c9eebd00e..c8111ea5df 100644 --- a/src/test/test_voting_flags.c +++ b/src/test/test_voting_flags.c @@ -60,7 +60,7 @@ check_result(flag_vote_test_cfg_t *c) bool result = false; routerstatus_t rs; memset(&rs, 0, sizeof(rs)); - set_routerstatus_from_routerinfo(&rs, &c->node, &c->ri, c->now, 0); + dirauth_set_routerstatus_from_routerinfo(&rs, &c->node, &c->ri, c->now, 0); tt_i64_op(rs.published_on, OP_EQ, c->expected.published_on); tt_str_op(rs.nickname, OP_EQ, c->expected.nickname); From 31fb4a78451a98a8d62e1f52e75e372a8a2dd48b Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 30 Apr 2019 12:48:00 -0400 Subject: [PATCH 0939/2557] Make the bwauth.c module dirauth-only. --- src/app/config/config.c | 2 +- src/app/main/shutdown.c | 1 - src/core/include.am | 7 +------ src/feature/dirauth/dirauth_sys.c | 2 ++ 4 files changed, 4 insertions(+), 8 deletions(-) diff --git a/src/app/config/config.c b/src/app/config/config.c index 3e0683eb56..26a3061a26 100644 --- a/src/app/config/config.c +++ b/src/app/config/config.c @@ -3556,12 +3556,12 @@ options_validate(or_options_t *old_options, or_options_t *options, options->V3AuthoritativeDir)) REJECT("AuthoritativeDir is set, but none of " "(Bridge/V3)AuthoritativeDir is set."); +#ifdef HAVE_MODULE_DIRAUTH /* If we have a v3bandwidthsfile and it's broken, complain on startup */ if (options->V3BandwidthsFile && !old_options) { dirserv_read_measured_bandwidths(options->V3BandwidthsFile, NULL, NULL, NULL); } -#ifdef HAVE_MODULE_DIRAUTH /* same for guardfraction file */ if (options->GuardfractionFile && !old_options) { dirserv_read_guardfraction_file(options->GuardfractionFile, NULL); diff --git a/src/app/main/shutdown.c b/src/app/main/shutdown.c index fd9512ce4b..92cd9c6f7b 100644 --- a/src/app/main/shutdown.c +++ b/src/app/main/shutdown.c @@ -128,7 +128,6 @@ tor_free_all(int postfork) networkstatus_free_all(); addressmap_free_all(); dirserv_free_all(); - dirserv_clear_measured_bw_cache(); rend_cache_free_all(); rend_service_authorization_free_all(); rep_hist_free_all(); diff --git a/src/core/include.am b/src/core/include.am index 63d4264210..18b6046bfc 100644 --- a/src/core/include.am +++ b/src/core/include.am @@ -150,12 +150,6 @@ LIBTOR_APP_A_SOURCES = \ src/feature/stats/rephist.c \ src/feature/stats/predict_ports.c -# These should eventually move into module_dirauth_sources, but for now -# the separation is only in the code location. -LIBTOR_APP_A_SOURCES += \ - src/feature/dirauth/bwauth.c - - if BUILD_NT_SERVICES LIBTOR_APP_A_SOURCES += src/app/main/ntmain.c endif @@ -172,6 +166,7 @@ LIBTOR_APP_TESTING_A_SOURCES = $(LIBTOR_APP_A_SOURCES) MODULE_DIRAUTH_SOURCES = \ src/feature/dirauth/authmode.c \ src/feature/dirauth/bridgeauth.c \ + src/feature/dirauth/bwauth.c \ src/feature/dirauth/dirauth_periodic.c \ src/feature/dirauth/dirauth_sys.c \ src/feature/dirauth/dircollate.c \ diff --git a/src/feature/dirauth/dirauth_sys.c b/src/feature/dirauth/dirauth_sys.c index b87fa5dc29..f691d5618a 100644 --- a/src/feature/dirauth/dirauth_sys.c +++ b/src/feature/dirauth/dirauth_sys.c @@ -6,6 +6,7 @@ #include "core/or/or.h" +#include "feature/dirauth/bwauth.h" #include "feature/dirauth/dirauth_sys.h" #include "feature/dirauth/dirvote.h" #include "feature/dirauth/dirauth_periodic.h" @@ -25,6 +26,7 @@ subsys_dirauth_shutdown(void) { dirserv_free_fingerprint_list(); dirvote_free_all(); + dirserv_clear_measured_bw_cache(); } const struct subsys_fns_t sys_dirauth = { From a45413e7d58363fc4df71376b7cfc56d52534f86 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 30 Apr 2019 12:52:00 -0400 Subject: [PATCH 0940/2557] Make keypin.c dirauth-only --- src/app/main/shutdown.c | 1 - src/core/include.am | 3 +-- src/feature/dirauth/dirauth_sys.c | 2 ++ 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/app/main/shutdown.c b/src/app/main/shutdown.c index 92cd9c6f7b..390a512c13 100644 --- a/src/app/main/shutdown.c +++ b/src/app/main/shutdown.c @@ -97,7 +97,6 @@ tor_cleanup(void) } if (authdir_mode_tests_reachability(options)) rep_hist_record_mtbf_data(now, 0); - keypin_close_journal(); } timers_shutdown(); diff --git a/src/core/include.am b/src/core/include.am index 18b6046bfc..dc7371fedd 100644 --- a/src/core/include.am +++ b/src/core/include.am @@ -81,7 +81,6 @@ LIBTOR_APP_A_SOURCES = \ src/feature/control/control_getinfo.c \ src/feature/control/fmt_serverstatus.c \ src/feature/control/getinfo_geoip.c \ - src/feature/dirauth/keypin.c \ src/feature/dircache/conscache.c \ src/feature/dircache/consdiffmgr.c \ src/feature/dircache/dircache.c \ @@ -118,7 +117,6 @@ LIBTOR_APP_A_SOURCES = \ src/feature/hs_common/replaycache.c \ src/feature/hs_common/shared_random_client.c \ src/feature/keymgt/loadkey.c \ - src/feature/dirauth/keypin.c \ src/feature/nodelist/authcert.c \ src/feature/nodelist/describe.c \ src/feature/nodelist/dirlist.c \ @@ -173,6 +171,7 @@ MODULE_DIRAUTH_SOURCES = \ src/feature/dirauth/dirvote.c \ src/feature/dirauth/dsigs_parse.c \ src/feature/dirauth/guardfraction.c \ + src/feature/dirauth/keypin.c \ src/feature/dirauth/process_descs.c \ src/feature/dirauth/reachability.c \ src/feature/dirauth/recommend_pkg.c \ diff --git a/src/feature/dirauth/dirauth_sys.c b/src/feature/dirauth/dirauth_sys.c index f691d5618a..e38d391300 100644 --- a/src/feature/dirauth/dirauth_sys.c +++ b/src/feature/dirauth/dirauth_sys.c @@ -10,6 +10,7 @@ #include "feature/dirauth/dirauth_sys.h" #include "feature/dirauth/dirvote.h" #include "feature/dirauth/dirauth_periodic.h" +#include "feature/dirauth/keypin.h" #include "feature/dirauth/process_descs.h" #include "lib/subsys/subsys.h" @@ -27,6 +28,7 @@ subsys_dirauth_shutdown(void) dirserv_free_fingerprint_list(); dirvote_free_all(); dirserv_clear_measured_bw_cache(); + keypin_close_journal(); } const struct subsys_fns_t sys_dirauth = { From 9c3aa22740b79cb55d4c2d523ffb3adb09e02479 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 30 Apr 2019 12:58:00 -0400 Subject: [PATCH 0941/2557] Remove some now-needless dirauth includes --- src/app/main/shutdown.c | 3 -- src/core/mainloop/mainloop.c | 1 - src/feature/nodelist/fmt_routerstatus.c | 41 ------------------------- 3 files changed, 45 deletions(-) diff --git a/src/app/main/shutdown.c b/src/app/main/shutdown.c index 390a512c13..c302ce455c 100644 --- a/src/app/main/shutdown.c +++ b/src/app/main/shutdown.c @@ -36,9 +36,6 @@ #include "feature/control/control.h" #include "feature/control/control_auth.h" #include "feature/dirauth/authmode.h" -#include "feature/dirauth/bwauth.h" -#include "feature/dirauth/dirvote.h" -#include "feature/dirauth/keypin.h" #include "feature/dirauth/shared_random.h" #include "feature/dircache/consdiffmgr.h" #include "feature/dircache/dirserv.h" diff --git a/src/core/mainloop/mainloop.c b/src/core/mainloop/mainloop.c index eb875df023..4401f805d9 100644 --- a/src/core/mainloop/mainloop.c +++ b/src/core/mainloop/mainloop.c @@ -75,7 +75,6 @@ #include "feature/control/control.h" #include "feature/control/control_events.h" #include "feature/dirauth/authmode.h" -#include "feature/dirauth/bridgeauth.h" #include "feature/dircache/consdiffmgr.h" #include "feature/dircache/dirserv.h" #include "feature/dircommon/directory.h" diff --git a/src/feature/nodelist/fmt_routerstatus.c b/src/feature/nodelist/fmt_routerstatus.c index 8c9212e05c..2ada4a7ed5 100644 --- a/src/feature/nodelist/fmt_routerstatus.c +++ b/src/feature/nodelist/fmt_routerstatus.c @@ -14,55 +14,14 @@ #include "core/or/or.h" #include "feature/nodelist/fmt_routerstatus.h" -/* #include "lib/container/buffers.h" */ -/* #include "app/config/config.h" */ -/* #include "app/config/confparse.h" */ -/* #include "core/or/channel.h" */ -/* #include "core/or/channeltls.h" */ -/* #include "core/or/command.h" */ -/* #include "core/mainloop/connection.h" */ -/* #include "core/or/connection_or.h" */ -/* #include "feature/dircache/conscache.h" */ -/* #include "feature/dircache/consdiffmgr.h" */ -/* #include "feature/control/control.h" */ -/* #include "feature/dircache/directory.h" */ -/* #include "feature/dircache/dirserv.h" */ -/* #include "feature/hibernate/hibernate.h" */ -/* #include "feature/dirauth/keypin.h" */ -/* #include "core/mainloop/mainloop.h" */ -/* #include "feature/nodelist/microdesc.h" */ -/* #include "feature/nodelist/networkstatus.h" */ -/* #include "feature/nodelist/nodelist.h" */ #include "core/or/policies.h" -/* #include "core/or/protover.h" */ -/* #include "feature/stats/rephist.h" */ -/* #include "feature/relay/router.h" */ -/* #include "feature/nodelist/dirlist.h" */ #include "feature/nodelist/routerlist.h" - -/* #include "feature/nodelist/routerparse.h" */ -/* #include "feature/nodelist/routerset.h" */ -/* #include "feature/nodelist/torcert.h" */ -/* #include "feature/dircommon/voting_schedule.h" */ - #include "feature/dirauth/dirvote.h" -/* #include "feature/dircache/cached_dir_st.h" */ -/* #include "feature/dircommon/dir_connection_st.h" */ -/* #include "feature/nodelist/extrainfo_st.h" */ -/* #include "feature/nodelist/microdesc_st.h" */ -/* #include "feature/nodelist/node_st.h" */ #include "feature/nodelist/routerinfo_st.h" -/* #include "feature/nodelist/routerlist_st.h" */ -/* #include "core/or/tor_version_st.h" */ #include "feature/nodelist/vote_routerstatus_st.h" -/* #include "lib/compress/compress.h" */ -/* #include "lib/container/order.h" */ #include "lib/crypt_ops/crypto_format.h" -/* #include "lib/encoding/confline.h" */ - -/* #include "lib/encoding/keyval.h" */ /** Helper: write the router-status information in rs into a newly * allocated character buffer. Use the same format as in network-status From 0f365e2f46094590cbf0d22b9e78bbe79d1051ed Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 30 Apr 2019 13:04:18 -0400 Subject: [PATCH 0942/2557] practracker updates. --- scripts/maint/practracker/exceptions.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/maint/practracker/exceptions.txt b/scripts/maint/practracker/exceptions.txt index d90ed1f4bd..c41b0a0b10 100644 --- a/scripts/maint/practracker/exceptions.txt +++ b/scripts/maint/practracker/exceptions.txt @@ -29,12 +29,12 @@ # # Remember: It is better to fix the problem than to add a new exception! -problem file-size /src/app/config/config.c 8492 +problem file-size /src/app/config/config.c 8494 problem include-count /src/app/config/config.c 87 problem function-size /src/app/config/config.c:options_act_reversible() 296 problem function-size /src/app/config/config.c:options_act() 588 problem function-size /src/app/config/config.c:resolve_my_address() 192 -problem function-size /src/app/config/config.c:options_validate() 1207 +problem function-size /src/app/config/config.c:options_validate() 1209 problem function-size /src/app/config/config.c:options_init_from_torrc() 202 problem function-size /src/app/config/config.c:options_init_from_string() 173 problem function-size /src/app/config/config.c:options_init_logs() 146 @@ -221,7 +221,7 @@ problem function-size /src/feature/nodelist/node_select.c:router_pick_directory_ problem function-size /src/feature/nodelist/node_select.c:compute_weighted_bandwidths() 206 problem function-size /src/feature/nodelist/node_select.c:router_pick_trusteddirserver_impl() 114 problem function-size /src/feature/nodelist/nodelist.c:compute_frac_paths_available() 193 -problem file-size /src/feature/nodelist/routerlist.c 3234 +problem file-size /src/feature/nodelist/routerlist.c 3238 problem function-size /src/feature/nodelist/routerlist.c:router_rebuild_store() 148 problem function-size /src/feature/nodelist/routerlist.c:router_add_to_routerlist() 169 problem function-size /src/feature/nodelist/routerlist.c:routerlist_remove_old_routers() 121 From ee36bfa6de95bf3d2214961d07edb001eb1ca150 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 30 Apr 2019 14:59:28 -0400 Subject: [PATCH 0943/2557] Changes file for improved dirauth modularity (ticket 30345) --- changes/ticket30345 | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 changes/ticket30345 diff --git a/changes/ticket30345 b/changes/ticket30345 new file mode 100644 index 0000000000..639db8d7ee --- /dev/null +++ b/changes/ticket30345 @@ -0,0 +1,3 @@ + o Minor features (modularity): + - The --disable-module-dirauth compile-time option now disables + even more dirauth-only code. Closes ticket 30345. From 721e65a1d5a5821149fc0c10e6c82398d953baf4 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 2 May 2019 09:37:18 -0400 Subject: [PATCH 0944/2557] Add comments to include.am files to note where new sources go This mechanism isn't perfect, and sometimes it will guess wrong, but it will help our automation. --- src/core/include.am | 2 ++ src/lib/arch/include.am | 1 + src/lib/buf/include.am | 2 ++ src/lib/cc/include.am | 1 + src/lib/compress/include.am | 2 ++ src/lib/container/include.am | 2 ++ src/lib/crypt_ops/include.am | 2 ++ src/lib/ctime/include.am | 2 ++ src/lib/defs/include.am | 1 + src/lib/dispatch/include.am | 2 ++ src/lib/encoding/include.am | 2 ++ src/lib/err/include.am | 2 ++ src/lib/evloop/include.am | 3 ++- src/lib/fdio/include.am | 2 ++ src/lib/fs/include.am | 2 ++ src/lib/geoip/include.am | 2 ++ src/lib/intmath/include.am | 2 ++ src/lib/lock/include.am | 2 ++ src/lib/log/include.am | 2 ++ src/lib/malloc/include.am | 2 ++ src/lib/math/include.am | 3 ++- src/lib/memarea/include.am | 2 ++ src/lib/meminfo/include.am | 2 ++ src/lib/net/include.am | 2 ++ src/lib/osinfo/include.am | 2 ++ src/lib/process/include.am | 2 ++ src/lib/pubsub/include.am | 2 ++ src/lib/sandbox/include.am | 2 ++ src/lib/smartlist_core/include.am | 2 ++ src/lib/string/include.am | 2 ++ src/lib/subsys/include.am | 1 + src/lib/term/include.am | 2 ++ src/lib/testsupport/include.am | 1 + src/lib/thread/include.am | 2 ++ src/lib/time/include.am | 2 ++ src/lib/tls/include.am | 2 ++ src/lib/trace/include.am | 3 ++- src/lib/version/include.am | 2 ++ src/lib/wallclock/include.am | 2 ++ src/test/include.am | 3 +++ 40 files changed, 76 insertions(+), 3 deletions(-) diff --git a/src/core/include.am b/src/core/include.am index 4ec42182a6..7c548fdb9a 100644 --- a/src/core/include.am +++ b/src/core/include.am @@ -6,6 +6,7 @@ noinst_LIBRARIES += \ src/core/libtor-app-testing.a endif +# ADD_C_FILE: INSERT SOURCES HERE. LIBTOR_APP_A_SOURCES = \ src/app/config/config.c \ src/app/config/confparse.c \ @@ -205,6 +206,7 @@ AM_CPPFLAGS += -DSHARE_DATADIR="\"$(datadir)\"" \ src_core_libtor_app_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_core_libtor_app_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/app/config/config.h \ src/app/config/confparse.h \ diff --git a/src/lib/arch/include.am b/src/lib/arch/include.am index f92ee9222f..c5926c6330 100644 --- a/src/lib/arch/include.am +++ b/src/lib/arch/include.am @@ -1,3 +1,4 @@ +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/arch/bytes.h diff --git a/src/lib/buf/include.am b/src/lib/buf/include.am index 3338c3dbdb..27430d1d38 100644 --- a/src/lib/buf/include.am +++ b/src/lib/buf/include.am @@ -5,6 +5,7 @@ if UNITTESTS_ENABLED noinst_LIBRARIES += src/lib/libtor-buf-testing.a endif +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_buf_a_SOURCES = \ src/lib/buf/buffers.c @@ -13,5 +14,6 @@ src_lib_libtor_buf_testing_a_SOURCES = \ src_lib_libtor_buf_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_buf_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/buf/buffers.h diff --git a/src/lib/cc/include.am b/src/lib/cc/include.am index 52cf8a9f72..1aa722dd82 100644 --- a/src/lib/cc/include.am +++ b/src/lib/cc/include.am @@ -1,4 +1,5 @@ +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/cc/compat_compiler.h \ src/lib/cc/ctassert.h \ diff --git a/src/lib/compress/include.am b/src/lib/compress/include.am index b952779578..60dd447d4e 100644 --- a/src/lib/compress/include.am +++ b/src/lib/compress/include.am @@ -5,6 +5,7 @@ if UNITTESTS_ENABLED noinst_LIBRARIES += src/lib/libtor-compress-testing.a endif +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_compress_a_SOURCES = \ src/lib/compress/compress.c \ src/lib/compress/compress_buf.c \ @@ -18,6 +19,7 @@ src_lib_libtor_compress_testing_a_SOURCES = \ src_lib_libtor_compress_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_compress_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/compress/compress.h \ src/lib/compress/compress_lzma.h \ diff --git a/src/lib/container/include.am b/src/lib/container/include.am index 50d35e749b..00d7b8e587 100644 --- a/src/lib/container/include.am +++ b/src/lib/container/include.am @@ -5,6 +5,7 @@ if UNITTESTS_ENABLED noinst_LIBRARIES += src/lib/libtor-container-testing.a endif +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_container_a_SOURCES = \ src/lib/container/bloomfilt.c \ src/lib/container/map.c \ @@ -17,6 +18,7 @@ src_lib_libtor_container_testing_a_SOURCES = \ src_lib_libtor_container_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_container_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/container/bitarray.h \ src/lib/container/bloomfilt.h \ diff --git a/src/lib/crypt_ops/include.am b/src/lib/crypt_ops/include.am index c90ef6eca8..1f58a33d38 100644 --- a/src/lib/crypt_ops/include.am +++ b/src/lib/crypt_ops/include.am @@ -5,6 +5,7 @@ if UNITTESTS_ENABLED noinst_LIBRARIES += src/lib/libtor-crypt-ops-testing.a endif +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_crypt_ops_a_SOURCES = \ src/lib/crypt_ops/crypto_cipher.c \ src/lib/crypt_ops/crypto_curve25519.c \ @@ -52,6 +53,7 @@ src_lib_libtor_crypt_ops_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_crypt_ops_testing_a_CFLAGS = \ $(AM_CFLAGS) $(TOR_CFLAGS_CRYPTLIB) $(TEST_CFLAGS) +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/crypt_ops/aes.h \ src/lib/crypt_ops/compat_openssl.h \ diff --git a/src/lib/ctime/include.am b/src/lib/ctime/include.am index b46c43ba0c..83942ca4e0 100644 --- a/src/lib/ctime/include.am +++ b/src/lib/ctime/include.am @@ -11,6 +11,7 @@ else mulodi4_source= endif +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_ctime_a_SOURCES = \ $(mulodi4_source) \ src/ext/csiphash.c \ @@ -21,5 +22,6 @@ src_lib_libtor_ctime_testing_a_SOURCES = \ src_lib_libtor_ctime_a_CFLAGS = @CFLAGS_CONSTTIME@ src_lib_libtor_ctime_testing_a_CFLAGS = @CFLAGS_CONSTTIME@ $(TEST_CFLAGS) +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/ctime/di_ops.h diff --git a/src/lib/defs/include.am b/src/lib/defs/include.am index 6a7f9114ea..dfddc92e55 100644 --- a/src/lib/defs/include.am +++ b/src/lib/defs/include.am @@ -1,4 +1,5 @@ +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/defs/dh_sizes.h \ src/lib/defs/digest_sizes.h \ diff --git a/src/lib/dispatch/include.am b/src/lib/dispatch/include.am index 4ec5b75cd1..4a0e0dfd90 100644 --- a/src/lib/dispatch/include.am +++ b/src/lib/dispatch/include.am @@ -5,6 +5,7 @@ if UNITTESTS_ENABLED noinst_LIBRARIES += src/lib/libtor-dispatch-testing.a endif +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_dispatch_a_SOURCES = \ src/lib/dispatch/dispatch_cfg.c \ src/lib/dispatch/dispatch_core.c \ @@ -16,6 +17,7 @@ src_lib_libtor_dispatch_testing_a_SOURCES = \ src_lib_libtor_dispatch_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_dispatch_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/dispatch/dispatch.h \ src/lib/dispatch/dispatch_cfg.h \ diff --git a/src/lib/encoding/include.am b/src/lib/encoding/include.am index 8272e4e5fa..48d0120bfc 100644 --- a/src/lib/encoding/include.am +++ b/src/lib/encoding/include.am @@ -4,6 +4,7 @@ if UNITTESTS_ENABLED noinst_LIBRARIES += src/lib/libtor-encoding-testing.a endif +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_encoding_a_SOURCES = \ src/lib/encoding/binascii.c \ src/lib/encoding/confline.c \ @@ -19,6 +20,7 @@ src_lib_libtor_encoding_testing_a_SOURCES = \ src_lib_libtor_encoding_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_encoding_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/encoding/binascii.h \ src/lib/encoding/confline.h \ diff --git a/src/lib/err/include.am b/src/lib/err/include.am index 43adcd2694..883ac91511 100644 --- a/src/lib/err/include.am +++ b/src/lib/err/include.am @@ -5,6 +5,7 @@ if UNITTESTS_ENABLED noinst_LIBRARIES += src/lib/libtor-err-testing.a endif +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_err_a_SOURCES = \ src/lib/err/backtrace.c \ src/lib/err/torerr.c \ @@ -15,6 +16,7 @@ src_lib_libtor_err_testing_a_SOURCES = \ src_lib_libtor_err_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_err_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/err/backtrace.h \ src/lib/err/torerr.h \ diff --git a/src/lib/evloop/include.am b/src/lib/evloop/include.am index 6b0076272a..6595b3a34b 100644 --- a/src/lib/evloop/include.am +++ b/src/lib/evloop/include.am @@ -5,6 +5,7 @@ if UNITTESTS_ENABLED noinst_LIBRARIES += src/lib/libtor-evloop-testing.a endif +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_evloop_a_SOURCES = \ src/lib/evloop/compat_libevent.c \ src/lib/evloop/procmon.c \ @@ -12,12 +13,12 @@ src_lib_libtor_evloop_a_SOURCES = \ src/lib/evloop/token_bucket.c \ src/lib/evloop/workqueue.c - src_lib_libtor_evloop_testing_a_SOURCES = \ $(src_lib_libtor_evloop_a_SOURCES) src_lib_libtor_evloop_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_evloop_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/evloop/compat_libevent.h \ src/lib/evloop/procmon.h \ diff --git a/src/lib/fdio/include.am b/src/lib/fdio/include.am index 6c18f00a0d..545bbc929e 100644 --- a/src/lib/fdio/include.am +++ b/src/lib/fdio/include.am @@ -5,6 +5,7 @@ if UNITTESTS_ENABLED noinst_LIBRARIES += src/lib/libtor-fdio-testing.a endif +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_fdio_a_SOURCES = \ src/lib/fdio/fdio.c @@ -13,5 +14,6 @@ src_lib_libtor_fdio_testing_a_SOURCES = \ src_lib_libtor_fdio_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_fdio_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/fdio/fdio.h diff --git a/src/lib/fs/include.am b/src/lib/fs/include.am index f33e4d6430..493db8f044 100644 --- a/src/lib/fs/include.am +++ b/src/lib/fs/include.am @@ -5,6 +5,7 @@ if UNITTESTS_ENABLED noinst_LIBRARIES += src/lib/libtor-fs-testing.a endif +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_fs_a_SOURCES = \ src/lib/fs/conffile.c \ src/lib/fs/dir.c \ @@ -25,6 +26,7 @@ src_lib_libtor_fs_testing_a_SOURCES = \ src_lib_libtor_fs_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_fs_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/fs/conffile.h \ src/lib/fs/dir.h \ diff --git a/src/lib/geoip/include.am b/src/lib/geoip/include.am index 9710d75ac7..ea426d14bc 100644 --- a/src/lib/geoip/include.am +++ b/src/lib/geoip/include.am @@ -4,6 +4,7 @@ if UNITTESTS_ENABLED noinst_LIBRARIES += src/lib/libtor-geoip-testing.a endif +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_geoip_a_SOURCES = \ src/lib/geoip/geoip.c @@ -12,6 +13,7 @@ src_lib_libtor_geoip_testing_a_SOURCES = \ src_lib_libtor_geoip_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_geoip_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/geoip/geoip.h \ src/lib/geoip/country.h diff --git a/src/lib/intmath/include.am b/src/lib/intmath/include.am index 45ee3bd53b..155ffa145a 100644 --- a/src/lib/intmath/include.am +++ b/src/lib/intmath/include.am @@ -5,6 +5,7 @@ if UNITTESTS_ENABLED noinst_LIBRARIES += src/lib/libtor-intmath-testing.a endif +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_intmath_a_SOURCES = \ src/lib/intmath/addsub.c \ src/lib/intmath/bits.c \ @@ -16,6 +17,7 @@ src_lib_libtor_intmath_testing_a_SOURCES = \ src_lib_libtor_intmath_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_intmath_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/intmath/addsub.h \ src/lib/intmath/cmp.h \ diff --git a/src/lib/lock/include.am b/src/lib/lock/include.am index 4e6f444347..1475b9911b 100644 --- a/src/lib/lock/include.am +++ b/src/lib/lock/include.am @@ -5,6 +5,7 @@ if UNITTESTS_ENABLED noinst_LIBRARIES += src/lib/libtor-lock-testing.a endif +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_lock_a_SOURCES = \ src/lib/lock/compat_mutex.c @@ -20,5 +21,6 @@ src_lib_libtor_lock_testing_a_SOURCES = \ src_lib_libtor_lock_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_lock_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/lock/compat_mutex.h diff --git a/src/lib/log/include.am b/src/lib/log/include.am index 9d3dbe3104..5b9f7113ba 100644 --- a/src/lib/log/include.am +++ b/src/lib/log/include.am @@ -5,6 +5,7 @@ if UNITTESTS_ENABLED noinst_LIBRARIES += src/lib/libtor-log-testing.a endif +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_log_a_SOURCES = \ src/lib/log/escape.c \ src/lib/log/ratelim.c \ @@ -21,6 +22,7 @@ src_lib_libtor_log_testing_a_SOURCES = \ src_lib_libtor_log_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_log_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/log/escape.h \ src/lib/log/ratelim.h \ diff --git a/src/lib/malloc/include.am b/src/lib/malloc/include.am index 95d96168e1..b74292bc6e 100644 --- a/src/lib/malloc/include.am +++ b/src/lib/malloc/include.am @@ -5,6 +5,7 @@ if UNITTESTS_ENABLED noinst_LIBRARIES += src/lib/libtor-malloc-testing.a endif +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_malloc_a_SOURCES = \ src/lib/malloc/malloc.c \ src/lib/malloc/map_anon.c @@ -18,6 +19,7 @@ src_lib_libtor_malloc_testing_a_SOURCES = \ src_lib_libtor_malloc_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_malloc_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/malloc/malloc.h \ src/lib/malloc/map_anon.h diff --git a/src/lib/math/include.am b/src/lib/math/include.am index 6d65ce90a7..b2ca280f47 100644 --- a/src/lib/math/include.am +++ b/src/lib/math/include.am @@ -5,17 +5,18 @@ if UNITTESTS_ENABLED noinst_LIBRARIES += src/lib/libtor-math-testing.a endif +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_math_a_SOURCES = \ src/lib/math/fp.c \ src/lib/math/laplace.c \ src/lib/math/prob_distr.c - src_lib_libtor_math_testing_a_SOURCES = \ $(src_lib_libtor_math_a_SOURCES) src_lib_libtor_math_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_math_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/math/fp.h \ src/lib/math/laplace.h \ diff --git a/src/lib/memarea/include.am b/src/lib/memarea/include.am index 94343dcead..83fb99ec73 100644 --- a/src/lib/memarea/include.am +++ b/src/lib/memarea/include.am @@ -5,6 +5,7 @@ if UNITTESTS_ENABLED noinst_LIBRARIES += src/lib/libtor-memarea-testing.a endif +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_memarea_a_SOURCES = \ src/lib/memarea/memarea.c @@ -13,5 +14,6 @@ src_lib_libtor_memarea_testing_a_SOURCES = \ src_lib_libtor_memarea_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_memarea_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/memarea/memarea.h diff --git a/src/lib/meminfo/include.am b/src/lib/meminfo/include.am index d1fdde6313..12c1bff72d 100644 --- a/src/lib/meminfo/include.am +++ b/src/lib/meminfo/include.am @@ -5,6 +5,7 @@ if UNITTESTS_ENABLED noinst_LIBRARIES += src/lib/libtor-meminfo-testing.a endif +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_meminfo_a_SOURCES = \ src/lib/meminfo/meminfo.c @@ -13,5 +14,6 @@ src_lib_libtor_meminfo_testing_a_SOURCES = \ src_lib_libtor_meminfo_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_meminfo_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/meminfo/meminfo.h diff --git a/src/lib/net/include.am b/src/lib/net/include.am index 8a88f0f2ae..485019f4b7 100644 --- a/src/lib/net/include.am +++ b/src/lib/net/include.am @@ -5,6 +5,7 @@ if UNITTESTS_ENABLED noinst_LIBRARIES += src/lib/libtor-net-testing.a endif +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_net_a_SOURCES = \ src/lib/net/address.c \ src/lib/net/alertsock.c \ @@ -21,6 +22,7 @@ src_lib_libtor_net_testing_a_SOURCES = \ src_lib_libtor_net_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_net_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/net/address.h \ src/lib/net/alertsock.h \ diff --git a/src/lib/osinfo/include.am b/src/lib/osinfo/include.am index 16c5812604..84bd7feb00 100644 --- a/src/lib/osinfo/include.am +++ b/src/lib/osinfo/include.am @@ -5,6 +5,7 @@ if UNITTESTS_ENABLED noinst_LIBRARIES += src/lib/libtor-osinfo-testing.a endif +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_osinfo_a_SOURCES = \ src/lib/osinfo/uname.c @@ -13,5 +14,6 @@ src_lib_libtor_osinfo_testing_a_SOURCES = \ src_lib_libtor_osinfo_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_osinfo_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/osinfo/uname.h diff --git a/src/lib/process/include.am b/src/lib/process/include.am index 83b67bf029..af5f99617b 100644 --- a/src/lib/process/include.am +++ b/src/lib/process/include.am @@ -5,6 +5,7 @@ if UNITTESTS_ENABLED noinst_LIBRARIES += src/lib/libtor-process-testing.a endif +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_process_a_SOURCES = \ src/lib/process/daemon.c \ src/lib/process/env.c \ @@ -23,6 +24,7 @@ src_lib_libtor_process_testing_a_SOURCES = \ src_lib_libtor_process_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_process_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/process/daemon.h \ src/lib/process/env.h \ diff --git a/src/lib/pubsub/include.am b/src/lib/pubsub/include.am index c0ec13d039..e2abebcd40 100644 --- a/src/lib/pubsub/include.am +++ b/src/lib/pubsub/include.am @@ -5,6 +5,7 @@ if UNITTESTS_ENABLED noinst_LIBRARIES += src/lib/libtor-pubsub-testing.a endif +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_pubsub_a_SOURCES = \ src/lib/pubsub/pubsub_build.c \ src/lib/pubsub/pubsub_check.c \ @@ -15,6 +16,7 @@ src_lib_libtor_pubsub_testing_a_SOURCES = \ src_lib_libtor_pubsub_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_pubsub_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/pubsub/pub_binding_st.h \ src/lib/pubsub/pubsub.h \ diff --git a/src/lib/sandbox/include.am b/src/lib/sandbox/include.am index adfda6bde5..e81f14b55f 100644 --- a/src/lib/sandbox/include.am +++ b/src/lib/sandbox/include.am @@ -5,6 +5,7 @@ if UNITTESTS_ENABLED noinst_LIBRARIES += src/lib/libtor-sandbox-testing.a endif +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_sandbox_a_SOURCES = \ src/lib/sandbox/sandbox.c @@ -13,6 +14,7 @@ src_lib_libtor_sandbox_testing_a_SOURCES = \ src_lib_libtor_sandbox_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_sandbox_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/sandbox/linux_syscalls.inc \ src/lib/sandbox/sandbox.h diff --git a/src/lib/smartlist_core/include.am b/src/lib/smartlist_core/include.am index 99d65f0b23..548179bc4f 100644 --- a/src/lib/smartlist_core/include.am +++ b/src/lib/smartlist_core/include.am @@ -5,6 +5,7 @@ if UNITTESTS_ENABLED noinst_LIBRARIES += src/lib/libtor-smartlist-core-testing.a endif +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_smartlist_core_a_SOURCES = \ src/lib/smartlist_core/smartlist_core.c \ src/lib/smartlist_core/smartlist_split.c @@ -15,6 +16,7 @@ src_lib_libtor_smartlist_core_testing_a_CPPFLAGS = \ $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_smartlist_core_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/smartlist_core/smartlist_core.h \ src/lib/smartlist_core/smartlist_foreach.h \ diff --git a/src/lib/string/include.am b/src/lib/string/include.am index edd74b8a3e..82d35cc5af 100644 --- a/src/lib/string/include.am +++ b/src/lib/string/include.am @@ -5,6 +5,7 @@ if UNITTESTS_ENABLED noinst_LIBRARIES += src/lib/libtor-string-testing.a endif +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_string_a_SOURCES = \ src/lib/string/compat_ctype.c \ src/lib/string/compat_string.c \ @@ -18,6 +19,7 @@ src_lib_libtor_string_testing_a_SOURCES = \ src_lib_libtor_string_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_string_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/string/compat_ctype.h \ src/lib/string/compat_string.h \ diff --git a/src/lib/subsys/include.am b/src/lib/subsys/include.am index 4741126b14..c9ab54ca73 100644 --- a/src/lib/subsys/include.am +++ b/src/lib/subsys/include.am @@ -1,3 +1,4 @@ +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/subsys/subsys.h diff --git a/src/lib/term/include.am b/src/lib/term/include.am index 55fe548ebc..a120bba0cb 100644 --- a/src/lib/term/include.am +++ b/src/lib/term/include.am @@ -11,6 +11,7 @@ else readpassphrase_source= endif +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_term_a_SOURCES = \ src/lib/term/getpass.c \ $(readpassphrase_source) @@ -20,5 +21,6 @@ src_lib_libtor_term_testing_a_SOURCES = \ src_lib_libtor_term_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_term_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/term/getpass.h diff --git a/src/lib/testsupport/include.am b/src/lib/testsupport/include.am index b2aa620985..a5ed46eb67 100644 --- a/src/lib/testsupport/include.am +++ b/src/lib/testsupport/include.am @@ -1,3 +1,4 @@ +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/testsupport/testsupport.h diff --git a/src/lib/thread/include.am b/src/lib/thread/include.am index 695795a2c8..cd8016b5df 100644 --- a/src/lib/thread/include.am +++ b/src/lib/thread/include.am @@ -12,6 +12,7 @@ if THREADS_WIN32 threads_impl_source=src/lib/thread/compat_winthreads.c endif +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_thread_a_SOURCES = \ src/lib/thread/compat_threads.c \ src/lib/thread/numcpus.c \ @@ -22,6 +23,7 @@ src_lib_libtor_thread_testing_a_SOURCES = \ src_lib_libtor_thread_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_thread_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/thread/numcpus.h \ src/lib/thread/thread_sys.h \ diff --git a/src/lib/time/include.am b/src/lib/time/include.am index dae16f49ac..dcb199b142 100644 --- a/src/lib/time/include.am +++ b/src/lib/time/include.am @@ -5,6 +5,7 @@ if UNITTESTS_ENABLED noinst_LIBRARIES += src/lib/libtor-time-testing.a endif +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_time_a_SOURCES = \ src/lib/time/compat_time.c \ src/lib/time/time_sys.c \ @@ -15,6 +16,7 @@ src_lib_libtor_time_testing_a_SOURCES = \ src_lib_libtor_time_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_time_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/time/compat_time.h \ src/lib/time/time_sys.h \ diff --git a/src/lib/tls/include.am b/src/lib/tls/include.am index 1817739eef..7e05ef4f8c 100644 --- a/src/lib/tls/include.am +++ b/src/lib/tls/include.am @@ -5,6 +5,7 @@ if UNITTESTS_ENABLED noinst_LIBRARIES += src/lib/libtor-tls-testing.a endif +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_tls_a_SOURCES = \ src/lib/tls/buffers_tls.c \ src/lib/tls/tortls.c \ @@ -29,6 +30,7 @@ src_lib_libtor_tls_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_tls_testing_a_CFLAGS = \ $(AM_CFLAGS) $(TOR_CFLAGS_CRYPTLIB) $(TEST_CFLAGS) +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/tls/ciphers.inc \ src/lib/tls/buffers_tls.h \ diff --git a/src/lib/trace/include.am b/src/lib/trace/include.am index 6f10c98744..98098c87f4 100644 --- a/src/lib/trace/include.am +++ b/src/lib/trace/include.am @@ -2,6 +2,7 @@ noinst_LIBRARIES += \ src/lib/libtor-trace.a +# ADD_C_FILE: INSERT HEADERS HERE. TRACEHEADERS = \ src/lib/trace/trace.h \ src/lib/trace/events.h @@ -11,7 +12,7 @@ TRACEHEADERS += \ src/lib/trace/debug.h endif -# Library source files. +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_trace_a_SOURCES = \ src/lib/trace/trace.c diff --git a/src/lib/version/include.am b/src/lib/version/include.am index 6944eb05e3..0ae31be1b2 100644 --- a/src/lib/version/include.am +++ b/src/lib/version/include.am @@ -5,6 +5,7 @@ if UNITTESTS_ENABLED noinst_LIBRARIES += src/lib/libtor-version-testing.a endif +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_version_a_SOURCES = \ src/lib/version/git_revision.c \ src/lib/version/version.c @@ -20,6 +21,7 @@ src/lib/version/git_revision.$(OBJEXT) \ src/lib/version/src_lib_libtor_version_testing_a-git_revision.$(OBJEXT): \ micro-revision.i +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/version/git_revision.h \ src/lib/version/torversion.h diff --git a/src/lib/wallclock/include.am b/src/lib/wallclock/include.am index 2351252e0c..2b50d6ccbb 100644 --- a/src/lib/wallclock/include.am +++ b/src/lib/wallclock/include.am @@ -5,6 +5,7 @@ if UNITTESTS_ENABLED noinst_LIBRARIES += src/lib/libtor-wallclock-testing.a endif +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_wallclock_a_SOURCES = \ src/lib/wallclock/approx_time.c \ src/lib/wallclock/time_to_tm.c \ @@ -15,6 +16,7 @@ src_lib_libtor_wallclock_testing_a_SOURCES = \ src_lib_libtor_wallclock_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_wallclock_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/wallclock/approx_time.h \ src/lib/wallclock/timeval.h \ diff --git a/src/test/include.am b/src/test/include.am index 022cdbe035..5d991f8399 100644 --- a/src/test/include.am +++ b/src/test/include.am @@ -85,6 +85,8 @@ src_test_AM_CPPFLAGS = -DSHARE_DATADIR="\"$(datadir)\"" \ src_test_test_SOURCES = if UNITTESTS_ENABLED + +# ADD_C_FILE: INSERT SOURCES HERE. src_test_test_SOURCES += \ src/test/log_test_helpers.c \ src/test/hs_test_helpers.c \ @@ -316,6 +318,7 @@ src_test_test_timers_LDADD = \ @TOR_LZMA_LIBS@ src_test_test_timers_LDFLAGS = $(src_test_test_LDFLAGS) +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS+= \ src/test/fakechans.h \ src/test/hs_test_helpers.h \ From 562bcbcfc216df9f25cd72f1e227b9313c872172 Mon Sep 17 00:00:00 2001 From: David Goulet Date: Thu, 2 May 2019 11:10:41 -0400 Subject: [PATCH 0945/2557] sendme: Add changes file for prop289 Signed-off-by: David Goulet --- changes/ticket26288 | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 changes/ticket26288 diff --git a/changes/ticket26288 b/changes/ticket26288 new file mode 100644 index 0000000000..59bb856dd2 --- /dev/null +++ b/changes/ticket26288 @@ -0,0 +1,6 @@ + o Major features (flow control): + - Implement authenticated SENDMEs detailed in proposal 289. A SENDME cell + now includes the digest of the last cell received so once the end point + receives the SENDME, it can confirm the other side's knowledge of the + previous cells that were sent. This behavior is controlled by two new + consensus parameters, see proposal for more details. Fixes ticket 26288. From 2fca2ed499bf2378684ba0cc3390818f52792728 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 2 May 2019 14:32:24 -0400 Subject: [PATCH 0946/2557] forward-port changelog and releasenotes from 0.4.0.5 --- ChangeLog | 48 +++++ ReleaseNotes | 599 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 647 insertions(+) diff --git a/ChangeLog b/ChangeLog index accde75b97..a69a7253b0 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,51 @@ +Changes in version 0.4.0.5 - 2019-05-02 + This is the first stable release in the 0.4.0.x series. It contains + improvements for power management and bootstrap reporting, as well as + preliminary backend support for circuit padding to prevent some kinds + of traffic analysis. It also continues our work in refactoring Tor for + long-term maintainability. + + Per our support policy, we will support the 0.4.0.x series for nine + months, or until three months after the release of a stable 0.4.1.x: + whichever is longer. If you need longer-term support, please stick + with 0.3.5.x, which will we plan to support until Feb 2022. + + Below are the changes since 0.4.0.4-rc. For a complete list of changes + since 0.3.5.7, see the ReleaseNotes file. + + o Minor features (continuous integration): + - In Travis, tell timelimit to use stem's backtrace signals, and + launch python directly from timelimit, so python receives the + signals from timelimit, rather than make. Closes ticket 30117. + + o Minor features (diagnostic): + - Add more diagnostic log messages in an attempt to solve the issue + of NUL bytes appearing in a microdescriptor cache. Related to + ticket 28223. + + o Minor features (testing): + - Use the approx_time() function when setting the "Expires" header + in directory replies, to make them more testable. Needed for + ticket 30001. + + o Minor bugfixes (rust): + - Abort on panic in all build profiles, instead of potentially + unwinding into C code. Fixes bug 27199; bugfix on 0.3.3.1-alpha. + + o Minor bugfixes (shellcheck): + - Look for scripts in their correct locations during "make + shellcheck". Previously we had looked in the wrong place during + out-of-tree builds. Fixes bug 30263; bugfix on 0.4.0.1-alpha. + + o Minor bugfixes (testing): + - Check the time in the "Expires" header using approx_time(). Fixes + bug 30001; bugfix on 0.4.0.4-rc. + + o Minor bugfixes (UI): + - Lower log level of unlink() errors during bootstrap. Fixes bug + 29930; bugfix on 0.4.0.1-alpha. + + Changes in version 0.4.0.4-rc - 2019-04-11 Tor 0.4.0.4-rc is the first release candidate in its series; it fixes several bugs from earlier versions, including some that had affected diff --git a/ReleaseNotes b/ReleaseNotes index 93dad1673b..badc5e6d0c 100644 --- a/ReleaseNotes +++ b/ReleaseNotes @@ -2,6 +2,605 @@ This document summarizes new features and bugfixes in each stable release of Tor. If you want to see more detailed descriptions of the changes in each development snapshot, see the ChangeLog file. +Changes in version 0.4.0.5 - 2019-05-02 + This is the first stable release in the 0.4.0.x series. It contains + improvements for power management and bootstrap reporting, as well as + preliminary backend support for circuit padding to prevent some kinds + of traffic analysis. It also continues our work in refactoring Tor for + long-term maintainability. + + Per our support policy, we will support the 0.4.0.x series for nine + months, or until three months after the release of a stable 0.4.1.x: + whichever is longer. If you need longer-term support, please stick + with 0.3.5.x, which will we plan to support until Feb 2022. + + Below are the changes since 0.3.5.7. For a complete list of changes + since 0.4.0.4-rc, see the ChangeLog file. + + o Major features (battery management, client, dormant mode): + - When Tor is running as a client, and it is unused for a long time, + it can now enter a "dormant" state. When Tor is dormant, it avoids + network and CPU activity until it is reawoken either by a user + request or by a controller command. For more information, see the + configuration options starting with "Dormant". Implements tickets + 2149 and 28335. + - The client's memory of whether it is "dormant", and how long it + has spent idle, persists across invocations. Implements + ticket 28624. + - There is a DormantOnFirstStartup option that integrators can use + if they expect that in many cases, Tor will be installed but + not used. + + o Major features (bootstrap reporting): + - When reporting bootstrap progress, report the first connection + uniformly, regardless of whether it's a connection for building + application circuits. This allows finer-grained reporting of early + progress than previously possible, with the improvements of ticket + 27169. Closes tickets 27167 and 27103. Addresses ticket 27308. + - When reporting bootstrap progress, treat connecting to a proxy or + pluggable transport as separate from having successfully used that + proxy or pluggable transport to connect to a relay. Closes tickets + 27100 and 28884. + + o Major features (circuit padding): + - Implement preliminary support for the circuit padding portion of + Proposal 254. The implementation supports Adaptive Padding (aka + WTF-PAD) state machines for use between experimental clients and + relays. Support is also provided for APE-style state machines that + use probability distributions instead of histograms to specify + inter-packet delay. At the moment, Tor does not provide any + padding state machines that are used in normal operation: for now, + this feature exists solely for experimentation. Closes + ticket 28142. + + o Major features (refactoring): + - Tor now uses an explicit list of its own subsystems when + initializing and shutting down. Previously, these systems were + managed implicitly in various places throughout the codebase. + (There may still be some subsystems using the old system.) Closes + ticket 28330. + + o Major bugfixes (cell scheduler, KIST, security): + - Make KIST consider the outbuf length when computing what it can + put in the outbuf. Previously, KIST acted as though the outbuf + were empty, which could lead to the outbuf becoming too full. It + is possible that an attacker could exploit this bug to cause a Tor + client or relay to run out of memory and crash. Fixes bug 29168; + bugfix on 0.3.2.1-alpha. This issue is also being tracked as + TROVE-2019-001 and CVE-2019-8955. + + o Major bugfixes (networking): + - Gracefully handle empty username/password fields in SOCKS5 + username/password auth messsage and allow SOCKS5 handshake to + continue. Previously, we had rejected these handshakes, breaking + certain applications. Fixes bug 29175; bugfix on 0.3.5.1-alpha. + + o Major bugfixes (NSS, relay): + - When running with NSS, disable TLS 1.2 ciphersuites that use + SHA384 for their PRF. Due to an NSS bug, the TLS key exporters for + these ciphersuites don't work -- which caused relays to fail to + handshake with one another when these ciphersuites were enabled. + Fixes bug 29241; bugfix on 0.3.5.1-alpha. + + o Major bugfixes (windows, startup): + - When reading a consensus file from disk, detect whether it was + written in text mode, and re-read it in text mode if so. Always + write consensus files in binary mode so that we can map them into + memory later. Previously, we had written in text mode, which + confused us when we tried to map the file on windows. Fixes bug + 28614; bugfix on 0.4.0.1-alpha. + + o Minor features (address selection): + - Treat the subnet 100.64.0.0/10 as public for some purposes; + private for others. This subnet is the RFC 6598 (Carrier Grade + NAT) IP range, and is deployed by many ISPs as an alternative to + RFC 1918 that does not break existing internal networks. Tor now + blocks SOCKS and control ports on these addresses and warns users + if client ports or ExtORPorts are listening on a RFC 6598 address. + Closes ticket 28525. Patch by Neel Chauhan. + + o Minor features (bandwidth authority): + - Make bandwidth authorities ignore relays that are reported in the + bandwidth file with the flag "vote=0". This change allows us to + report unmeasured relays for diagnostic reasons without including + their bandwidth in the bandwidth authorities' vote. Closes + ticket 29806. + - When a directory authority is using a bandwidth file to obtain the + bandwidth values that will be included in the next vote, serve + this bandwidth file at /tor/status-vote/next/bandwidth. Closes + ticket 21377. + + o Minor features (bootstrap reporting): + - When reporting bootstrap progress, stop distinguishing between + situations where only internal paths are available and situations + where external paths are available. Previously, Tor would often + erroneously report that it had only internal paths. Closes + ticket 27402. + + o Minor features (compilation): + - Compile correctly when OpenSSL is built with engine support + disabled, or with deprecated APIs disabled. Closes ticket 29026. + Patches from "Mangix". + + o Minor features (continuous integration): + - On Travis Rust builds, cleanup Rust registry and refrain from + caching the "target/" directory to speed up builds. Resolves + issue 29962. + - Log Python version during each Travis CI job. Resolves + issue 28551. + - In Travis, tell timelimit to use stem's backtrace signals, and + launch python directly from timelimit, so python receives the + signals from timelimit, rather than make. Closes ticket 30117. + + o Minor features (controller): + - Add a DROPOWNERSHIP command to undo the effects of TAKEOWNERSHIP. + Implements ticket 28843. + + o Minor features (developer tooling): + - Check that bugfix versions in changes files look like Tor versions + from the versions spec. Warn when bugfixes claim to be on a future + release. Closes ticket 27761. + - Provide a git pre-commit hook that disallows commiting if we have + any failures in our code and changelog formatting checks. It is + now available in scripts/maint/pre-commit.git-hook. Implements + feature 28976. + - Provide a git hook script to prevent "fixup!" and "squash!" + commits from ending up in the master branch, as scripts/main/pre- + push.git-hook. Closes ticket 27993. + + o Minor features (diagnostic): + - Add more diagnostic log messages in an attempt to solve the issue + of NUL bytes appearing in a microdescriptor cache. Related to + ticket 28223. + + o Minor features (directory authority): + - When a directory authority is using a bandwidth file to obtain + bandwidth values, include the digest of that file in the vote. + Closes ticket 26698. + - Directory authorities support a new consensus algorithm, under + which the family lines in microdescriptors are encoded in a + canonical form. This change makes family lines more compressible + in transit, and on the client. Closes ticket 28266; implements + proposal 298. + + o Minor features (directory authority, relay): + - Authorities now vote on a "StaleDesc" flag to indicate that a + relay's descriptor is so old that the relay should upload again + soon. Relays treat this flag as a signal to upload a new + descriptor. This flag will eventually let us remove the + 'published' date from routerstatus entries, and make our consensus + diffs much smaller. Closes ticket 26770; implements proposal 293. + + o Minor features (dormant mode): + - Add a DormantCanceledByStartup option to tell Tor that it should + treat a startup event as cancelling any previous dormant state. + Integrators should use this option with caution: it should only be + used if Tor is being started because of something that the user + did, and not if Tor is being automatically started in the + background. Closes ticket 29357. + + o Minor features (fallback directory mirrors): + - Update the fallback whitelist based on operator opt-ins and opt- + outs. Closes ticket 24805, patch by Phoul. + + o Minor features (FreeBSD): + - On FreeBSD-based systems, warn relay operators if the + "net.inet.ip.random_id" sysctl (IP ID randomization) is disabled. + Closes ticket 28518. + + o Minor features (geoip): + - Update geoip and geoip6 to the April 2 2019 Maxmind GeoLite2 + Country database. Closes ticket 29992. + + o Minor features (HTTP standards compliance): + - Stop sending the header "Content-type: application/octet-stream" + along with transparently compressed documents: this confused + browsers. Closes ticket 28100. + + o Minor features (IPv6): + - We add an option ClientAutoIPv6ORPort, to make clients randomly + prefer a node's IPv4 or IPv6 ORPort. The random preference is set + every time a node is loaded from a new consensus or bridge config. + We expect that this option will enable clients to bootstrap more + quickly without having to determine whether they support IPv4, + IPv6, or both. Closes ticket 27490. Patch by Neel Chauhan. + - When using addrs_in_same_network_family(), avoid choosing circuit + paths that pass through the same IPv6 subnet more than once. + Previously, we only checked IPv4 subnets. Closes ticket 24393. + Patch by Neel Chauhan. + + o Minor features (log messages): + - Improve log message in v3 onion services that could print out + negative revision counters. Closes ticket 27707. Patch + by "ffmancera". + + o Minor features (memory usage): + - Save memory by storing microdescriptor family lists with a more + compact representation. Closes ticket 27359. + - Tor clients now use mmap() to read consensus files from disk, so + that they no longer need keep the full text of a consensus in + memory when parsing it or applying a diff. Closes ticket 27244. + + o Minor features (NSS, diagnostic): + - Try to log an error from NSS (if there is any) and a more useful + description of our situation if we are using NSS and a call to + SSL_ExportKeyingMaterial() fails. Diagnostic for ticket 29241. + + o Minor features (parsing): + - Directory authorities now validate that router descriptors and + ExtraInfo documents are in a valid subset of UTF-8, and reject + them if they are not. Closes ticket 27367. + + o Minor features (performance): + - Cache the results of summarize_protocol_flags(), so that we don't + have to parse the same protocol-versions string over and over. + This should save us a huge number of malloc calls on startup, and + may reduce memory fragmentation with some allocators. Closes + ticket 27225. + - Remove a needless memset() call from get_token_arguments, thereby + speeding up the tokenization of directory objects by about 20%. + Closes ticket 28852. + - Replace parse_short_policy() with a faster implementation, to + improve microdescriptor parsing time. Closes ticket 28853. + - Speed up directory parsing a little by avoiding use of the non- + inlined strcmp_len() function. Closes ticket 28856. + - Speed up microdescriptor parsing by about 30%, to help improve + startup time. Closes ticket 28839. + + o Minor features (pluggable transports): + - Add support for emitting STATUS updates to Tor's control port from + a pluggable transport process. Closes ticket 28846. + - Add support for logging to Tor's logging subsystem from a + pluggable transport process. Closes ticket 28180. + + o Minor features (process management): + - Add a new process API for handling child processes. This new API + allows Tor to have bi-directional communication with child + processes on both Unix and Windows. Closes ticket 28179. + - Use the subsystem manager to initialize and shut down the process + module. Closes ticket 28847. + + o Minor features (relay): + - When listing relay families, list them in canonical form including + the relay's own identity, and try to give a more useful set of + warnings. Part of ticket 28266 and proposal 298. + + o Minor features (required protocols): + - Before exiting because of a missing required protocol, Tor will + now check the publication time of the consensus, and not exit + unless the consensus is newer than the Tor program's own release + date. Previously, Tor would not check the consensus publication + time, and so might exit because of a missing protocol that might + no longer be required in a current consensus. Implements proposal + 297; closes ticket 27735. + + o Minor features (testing): + - Treat all unexpected ERR and BUG messages as test failures. Closes + ticket 28668. + - Allow a HeartbeatPeriod of less than 30 minutes in testing Tor + networks. Closes ticket 28840. Patch by Rob Jansen. + - Use the approx_time() function when setting the "Expires" header + in directory replies, to make them more testable. Needed for + ticket 30001. + + o Minor bugfixes (security): + - Fix a potential double free bug when reading huge bandwidth files. + The issue is not exploitable in the current Tor network because + the vulnerable code is only reached when directory authorities + read bandwidth files, but bandwidth files come from a trusted + source (usually the authorities themselves). Furthermore, the + issue is only exploitable in rare (non-POSIX) 32-bit architectures, + which are not used by any of the current authorities. Fixes bug + 30040; bugfix on 0.3.5.1-alpha. Bug found and fixed by + Tobias Stoeckmann. + - Verify in more places that we are not about to create a buffer + with more than INT_MAX bytes, to avoid possible OOB access in the + event of bugs. Fixes bug 30041; bugfix on 0.2.0.16. Found and + fixed by Tobias Stoeckmann. + + o Minor bugfix (continuous integration): + - Reset coverage state on disk after Travis CI has finished. This + should prevent future coverage merge errors from causing the test + suite for the "process" subsystem to fail. The process subsystem + was introduced in 0.4.0.1-alpha. Fixes bug 29036; bugfix + on 0.2.9.15. + - Terminate test-stem if it takes more than 9.5 minutes to run. + (Travis terminates the job after 10 minutes of no output.) + Diagnostic for 29437. Fixes bug 30011; bugfix on 0.3.5.4-alpha. + + o Minor bugfixes (build, compatibility, rust): + - Update Cargo.lock file to match the version made by the latest + version of Rust, so that "make distcheck" will pass again. Fixes + bug 29244; bugfix on 0.3.3.4-alpha. + + o Minor bugfixes (C correctness): + - Fix an unlikely memory leak in consensus_diff_apply(). Fixes bug + 29824; bugfix on 0.3.1.1-alpha. This is Coverity warning + CID 1444119. + + o Minor bugfixes (client, clock skew): + - Bootstrap successfully even when Tor's clock is behind the clocks + on the authorities. Fixes bug 28591; bugfix on 0.2.0.9-alpha. + - Select guards even if the consensus has expired, as long as the + consensus is still reasonably live. Fixes bug 24661; bugfix + on 0.3.0.1-alpha. + + o Minor bugfixes (compilation): + - Fix compilation warnings in test_circuitpadding.c. Fixes bug + 29169; bugfix on 0.4.0.1-alpha. + - Silence a compiler warning in test-memwipe.c on OpenBSD. Fixes bug + 29145; bugfix on 0.2.9.3-alpha. Patch from Kris Katterjohn. + - Compile correctly on OpenBSD; previously, we were missing some + headers required in order to detect it properly. Fixes bug 28938; + bugfix on 0.3.5.1-alpha. Patch from Kris Katterjohn. + + o Minor bugfixes (directory clients): + - Mark outdated dirservers when Tor only has a reasonably live + consensus. Fixes bug 28569; bugfix on 0.3.2.5-alpha. + + o Minor bugfixes (directory mirrors): + - Even when a directory mirror's clock is behind the clocks on the + authorities, we now allow the mirror to serve "future" + consensuses. Fixes bug 28654; bugfix on 0.3.0.1-alpha. + + o Minor bugfixes (DNS): + - Gracefully handle an empty or absent resolve.conf file by falling + back to using "localhost" as a DNS server (and hoping it works). + Previously, we would just stop running as an exit. Fixes bug + 21900; bugfix on 0.2.1.10-alpha. + + o Minor bugfixes (documentation): + - Describe the contents of the v3 onion service client authorization + files correctly: They hold public keys, not private keys. Fixes + bug 28979; bugfix on 0.3.5.1-alpha. Spotted by "Felixix". + + o Minor bugfixes (guards): + - In count_acceptable_nodes(), the minimum number is now one bridge + or guard node, and two non-guard nodes for a circuit. Previously, + we had added up the sum of all nodes with a descriptor, but that + could cause us to build failing circuits when we had either too + many bridges or not enough guard nodes. Fixes bug 25885; bugfix on + 0.3.6.1-alpha. Patch by Neel Chauhan. + + o Minor bugfixes (IPv6): + - Fix tor_ersatz_socketpair on IPv6-only systems. Previously, the + IPv6 socket was bound using an address family of AF_INET instead + of AF_INET6. Fixes bug 28995; bugfix on 0.3.5.1-alpha. Patch from + Kris Katterjohn. + + o Minor bugfixes (linux seccomp sandbox): + - Fix startup crash when experimental sandbox support is enabled. + Fixes bug 29150; bugfix on 0.4.0.1-alpha. Patch by Peter Gerber. + + o Minor bugfixes (logging): + - Correct a misleading error message when IPv4Only or IPv6Only is + used but the resolved address can not be interpreted as an address + of the specified IP version. Fixes bug 13221; bugfix on + 0.2.3.9-alpha. Patch from Kris Katterjohn. + - Log the correct port number for listening sockets when "auto" is + used to let Tor pick the port number. Previously, port 0 was + logged instead of the actual port number. Fixes bug 29144; bugfix + on 0.3.5.1-alpha. Patch from Kris Katterjohn. + - Stop logging a BUG() warning when Tor is waiting for exit + descriptors. Fixes bug 28656; bugfix on 0.3.5.1-alpha. + - Avoid logging that we are relaxing a circuit timeout when that + timeout is fixed. Fixes bug 28698; bugfix on 0.2.4.7-alpha. + - Log more information at "warning" level when unable to read a + private key; log more information at "info" level when unable to + read a public key. We had warnings here before, but they were lost + during our NSS work. Fixes bug 29042; bugfix on 0.3.5.1-alpha. + - Rework rep_hist_log_link_protocol_counts() to iterate through all + link protocol versions when logging incoming/outgoing connection + counts. Tor no longer skips version 5, and we won't have to + remember to update this function when new link protocol version is + developed. Fixes bug 28920; bugfix on 0.2.6.10. + + o Minor bugfixes (memory management): + - Refactor the shared random state's memory management so that it + actually takes ownership of the shared random value pointers. + Fixes bug 29706; bugfix on 0.2.9.1-alpha. + - Stop leaking parts of the shared random state in the shared-random + unit tests. Fixes bug 29599; bugfix on 0.2.9.1-alpha. + + o Minor bugfixes (misc): + - The amount of total available physical memory is now determined + using the sysctl identifier HW_PHYSMEM (rather than HW_USERMEM) + when it is defined and a 64-bit variant is not available. Fixes + bug 28981; bugfix on 0.2.5.4-alpha. Patch from Kris Katterjohn. + + o Minor bugfixes (networking): + - Introduce additional checks into tor_addr_parse() to reject + certain incorrect inputs that previously were not detected. Fixes + bug 23082; bugfix on 0.2.0.10-alpha. + + o Minor bugfixes (onion service v3, client): + - Stop logging a "BUG()" warning and stacktrace when we find a SOCKS + connection waiting for a descriptor that we actually have in the + cache. It turns out that this can actually happen, though it is + rare. Now, tor will recover and retry the descriptor. Fixes bug + 28669; bugfix on 0.3.2.4-alpha. + + o Minor bugfixes (onion services): + - Avoid crashing if ClientOnionAuthDir (incorrectly) contains more + than one private key for a hidden service. Fixes bug 29040; bugfix + on 0.3.5.1-alpha. + - In hs_cache_store_as_client() log an HSDesc we failed to parse at + "debug" level. Tor used to log it as a warning, which caused very + long log lines to appear for some users. Fixes bug 29135; bugfix + on 0.3.2.1-alpha. + - Stop logging "Tried to establish rendezvous on non-OR circuit..." + as a warning. Instead, log it as a protocol warning, because there + is nothing that relay operators can do to fix it. Fixes bug 29029; + bugfix on 0.2.5.7-rc. + + o Minor bugfixes (periodic events): + - Refrain from calling routerlist_remove_old_routers() from + check_descriptor_callback(). Instead, create a new hourly periodic + event. Fixes bug 27929; bugfix on 0.2.8.1-alpha. + + o Minor bugfixes (pluggable transports): + - Make sure that data is continously read from standard output and + standard error pipes of a pluggable transport child-process, to + avoid deadlocking when a pipe's buffer is full. Fixes bug 26360; + bugfix on 0.2.3.6-alpha. + + o Minor bugfixes (rust): + - Abort on panic in all build profiles, instead of potentially + unwinding into C code. Fixes bug 27199; bugfix on 0.3.3.1-alpha. + + o Minor bugfixes (scheduler): + - When re-adding channels to the pending list, check the correct + channel's sched_heap_idx. This issue has had no effect in mainline + Tor, but could have led to bugs down the road in improved versions + of our circuit scheduling code. Fixes bug 29508; bugfix + on 0.3.2.10. + + o Minor bugfixes (shellcheck): + - Look for scripts in their correct locations during "make + shellcheck". Previously we had looked in the wrong place during + out-of-tree builds. Fixes bug 30263; bugfix on 0.4.0.1-alpha. + + o Minor bugfixes (single onion services): + - Allow connections to single onion services to remain idle without + being disconnected. Previously, relays acting as rendezvous points + for single onion services were mistakenly closing idle rendezvous + circuits after 60 seconds, thinking that they were unused + directory-fetching circuits that had served their purpose. Fixes + bug 29665; bugfix on 0.2.1.26. + + o Minor bugfixes (stats): + - When ExtraInfoStatistics is 0, stop including PaddingStatistics in + relay and bridge extra-info documents. Fixes bug 29017; bugfix + on 0.3.1.1-alpha. + + o Minor bugfixes (testing): + - Backport the 0.3.4 src/test/test-network.sh to 0.2.9. We need a + recent test-network.sh to use new chutney features in CI. Fixes + bug 29703; bugfix on 0.2.9.1-alpha. + - Fix a test failure on Windows caused by an unexpected "BUG" + warning in our tests for tor_gmtime_r(-1). Fixes bug 29922; bugfix + on 0.2.9.3-alpha. + - Downgrade some LOG_ERR messages in the address/* tests to + warnings. The LOG_ERR messages were occurring when we had no + configured network. We were failing the unit tests, because we + backported 28668 to 0.3.5.8, but did not backport 29530. Fixes bug + 29530; bugfix on 0.3.5.8. + - Fix our gcov wrapper script to look for object files at the + correct locations. Fixes bug 29435; bugfix on 0.3.5.1-alpha. + - Decrease the false positive rate of stochastic probability + distribution tests. Fixes bug 29693; bugfix on 0.4.0.1-alpha. + - Fix intermittent failures on an adaptive padding test. Fixes one + case of bug 29122; bugfix on 0.4.0.1-alpha. + - Disable an unstable circuit-padding test that was failing + intermittently because of an ill-defined small histogram. Such + histograms will be allowed again after 29298 is implemented. Fixes + a second case of bug 29122; bugfix on 0.4.0.1-alpha. + - Detect and suppress "bug" warnings from the util/time test on + Windows. Fixes bug 29161; bugfix on 0.2.9.3-alpha. + - Do not log an error-level message if we fail to find an IPv6 + network interface from the unit tests. Fixes bug 29160; bugfix + on 0.2.7.3-rc. + - Instead of relying on hs_free_all() to clean up all onion service + objects in test_build_descriptors(), we now deallocate them one by + one. This lets Coverity know that we are not leaking memory there + and fixes CID 1442277. Fixes bug 28989; bugfix on 0.3.5.1-alpha. + - Check the time in the "Expires" header using approx_time(). Fixes + bug 30001; bugfix on 0.4.0.4-rc. + + o Minor bugfixes (TLS protocol): + - When classifying a client's selection of TLS ciphers, if the + client ciphers are not yet available, do not cache the result. + Previously, we had cached the unavailability of the cipher list + and never looked again, which in turn led us to assume that the + client only supported the ancient V1 link protocol. This, in turn, + was causing Stem integration tests to stall in some cases. Fixes + bug 30021; bugfix on 0.2.4.8-alpha. + + o Minor bugfixes (UI): + - Lower log level of unlink() errors during bootstrap. Fixes bug + 29930; bugfix on 0.4.0.1-alpha. + + o Minor bugfixes (usability): + - Stop saying "Your Guard ..." in pathbias_measure_{use,close}_rate(). + Some users took this phrasing to mean that the mentioned guard was + under their control or responsibility, which it is not. Fixes bug + 28895; bugfix on Tor 0.3.0.1-alpha. + + o Minor bugfixes (Windows, CI): + - Skip the Appveyor 32-bit Windows Server 2016 job, and 64-bit + Windows Server 2012 R2 job. The remaining 2 jobs still provide + coverage of 64/32-bit, and Windows Server 2016/2012 R2. Also set + fast_finish, so failed jobs terminate the build immediately. Fixes + bug 29601; bugfix on 0.3.5.4-alpha. + + o Code simplification and refactoring: + - Introduce a connection_dir_buf_add() helper function that detects + whether compression is in use, and adds a string accordingly. + Resolves issue 28816. + - Refactor handle_get_next_bandwidth() to use + connection_dir_buf_add(). Implements ticket 29897. + - Reimplement NETINFO cell parsing and generation to rely on + trunnel-generated wire format handling code. Closes ticket 27325. + - Remove unnecessary unsafe code from the Rust macro "cstr!". Closes + ticket 28077. + - Rework SOCKS wire format handling to rely on trunnel-generated + parsing/generation code. Resolves ticket 27620. + - Split out bootstrap progress reporting from control.c into a + separate file. Part of ticket 27402. + - The .may_include files that we use to describe our directory-by- + directory dependency structure now describe a noncircular + dependency graph over the directories that they cover. Our + checkIncludes.py tool now enforces this noncircularity. Closes + ticket 28362. + + o Documentation: + - Clarify that Tor performs stream isolation among *Port listeners + by default. Resolves issue 29121. + - In the manpage entry describing MapAddress torrc setting, use + example IP addresses from ranges specified for use in documentation + by RFC 5737. Resolves issue 28623. + - Mention that you cannot add a new onion service if Tor is already + running with Sandbox enabled. Closes ticket 28560. + - Improve ControlPort documentation. Mention that it accepts + address:port pairs, and can be used multiple times. Closes + ticket 28805. + - Document the exact output of "tor --version". Closes ticket 28889. + + o Removed features: + - Remove the old check-tor script. Resolves issue 29072. + - Stop responding to the 'GETINFO status/version/num-concurring' and + 'GETINFO status/version/num-versioning' control port commands, as + those were deprecated back in 0.2.0.30. Also stop listing them in + output of 'GETINFO info/names'. Resolves ticket 28757. + - The scripts used to generate and maintain the list of fallback + directories have been extracted into a new "fallback-scripts" + repository. Closes ticket 27914. + + o Testing: + - Run shellcheck for scripts in the in scripts/ directory. Closes + ticket 28058. + - Add unit tests for tokenize_string() and get_next_token() + functions. Resolves ticket 27625. + + o Code simplification and refactoring (onion service v3): + - Consolidate the authorized client descriptor cookie computation + code from client and service into one function. Closes + ticket 27549. + + o Code simplification and refactoring (shell scripts): + - Cleanup scan-build.sh to silence shellcheck warnings. Closes + ticket 28007. + - Fix issues that shellcheck found in chutney-git-bisect.sh. + Resolves ticket 28006. + - Fix issues that shellcheck found in updateRustDependencies.sh. + Resolves ticket 28012. + - Fix shellcheck warnings in cov-diff script. Resolves issue 28009. + - Fix shellcheck warnings in run_calltool.sh. Resolves ticket 28011. + - Fix shellcheck warnings in run_trunnel.sh. Resolves issue 28010. + - Fix shellcheck warnings in scripts/test/coverage. Resolves + issue 28008. + + Changes in version 0.3.5.8 - 2019-02-21 Tor 0.3.5.8 backports serveral fixes from later releases, including fixes for an annoying SOCKS-parsing bug that affected users in earlier 0.3.5.x From 2b54733dc0bb055b609502cede2a6055b3c3ef15 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 2 May 2019 14:32:55 -0400 Subject: [PATCH 0947/2557] Bump to 0.4.0.5-dev --- configure.ac | 4 ++-- contrib/win32build/tor-mingw.nsi.in | 2 +- src/win32/orconfig.h | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/configure.ac b/configure.ac index 28bf5d1239..4b69b3c89d 100644 --- a/configure.ac +++ b/configure.ac @@ -4,7 +4,7 @@ dnl Copyright (c) 2007-2019, The Tor Project, Inc. dnl See LICENSE for licensing information AC_PREREQ([2.63]) -AC_INIT([tor],[0.4.0.5]) +AC_INIT([tor],[0.4.0.5-dev]) AC_CONFIG_SRCDIR([src/app/main/tor_main.c]) AC_CONFIG_MACRO_DIR([m4]) @@ -14,7 +14,7 @@ AC_CONFIG_MACRO_DIR([m4]) # version number changes. Tor uses it to make sure that it # only shuts down for missing "required protocols" when those protocols # are listed as required by a consensus after this date. -AC_DEFINE(APPROX_RELEASE_DATE, ["2019-04-30"], # for 0.4.0.5 +AC_DEFINE(APPROX_RELEASE_DATE, ["2019-05-02"], # for 0.4.0.5-dev [Approximate date when this software was released. (Updated when the version changes.)]) # "foreign" means we don't follow GNU package layout standards diff --git a/contrib/win32build/tor-mingw.nsi.in b/contrib/win32build/tor-mingw.nsi.in index 3538b7caaa..7b3cf21160 100644 --- a/contrib/win32build/tor-mingw.nsi.in +++ b/contrib/win32build/tor-mingw.nsi.in @@ -8,7 +8,7 @@ !include "LogicLib.nsh" !include "FileFunc.nsh" !insertmacro GetParameters -!define VERSION "0.4.0.5" +!define VERSION "0.4.0.5-dev" !define INSTALLER "tor-${VERSION}-win32.exe" !define WEBSITE "https://www.torproject.org/" !define LICENSE "LICENSE" diff --git a/src/win32/orconfig.h b/src/win32/orconfig.h index 23722d6174..adac422687 100644 --- a/src/win32/orconfig.h +++ b/src/win32/orconfig.h @@ -218,7 +218,7 @@ #define USING_TWOS_COMPLEMENT /* Version number of package */ -#define VERSION "0.4.0.5" +#define VERSION "0.4.0.5-dev" From d20cfc00ce435d78bbf386e5f5505e0989439c11 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 3 May 2019 08:51:28 -0400 Subject: [PATCH 0948/2557] Note how to update the new website in the ReleasingTor.md instructions --- doc/HACKING/ReleasingTor.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/HACKING/ReleasingTor.md b/doc/HACKING/ReleasingTor.md index 7334b1b34a..4c87a366cc 100644 --- a/doc/HACKING/ReleasingTor.md +++ b/doc/HACKING/ReleasingTor.md @@ -176,7 +176,8 @@ new Tor release: `/srv/dist-master.torproject.org/htdocs/` on dist-master. Run "static-update-component dist.torproject.org" on dist-master. - In the webwml.git repository, `include/versions.wmi` and `Makefile` + In the webwml.git repository, `include/versions.wmi` and `Makefile`. + In the project/web/tpo.git repository, update `databags/versions.ini` to note the new version. Push these changes to master. (NOTE: Due to #17805, there can only be one stable version listed at From c3a5e6b4363eba5157c7cccc049f31f6ae144fcf Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Mon, 8 Apr 2019 14:35:02 +0300 Subject: [PATCH 0949/2557] Hiding crypt_path_t: Introduce opaque crypt_path_private_t . This will be our base for incrementally hiding crypt_path_t. --- src/core/or/crypt_path_st.h | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/core/or/crypt_path_st.h b/src/core/or/crypt_path_st.h index 429480f8ab..90f6a37881 100644 --- a/src/core/or/crypt_path_st.h +++ b/src/core/or/crypt_path_st.h @@ -24,6 +24,15 @@ struct onion_handshake_state_t { } u; }; +#ifdef CRYPT_PATH_PRIVATE + +/* The private parts of crypt path that don't need to be exposed to all the + * modules. */ +struct crypt_path_private_t { +}; + +#endif + /** Holds accounting information for a single step in the layered encryption * performed by a circuit. Used only at the client edge of a circuit. */ struct crypt_path_t { @@ -65,6 +74,10 @@ struct crypt_path_t { * at this step? */ int deliver_window; /**< How many cells are we willing to deliver originating * at this step? */ + + /* Private parts of the crypt_path. Eventually everything should be + * private. */ + struct crypt_path_private_t *private; }; #endif From 9584798e57f2e5525e01b8bec51de61ff0c256b9 Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Mon, 8 Apr 2019 12:46:40 +0300 Subject: [PATCH 0950/2557] Hiding crypt_path_t: Move assert functions in crypt_path.c. This commit only moves code, and makes one function public. --- src/core/include.am | 2 + src/core/mainloop/connection.c | 1 + src/core/or/circuitlist.c | 54 +------------------------ src/core/or/circuitlist.h | 1 - src/core/or/crypt_path.c | 74 ++++++++++++++++++++++++++++++++++ src/core/or/crypt_path.h | 11 +++++ 6 files changed, 89 insertions(+), 54 deletions(-) create mode 100644 src/core/or/crypt_path.c create mode 100644 src/core/or/crypt_path.h diff --git a/src/core/include.am b/src/core/include.am index 9493f79552..8435ce0415 100644 --- a/src/core/include.am +++ b/src/core/include.am @@ -39,6 +39,7 @@ LIBTOR_APP_A_SOURCES = \ src/core/or/circuitpadding.c \ src/core/or/circuitstats.c \ src/core/or/circuituse.c \ + src/core/or/crypt_path.c \ src/core/or/command.c \ src/core/or/connection_edge.c \ src/core/or/connection_or.c \ @@ -247,6 +248,7 @@ noinst_HEADERS += \ src/core/or/connection_edge.h \ src/core/or/connection_or.h \ src/core/or/connection_st.h \ + src/core/or/crypt_path.h \ src/core/or/cpath_build_state_st.h \ src/core/or/crypt_path_reference_st.h \ src/core/or/crypt_path_st.h \ diff --git a/src/core/mainloop/connection.c b/src/core/mainloop/connection.c index 30504e4edb..f6adfa765a 100644 --- a/src/core/mainloop/connection.c +++ b/src/core/mainloop/connection.c @@ -82,6 +82,7 @@ #include "core/or/policies.h" #include "core/or/reasons.h" #include "core/or/relay.h" +#include "core/or/crypt_path.h" #include "core/proto/proto_http.h" #include "core/proto/proto_socks.h" #include "feature/client/dnsserv.h" diff --git a/src/core/or/circuitlist.c b/src/core/or/circuitlist.c index 6428cdb8a7..ee9e89f380 100644 --- a/src/core/or/circuitlist.c +++ b/src/core/or/circuitlist.c @@ -63,6 +63,7 @@ #include "core/or/circuituse.h" #include "core/or/circuitstats.h" #include "core/or/circuitpadding.h" +#include "core/or/crypt_path.h" #include "core/mainloop/connection.h" #include "app/config/config.h" #include "core/or/connection_edge.h" @@ -2785,59 +2786,6 @@ circuits_handle_oom(size_t current_allocation) n_dirconns_killed); } -/** Verify that cpath layer cp has all of its invariants - * correct. Trigger an assert if anything is invalid. - */ -void -assert_cpath_layer_ok(const crypt_path_t *cp) -{ -// tor_assert(cp->addr); /* these are zero for rendezvous extra-hops */ -// tor_assert(cp->port); - tor_assert(cp); - tor_assert(cp->magic == CRYPT_PATH_MAGIC); - switch (cp->state) - { - case CPATH_STATE_OPEN: - relay_crypto_assert_ok(&cp->crypto); - /* fall through */ - case CPATH_STATE_CLOSED: - /*XXXX Assert that there's no handshake_state either. */ - tor_assert(!cp->rend_dh_handshake_state); - break; - case CPATH_STATE_AWAITING_KEYS: - /* tor_assert(cp->dh_handshake_state); */ - break; - default: - log_fn(LOG_ERR, LD_BUG, "Unexpected state %d", cp->state); - tor_assert(0); - } - tor_assert(cp->package_window >= 0); - tor_assert(cp->deliver_window >= 0); -} - -/** Verify that cpath cp has all of its invariants - * correct. Trigger an assert if anything is invalid. - */ -static void -assert_cpath_ok(const crypt_path_t *cp) -{ - const crypt_path_t *start = cp; - - do { - assert_cpath_layer_ok(cp); - /* layers must be in sequence of: "open* awaiting? closed*" */ - if (cp != start) { - if (cp->state == CPATH_STATE_AWAITING_KEYS) { - tor_assert(cp->prev->state == CPATH_STATE_OPEN); - } else if (cp->state == CPATH_STATE_OPEN) { - tor_assert(cp->prev->state == CPATH_STATE_OPEN); - } - } - cp = cp->next; - tor_assert(cp); - } while (cp != start); -} - /** Verify that circuit c has all of its invariants * correct. Trigger an assert if anything is invalid. */ diff --git a/src/core/or/circuitlist.h b/src/core/or/circuitlist.h index f34f4ed6b7..a50e23716a 100644 --- a/src/core/or/circuitlist.h +++ b/src/core/or/circuitlist.h @@ -228,7 +228,6 @@ int circuit_count_pending_on_channel(channel_t *chan); #define circuit_mark_for_close(c, reason) \ circuit_mark_for_close_((c), (reason), __LINE__, SHORT_FILE__) -void assert_cpath_layer_ok(const crypt_path_t *cp); MOCK_DECL(void, assert_circuit_ok,(const circuit_t *c)); void circuit_free_all(void); void circuits_handle_oom(size_t current_allocation); diff --git a/src/core/or/crypt_path.c b/src/core/or/crypt_path.c new file mode 100644 index 0000000000..d4fc59630a --- /dev/null +++ b/src/core/or/crypt_path.c @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file crypt_path.c + * + * \brief Functions dealing with layered circuit encryption. This file aims to + * provide an API around the crypt_path_t structure which holds crypto + * information about a specific hop of a circuit. + **/ + +#define CRYPT_PATH_PRIVATE + +#include "core/or/or.h" +#include "core/or/crypt_path.h" + +#include "core/crypto/relay_crypto.h" + +#include "core/or/crypt_path_st.h" + +/** Verify that cpath cp has all of its invariants + * correct. Trigger an assert if anything is invalid. + */ +void +assert_cpath_ok(const crypt_path_t *cp) +{ + const crypt_path_t *start = cp; + + do { + assert_cpath_layer_ok(cp); + /* layers must be in sequence of: "open* awaiting? closed*" */ + if (cp != start) { + if (cp->state == CPATH_STATE_AWAITING_KEYS) { + tor_assert(cp->prev->state == CPATH_STATE_OPEN); + } else if (cp->state == CPATH_STATE_OPEN) { + tor_assert(cp->prev->state == CPATH_STATE_OPEN); + } + } + cp = cp->next; + tor_assert(cp); + } while (cp != start); +} + +/** Verify that cpath layer cp has all of its invariants + * correct. Trigger an assert if anything is invalid. + */ +void +assert_cpath_layer_ok(const crypt_path_t *cp) +{ +// tor_assert(cp->addr); /* these are zero for rendezvous extra-hops */ +// tor_assert(cp->port); + tor_assert(cp); + tor_assert(cp->magic == CRYPT_PATH_MAGIC); + switch (cp->state) + { + case CPATH_STATE_OPEN: + relay_crypto_assert_ok(&cp->crypto); + /* fall through */ + case CPATH_STATE_CLOSED: + /*XXXX Assert that there's no handshake_state either. */ + tor_assert(!cp->rend_dh_handshake_state); + break; + case CPATH_STATE_AWAITING_KEYS: + /* tor_assert(cp->dh_handshake_state); */ + break; + default: + log_fn(LOG_ERR, LD_BUG, "Unexpected state %d", cp->state); + tor_assert(0); + } + tor_assert(cp->package_window >= 0); + tor_assert(cp->deliver_window >= 0); +} + diff --git a/src/core/or/crypt_path.h b/src/core/or/crypt_path.h new file mode 100644 index 0000000000..a9b9aae43d --- /dev/null +++ b/src/core/or/crypt_path.h @@ -0,0 +1,11 @@ +/** + * \file crypt_path.h + * \brief Header file for crypt_path.c. + **/ + +/* rename */ +void assert_cpath_layer_ok(const crypt_path_t *cp); + +/* rename */ +void assert_cpath_ok(const crypt_path_t *cp); + From f74a80dc3b2ada940e72cd174af5779cac3c3948 Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Mon, 8 Apr 2019 13:01:18 +0300 Subject: [PATCH 0951/2557] Hiding crypt_path_t: Move init functions to crypt_path.c. This commit only moves code. --- src/core/or/circuitbuild.c | 41 +--------------------------------- src/core/or/circuitbuild.h | 1 - src/core/or/crypt_path.c | 41 ++++++++++++++++++++++++++++++++++ src/core/or/crypt_path.h | 6 +++++ src/feature/hs/hs_circuit.c | 1 + src/feature/rend/rendservice.c | 1 + src/test/test_circuitpadding.c | 1 + src/test/test_relaycrypt.c | 2 +- 8 files changed, 52 insertions(+), 42 deletions(-) diff --git a/src/core/or/circuitbuild.c b/src/core/or/circuitbuild.c index cfe0a97bcf..7216b813bd 100644 --- a/src/core/or/circuitbuild.c +++ b/src/core/or/circuitbuild.c @@ -51,6 +51,7 @@ #include "core/or/ocirc_event.h" #include "core/or/policies.h" #include "core/or/relay.h" +#include "core/or/crypt_path.h" #include "feature/client/bridges.h" #include "feature/client/circpathbias.h" #include "feature/client/entrynodes.h" @@ -91,7 +92,6 @@ static int circuit_deliver_create_cell(circuit_t *circ, const create_cell_t *create_cell, int relayed); static crypt_path_t *onion_next_hop_in_cpath(crypt_path_t *cpath); -STATIC int onion_append_hop(crypt_path_t **head_ptr, extend_info_t *choice); static int circuit_send_first_onion_skin(origin_circuit_t *circ); static int circuit_build_no_more_hops(origin_circuit_t *circ); static int circuit_send_intermediate_onion_skin(origin_circuit_t *circ, @@ -2373,23 +2373,6 @@ count_acceptable_nodes, (const smartlist_t *nodes, int direct)) return num; } -/** Add new_hop to the end of the doubly-linked-list head_ptr. - * This function is used to extend cpath by another hop. - */ -void -onion_append_to_cpath(crypt_path_t **head_ptr, crypt_path_t *new_hop) -{ - if (*head_ptr) { - new_hop->next = (*head_ptr); - new_hop->prev = (*head_ptr)->prev; - (*head_ptr)->prev->next = new_hop; - (*head_ptr)->prev = new_hop; - } else { - *head_ptr = new_hop; - new_hop->prev = new_hop->next = new_hop; - } -} - #ifdef TOR_UNIT_TESTS /** Unittest helper function: Count number of hops in cpath linked list. */ @@ -2763,28 +2746,6 @@ onion_extend_cpath(origin_circuit_t *circ) return 0; } -/** Create a new hop, annotate it with information about its - * corresponding router choice, and append it to the - * end of the cpath head_ptr. */ -STATIC int -onion_append_hop(crypt_path_t **head_ptr, extend_info_t *choice) -{ - crypt_path_t *hop = tor_malloc_zero(sizeof(crypt_path_t)); - - /* link hop into the cpath, at the end. */ - onion_append_to_cpath(head_ptr, hop); - - hop->magic = CRYPT_PATH_MAGIC; - hop->state = CPATH_STATE_CLOSED; - - hop->extend_info = extend_info_dup(choice); - - hop->package_window = circuit_initial_package_window(); - hop->deliver_window = CIRCWINDOW_START; - - return 0; -} - /** Allocate a new extend_info object based on the various arguments. */ extend_info_t * extend_info_new(const char *nickname, diff --git a/src/core/or/circuitbuild.h b/src/core/or/circuitbuild.h index b45bc816a3..e6f4f4b496 100644 --- a/src/core/or/circuitbuild.h +++ b/src/core/or/circuitbuild.h @@ -51,7 +51,6 @@ MOCK_DECL(int, circuit_all_predicted_ports_handled, (time_t now, int circuit_append_new_exit(origin_circuit_t *circ, extend_info_t *info); int circuit_extend_to_new_exit(origin_circuit_t *circ, extend_info_t *info); -void onion_append_to_cpath(crypt_path_t **head_ptr, crypt_path_t *new_hop); extend_info_t *extend_info_new(const char *nickname, const char *rsa_id_digest, const struct ed25519_public_key_t *ed_id, diff --git a/src/core/or/crypt_path.c b/src/core/or/crypt_path.c index d4fc59630a..ad1255c865 100644 --- a/src/core/or/crypt_path.c +++ b/src/core/or/crypt_path.c @@ -16,9 +16,50 @@ #include "core/or/crypt_path.h" #include "core/crypto/relay_crypto.h" +#include "core/or/circuitbuild.h" +#include "core/or/circuitlist.h" #include "core/or/crypt_path_st.h" +/** Add new_hop to the end of the doubly-linked-list head_ptr. + * This function is used to extend cpath by another hop. + */ +void +onion_append_to_cpath(crypt_path_t **head_ptr, crypt_path_t *new_hop) +{ + if (*head_ptr) { + new_hop->next = (*head_ptr); + new_hop->prev = (*head_ptr)->prev; + (*head_ptr)->prev->next = new_hop; + (*head_ptr)->prev = new_hop; + } else { + *head_ptr = new_hop; + new_hop->prev = new_hop->next = new_hop; + } +} + +/** Create a new hop, annotate it with information about its + * corresponding router choice, and append it to the + * end of the cpath head_ptr. */ +int +onion_append_hop(crypt_path_t **head_ptr, extend_info_t *choice) +{ + crypt_path_t *hop = tor_malloc_zero(sizeof(crypt_path_t)); + + /* link hop into the cpath, at the end. */ + onion_append_to_cpath(head_ptr, hop); + + hop->magic = CRYPT_PATH_MAGIC; + hop->state = CPATH_STATE_CLOSED; + + hop->extend_info = extend_info_dup(choice); + + hop->package_window = circuit_initial_package_window(); + hop->deliver_window = CIRCWINDOW_START; + + return 0; +} + /** Verify that cpath cp has all of its invariants * correct. Trigger an assert if anything is invalid. */ diff --git a/src/core/or/crypt_path.h b/src/core/or/crypt_path.h index a9b9aae43d..7614aaff28 100644 --- a/src/core/or/crypt_path.h +++ b/src/core/or/crypt_path.h @@ -9,3 +9,9 @@ void assert_cpath_layer_ok(const crypt_path_t *cp); /* rename */ void assert_cpath_ok(const crypt_path_t *cp); +/* rename */ +int onion_append_hop(crypt_path_t **head_ptr, extend_info_t *choice); + +/* rename */ +void onion_append_to_cpath(crypt_path_t **head_ptr, crypt_path_t *new_hop); + diff --git a/src/feature/hs/hs_circuit.c b/src/feature/hs/hs_circuit.c index 253c24d643..a42228d362 100644 --- a/src/feature/hs/hs_circuit.c +++ b/src/feature/hs/hs_circuit.c @@ -15,6 +15,7 @@ #include "core/or/circuituse.h" #include "core/or/policies.h" #include "core/or/relay.h" +#include "core/or/crypt_path.h" #include "feature/client/circpathbias.h" #include "feature/hs/hs_cell.h" #include "feature/hs/hs_circuit.h" diff --git a/src/feature/rend/rendservice.c b/src/feature/rend/rendservice.c index 996e7b9a28..5c267f8e34 100644 --- a/src/feature/rend/rendservice.c +++ b/src/feature/rend/rendservice.c @@ -18,6 +18,7 @@ #include "core/or/circuituse.h" #include "core/or/policies.h" #include "core/or/relay.h" +#include "core/or/crypt_path.h" #include "feature/client/circpathbias.h" #include "feature/control/control_events.h" #include "feature/dirclient/dirclient.h" diff --git a/src/test/test_circuitpadding.c b/src/test/test_circuitpadding.c index 3289c866cf..e24506d9bb 100644 --- a/src/test/test_circuitpadding.c +++ b/src/test/test_circuitpadding.c @@ -9,6 +9,7 @@ #include "core/or/connection_or.h" #include "core/or/channel.h" #include "core/or/channeltls.h" +#include "core/or/crypt_path.h" #include #include "lib/evloop/compat_libevent.h" #include "lib/time/compat_time.h" diff --git a/src/test/test_relaycrypt.c b/src/test/test_relaycrypt.c index fe6889e521..cd58094b10 100644 --- a/src/test/test_relaycrypt.c +++ b/src/test/test_relaycrypt.c @@ -10,7 +10,7 @@ #include "lib/crypt_ops/crypto_rand.h" #include "core/or/relay.h" #include "core/crypto/relay_crypto.h" - +#include "core/or/crypt_path.h" #include "core/or/cell_st.h" #include "core/or/or_circuit_st.h" #include "core/or/origin_circuit_st.h" From 0c5176d00cfe44e645175c23ed48eccbc74b4842 Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Mon, 8 Apr 2019 15:16:37 +0300 Subject: [PATCH 0952/2557] Hiding crypt_path_t: Start with crypt_path.crypto . Create some functions to eventually be able to hide crypt_path_t.crypto. --- src/core/crypto/relay_crypto.c | 13 +++++++------ src/core/crypto/relay_crypto.h | 5 +++++ src/core/or/crypt_path.c | 31 ++++++++++++++++++++++++++++++- src/core/or/crypt_path.h | 11 +++++++++++ src/core/or/crypt_path_st.h | 7 +++---- 5 files changed, 56 insertions(+), 11 deletions(-) diff --git a/src/core/crypto/relay_crypto.c b/src/core/crypto/relay_crypto.c index 8931163161..96b1002cab 100644 --- a/src/core/crypto/relay_crypto.c +++ b/src/core/crypto/relay_crypto.c @@ -6,6 +6,7 @@ #include "core/or/or.h" #include "core/or/circuitlist.h" +#include "core/or/crypt_path.h" #include "app/config/config.h" #include "lib/crypt_ops/crypto_cipher.h" #include "lib/crypt_ops/crypto_util.h" @@ -21,7 +22,7 @@ /** Update digest from the payload of cell. Assign integrity part to * cell. */ -static void +void relay_set_digest(crypto_digest_t *digest, cell_t *cell) { char integrity[4]; @@ -85,7 +86,7 @@ relay_digest_matches(crypto_digest_t *digest, cell_t *cell) * * Note that we use the same operation for encrypting and for decrypting. */ -static void +void relay_crypt_one_payload(crypto_cipher_t *cipher, uint8_t *in) { crypto_cipher_crypt_inplace(cipher, (char*) in, CELL_PAYLOAD_SIZE); @@ -152,12 +153,12 @@ relay_decrypt_cell(circuit_t *circ, cell_t *cell, tor_assert(thishop); /* decrypt one layer */ - relay_crypt_one_payload(thishop->crypto.b_crypto, cell->payload); + cpath_crypt_cell(thishop, cell->payload, true); relay_header_unpack(&rh, cell->payload); if (rh.recognized == 0) { /* it's possibly recognized. have to check digest to be sure. */ - if (relay_digest_matches(thishop->crypto.b_digest, cell)) { + if (relay_digest_matches(cpath_get_incoming_digest(thishop), cell)) { *recognized = 1; *layer_hint = thishop; /* This cell is for us. Keep a record of this cell because we will @@ -210,14 +211,14 @@ relay_encrypt_cell_outbound(cell_t *cell, crypt_path_t *layer_hint) { crypt_path_t *thishop; /* counter for repeated crypts */ - relay_set_digest(layer_hint->crypto.f_digest, cell); + cpath_set_cell_forward_digest(layer_hint, cell); thishop = layer_hint; /* moving from farthest to nearest hop */ do { tor_assert(thishop); log_debug(LD_OR,"encrypting a layer of the relay cell."); - relay_crypt_one_payload(thishop->crypto.f_crypto, cell->payload); + cpath_crypt_cell(thishop, cell->payload, false); thishop = thishop->prev; } while (thishop != circ->cpath->prev); diff --git a/src/core/crypto/relay_crypto.h b/src/core/crypto/relay_crypto.h index bcc1531838..7f09219c7f 100644 --- a/src/core/crypto/relay_crypto.h +++ b/src/core/crypto/relay_crypto.h @@ -29,6 +29,11 @@ void relay_crypto_assert_ok(const relay_crypto_t *crypto); uint8_t *relay_crypto_get_sendme_digest(relay_crypto_t *crypto); void relay_crypto_record_sendme_digest(relay_crypto_t *crypto); +void +relay_crypt_one_payload(crypto_cipher_t *cipher, uint8_t *in); + +void +relay_set_digest(crypto_digest_t *digest, cell_t *cell); #endif /* !defined(TOR_RELAY_CRYPTO_H) */ diff --git a/src/core/or/crypt_path.c b/src/core/or/crypt_path.c index ad1255c865..9fc3e013b2 100644 --- a/src/core/or/crypt_path.c +++ b/src/core/or/crypt_path.c @@ -20,6 +20,7 @@ #include "core/or/circuitlist.h" #include "core/or/crypt_path_st.h" +#include "core/or/cell_st.h" /** Add new_hop to the end of the doubly-linked-list head_ptr. * This function is used to extend cpath by another hop. @@ -96,7 +97,7 @@ assert_cpath_layer_ok(const crypt_path_t *cp) switch (cp->state) { case CPATH_STATE_OPEN: - relay_crypto_assert_ok(&cp->crypto); + relay_crypto_assert_ok(&cp->private->crypto); /* fall through */ case CPATH_STATE_CLOSED: /*XXXX Assert that there's no handshake_state either. */ @@ -113,3 +114,31 @@ assert_cpath_layer_ok(const crypt_path_t *cp) tor_assert(cp->deliver_window >= 0); } +/********************** cpath crypto API *******************************/ + +/** Encrypt or decrypt payload using the crypto of cpath. Actual + * operation decided by is_decrypt. */ +void +cpath_crypt_cell(const crypt_path_t *cpath, uint8_t *payload, bool is_decrypt) +{ + if (is_decrypt) { + relay_crypt_one_payload(cpath->private->crypto.b_crypto, payload); + } else { + relay_crypt_one_payload(cpath->private->crypto.f_crypto, payload); + } +} + +/** Getter for the incoming digest of cpath. */ +struct crypto_digest_t * +cpath_get_incoming_digest(const crypt_path_t *cpath) +{ + return cpath->private->crypto.b_digest; +} + +/** Set the right integrity digest on the outgoing cell based on the + * cell payload and update the forward digest of cpath. */ +void +cpath_set_cell_forward_digest(crypt_path_t *cpath, cell_t *cell) +{ + relay_set_digest(cpath->private->crypto.f_digest, cell); +} diff --git a/src/core/or/crypt_path.h b/src/core/or/crypt_path.h index 7614aaff28..fe25d85cfe 100644 --- a/src/core/or/crypt_path.h +++ b/src/core/or/crypt_path.h @@ -15,3 +15,14 @@ int onion_append_hop(crypt_path_t **head_ptr, extend_info_t *choice); /* rename */ void onion_append_to_cpath(crypt_path_t **head_ptr, crypt_path_t *new_hop); + +void +cpath_crypt_cell(const crypt_path_t *cpath, uint8_t *payload, bool is_decrypt); + +struct crypto_digest_t * +cpath_get_incoming_digest(const crypt_path_t *cpath); + +void +cpath_set_cell_forward_digest(crypt_path_t *cpath, cell_t *cell); + + diff --git a/src/core/or/crypt_path_st.h b/src/core/or/crypt_path_st.h index 90f6a37881..833cfefad1 100644 --- a/src/core/or/crypt_path_st.h +++ b/src/core/or/crypt_path_st.h @@ -29,6 +29,9 @@ struct onion_handshake_state_t { /* The private parts of crypt path that don't need to be exposed to all the * modules. */ struct crypt_path_private_t { + /** Cryptographic state used for encrypting and authenticating relay + * cells to and from this hop. */ + relay_crypto_t crypto; }; #endif @@ -38,10 +41,6 @@ struct crypt_path_private_t { struct crypt_path_t { uint32_t magic; - /** Cryptographic state used for encrypting and authenticating relay - * cells to and from this hop. */ - relay_crypto_t crypto; - /** Current state of the handshake as performed with the OR at this * step. */ onion_handshake_state_t handshake_state; From 5f96b7abccc8e393c7f5e370ab3bf838dc3f8d4f Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Mon, 8 Apr 2019 15:32:23 +0300 Subject: [PATCH 0953/2557] Hiding crypt_path_t: Move some more init funcs in crypt_path.c. Everything is moved, but the argument of the function is edited to access ->private->crypto. --- src/core/or/circuitbuild.c | 28 ---------------------------- src/core/or/circuitbuild.h | 3 --- src/core/or/crypt_path.c | 30 ++++++++++++++++++++++++++++++ src/core/or/crypt_path.h | 4 ++++ 4 files changed, 34 insertions(+), 31 deletions(-) diff --git a/src/core/or/circuitbuild.c b/src/core/or/circuitbuild.c index 7216b813bd..1ceb77c4ad 100644 --- a/src/core/or/circuitbuild.c +++ b/src/core/or/circuitbuild.c @@ -1360,34 +1360,6 @@ circuit_extend(cell_t *cell, circuit_t *circ) return 0; } -/** Initialize cpath-\>{f|b}_{crypto|digest} from the key material in key_data. - * - * If is_hs_v3 is set, this cpath will be used for next gen hidden - * service circuits and key_data must be at least - * HS_NTOR_KEY_EXPANSION_KDF_OUT_LEN bytes in length. - * - * If is_hs_v3 is not set, key_data must contain CPATH_KEY_MATERIAL_LEN - * bytes, which are used as follows: - * - 20 to initialize f_digest - * - 20 to initialize b_digest - * - 16 to key f_crypto - * - 16 to key b_crypto - * - * (If 'reverse' is true, then f_XX and b_XX are swapped.) - * - * Return 0 if init was successful, else -1 if it failed. - */ -int -circuit_init_cpath_crypto(crypt_path_t *cpath, - const char *key_data, size_t key_data_len, - int reverse, int is_hs_v3) -{ - - tor_assert(cpath); - return relay_crypto_init(&cpath->crypto, key_data, key_data_len, reverse, - is_hs_v3); -} - /** A "created" cell reply came back to us on circuit circ. * (The body of reply varies depending on what sort of handshake * this is.) diff --git a/src/core/or/circuitbuild.h b/src/core/or/circuitbuild.h index e6f4f4b496..f6403955ba 100644 --- a/src/core/or/circuitbuild.h +++ b/src/core/or/circuitbuild.h @@ -34,9 +34,6 @@ int circuit_timeout_want_to_count_circ(const origin_circuit_t *circ); int circuit_send_next_onion_skin(origin_circuit_t *circ); void circuit_note_clock_jumped(int64_t seconds_elapsed, bool was_idle); int circuit_extend(cell_t *cell, circuit_t *circ); -int circuit_init_cpath_crypto(crypt_path_t *cpath, - const char *key_data, size_t key_data_len, - int reverse, int is_hs_v3); struct created_cell_t; int circuit_finish_handshake(origin_circuit_t *circ, const struct created_cell_t *created_cell); diff --git a/src/core/or/crypt_path.c b/src/core/or/crypt_path.c index 9fc3e013b2..77f129eff3 100644 --- a/src/core/or/crypt_path.c +++ b/src/core/or/crypt_path.c @@ -114,6 +114,36 @@ assert_cpath_layer_ok(const crypt_path_t *cp) tor_assert(cp->deliver_window >= 0); } +/** Initialize cpath-\>{f|b}_{crypto|digest} from the key material in key_data. + * + * If is_hs_v3 is set, this cpath will be used for next gen hidden + * service circuits and key_data must be at least + * HS_NTOR_KEY_EXPANSION_KDF_OUT_LEN bytes in length. + * + * If is_hs_v3 is not set, key_data must contain CPATH_KEY_MATERIAL_LEN + * bytes, which are used as follows: + * - 20 to initialize f_digest + * - 20 to initialize b_digest + * - 16 to key f_crypto + * - 16 to key b_crypto + * + * (If 'reverse' is true, then f_XX and b_XX are swapped.) + * + * Return 0 if init was successful, else -1 if it failed. + */ +int +circuit_init_cpath_crypto(crypt_path_t *cpath, + const char *key_data, size_t key_data_len, + int reverse, int is_hs_v3) +{ + + tor_assert(cpath); + return relay_crypto_init(&cpath->private->crypto, key_data, key_data_len, reverse, + is_hs_v3); +} + + + /********************** cpath crypto API *******************************/ /** Encrypt or decrypt payload using the crypto of cpath. Actual diff --git a/src/core/or/crypt_path.h b/src/core/or/crypt_path.h index fe25d85cfe..a7ebe604f5 100644 --- a/src/core/or/crypt_path.h +++ b/src/core/or/crypt_path.h @@ -12,6 +12,10 @@ void assert_cpath_ok(const crypt_path_t *cp); /* rename */ int onion_append_hop(crypt_path_t **head_ptr, extend_info_t *choice); +int circuit_init_cpath_crypto(crypt_path_t *cpath, + const char *key_data, size_t key_data_len, + int reverse, int is_hs_v3); + /* rename */ void onion_append_to_cpath(crypt_path_t **head_ptr, crypt_path_t *new_hop); From 4bd0c4852aad724fd9639f5250c5893341cd5935 Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Mon, 8 Apr 2019 15:37:02 +0300 Subject: [PATCH 0954/2557] Hiding crypt_path_t: Move the free func in crypt_path.c. Again everything is moved, apart from a free line using ->private. --- src/core/or/circuitlist.c | 17 ----------------- src/core/or/crypt_path.c | 19 +++++++++++++++++++ src/core/or/crypt_path.h | 3 +++ 3 files changed, 22 insertions(+), 17 deletions(-) diff --git a/src/core/or/circuitlist.c b/src/core/or/circuitlist.c index ee9e89f380..83c651ff17 100644 --- a/src/core/or/circuitlist.c +++ b/src/core/or/circuitlist.c @@ -133,7 +133,6 @@ static smartlist_t *circuits_pending_other_guards = NULL; * circuit_mark_for_close and which are waiting for circuit_about_to_free. */ static smartlist_t *circuits_pending_close = NULL; -static void circuit_free_cpath_node(crypt_path_t *victim); static void cpath_ref_decref(crypt_path_reference_t *cpath_ref); static void circuit_about_to_free_atexit(circuit_t *circ); static void circuit_about_to_free(circuit_t *circ); @@ -1333,22 +1332,6 @@ circuit_free_all(void) HT_CLEAR(chan_circid_map, &chan_circid_map); } -/** Deallocate space associated with the cpath node victim. */ -static void -circuit_free_cpath_node(crypt_path_t *victim) -{ - if (!victim) - return; - - relay_crypto_clear(&victim->crypto); - onion_handshake_state_release(&victim->handshake_state); - crypto_dh_free(victim->rend_dh_handshake_state); - extend_info_free(victim->extend_info); - - memwipe(victim, 0xBB, sizeof(crypt_path_t)); /* poison memory */ - tor_free(victim); -} - /** Release a crypt_path_reference_t*, which may be NULL. */ static void cpath_ref_decref(crypt_path_reference_t *cpath_ref) diff --git a/src/core/or/crypt_path.c b/src/core/or/crypt_path.c index 77f129eff3..54f5623d32 100644 --- a/src/core/or/crypt_path.c +++ b/src/core/or/crypt_path.c @@ -16,9 +16,13 @@ #include "core/or/crypt_path.h" #include "core/crypto/relay_crypto.h" +#include "core/crypto/onion_crypto.h" #include "core/or/circuitbuild.h" #include "core/or/circuitlist.h" +#include "lib/crypt_ops/crypto_dh.h" +#include "lib/crypt_ops/crypto_util.h" + #include "core/or/crypt_path_st.h" #include "core/or/cell_st.h" @@ -143,6 +147,21 @@ circuit_init_cpath_crypto(crypt_path_t *cpath, } +/** Deallocate space associated with the cpath node victim. */ +void +circuit_free_cpath_node(crypt_path_t *victim) +{ + if (!victim) + return; + + relay_crypto_clear(&victim->private->crypto); + onion_handshake_state_release(&victim->handshake_state); + crypto_dh_free(victim->rend_dh_handshake_state); + extend_info_free(victim->extend_info); + + memwipe(victim, 0xBB, sizeof(crypt_path_t)); /* poison memory */ + tor_free(victim); +} /********************** cpath crypto API *******************************/ diff --git a/src/core/or/crypt_path.h b/src/core/or/crypt_path.h index a7ebe604f5..e8455c6326 100644 --- a/src/core/or/crypt_path.h +++ b/src/core/or/crypt_path.h @@ -16,6 +16,9 @@ int circuit_init_cpath_crypto(crypt_path_t *cpath, const char *key_data, size_t key_data_len, int reverse, int is_hs_v3); +void +circuit_free_cpath_node(crypt_path_t *victim); + /* rename */ void onion_append_to_cpath(crypt_path_t **head_ptr, crypt_path_t *new_hop); From 18d61c0e6e71dace189384c8af7f4fec158969b3 Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Mon, 8 Apr 2019 15:43:23 +0300 Subject: [PATCH 0955/2557] Hiding crypt_path_t: Fixup broken unittests. --- src/test/test_circuitpadding.c | 3 ++- src/test/test_hs_client.c | 17 +++++++++-------- src/test/test_hs_service.c | 9 +++++---- src/test/test_relaycrypt.c | 4 +++- 4 files changed, 19 insertions(+), 14 deletions(-) diff --git a/src/test/test_circuitpadding.c b/src/test/test_circuitpadding.c index e24506d9bb..8a2667e802 100644 --- a/src/test/test_circuitpadding.c +++ b/src/test/test_circuitpadding.c @@ -2,6 +2,7 @@ #define TOR_TIMERS_PRIVATE #define CIRCUITPADDING_PRIVATE #define NETWORKSTATUS_PRIVATE +#define CRYPT_PATH_PRIVATE #include "core/or/or.h" #include "test.h" @@ -149,7 +150,7 @@ new_fake_orcirc(channel_t *nchan, channel_t *pchan) log_warn(LD_BUG,"Circuit initialization failed"); return NULL; } - orcirc->crypto = tmp_cpath.crypto; + orcirc->crypto = tmp_cpath.private->crypto; return orcirc; } diff --git a/src/test/test_hs_client.c b/src/test/test_hs_client.c index 8362b6cbda..607be339a9 100644 --- a/src/test/test_hs_client.c +++ b/src/test/test_hs_client.c @@ -14,6 +14,7 @@ #define CIRCUITBUILD_PRIVATE #define CIRCUITLIST_PRIVATE #define CONNECTION_PRIVATE +#define CRYPT_PATH_PRIVATE #include "test/test.h" #include "test/test_helpers.h" @@ -241,12 +242,12 @@ test_e2e_rend_circuit_setup_legacy(void *arg) tt_int_op(retval, OP_EQ, 1); /* Check the digest algo */ - tt_int_op(crypto_digest_get_algorithm(or_circ->cpath->crypto.f_digest), + tt_int_op(crypto_digest_get_algorithm(or_circ->cpath->private->crypto.f_digest), OP_EQ, DIGEST_SHA1); - tt_int_op(crypto_digest_get_algorithm(or_circ->cpath->crypto.b_digest), + tt_int_op(crypto_digest_get_algorithm(or_circ->cpath->private->crypto.b_digest), OP_EQ, DIGEST_SHA1); - tt_assert(or_circ->cpath->crypto.f_crypto); - tt_assert(or_circ->cpath->crypto.b_crypto); + tt_assert(or_circ->cpath->private->crypto.f_crypto); + tt_assert(or_circ->cpath->private->crypto.b_crypto); /* Ensure that circ purpose was changed */ tt_int_op(or_circ->base_.purpose, OP_EQ, CIRCUIT_PURPOSE_C_REND_JOINED); @@ -311,12 +312,12 @@ test_e2e_rend_circuit_setup(void *arg) tt_int_op(retval, OP_EQ, 1); /* Check that the crypt path has prop224 algorithm parameters */ - tt_int_op(crypto_digest_get_algorithm(or_circ->cpath->crypto.f_digest), + tt_int_op(crypto_digest_get_algorithm(or_circ->cpath->private->crypto.f_digest), OP_EQ, DIGEST_SHA3_256); - tt_int_op(crypto_digest_get_algorithm(or_circ->cpath->crypto.b_digest), + tt_int_op(crypto_digest_get_algorithm(or_circ->cpath->private->crypto.b_digest), OP_EQ, DIGEST_SHA3_256); - tt_assert(or_circ->cpath->crypto.f_crypto); - tt_assert(or_circ->cpath->crypto.b_crypto); + tt_assert(or_circ->cpath->private->crypto.f_crypto); + tt_assert(or_circ->cpath->private->crypto.b_crypto); /* Ensure that circ purpose was changed */ tt_int_op(or_circ->base_.purpose, OP_EQ, CIRCUIT_PURPOSE_C_REND_JOINED); diff --git a/src/test/test_hs_service.c b/src/test/test_hs_service.c index 57132e6197..bfa66f551a 100644 --- a/src/test/test_hs_service.c +++ b/src/test/test_hs_service.c @@ -21,6 +21,7 @@ #define STATEFILE_PRIVATE #define TOR_CHANNEL_INTERNAL_ #define HS_CLIENT_PRIVATE +#define CRYPT_PATH_PRIVATE #include "test/test.h" #include "test/test_helpers.h" @@ -193,12 +194,12 @@ test_e2e_rend_circuit_setup(void *arg) tt_int_op(retval, OP_EQ, 1); /* Check the digest algo */ - tt_int_op(crypto_digest_get_algorithm(or_circ->cpath->crypto.f_digest), + tt_int_op(crypto_digest_get_algorithm(or_circ->cpath->private->crypto.f_digest), OP_EQ, DIGEST_SHA3_256); - tt_int_op(crypto_digest_get_algorithm(or_circ->cpath->crypto.b_digest), + tt_int_op(crypto_digest_get_algorithm(or_circ->cpath->private->crypto.b_digest), OP_EQ, DIGEST_SHA3_256); - tt_assert(or_circ->cpath->crypto.f_crypto); - tt_assert(or_circ->cpath->crypto.b_crypto); + tt_assert(or_circ->cpath->private->crypto.f_crypto); + tt_assert(or_circ->cpath->private->crypto.b_crypto); /* Ensure that circ purpose was changed */ tt_int_op(or_circ->base_.purpose, OP_EQ, CIRCUIT_PURPOSE_S_REND_JOINED); diff --git a/src/test/test_relaycrypt.c b/src/test/test_relaycrypt.c index cd58094b10..b94ee07abc 100644 --- a/src/test/test_relaycrypt.c +++ b/src/test/test_relaycrypt.c @@ -3,6 +3,8 @@ * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ +#define CRYPT_PATH_PRIVATE + #include "core/or/or.h" #include "core/or/circuitbuild.h" #define CIRCUITLIST_PRIVATE @@ -49,7 +51,7 @@ testing_circuitset_setup(const struct testcase_t *testcase) cs->origin_circ->base_.purpose = CIRCUIT_PURPOSE_C_GENERAL; for (i=0; i<3; ++i) { crypt_path_t *hop = tor_malloc_zero(sizeof(*hop)); - relay_crypto_init(&hop->crypto, KEY_MATERIAL[i], sizeof(KEY_MATERIAL[i]), + relay_crypto_init(&hop->private->crypto, KEY_MATERIAL[i], sizeof(KEY_MATERIAL[i]), 0, 0); hop->state = CPATH_STATE_OPEN; onion_append_to_cpath(&cs->origin_circ->cpath, hop); From f5635989b06260710b282e75be7b731e2846f700 Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Mon, 8 Apr 2019 16:18:44 +0300 Subject: [PATCH 0956/2557] Hiding crypt_path_t: Create a constructor for crypt_path_t. We are using an opaque pointer so the structure needs to be allocated on the heap. This means we now need a constructor for crypt_path_t. Also modify all places initializing a crypt_path_t to use the constructor. --- src/core/or/crypt_path.c | 15 +++++++++++++-- src/core/or/crypt_path.h | 2 ++ src/core/or/crypt_path_st.h | 5 ++--- src/feature/hs/hs_circuit.c | 3 +-- src/feature/rend/rendclient.c | 5 ++--- src/feature/rend/rendservice.c | 3 +-- src/test/test_circuitpadding.c | 13 +++++++------ src/test/test_hs_client.c | 5 ++--- src/test/test_hs_service.c | 4 ++-- src/test/test_relaycell.c | 4 ++-- src/test/test_relaycrypt.c | 2 +- 11 files changed, 35 insertions(+), 26 deletions(-) diff --git a/src/core/or/crypt_path.c b/src/core/or/crypt_path.c index 54f5623d32..975af6c16d 100644 --- a/src/core/or/crypt_path.c +++ b/src/core/or/crypt_path.c @@ -26,6 +26,17 @@ #include "core/or/crypt_path_st.h" #include "core/or/cell_st.h" +/** Initialize and return a minimal crypt_path_t */ +crypt_path_t * +crypt_path_new(void) +{ + crypt_path_t *cpath = tor_malloc_zero(sizeof(crypt_path_t)); + cpath->magic = CRYPT_PATH_MAGIC; + cpath->private = tor_malloc_zero(sizeof(struct crypt_path_private_t)); + + return cpath; +} + /** Add new_hop to the end of the doubly-linked-list head_ptr. * This function is used to extend cpath by another hop. */ @@ -49,12 +60,11 @@ onion_append_to_cpath(crypt_path_t **head_ptr, crypt_path_t *new_hop) int onion_append_hop(crypt_path_t **head_ptr, extend_info_t *choice) { - crypt_path_t *hop = tor_malloc_zero(sizeof(crypt_path_t)); + crypt_path_t *hop = crypt_path_new(); /* link hop into the cpath, at the end. */ onion_append_to_cpath(head_ptr, hop); - hop->magic = CRYPT_PATH_MAGIC; hop->state = CPATH_STATE_CLOSED; hop->extend_info = extend_info_dup(choice); @@ -158,6 +168,7 @@ circuit_free_cpath_node(crypt_path_t *victim) onion_handshake_state_release(&victim->handshake_state); crypto_dh_free(victim->rend_dh_handshake_state); extend_info_free(victim->extend_info); + tor_free(victim->private); memwipe(victim, 0xBB, sizeof(crypt_path_t)); /* poison memory */ tor_free(victim); diff --git a/src/core/or/crypt_path.h b/src/core/or/crypt_path.h index e8455c6326..c6d1cd1400 100644 --- a/src/core/or/crypt_path.h +++ b/src/core/or/crypt_path.h @@ -3,6 +3,8 @@ * \brief Header file for crypt_path.c. **/ +crypt_path_t *crypt_path_new(void); + /* rename */ void assert_cpath_layer_ok(const crypt_path_t *cp); diff --git a/src/core/or/crypt_path_st.h b/src/core/or/crypt_path_st.h index 833cfefad1..7da3c57f49 100644 --- a/src/core/or/crypt_path_st.h +++ b/src/core/or/crypt_path_st.h @@ -8,9 +8,6 @@ #define CRYPT_PATH_ST_H #include "core/or/relay_crypto_st.h" -struct crypto_dh_t; - -#define CRYPT_PATH_MAGIC 0x70127012u struct fast_handshake_state_t; struct ntor_handshake_state_t; @@ -26,6 +23,8 @@ struct onion_handshake_state_t { #ifdef CRYPT_PATH_PRIVATE +#define CRYPT_PATH_MAGIC 0x70127012u + /* The private parts of crypt path that don't need to be exposed to all the * modules. */ struct crypt_path_private_t { diff --git a/src/feature/hs/hs_circuit.c b/src/feature/hs/hs_circuit.c index a42228d362..3356db9d90 100644 --- a/src/feature/hs/hs_circuit.c +++ b/src/feature/hs/hs_circuit.c @@ -87,8 +87,7 @@ create_rend_cpath(const uint8_t *ntor_key_seed, size_t seed_len, } /* Setup the cpath */ - cpath = tor_malloc_zero(sizeof(crypt_path_t)); - cpath->magic = CRYPT_PATH_MAGIC; + cpath = crypt_path_new(); if (circuit_init_cpath_crypto(cpath, (char*)keys, sizeof(keys), is_service_side, 1) < 0) { diff --git a/src/feature/rend/rendclient.c b/src/feature/rend/rendclient.c index f84d221b1a..c6e9dde878 100644 --- a/src/feature/rend/rendclient.c +++ b/src/feature/rend/rendclient.c @@ -16,6 +16,7 @@ #include "core/or/circuituse.h" #include "core/or/connection_edge.h" #include "core/or/relay.h" +#include "core/or/crypt_path.h" #include "feature/client/circpathbias.h" #include "feature/control/control_events.h" #include "feature/dirclient/dirclient.h" @@ -194,9 +195,7 @@ rend_client_send_introduction(origin_circuit_t *introcirc, /* Initialize the pending_final_cpath and start the DH handshake. */ cpath = rendcirc->build_state->pending_final_cpath; if (!cpath) { - cpath = rendcirc->build_state->pending_final_cpath = - tor_malloc_zero(sizeof(crypt_path_t)); - cpath->magic = CRYPT_PATH_MAGIC; + cpath = rendcirc->build_state->pending_final_cpath = crypt_path_new(); if (!(cpath->rend_dh_handshake_state = crypto_dh_new(DH_TYPE_REND))) { log_warn(LD_BUG, "Internal error: couldn't allocate DH."); status = -2; diff --git a/src/feature/rend/rendservice.c b/src/feature/rend/rendservice.c index 5c267f8e34..38da4cfe7a 100644 --- a/src/feature/rend/rendservice.c +++ b/src/feature/rend/rendservice.c @@ -2158,8 +2158,7 @@ rend_service_receive_introduction(origin_circuit_t *circuit, launched->build_state->service_pending_final_cpath_ref->refcount = 1; launched->build_state->service_pending_final_cpath_ref->cpath = cpath = - tor_malloc_zero(sizeof(crypt_path_t)); - cpath->magic = CRYPT_PATH_MAGIC; + crypt_path_new(); launched->build_state->expiry_time = now + MAX_REND_TIMEOUT; cpath->rend_dh_handshake_state = dh; diff --git a/src/test/test_circuitpadding.c b/src/test/test_circuitpadding.c index 8a2667e802..6fa790c40d 100644 --- a/src/test/test_circuitpadding.c +++ b/src/test/test_circuitpadding.c @@ -115,7 +115,7 @@ new_fake_orcirc(channel_t *nchan, channel_t *pchan) { or_circuit_t *orcirc = NULL; circuit_t *circ = NULL; - crypt_path_t tmp_cpath; + crypt_path_t *tmp_cpath; char whatevs_key[CPATH_KEY_MATERIAL_LEN]; orcirc = tor_malloc_zero(sizeof(*orcirc)); @@ -144,13 +144,15 @@ new_fake_orcirc(channel_t *nchan, channel_t *pchan) circuit_set_p_circid_chan(orcirc, orcirc->p_circ_id, pchan); circuit_set_n_circid_chan(circ, circ->n_circ_id, nchan); - memset(&tmp_cpath, 0, sizeof(tmp_cpath)); - if (circuit_init_cpath_crypto(&tmp_cpath, whatevs_key, + tmp_cpath = crypt_path_new(); + if (circuit_init_cpath_crypto(tmp_cpath, whatevs_key, sizeof(whatevs_key), 0, 0)<0) { log_warn(LD_BUG,"Circuit initialization failed"); return NULL; } - orcirc->crypto = tmp_cpath.private->crypto; + orcirc->crypto = tmp_cpath->private->crypto; + tor_free(tmp_cpath->private); + tor_free(tmp_cpath); return orcirc; } @@ -1618,10 +1620,9 @@ simulate_single_hop_extend(circuit_t *client, circuit_t *mid_relay, circpad_cell_event_nonpadding_received((circuit_t*)client); // Add a hop to cpath - crypt_path_t *hop = tor_malloc_zero(sizeof(crypt_path_t)); + crypt_path_t *hop = crypt_path_new(); onion_append_to_cpath(&TO_ORIGIN_CIRCUIT(client)->cpath, hop); - hop->magic = CRYPT_PATH_MAGIC; hop->state = CPATH_STATE_OPEN; // add an extend info to indicate if this node supports padding or not. diff --git a/src/test/test_hs_client.c b/src/test/test_hs_client.c index 607be339a9..9e1d73a855 100644 --- a/src/test/test_hs_client.c +++ b/src/test/test_hs_client.c @@ -39,6 +39,7 @@ #include "feature/hs/hs_cache.h" #include "core/or/circuitlist.h" #include "core/or/circuitbuild.h" +#include "core/or/crypt_path.h" #include "core/mainloop/connection.h" #include "core/or/connection_edge.h" #include "feature/nodelist/networkstatus.h" @@ -145,9 +146,7 @@ helper_get_circ_and_stream_for_test(origin_circuit_t **circ_out, if (is_legacy) { /* Legacy: Setup rend data and final cpath */ - or_circ->build_state->pending_final_cpath = - tor_malloc_zero(sizeof(crypt_path_t)); - or_circ->build_state->pending_final_cpath->magic = CRYPT_PATH_MAGIC; + or_circ->build_state->pending_final_cpath = crypt_path_new(); or_circ->build_state->pending_final_cpath->rend_dh_handshake_state = crypto_dh_new(DH_TYPE_REND); tt_assert( diff --git a/src/test/test_hs_service.c b/src/test/test_hs_service.c index bfa66f551a..357db89040 100644 --- a/src/test/test_hs_service.c +++ b/src/test/test_hs_service.c @@ -38,6 +38,7 @@ #include "core/or/circuitbuild.h" #include "core/or/circuitlist.h" #include "core/or/circuituse.h" +#include "core/or/crypt_path.h" #include "core/or/connection_edge.h" #include "core/or/edge_connection_st.h" #include "core/or/relay.h" @@ -218,8 +219,7 @@ helper_create_origin_circuit(int purpose, int flags) circ = origin_circuit_init(purpose, flags); tor_assert(circ); - circ->cpath = tor_malloc_zero(sizeof(crypt_path_t)); - circ->cpath->magic = CRYPT_PATH_MAGIC; + circ->cpath = crypt_path_new(); circ->cpath->state = CPATH_STATE_OPEN; circ->cpath->package_window = circuit_initial_package_window(); circ->cpath->deliver_window = CIRCWINDOW_START; diff --git a/src/test/test_relaycell.c b/src/test/test_relaycell.c index 0623583511..b48c7ca8ac 100644 --- a/src/test/test_relaycell.c +++ b/src/test/test_relaycell.c @@ -16,6 +16,7 @@ #include "lib/crypt_ops/crypto_rand.h" #include "core/or/circuitbuild.h" #include "core/or/circuitlist.h" +#include "core/or/crypt_path.h" #include "core/or/connection_edge.h" #include "core/or/relay.h" #include "test/test.h" @@ -90,8 +91,7 @@ helper_create_origin_circuit(int purpose, int flags) circ = origin_circuit_init(purpose, flags); tor_assert(circ); - circ->cpath = tor_malloc_zero(sizeof(crypt_path_t)); - circ->cpath->magic = CRYPT_PATH_MAGIC; + circ->cpath = crypt_path_new(); circ->cpath->state = CPATH_STATE_OPEN; circ->cpath->package_window = circuit_initial_package_window(); circ->cpath->deliver_window = CIRCWINDOW_START; diff --git a/src/test/test_relaycrypt.c b/src/test/test_relaycrypt.c index b94ee07abc..1fe5df96ed 100644 --- a/src/test/test_relaycrypt.c +++ b/src/test/test_relaycrypt.c @@ -50,7 +50,7 @@ testing_circuitset_setup(const struct testcase_t *testcase) cs->origin_circ = origin_circuit_new(); cs->origin_circ->base_.purpose = CIRCUIT_PURPOSE_C_GENERAL; for (i=0; i<3; ++i) { - crypt_path_t *hop = tor_malloc_zero(sizeof(*hop)); + crypt_path_t *hop = crypt_path_new(); relay_crypto_init(&hop->private->crypto, KEY_MATERIAL[i], sizeof(KEY_MATERIAL[i]), 0, 0); hop->state = CPATH_STATE_OPEN; From cd38e41620120a11a70ebe059f3adbaa05e4c1ff Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Wed, 10 Apr 2019 16:28:29 +0300 Subject: [PATCH 0957/2557] Hiding crypt_path_t: Ensure that ->private is initialized. Now that we are using a constructor we should be more careful that we are always using the constructor to initialize crypt_path_t, so make sure that ->private is initialized. --- src/core/or/crypt_path.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/core/or/crypt_path.c b/src/core/or/crypt_path.c index 975af6c16d..e24712ed89 100644 --- a/src/core/or/crypt_path.c +++ b/src/core/or/crypt_path.c @@ -108,6 +108,7 @@ assert_cpath_layer_ok(const crypt_path_t *cp) // tor_assert(cp->port); tor_assert(cp); tor_assert(cp->magic == CRYPT_PATH_MAGIC); + tor_assert(cp->private); switch (cp->state) { case CPATH_STATE_OPEN: @@ -152,6 +153,7 @@ circuit_init_cpath_crypto(crypt_path_t *cpath, { tor_assert(cpath); + tor_assert(cpath->private); return relay_crypto_init(&cpath->private->crypto, key_data, key_data_len, reverse, is_hs_v3); } @@ -161,7 +163,7 @@ circuit_init_cpath_crypto(crypt_path_t *cpath, void circuit_free_cpath_node(crypt_path_t *victim) { - if (!victim) + if (!victim || BUG(!victim->private)) return; relay_crypto_clear(&victim->private->crypto); @@ -181,6 +183,9 @@ circuit_free_cpath_node(crypt_path_t *victim) void cpath_crypt_cell(const crypt_path_t *cpath, uint8_t *payload, bool is_decrypt) { + tor_assert(cpath); + tor_assert(cpath->private); + if (is_decrypt) { relay_crypt_one_payload(cpath->private->crypto.b_crypto, payload); } else { @@ -192,6 +197,8 @@ cpath_crypt_cell(const crypt_path_t *cpath, uint8_t *payload, bool is_decrypt) struct crypto_digest_t * cpath_get_incoming_digest(const crypt_path_t *cpath) { + tor_assert(cpath); + tor_assert(cpath->private); return cpath->private->crypto.b_digest; } @@ -200,5 +207,7 @@ cpath_get_incoming_digest(const crypt_path_t *cpath) void cpath_set_cell_forward_digest(crypt_path_t *cpath, cell_t *cell) { + tor_assert(cpath); + tor_assert(cpath->private); relay_set_digest(cpath->private->crypto.f_digest, cell); } From 593b7726e98fd68cccadb3da219d9f31692e8c80 Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Mon, 8 Apr 2019 16:36:12 +0300 Subject: [PATCH 0958/2557] Hiding crypt_path_t: Trivial changes to satisfy check-local. --- scripts/maint/practracker/exceptions.txt | 7 ++++--- src/core/or/crypt_path.c | 5 ++--- src/core/or/crypt_path.h | 6 ++++-- src/test/test_hs_client.c | 12 ++++++++---- src/test/test_hs_service.c | 6 ++++-- src/test/test_relaycrypt.c | 4 ++-- 6 files changed, 24 insertions(+), 16 deletions(-) diff --git a/scripts/maint/practracker/exceptions.txt b/scripts/maint/practracker/exceptions.txt index 21fe9ec351..a2b6d36ea8 100644 --- a/scripts/maint/practracker/exceptions.txt +++ b/scripts/maint/practracker/exceptions.txt @@ -55,8 +55,8 @@ problem function-size /src/app/main/main.c:run_tor_main_loop() 105 problem function-size /src/app/main/ntmain.c:nt_service_install() 125 problem include-count /src/app/main/shutdown.c 52 problem file-size /src/core/mainloop/connection.c 5559 -problem include-count /src/core/mainloop/connection.c 61 -problem function-size /src/core/mainloop/connection.c:connection_free_minimal() 185 +problem include-count /src/core/mainloop/connection.c 62 +problem function-size /src/core/mainloop/connection.c:connection_free_minimal() 184 problem function-size /src/core/mainloop/connection.c:connection_listener_new() 328 problem function-size /src/core/mainloop/connection.c:connection_handle_listener_read() 161 problem function-size /src/core/mainloop/connection.c:connection_connect_sockaddr() 103 @@ -80,10 +80,11 @@ problem function-size /src/core/or/channeltls.c:channel_tls_process_certs_cell() problem function-size /src/core/or/channeltls.c:channel_tls_process_authenticate_cell() 202 problem file-size /src/core/or/circuitbuild.c 3061 problem include-count /src/core/or/circuitbuild.c 53 +problem include-count /src/core/or/circuitbuild.c 54 problem function-size /src/core/or/circuitbuild.c:get_unique_circ_id_by_chan() 128 problem function-size /src/core/or/circuitbuild.c:circuit_extend() 147 problem function-size /src/core/or/circuitbuild.c:choose_good_exit_server_general() 206 -problem include-count /src/core/or/circuitlist.c 54 +problem include-count /src/core/or/circuitlist.c 55 problem function-size /src/core/or/circuitlist.c:HT_PROTOTYPE() 128 problem function-size /src/core/or/circuitlist.c:circuit_free_() 143 problem function-size /src/core/or/circuitlist.c:circuit_find_to_cannibalize() 102 diff --git a/src/core/or/crypt_path.c b/src/core/or/crypt_path.c index e24712ed89..ea07ec495f 100644 --- a/src/core/or/crypt_path.c +++ b/src/core/or/crypt_path.c @@ -154,11 +154,10 @@ circuit_init_cpath_crypto(crypt_path_t *cpath, tor_assert(cpath); tor_assert(cpath->private); - return relay_crypto_init(&cpath->private->crypto, key_data, key_data_len, reverse, - is_hs_v3); + return relay_crypto_init(&cpath->private->crypto, key_data, key_data_len, + reverse, is_hs_v3); } - /** Deallocate space associated with the cpath node victim. */ void circuit_free_cpath_node(crypt_path_t *victim) diff --git a/src/core/or/crypt_path.h b/src/core/or/crypt_path.h index c6d1cd1400..874ff2b2ad 100644 --- a/src/core/or/crypt_path.h +++ b/src/core/or/crypt_path.h @@ -3,6 +3,9 @@ * \brief Header file for crypt_path.c. **/ +#ifndef CRYPT_PATH_H +#define CRYPT_PATH_H + crypt_path_t *crypt_path_new(void); /* rename */ @@ -24,7 +27,6 @@ circuit_free_cpath_node(crypt_path_t *victim); /* rename */ void onion_append_to_cpath(crypt_path_t **head_ptr, crypt_path_t *new_hop); - void cpath_crypt_cell(const crypt_path_t *cpath, uint8_t *payload, bool is_decrypt); @@ -34,4 +36,4 @@ cpath_get_incoming_digest(const crypt_path_t *cpath); void cpath_set_cell_forward_digest(crypt_path_t *cpath, cell_t *cell); - +#endif diff --git a/src/test/test_hs_client.c b/src/test/test_hs_client.c index 9e1d73a855..cd049b7c47 100644 --- a/src/test/test_hs_client.c +++ b/src/test/test_hs_client.c @@ -241,9 +241,11 @@ test_e2e_rend_circuit_setup_legacy(void *arg) tt_int_op(retval, OP_EQ, 1); /* Check the digest algo */ - tt_int_op(crypto_digest_get_algorithm(or_circ->cpath->private->crypto.f_digest), + tt_int_op( + crypto_digest_get_algorithm(or_circ->cpath->private->crypto.f_digest), OP_EQ, DIGEST_SHA1); - tt_int_op(crypto_digest_get_algorithm(or_circ->cpath->private->crypto.b_digest), + tt_int_op( + crypto_digest_get_algorithm(or_circ->cpath->private->crypto.b_digest), OP_EQ, DIGEST_SHA1); tt_assert(or_circ->cpath->private->crypto.f_crypto); tt_assert(or_circ->cpath->private->crypto.b_crypto); @@ -311,9 +313,11 @@ test_e2e_rend_circuit_setup(void *arg) tt_int_op(retval, OP_EQ, 1); /* Check that the crypt path has prop224 algorithm parameters */ - tt_int_op(crypto_digest_get_algorithm(or_circ->cpath->private->crypto.f_digest), + tt_int_op( + crypto_digest_get_algorithm(or_circ->cpath->private->crypto.f_digest), OP_EQ, DIGEST_SHA3_256); - tt_int_op(crypto_digest_get_algorithm(or_circ->cpath->private->crypto.b_digest), + tt_int_op( + crypto_digest_get_algorithm(or_circ->cpath->private->crypto.b_digest), OP_EQ, DIGEST_SHA3_256); tt_assert(or_circ->cpath->private->crypto.f_crypto); tt_assert(or_circ->cpath->private->crypto.b_crypto); diff --git a/src/test/test_hs_service.c b/src/test/test_hs_service.c index 357db89040..08dac04d21 100644 --- a/src/test/test_hs_service.c +++ b/src/test/test_hs_service.c @@ -195,9 +195,11 @@ test_e2e_rend_circuit_setup(void *arg) tt_int_op(retval, OP_EQ, 1); /* Check the digest algo */ - tt_int_op(crypto_digest_get_algorithm(or_circ->cpath->private->crypto.f_digest), + tt_int_op( + crypto_digest_get_algorithm(or_circ->cpath->private->crypto.f_digest), OP_EQ, DIGEST_SHA3_256); - tt_int_op(crypto_digest_get_algorithm(or_circ->cpath->private->crypto.b_digest), + tt_int_op( + crypto_digest_get_algorithm(or_circ->cpath->private->crypto.b_digest), OP_EQ, DIGEST_SHA3_256); tt_assert(or_circ->cpath->private->crypto.f_crypto); tt_assert(or_circ->cpath->private->crypto.b_crypto); diff --git a/src/test/test_relaycrypt.c b/src/test/test_relaycrypt.c index 1fe5df96ed..a3a102e73b 100644 --- a/src/test/test_relaycrypt.c +++ b/src/test/test_relaycrypt.c @@ -51,8 +51,8 @@ testing_circuitset_setup(const struct testcase_t *testcase) cs->origin_circ->base_.purpose = CIRCUIT_PURPOSE_C_GENERAL; for (i=0; i<3; ++i) { crypt_path_t *hop = crypt_path_new(); - relay_crypto_init(&hop->private->crypto, KEY_MATERIAL[i], sizeof(KEY_MATERIAL[i]), - 0, 0); + relay_crypto_init(&hop->private->crypto, KEY_MATERIAL[i], + sizeof(KEY_MATERIAL[i]), 0, 0); hop->state = CPATH_STATE_OPEN; onion_append_to_cpath(&cs->origin_circ->cpath, hop); tt_ptr_op(hop, OP_EQ, cs->origin_circ->cpath->prev); From 58fbbc1409f65bbb65c9da03a035a5767820146b Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Tue, 9 Apr 2019 12:38:19 +0300 Subject: [PATCH 0959/2557] Hiding crypt_path_t: Rename some functions to fit the crypt_path API. Some of these functions are now public and cpath-specific so their name should signify the fact they are part of the cpath module: assert_cpath_layer_ok -> cpath_assert_layer_ok assert_cpath_ok -> cpath_assert_ok onion_append_hop -> cpath_append_hop circuit_init_cpath_crypto -> cpath_init_circuit_crypto circuit_free_cpath_node -> cpath_free onion_append_to_cpath -> cpath_extend_linked_list --- src/core/mainloop/connection.c | 2 +- src/core/or/circuitbuild.c | 8 ++++---- src/core/or/circuitlist.c | 10 +++++----- src/core/or/crypt_path.c | 16 ++++++++-------- src/core/or/crypt_path.h | 16 ++++++---------- src/feature/hs/hs_circuit.c | 6 +++--- src/feature/rend/rendservice.c | 4 ++-- src/test/test_circuitpadding.c | 6 +++--- src/test/test_circuitstats.c | 16 ++++++++-------- src/test/test_relaycrypt.c | 2 +- 10 files changed, 41 insertions(+), 45 deletions(-) diff --git a/src/core/mainloop/connection.c b/src/core/mainloop/connection.c index f6adfa765a..de49a1b7ef 100644 --- a/src/core/mainloop/connection.c +++ b/src/core/mainloop/connection.c @@ -5331,7 +5331,7 @@ assert_connection_ok(connection_t *conn, time_t now) tor_assert(entry_conn->socks_request->has_finished); if (!conn->marked_for_close) { tor_assert(ENTRY_TO_EDGE_CONN(entry_conn)->cpath_layer); - assert_cpath_layer_ok(ENTRY_TO_EDGE_CONN(entry_conn)->cpath_layer); + cpath_assert_layer_ok(ENTRY_TO_EDGE_CONN(entry_conn)->cpath_layer); } } } diff --git a/src/core/or/circuitbuild.c b/src/core/or/circuitbuild.c index 1ceb77c4ad..b445b94637 100644 --- a/src/core/or/circuitbuild.c +++ b/src/core/or/circuitbuild.c @@ -1409,7 +1409,7 @@ circuit_finish_handshake(origin_circuit_t *circ, onion_handshake_state_release(&hop->handshake_state); - if (circuit_init_cpath_crypto(hop, keys, sizeof(keys), 0, 0)<0) { + if (cpath_init_circuit_crypto(hop, keys, sizeof(keys), 0, 0)<0) { return -END_CIRC_REASON_TORPROTOCOL; } @@ -1461,7 +1461,7 @@ circuit_truncated(origin_circuit_t *circ, int reason) } layer->next = victim->next; - circuit_free_cpath_node(victim); + cpath_free(victim); } log_info(LD_CIRC, "finished"); @@ -2280,7 +2280,7 @@ circuit_append_new_exit(origin_circuit_t *circ, extend_info_t *exit_ei) state->chosen_exit = extend_info_dup(exit_ei); ++circ->build_state->desired_path_len; - onion_append_hop(&circ->cpath, exit_ei); + cpath_append_hop(&circ->cpath, exit_ei); return 0; } @@ -2713,7 +2713,7 @@ onion_extend_cpath(origin_circuit_t *circ) extend_info_describe(info), cur_len+1, build_state_get_exit_nickname(state)); - onion_append_hop(&circ->cpath, info); + cpath_append_hop(&circ->cpath, info); extend_info_free(info); return 0; } diff --git a/src/core/or/circuitlist.c b/src/core/or/circuitlist.c index 83c651ff17..cd2259c98d 100644 --- a/src/core/or/circuitlist.c +++ b/src/core/or/circuitlist.c @@ -1148,7 +1148,7 @@ circuit_free_(circuit_t *circ) if (ocirc->build_state) { extend_info_free(ocirc->build_state->chosen_exit); - circuit_free_cpath_node(ocirc->build_state->pending_final_cpath); + cpath_free(ocirc->build_state->pending_final_cpath); cpath_ref_decref(ocirc->build_state->service_pending_final_cpath_ref); } tor_free(ocirc->build_state); @@ -1272,10 +1272,10 @@ circuit_clear_cpath(origin_circuit_t *circ) while (cpath->next && cpath->next != head) { victim = cpath; cpath = victim->next; - circuit_free_cpath_node(victim); + cpath_free(victim); } - circuit_free_cpath_node(cpath); + cpath_free(cpath); circ->cpath = NULL; } @@ -1338,7 +1338,7 @@ cpath_ref_decref(crypt_path_reference_t *cpath_ref) { if (cpath_ref != NULL) { if (--(cpath_ref->refcount) == 0) { - circuit_free_cpath_node(cpath_ref->cpath); + cpath_free(cpath_ref->cpath); tor_free(cpath_ref); } } @@ -2830,7 +2830,7 @@ assert_circuit_ok,(const circuit_t *c)) !smartlist_contains(circuits_pending_chans, c)); } if (origin_circ && origin_circ->cpath) { - assert_cpath_ok(origin_circ->cpath); + cpath_assert_ok(origin_circ->cpath); } if (c->purpose == CIRCUIT_PURPOSE_REND_ESTABLISHED) { tor_assert(or_circ); diff --git a/src/core/or/crypt_path.c b/src/core/or/crypt_path.c index ea07ec495f..13063e5da8 100644 --- a/src/core/or/crypt_path.c +++ b/src/core/or/crypt_path.c @@ -41,7 +41,7 @@ crypt_path_new(void) * This function is used to extend cpath by another hop. */ void -onion_append_to_cpath(crypt_path_t **head_ptr, crypt_path_t *new_hop) +cpath_extend_linked_list(crypt_path_t **head_ptr, crypt_path_t *new_hop) { if (*head_ptr) { new_hop->next = (*head_ptr); @@ -58,12 +58,12 @@ onion_append_to_cpath(crypt_path_t **head_ptr, crypt_path_t *new_hop) * corresponding router choice, and append it to the * end of the cpath head_ptr. */ int -onion_append_hop(crypt_path_t **head_ptr, extend_info_t *choice) +cpath_append_hop(crypt_path_t **head_ptr, extend_info_t *choice) { crypt_path_t *hop = crypt_path_new(); /* link hop into the cpath, at the end. */ - onion_append_to_cpath(head_ptr, hop); + cpath_extend_linked_list(head_ptr, hop); hop->state = CPATH_STATE_CLOSED; @@ -79,12 +79,12 @@ onion_append_hop(crypt_path_t **head_ptr, extend_info_t *choice) * correct. Trigger an assert if anything is invalid. */ void -assert_cpath_ok(const crypt_path_t *cp) +cpath_assert_ok(const crypt_path_t *cp) { const crypt_path_t *start = cp; do { - assert_cpath_layer_ok(cp); + cpath_assert_layer_ok(cp); /* layers must be in sequence of: "open* awaiting? closed*" */ if (cp != start) { if (cp->state == CPATH_STATE_AWAITING_KEYS) { @@ -102,7 +102,7 @@ assert_cpath_ok(const crypt_path_t *cp) * correct. Trigger an assert if anything is invalid. */ void -assert_cpath_layer_ok(const crypt_path_t *cp) +cpath_assert_layer_ok(const crypt_path_t *cp) { // tor_assert(cp->addr); /* these are zero for rendezvous extra-hops */ // tor_assert(cp->port); @@ -147,7 +147,7 @@ assert_cpath_layer_ok(const crypt_path_t *cp) * Return 0 if init was successful, else -1 if it failed. */ int -circuit_init_cpath_crypto(crypt_path_t *cpath, +cpath_init_circuit_crypto(crypt_path_t *cpath, const char *key_data, size_t key_data_len, int reverse, int is_hs_v3) { @@ -160,7 +160,7 @@ circuit_init_cpath_crypto(crypt_path_t *cpath, /** Deallocate space associated with the cpath node victim. */ void -circuit_free_cpath_node(crypt_path_t *victim) +cpath_free(crypt_path_t *victim) { if (!victim || BUG(!victim->private)) return; diff --git a/src/core/or/crypt_path.h b/src/core/or/crypt_path.h index 874ff2b2ad..4a0117360e 100644 --- a/src/core/or/crypt_path.h +++ b/src/core/or/crypt_path.h @@ -8,24 +8,20 @@ crypt_path_t *crypt_path_new(void); -/* rename */ -void assert_cpath_layer_ok(const crypt_path_t *cp); +void cpath_assert_layer_ok(const crypt_path_t *cp); -/* rename */ -void assert_cpath_ok(const crypt_path_t *cp); +void cpath_assert_ok(const crypt_path_t *cp); -/* rename */ -int onion_append_hop(crypt_path_t **head_ptr, extend_info_t *choice); +int cpath_append_hop(crypt_path_t **head_ptr, extend_info_t *choice); -int circuit_init_cpath_crypto(crypt_path_t *cpath, +int cpath_init_circuit_crypto(crypt_path_t *cpath, const char *key_data, size_t key_data_len, int reverse, int is_hs_v3); void -circuit_free_cpath_node(crypt_path_t *victim); +cpath_free(crypt_path_t *victim); -/* rename */ -void onion_append_to_cpath(crypt_path_t **head_ptr, crypt_path_t *new_hop); +void cpath_extend_linked_list(crypt_path_t **head_ptr, crypt_path_t *new_hop); void cpath_crypt_cell(const crypt_path_t *cpath, uint8_t *payload, bool is_decrypt); diff --git a/src/feature/hs/hs_circuit.c b/src/feature/hs/hs_circuit.c index 3356db9d90..7d17aff72f 100644 --- a/src/feature/hs/hs_circuit.c +++ b/src/feature/hs/hs_circuit.c @@ -89,7 +89,7 @@ create_rend_cpath(const uint8_t *ntor_key_seed, size_t seed_len, /* Setup the cpath */ cpath = crypt_path_new(); - if (circuit_init_cpath_crypto(cpath, (char*)keys, sizeof(keys), + if (cpath_init_circuit_crypto(cpath, (char*)keys, sizeof(keys), is_service_side, 1) < 0) { tor_free(cpath); goto err; @@ -126,7 +126,7 @@ create_rend_cpath_legacy(origin_circuit_t *circ, const uint8_t *rend_cell_body) goto err; } /* ... and set up cpath. */ - if (circuit_init_cpath_crypto(hop, + if (cpath_init_circuit_crypto(hop, keys+DIGEST_LEN, sizeof(keys)-DIGEST_LEN, 0, 0) < 0) goto err; @@ -177,7 +177,7 @@ finalize_rend_circuit(origin_circuit_t *circ, crypt_path_t *hop, circ->hs_circ_has_timed_out = 0; /* Append the hop to the cpath of this circuit */ - onion_append_to_cpath(&circ->cpath, hop); + cpath_extend_linked_list(&circ->cpath, hop); /* In legacy code, 'pending_final_cpath' points to the final hop we just * appended to the cpath. We set the original pointer to NULL so that we diff --git a/src/feature/rend/rendservice.c b/src/feature/rend/rendservice.c index 38da4cfe7a..0ecd0e6ff6 100644 --- a/src/feature/rend/rendservice.c +++ b/src/feature/rend/rendservice.c @@ -2163,7 +2163,7 @@ rend_service_receive_introduction(origin_circuit_t *circuit, cpath->rend_dh_handshake_state = dh; dh = NULL; - if (circuit_init_cpath_crypto(cpath, + if (cpath_init_circuit_crypto(cpath, keys+DIGEST_LEN, sizeof(keys)-DIGEST_LEN, 1, 0)<0) goto err; @@ -3547,7 +3547,7 @@ rend_service_rendezvous_has_opened(origin_circuit_t *circuit) hop->package_window = circuit_initial_package_window(); hop->deliver_window = CIRCWINDOW_START; - onion_append_to_cpath(&circuit->cpath, hop); + cpath_extend_linked_list(&circuit->cpath, hop); circuit->build_state->pending_final_cpath = NULL; /* prevent double-free */ /* Change the circuit purpose. */ diff --git a/src/test/test_circuitpadding.c b/src/test/test_circuitpadding.c index 6fa790c40d..e33e56af3f 100644 --- a/src/test/test_circuitpadding.c +++ b/src/test/test_circuitpadding.c @@ -145,7 +145,7 @@ new_fake_orcirc(channel_t *nchan, channel_t *pchan) circuit_set_n_circid_chan(circ, circ->n_circ_id, nchan); tmp_cpath = crypt_path_new(); - if (circuit_init_cpath_crypto(tmp_cpath, whatevs_key, + if (cpath_init_circuit_crypto(tmp_cpath, whatevs_key, sizeof(whatevs_key), 0, 0)<0) { log_warn(LD_BUG,"Circuit initialization failed"); return NULL; @@ -1621,7 +1621,7 @@ simulate_single_hop_extend(circuit_t *client, circuit_t *mid_relay, // Add a hop to cpath crypt_path_t *hop = crypt_path_new(); - onion_append_to_cpath(&TO_ORIGIN_CIRCUIT(client)->cpath, hop); + cpath_extend_linked_list(&TO_ORIGIN_CIRCUIT(client)->cpath, hop); hop->state = CPATH_STATE_OPEN; @@ -1634,7 +1634,7 @@ simulate_single_hop_extend(circuit_t *client, circuit_t *mid_relay, digest, NULL, NULL, NULL, &addr, padding); - circuit_init_cpath_crypto(hop, whatevs_key, sizeof(whatevs_key), 0, 0); + cpath_init_circuit_crypto(hop, whatevs_key, sizeof(whatevs_key), 0, 0); hop->package_window = circuit_initial_package_window(); hop->deliver_window = CIRCWINDOW_START; diff --git a/src/test/test_circuitstats.c b/src/test/test_circuitstats.c index 1cbcb14f2b..2a09622f09 100644 --- a/src/test/test_circuitstats.c +++ b/src/test/test_circuitstats.c @@ -28,7 +28,7 @@ origin_circuit_t *subtest_fourhop_circuit(struct timeval, int); origin_circuit_t *add_opened_threehop(void); origin_circuit_t *build_unopened_fourhop(struct timeval); -int onion_append_hop(crypt_path_t **head_ptr, extend_info_t *choice); +int cpath_append_hop(crypt_path_t **head_ptr, extend_info_t *choice); static int marked_for_close; /* Mock function because we are not trying to test the close circuit that does @@ -57,9 +57,9 @@ add_opened_threehop(void) or_circ->build_state = tor_malloc_zero(sizeof(cpath_build_state_t)); or_circ->build_state->desired_path_len = DEFAULT_ROUTE_LEN; - onion_append_hop(&or_circ->cpath, &fakehop); - onion_append_hop(&or_circ->cpath, &fakehop); - onion_append_hop(&or_circ->cpath, &fakehop); + cpath_append_hop(&or_circ->cpath, &fakehop); + cpath_append_hop(&or_circ->cpath, &fakehop); + cpath_append_hop(&or_circ->cpath, &fakehop); or_circ->has_opened = 1; TO_CIRCUIT(or_circ)->state = CIRCUIT_STATE_OPEN; @@ -82,10 +82,10 @@ build_unopened_fourhop(struct timeval circ_start_time) or_circ->build_state = tor_malloc_zero(sizeof(cpath_build_state_t)); or_circ->build_state->desired_path_len = 4; - onion_append_hop(&or_circ->cpath, fakehop); - onion_append_hop(&or_circ->cpath, fakehop); - onion_append_hop(&or_circ->cpath, fakehop); - onion_append_hop(&or_circ->cpath, fakehop); + cpath_append_hop(&or_circ->cpath, fakehop); + cpath_append_hop(&or_circ->cpath, fakehop); + cpath_append_hop(&or_circ->cpath, fakehop); + cpath_append_hop(&or_circ->cpath, fakehop); tor_free(fakehop); diff --git a/src/test/test_relaycrypt.c b/src/test/test_relaycrypt.c index a3a102e73b..1977958d1f 100644 --- a/src/test/test_relaycrypt.c +++ b/src/test/test_relaycrypt.c @@ -54,7 +54,7 @@ testing_circuitset_setup(const struct testcase_t *testcase) relay_crypto_init(&hop->private->crypto, KEY_MATERIAL[i], sizeof(KEY_MATERIAL[i]), 0, 0); hop->state = CPATH_STATE_OPEN; - onion_append_to_cpath(&cs->origin_circ->cpath, hop); + cpath_extend_linked_list(&cs->origin_circ->cpath, hop); tt_ptr_op(hop, OP_EQ, cs->origin_circ->cpath->prev); } From 0ed5c6edf9c905276d462ed2402568216ecb1dee Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Tue, 9 Apr 2019 17:57:04 +0300 Subject: [PATCH 0960/2557] Hiding crypt_path_t: Move some more crypt_path-specific functions. - Move test-only cpath_get_n_hops() to crypt_path.c. - Move onion_next_hop_in_cpath() and rename to cpath_get_next_non_open_hop(). The latter function was directly accessing cpath->state, and it's a first step at hiding ->state. --- src/core/or/circuitbuild.c | 45 +++----------------------------------- src/core/or/circuitbuild.h | 5 ----- src/core/or/crypt_path.c | 41 ++++++++++++++++++++++++++++++++++ src/core/or/crypt_path.h | 6 +++++ 4 files changed, 50 insertions(+), 47 deletions(-) diff --git a/src/core/or/circuitbuild.c b/src/core/or/circuitbuild.c index b445b94637..e59aca0e25 100644 --- a/src/core/or/circuitbuild.c +++ b/src/core/or/circuitbuild.c @@ -91,7 +91,6 @@ static channel_t * channel_connect_for_circuit(const tor_addr_t *addr, static int circuit_deliver_create_cell(circuit_t *circ, const create_cell_t *create_cell, int relayed); -static crypt_path_t *onion_next_hop_in_cpath(crypt_path_t *cpath); static int circuit_send_first_onion_skin(origin_circuit_t *circ); static int circuit_build_no_more_hops(origin_circuit_t *circ); static int circuit_send_intermediate_onion_skin(origin_circuit_t *circ, @@ -547,7 +546,7 @@ circuit_handle_first_hop(origin_circuit_t *circ) int should_launch = 0; const or_options_t *options = get_options(); - firsthop = onion_next_hop_in_cpath(circ->cpath); + firsthop = cpath_get_next_non_open_hop(circ->cpath); tor_assert(firsthop); tor_assert(firsthop->extend_info); @@ -948,7 +947,7 @@ circuit_send_next_onion_skin(origin_circuit_t *circ) tor_assert(circ->cpath->state == CPATH_STATE_OPEN); tor_assert(circ->base_.state == CIRCUIT_STATE_BUILDING); - crypt_path_t *hop = onion_next_hop_in_cpath(circ->cpath); + crypt_path_t *hop = cpath_get_next_non_open_hop(circ->cpath); circuit_build_times_handle_completed_hop(circ); circpad_machine_event_circ_added_hop(circ); @@ -1385,7 +1384,7 @@ circuit_finish_handshake(origin_circuit_t *circ, if (circ->cpath->state == CPATH_STATE_AWAITING_KEYS) { hop = circ->cpath; } else { - hop = onion_next_hop_in_cpath(circ->cpath); + hop = cpath_get_next_non_open_hop(circ->cpath); if (!hop) { /* got an extended when we're all done? */ log_warn(LD_PROTOCOL,"got extended when circ already built? Closing."); return - END_CIRC_REASON_TORPROTOCOL; @@ -2345,30 +2344,6 @@ count_acceptable_nodes, (const smartlist_t *nodes, int direct)) return num; } -#ifdef TOR_UNIT_TESTS - -/** Unittest helper function: Count number of hops in cpath linked list. */ -unsigned int -cpath_get_n_hops(crypt_path_t **head_ptr) -{ - unsigned int n_hops = 0; - crypt_path_t *tmp; - - if (!*head_ptr) { - return 0; - } - - tmp = *head_ptr; - do { - n_hops++; - tmp = tmp->next; - } while (tmp != *head_ptr); - - return n_hops; -} - -#endif /* defined(TOR_UNIT_TESTS) */ - /** * Build the exclude list for vanguard circuits. * @@ -2643,20 +2618,6 @@ choose_good_entry_server(uint8_t purpose, cpath_build_state_t *state, return choice; } -/** Return the first non-open hop in cpath, or return NULL if all - * hops are open. */ -static crypt_path_t * -onion_next_hop_in_cpath(crypt_path_t *cpath) -{ - crypt_path_t *hop = cpath; - do { - if (hop->state != CPATH_STATE_OPEN) - return hop; - hop = hop->next; - } while (hop != cpath); - return NULL; -} - /** Choose a suitable next hop for the circuit circ. * Append the hop info to circ->cpath. * diff --git a/src/core/or/circuitbuild.h b/src/core/or/circuitbuild.h index f6403955ba..ad7d032cd4 100644 --- a/src/core/or/circuitbuild.h +++ b/src/core/or/circuitbuild.h @@ -89,11 +89,6 @@ STATIC int onion_pick_cpath_exit(origin_circuit_t *circ, extend_info_t *exit_ei, int is_hs_v3_rp_circuit); -#if defined(TOR_UNIT_TESTS) -unsigned int cpath_get_n_hops(crypt_path_t **head_ptr); - -#endif /* defined(TOR_UNIT_TESTS) */ - #endif /* defined(CIRCUITBUILD_PRIVATE) */ #endif /* !defined(TOR_CIRCUITBUILD_H) */ diff --git a/src/core/or/crypt_path.c b/src/core/or/crypt_path.c index 13063e5da8..8fcbcc2a12 100644 --- a/src/core/or/crypt_path.c +++ b/src/core/or/crypt_path.c @@ -210,3 +210,44 @@ cpath_set_cell_forward_digest(crypt_path_t *cpath, cell_t *cell) tor_assert(cpath->private); relay_set_digest(cpath->private->crypto.f_digest, cell); } + +/************ other cpath functions ***************************/ + +/** Return the first non-open hop in cpath, or return NULL if all + * hops are open. */ +crypt_path_t * +cpath_get_next_non_open_hop(crypt_path_t *cpath) +{ + crypt_path_t *hop = cpath; + do { + if (hop->state != CPATH_STATE_OPEN) + return hop; + hop = hop->next; + } while (hop != cpath); + return NULL; +} + +#ifdef TOR_UNIT_TESTS + +/** Unittest helper function: Count number of hops in cpath linked list. */ +unsigned int +cpath_get_n_hops(crypt_path_t **head_ptr) +{ + unsigned int n_hops = 0; + crypt_path_t *tmp; + + if (!*head_ptr) { + return 0; + } + + tmp = *head_ptr; + do { + n_hops++; + tmp = tmp->next; + } while (tmp != *head_ptr); + + return n_hops; +} + +#endif /* defined(TOR_UNIT_TESTS) */ + diff --git a/src/core/or/crypt_path.h b/src/core/or/crypt_path.h index 4a0117360e..ed59037760 100644 --- a/src/core/or/crypt_path.h +++ b/src/core/or/crypt_path.h @@ -32,4 +32,10 @@ cpath_get_incoming_digest(const crypt_path_t *cpath); void cpath_set_cell_forward_digest(crypt_path_t *cpath, cell_t *cell); +crypt_path_t *cpath_get_next_non_open_hop(crypt_path_t *cpath); + +#if defined(TOR_UNIT_TESTS) +unsigned int cpath_get_n_hops(crypt_path_t **head_ptr); +#endif /* defined(TOR_UNIT_TESTS) */ + #endif From 2e9e3e7d4198ff75e6bd12bc7a38c0f288fbe381 Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Tue, 9 Apr 2019 18:04:15 +0300 Subject: [PATCH 0961/2557] Hiding crypt_path_t: Some TODO notes for future directions. --- src/core/or/crypt_path.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/core/or/crypt_path.c b/src/core/or/crypt_path.c index 8fcbcc2a12..c7ff8690de 100644 --- a/src/core/or/crypt_path.c +++ b/src/core/or/crypt_path.c @@ -8,6 +8,17 @@ * \brief Functions dealing with layered circuit encryption. This file aims to * provide an API around the crypt_path_t structure which holds crypto * information about a specific hop of a circuit. + * + * TODO: We should eventually move all functions dealing and manipulating + * crypt_path_t to this file, so that eventually we encapsulate more and more + * of crypt_path_t. Here are some more functions that can be moved here with + * some more effort: + * + * - circuit_list_path_impl() + * - Functions dealing with cpaths in HSv2 create_rend_cpath() and + * create_rend_cpath_legacy() + * - The cpath related parts of rend_service_receive_introduction() and + * rend_client_send_introduction(). **/ #define CRYPT_PATH_PRIVATE From 4060b7623d3845a4d4ecdbf8f9c219e0148e1380 Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Fri, 26 Apr 2019 14:26:22 +0300 Subject: [PATCH 0962/2557] Revert "Hiding crypt_path_t: Create a constructor for crypt_path_t." This reverts commit ab8b80944967ee5a6a0c45dbf61839cf257bfe44. --- src/core/or/crypt_path.c | 15 ++------------- src/core/or/crypt_path.h | 2 -- src/core/or/crypt_path_st.h | 5 +++-- src/feature/hs/hs_circuit.c | 3 ++- src/feature/rend/rendclient.c | 5 +++-- src/feature/rend/rendservice.c | 3 ++- src/test/test_circuitpadding.c | 13 ++++++------- src/test/test_hs_client.c | 6 ++++-- src/test/test_hs_service.c | 5 +++-- src/test/test_relaycell.c | 4 ++-- src/test/test_relaycrypt.c | 2 +- 11 files changed, 28 insertions(+), 35 deletions(-) diff --git a/src/core/or/crypt_path.c b/src/core/or/crypt_path.c index c7ff8690de..c44d65231d 100644 --- a/src/core/or/crypt_path.c +++ b/src/core/or/crypt_path.c @@ -37,17 +37,6 @@ #include "core/or/crypt_path_st.h" #include "core/or/cell_st.h" -/** Initialize and return a minimal crypt_path_t */ -crypt_path_t * -crypt_path_new(void) -{ - crypt_path_t *cpath = tor_malloc_zero(sizeof(crypt_path_t)); - cpath->magic = CRYPT_PATH_MAGIC; - cpath->private = tor_malloc_zero(sizeof(struct crypt_path_private_t)); - - return cpath; -} - /** Add new_hop to the end of the doubly-linked-list head_ptr. * This function is used to extend cpath by another hop. */ @@ -71,11 +60,12 @@ cpath_extend_linked_list(crypt_path_t **head_ptr, crypt_path_t *new_hop) int cpath_append_hop(crypt_path_t **head_ptr, extend_info_t *choice) { - crypt_path_t *hop = crypt_path_new(); + crypt_path_t *hop = tor_malloc_zero(sizeof(crypt_path_t)); /* link hop into the cpath, at the end. */ cpath_extend_linked_list(head_ptr, hop); + hop->magic = CRYPT_PATH_MAGIC; hop->state = CPATH_STATE_CLOSED; hop->extend_info = extend_info_dup(choice); @@ -180,7 +170,6 @@ cpath_free(crypt_path_t *victim) onion_handshake_state_release(&victim->handshake_state); crypto_dh_free(victim->rend_dh_handshake_state); extend_info_free(victim->extend_info); - tor_free(victim->private); memwipe(victim, 0xBB, sizeof(crypt_path_t)); /* poison memory */ tor_free(victim); diff --git a/src/core/or/crypt_path.h b/src/core/or/crypt_path.h index ed59037760..19c8571d06 100644 --- a/src/core/or/crypt_path.h +++ b/src/core/or/crypt_path.h @@ -6,8 +6,6 @@ #ifndef CRYPT_PATH_H #define CRYPT_PATH_H -crypt_path_t *crypt_path_new(void); - void cpath_assert_layer_ok(const crypt_path_t *cp); void cpath_assert_ok(const crypt_path_t *cp); diff --git a/src/core/or/crypt_path_st.h b/src/core/or/crypt_path_st.h index 7da3c57f49..833cfefad1 100644 --- a/src/core/or/crypt_path_st.h +++ b/src/core/or/crypt_path_st.h @@ -8,6 +8,9 @@ #define CRYPT_PATH_ST_H #include "core/or/relay_crypto_st.h" +struct crypto_dh_t; + +#define CRYPT_PATH_MAGIC 0x70127012u struct fast_handshake_state_t; struct ntor_handshake_state_t; @@ -23,8 +26,6 @@ struct onion_handshake_state_t { #ifdef CRYPT_PATH_PRIVATE -#define CRYPT_PATH_MAGIC 0x70127012u - /* The private parts of crypt path that don't need to be exposed to all the * modules. */ struct crypt_path_private_t { diff --git a/src/feature/hs/hs_circuit.c b/src/feature/hs/hs_circuit.c index 7d17aff72f..a6e86c5ab3 100644 --- a/src/feature/hs/hs_circuit.c +++ b/src/feature/hs/hs_circuit.c @@ -87,7 +87,8 @@ create_rend_cpath(const uint8_t *ntor_key_seed, size_t seed_len, } /* Setup the cpath */ - cpath = crypt_path_new(); + cpath = tor_malloc_zero(sizeof(crypt_path_t)); + cpath->magic = CRYPT_PATH_MAGIC; if (cpath_init_circuit_crypto(cpath, (char*)keys, sizeof(keys), is_service_side, 1) < 0) { diff --git a/src/feature/rend/rendclient.c b/src/feature/rend/rendclient.c index c6e9dde878..f84d221b1a 100644 --- a/src/feature/rend/rendclient.c +++ b/src/feature/rend/rendclient.c @@ -16,7 +16,6 @@ #include "core/or/circuituse.h" #include "core/or/connection_edge.h" #include "core/or/relay.h" -#include "core/or/crypt_path.h" #include "feature/client/circpathbias.h" #include "feature/control/control_events.h" #include "feature/dirclient/dirclient.h" @@ -195,7 +194,9 @@ rend_client_send_introduction(origin_circuit_t *introcirc, /* Initialize the pending_final_cpath and start the DH handshake. */ cpath = rendcirc->build_state->pending_final_cpath; if (!cpath) { - cpath = rendcirc->build_state->pending_final_cpath = crypt_path_new(); + cpath = rendcirc->build_state->pending_final_cpath = + tor_malloc_zero(sizeof(crypt_path_t)); + cpath->magic = CRYPT_PATH_MAGIC; if (!(cpath->rend_dh_handshake_state = crypto_dh_new(DH_TYPE_REND))) { log_warn(LD_BUG, "Internal error: couldn't allocate DH."); status = -2; diff --git a/src/feature/rend/rendservice.c b/src/feature/rend/rendservice.c index 0ecd0e6ff6..98c7253bcc 100644 --- a/src/feature/rend/rendservice.c +++ b/src/feature/rend/rendservice.c @@ -2158,7 +2158,8 @@ rend_service_receive_introduction(origin_circuit_t *circuit, launched->build_state->service_pending_final_cpath_ref->refcount = 1; launched->build_state->service_pending_final_cpath_ref->cpath = cpath = - crypt_path_new(); + tor_malloc_zero(sizeof(crypt_path_t)); + cpath->magic = CRYPT_PATH_MAGIC; launched->build_state->expiry_time = now + MAX_REND_TIMEOUT; cpath->rend_dh_handshake_state = dh; diff --git a/src/test/test_circuitpadding.c b/src/test/test_circuitpadding.c index e33e56af3f..5550488d0f 100644 --- a/src/test/test_circuitpadding.c +++ b/src/test/test_circuitpadding.c @@ -115,7 +115,7 @@ new_fake_orcirc(channel_t *nchan, channel_t *pchan) { or_circuit_t *orcirc = NULL; circuit_t *circ = NULL; - crypt_path_t *tmp_cpath; + crypt_path_t tmp_cpath; char whatevs_key[CPATH_KEY_MATERIAL_LEN]; orcirc = tor_malloc_zero(sizeof(*orcirc)); @@ -144,15 +144,13 @@ new_fake_orcirc(channel_t *nchan, channel_t *pchan) circuit_set_p_circid_chan(orcirc, orcirc->p_circ_id, pchan); circuit_set_n_circid_chan(circ, circ->n_circ_id, nchan); - tmp_cpath = crypt_path_new(); - if (cpath_init_circuit_crypto(tmp_cpath, whatevs_key, + memset(&tmp_cpath, 0, sizeof(tmp_cpath)); + if (cpath_init_circuit_crypto(&tmp_cpath, whatevs_key, sizeof(whatevs_key), 0, 0)<0) { log_warn(LD_BUG,"Circuit initialization failed"); return NULL; } - orcirc->crypto = tmp_cpath->private->crypto; - tor_free(tmp_cpath->private); - tor_free(tmp_cpath); + orcirc->crypto = tmp_cpath.private->crypto; return orcirc; } @@ -1620,9 +1618,10 @@ simulate_single_hop_extend(circuit_t *client, circuit_t *mid_relay, circpad_cell_event_nonpadding_received((circuit_t*)client); // Add a hop to cpath - crypt_path_t *hop = crypt_path_new(); + crypt_path_t *hop = tor_malloc_zero(sizeof(crypt_path_t)); cpath_extend_linked_list(&TO_ORIGIN_CIRCUIT(client)->cpath, hop); + hop->magic = CRYPT_PATH_MAGIC; hop->state = CPATH_STATE_OPEN; // add an extend info to indicate if this node supports padding or not. diff --git a/src/test/test_hs_client.c b/src/test/test_hs_client.c index cd049b7c47..7f5f255076 100644 --- a/src/test/test_hs_client.c +++ b/src/test/test_hs_client.c @@ -39,13 +39,13 @@ #include "feature/hs/hs_cache.h" #include "core/or/circuitlist.h" #include "core/or/circuitbuild.h" -#include "core/or/crypt_path.h" #include "core/mainloop/connection.h" #include "core/or/connection_edge.h" #include "feature/nodelist/networkstatus.h" #include "core/or/cpath_build_state_st.h" #include "core/or/crypt_path_st.h" +#include "core/or/crypt_path.h" #include "feature/dircommon/dir_connection_st.h" #include "core/or/entry_connection_st.h" #include "core/or/extend_info_st.h" @@ -146,7 +146,9 @@ helper_get_circ_and_stream_for_test(origin_circuit_t **circ_out, if (is_legacy) { /* Legacy: Setup rend data and final cpath */ - or_circ->build_state->pending_final_cpath = crypt_path_new(); + or_circ->build_state->pending_final_cpath = + tor_malloc_zero(sizeof(crypt_path_t)); + or_circ->build_state->pending_final_cpath->magic = CRYPT_PATH_MAGIC; or_circ->build_state->pending_final_cpath->rend_dh_handshake_state = crypto_dh_new(DH_TYPE_REND); tt_assert( diff --git a/src/test/test_hs_service.c b/src/test/test_hs_service.c index 08dac04d21..8a22e4d590 100644 --- a/src/test/test_hs_service.c +++ b/src/test/test_hs_service.c @@ -38,7 +38,6 @@ #include "core/or/circuitbuild.h" #include "core/or/circuitlist.h" #include "core/or/circuituse.h" -#include "core/or/crypt_path.h" #include "core/or/connection_edge.h" #include "core/or/edge_connection_st.h" #include "core/or/relay.h" @@ -62,6 +61,7 @@ #include "core/or/cpath_build_state_st.h" #include "core/or/crypt_path_st.h" +#include "core/or/crypt_path.h" #include "feature/nodelist/networkstatus_st.h" #include "feature/nodelist/node_st.h" #include "core/or/origin_circuit_st.h" @@ -221,7 +221,8 @@ helper_create_origin_circuit(int purpose, int flags) circ = origin_circuit_init(purpose, flags); tor_assert(circ); - circ->cpath = crypt_path_new(); + circ->cpath = tor_malloc_zero(sizeof(crypt_path_t)); + circ->cpath->magic = CRYPT_PATH_MAGIC; circ->cpath->state = CPATH_STATE_OPEN; circ->cpath->package_window = circuit_initial_package_window(); circ->cpath->deliver_window = CIRCWINDOW_START; diff --git a/src/test/test_relaycell.c b/src/test/test_relaycell.c index b48c7ca8ac..0623583511 100644 --- a/src/test/test_relaycell.c +++ b/src/test/test_relaycell.c @@ -16,7 +16,6 @@ #include "lib/crypt_ops/crypto_rand.h" #include "core/or/circuitbuild.h" #include "core/or/circuitlist.h" -#include "core/or/crypt_path.h" #include "core/or/connection_edge.h" #include "core/or/relay.h" #include "test/test.h" @@ -91,7 +90,8 @@ helper_create_origin_circuit(int purpose, int flags) circ = origin_circuit_init(purpose, flags); tor_assert(circ); - circ->cpath = crypt_path_new(); + circ->cpath = tor_malloc_zero(sizeof(crypt_path_t)); + circ->cpath->magic = CRYPT_PATH_MAGIC; circ->cpath->state = CPATH_STATE_OPEN; circ->cpath->package_window = circuit_initial_package_window(); circ->cpath->deliver_window = CIRCWINDOW_START; diff --git a/src/test/test_relaycrypt.c b/src/test/test_relaycrypt.c index 1977958d1f..5dc6b47d74 100644 --- a/src/test/test_relaycrypt.c +++ b/src/test/test_relaycrypt.c @@ -50,7 +50,7 @@ testing_circuitset_setup(const struct testcase_t *testcase) cs->origin_circ = origin_circuit_new(); cs->origin_circ->base_.purpose = CIRCUIT_PURPOSE_C_GENERAL; for (i=0; i<3; ++i) { - crypt_path_t *hop = crypt_path_new(); + crypt_path_t *hop = tor_malloc_zero(sizeof(*hop)); relay_crypto_init(&hop->private->crypto, KEY_MATERIAL[i], sizeof(KEY_MATERIAL[i]), 0, 0); hop->state = CPATH_STATE_OPEN; From 2ef0324639dd2e2c551be039c7f449eb6cab6703 Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Fri, 26 Apr 2019 14:28:03 +0300 Subject: [PATCH 0963/2557] Revert "Hiding crypt_path_t: Ensure that ->private is initialized." This reverts commit 7497c9193a0f2c891a0802bf5fbe73cf7ec1ca99. --- src/core/or/crypt_path.c | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/src/core/or/crypt_path.c b/src/core/or/crypt_path.c index c44d65231d..b7068fd67a 100644 --- a/src/core/or/crypt_path.c +++ b/src/core/or/crypt_path.c @@ -109,7 +109,6 @@ cpath_assert_layer_ok(const crypt_path_t *cp) // tor_assert(cp->port); tor_assert(cp); tor_assert(cp->magic == CRYPT_PATH_MAGIC); - tor_assert(cp->private); switch (cp->state) { case CPATH_STATE_OPEN: @@ -154,7 +153,6 @@ cpath_init_circuit_crypto(crypt_path_t *cpath, { tor_assert(cpath); - tor_assert(cpath->private); return relay_crypto_init(&cpath->private->crypto, key_data, key_data_len, reverse, is_hs_v3); } @@ -163,7 +161,7 @@ cpath_init_circuit_crypto(crypt_path_t *cpath, void cpath_free(crypt_path_t *victim) { - if (!victim || BUG(!victim->private)) + if (!victim) return; relay_crypto_clear(&victim->private->crypto); @@ -182,9 +180,6 @@ cpath_free(crypt_path_t *victim) void cpath_crypt_cell(const crypt_path_t *cpath, uint8_t *payload, bool is_decrypt) { - tor_assert(cpath); - tor_assert(cpath->private); - if (is_decrypt) { relay_crypt_one_payload(cpath->private->crypto.b_crypto, payload); } else { @@ -196,8 +191,6 @@ cpath_crypt_cell(const crypt_path_t *cpath, uint8_t *payload, bool is_decrypt) struct crypto_digest_t * cpath_get_incoming_digest(const crypt_path_t *cpath) { - tor_assert(cpath); - tor_assert(cpath->private); return cpath->private->crypto.b_digest; } @@ -206,8 +199,6 @@ cpath_get_incoming_digest(const crypt_path_t *cpath) void cpath_set_cell_forward_digest(crypt_path_t *cpath, cell_t *cell) { - tor_assert(cpath); - tor_assert(cpath->private); relay_set_digest(cpath->private->crypto.f_digest, cell); } From 55d35c0caa4142f92e2efd85bffe52568c173100 Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Fri, 26 Apr 2019 14:19:14 +0300 Subject: [PATCH 0964/2557] Hiding crypt_path_t: Hiding 'crypto' using a macro. --- src/core/or/crypt_path_st.h | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/src/core/or/crypt_path_st.h b/src/core/or/crypt_path_st.h index 833cfefad1..d18d23e939 100644 --- a/src/core/or/crypt_path_st.h +++ b/src/core/or/crypt_path_st.h @@ -24,15 +24,16 @@ struct onion_handshake_state_t { } u; }; +/** Macro to encapsulate private members of a struct. + * + * Renames 'x' to 'x_crypt_path_private_field'. + */ +#define CRYPT_PATH_PRIV_FIELD(x) x ## _crypt_path_private_field + #ifdef CRYPT_PATH_PRIVATE -/* The private parts of crypt path that don't need to be exposed to all the - * modules. */ -struct crypt_path_private_t { - /** Cryptographic state used for encrypting and authenticating relay - * cells to and from this hop. */ - relay_crypto_t crypto; -}; +/* Helper macro to access private members of a struct. */ +#define pvt_crypto CRYPT_PATH_PRIV_FIELD(crypto) #endif @@ -74,9 +75,11 @@ struct crypt_path_t { int deliver_window; /**< How many cells are we willing to deliver originating * at this step? */ - /* Private parts of the crypt_path. Eventually everything should be - * private. */ - struct crypt_path_private_t *private; + /*********************** Private members ****************************/ + + /** Private member: Cryptographic state used for encrypting and + * authenticating relay cells to and from this hop. */ + relay_crypto_t CRYPT_PATH_PRIV_FIELD(crypto); }; #endif From ea5f355fc96b6c61b40dafaea05f147f6d9ba57b Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Fri, 26 Apr 2019 14:20:26 +0300 Subject: [PATCH 0965/2557] Hiding crypt_path_t: Change code to use the privatization macro. --- src/core/or/crypt_path.c | 14 +++++++------- src/test/test_circuitpadding.c | 2 +- src/test/test_hs_client.c | 16 ++++++++-------- src/test/test_hs_service.c | 8 ++++---- src/test/test_relaycrypt.c | 2 +- 5 files changed, 21 insertions(+), 21 deletions(-) diff --git a/src/core/or/crypt_path.c b/src/core/or/crypt_path.c index b7068fd67a..e2234cc2a6 100644 --- a/src/core/or/crypt_path.c +++ b/src/core/or/crypt_path.c @@ -112,7 +112,7 @@ cpath_assert_layer_ok(const crypt_path_t *cp) switch (cp->state) { case CPATH_STATE_OPEN: - relay_crypto_assert_ok(&cp->private->crypto); + relay_crypto_assert_ok(&cp->pvt_crypto); /* fall through */ case CPATH_STATE_CLOSED: /*XXXX Assert that there's no handshake_state either. */ @@ -153,7 +153,7 @@ cpath_init_circuit_crypto(crypt_path_t *cpath, { tor_assert(cpath); - return relay_crypto_init(&cpath->private->crypto, key_data, key_data_len, + return relay_crypto_init(&cpath->pvt_crypto, key_data, key_data_len, reverse, is_hs_v3); } @@ -164,7 +164,7 @@ cpath_free(crypt_path_t *victim) if (!victim) return; - relay_crypto_clear(&victim->private->crypto); + relay_crypto_clear(&victim->pvt_crypto); onion_handshake_state_release(&victim->handshake_state); crypto_dh_free(victim->rend_dh_handshake_state); extend_info_free(victim->extend_info); @@ -181,9 +181,9 @@ void cpath_crypt_cell(const crypt_path_t *cpath, uint8_t *payload, bool is_decrypt) { if (is_decrypt) { - relay_crypt_one_payload(cpath->private->crypto.b_crypto, payload); + relay_crypt_one_payload(cpath->pvt_crypto.b_crypto, payload); } else { - relay_crypt_one_payload(cpath->private->crypto.f_crypto, payload); + relay_crypt_one_payload(cpath->pvt_crypto.f_crypto, payload); } } @@ -191,7 +191,7 @@ cpath_crypt_cell(const crypt_path_t *cpath, uint8_t *payload, bool is_decrypt) struct crypto_digest_t * cpath_get_incoming_digest(const crypt_path_t *cpath) { - return cpath->private->crypto.b_digest; + return cpath->pvt_crypto.b_digest; } /** Set the right integrity digest on the outgoing cell based on the @@ -199,7 +199,7 @@ cpath_get_incoming_digest(const crypt_path_t *cpath) void cpath_set_cell_forward_digest(crypt_path_t *cpath, cell_t *cell) { - relay_set_digest(cpath->private->crypto.f_digest, cell); + relay_set_digest(cpath->pvt_crypto.f_digest, cell); } /************ other cpath functions ***************************/ diff --git a/src/test/test_circuitpadding.c b/src/test/test_circuitpadding.c index 5550488d0f..914bcb97d7 100644 --- a/src/test/test_circuitpadding.c +++ b/src/test/test_circuitpadding.c @@ -150,7 +150,7 @@ new_fake_orcirc(channel_t *nchan, channel_t *pchan) log_warn(LD_BUG,"Circuit initialization failed"); return NULL; } - orcirc->crypto = tmp_cpath.private->crypto; + orcirc->crypto = tmp_cpath.pvt_crypto; return orcirc; } diff --git a/src/test/test_hs_client.c b/src/test/test_hs_client.c index 7f5f255076..6cf0b68e98 100644 --- a/src/test/test_hs_client.c +++ b/src/test/test_hs_client.c @@ -244,13 +244,13 @@ test_e2e_rend_circuit_setup_legacy(void *arg) /* Check the digest algo */ tt_int_op( - crypto_digest_get_algorithm(or_circ->cpath->private->crypto.f_digest), + crypto_digest_get_algorithm(or_circ->cpath->pvt_crypto.f_digest), OP_EQ, DIGEST_SHA1); tt_int_op( - crypto_digest_get_algorithm(or_circ->cpath->private->crypto.b_digest), + crypto_digest_get_algorithm(or_circ->cpath->pvt_crypto.b_digest), OP_EQ, DIGEST_SHA1); - tt_assert(or_circ->cpath->private->crypto.f_crypto); - tt_assert(or_circ->cpath->private->crypto.b_crypto); + tt_assert(or_circ->cpath->pvt_crypto.f_crypto); + tt_assert(or_circ->cpath->pvt_crypto.b_crypto); /* Ensure that circ purpose was changed */ tt_int_op(or_circ->base_.purpose, OP_EQ, CIRCUIT_PURPOSE_C_REND_JOINED); @@ -316,13 +316,13 @@ test_e2e_rend_circuit_setup(void *arg) /* Check that the crypt path has prop224 algorithm parameters */ tt_int_op( - crypto_digest_get_algorithm(or_circ->cpath->private->crypto.f_digest), + crypto_digest_get_algorithm(or_circ->cpath->pvt_crypto.f_digest), OP_EQ, DIGEST_SHA3_256); tt_int_op( - crypto_digest_get_algorithm(or_circ->cpath->private->crypto.b_digest), + crypto_digest_get_algorithm(or_circ->cpath->pvt_crypto.b_digest), OP_EQ, DIGEST_SHA3_256); - tt_assert(or_circ->cpath->private->crypto.f_crypto); - tt_assert(or_circ->cpath->private->crypto.b_crypto); + tt_assert(or_circ->cpath->pvt_crypto.f_crypto); + tt_assert(or_circ->cpath->pvt_crypto.b_crypto); /* Ensure that circ purpose was changed */ tt_int_op(or_circ->base_.purpose, OP_EQ, CIRCUIT_PURPOSE_C_REND_JOINED); diff --git a/src/test/test_hs_service.c b/src/test/test_hs_service.c index 8a22e4d590..2a6aa5c63c 100644 --- a/src/test/test_hs_service.c +++ b/src/test/test_hs_service.c @@ -196,13 +196,13 @@ test_e2e_rend_circuit_setup(void *arg) /* Check the digest algo */ tt_int_op( - crypto_digest_get_algorithm(or_circ->cpath->private->crypto.f_digest), + crypto_digest_get_algorithm(or_circ->cpath->pvt_crypto.f_digest), OP_EQ, DIGEST_SHA3_256); tt_int_op( - crypto_digest_get_algorithm(or_circ->cpath->private->crypto.b_digest), + crypto_digest_get_algorithm(or_circ->cpath->pvt_crypto.b_digest), OP_EQ, DIGEST_SHA3_256); - tt_assert(or_circ->cpath->private->crypto.f_crypto); - tt_assert(or_circ->cpath->private->crypto.b_crypto); + tt_assert(or_circ->cpath->pvt_crypto.f_crypto); + tt_assert(or_circ->cpath->pvt_crypto.b_crypto); /* Ensure that circ purpose was changed */ tt_int_op(or_circ->base_.purpose, OP_EQ, CIRCUIT_PURPOSE_S_REND_JOINED); diff --git a/src/test/test_relaycrypt.c b/src/test/test_relaycrypt.c index 5dc6b47d74..4bbf07c3ec 100644 --- a/src/test/test_relaycrypt.c +++ b/src/test/test_relaycrypt.c @@ -51,7 +51,7 @@ testing_circuitset_setup(const struct testcase_t *testcase) cs->origin_circ->base_.purpose = CIRCUIT_PURPOSE_C_GENERAL; for (i=0; i<3; ++i) { crypt_path_t *hop = tor_malloc_zero(sizeof(*hop)); - relay_crypto_init(&hop->private->crypto, KEY_MATERIAL[i], + relay_crypto_init(&hop->pvt_crypto, KEY_MATERIAL[i], sizeof(KEY_MATERIAL[i]), 0, 0); hop->state = CPATH_STATE_OPEN; cpath_extend_linked_list(&cs->origin_circ->cpath, hop); From 7f2cd6545ce324b5241f002e7c412408ca5902b7 Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Fri, 3 May 2019 18:27:58 +0300 Subject: [PATCH 0966/2557] Hiding crypt_path_t: Hide 'crypto' usage in sendme.c --- scripts/maint/practracker/exceptions.txt | 7 +++---- src/core/crypto/relay_crypto.c | 2 +- src/core/or/crypt_path.c | 18 ++++++++++++++++++ src/core/or/crypt_path.h | 4 ++++ src/core/or/sendme.c | 12 ++---------- src/core/or/sendme.h | 1 - 6 files changed, 28 insertions(+), 16 deletions(-) diff --git a/scripts/maint/practracker/exceptions.txt b/scripts/maint/practracker/exceptions.txt index a2b6d36ea8..70176ad896 100644 --- a/scripts/maint/practracker/exceptions.txt +++ b/scripts/maint/practracker/exceptions.txt @@ -54,9 +54,9 @@ problem function-size /src/app/main/main.c:sandbox_init_filter() 291 problem function-size /src/app/main/main.c:run_tor_main_loop() 105 problem function-size /src/app/main/ntmain.c:nt_service_install() 125 problem include-count /src/app/main/shutdown.c 52 -problem file-size /src/core/mainloop/connection.c 5559 +problem file-size /src/core/mainloop/connection.c 5560 problem include-count /src/core/mainloop/connection.c 62 -problem function-size /src/core/mainloop/connection.c:connection_free_minimal() 184 +problem function-size /src/core/mainloop/connection.c:connection_free_minimal() 185 problem function-size /src/core/mainloop/connection.c:connection_listener_new() 328 problem function-size /src/core/mainloop/connection.c:connection_handle_listener_read() 161 problem function-size /src/core/mainloop/connection.c:connection_connect_sockaddr() 103 @@ -79,7 +79,6 @@ problem function-size /src/core/or/channeltls.c:channel_tls_process_netinfo_cell problem function-size /src/core/or/channeltls.c:channel_tls_process_certs_cell() 246 problem function-size /src/core/or/channeltls.c:channel_tls_process_authenticate_cell() 202 problem file-size /src/core/or/circuitbuild.c 3061 -problem include-count /src/core/or/circuitbuild.c 53 problem include-count /src/core/or/circuitbuild.c 54 problem function-size /src/core/or/circuitbuild.c:get_unique_circ_id_by_chan() 128 problem function-size /src/core/or/circuitbuild.c:circuit_extend() 147 @@ -246,7 +245,7 @@ problem function-size /src/feature/rend/rendmid.c:rend_mid_establish_intro_legac problem function-size /src/feature/rend/rendparse.c:rend_parse_v2_service_descriptor() 187 problem function-size /src/feature/rend/rendparse.c:rend_decrypt_introduction_points() 104 problem function-size /src/feature/rend/rendparse.c:rend_parse_introduction_points() 131 -problem file-size /src/feature/rend/rendservice.c 4510 +problem file-size /src/feature/rend/rendservice.c 4511 problem function-size /src/feature/rend/rendservice.c:rend_service_prune_list_impl_() 107 problem function-size /src/feature/rend/rendservice.c:rend_config_service() 164 problem function-size /src/feature/rend/rendservice.c:rend_service_load_auth_keys() 178 diff --git a/src/core/crypto/relay_crypto.c b/src/core/crypto/relay_crypto.c index 96b1002cab..74cccd2223 100644 --- a/src/core/crypto/relay_crypto.c +++ b/src/core/crypto/relay_crypto.c @@ -164,7 +164,7 @@ relay_decrypt_cell(circuit_t *circ, cell_t *cell, /* This cell is for us. Keep a record of this cell because we will * use it in the next SENDME cell. */ if (sendme_circuit_cell_is_next(thishop->deliver_window)) { - sendme_circuit_record_inbound_cell(thishop); + cpath_sendme_circuit_record_inbound_cell(thishop); } return 0; } diff --git a/src/core/or/crypt_path.c b/src/core/or/crypt_path.c index e2234cc2a6..a4b7190e21 100644 --- a/src/core/or/crypt_path.c +++ b/src/core/or/crypt_path.c @@ -202,6 +202,24 @@ cpath_set_cell_forward_digest(crypt_path_t *cpath, cell_t *cell) relay_set_digest(cpath->pvt_crypto.f_digest, cell); } +/************ cpath sendme API ***************************/ + +/** Keep the current inbound cell digest for the next SENDME digest. This part + * is only done by the client as the circuit came back from the Exit. */ +void +cpath_sendme_circuit_record_inbound_cell(crypt_path_t *cpath) +{ + tor_assert(cpath); + relay_crypto_record_sendme_digest(&cpath->pvt_crypto); +} + +/** Return the sendme_digest of this cpath. */ +uint8_t * +cpath_get_sendme_digest(crypt_path_t *cpath) +{ + return relay_crypto_get_sendme_digest(&cpath->pvt_crypto); +} + /************ other cpath functions ***************************/ /** Return the first non-open hop in cpath, or return NULL if all diff --git a/src/core/or/crypt_path.h b/src/core/or/crypt_path.h index 19c8571d06..30c14b3dce 100644 --- a/src/core/or/crypt_path.h +++ b/src/core/or/crypt_path.h @@ -32,6 +32,10 @@ cpath_set_cell_forward_digest(crypt_path_t *cpath, cell_t *cell); crypt_path_t *cpath_get_next_non_open_hop(crypt_path_t *cpath); +void cpath_sendme_circuit_record_inbound_cell(crypt_path_t *cpath); + +uint8_t *cpath_get_sendme_digest(crypt_path_t *cpath); + #if defined(TOR_UNIT_TESTS) unsigned int cpath_get_n_hops(crypt_path_t **head_ptr); #endif /* defined(TOR_UNIT_TESTS) */ diff --git a/src/core/or/sendme.c b/src/core/or/sendme.c index 70ff3798ba..46fdc3ca1c 100644 --- a/src/core/or/sendme.c +++ b/src/core/or/sendme.c @@ -15,6 +15,7 @@ #include "core/crypto/relay_crypto.h" #include "core/mainloop/connection.h" #include "core/or/cell_st.h" +#include "core/or/crypt_path.h" #include "core/or/circuitlist.h" #include "core/or/circuituse.h" #include "core/or/or_circuit_st.h" @@ -299,15 +300,6 @@ sendme_circuit_record_outbound_cell(or_circuit_t *or_circ) relay_crypto_record_sendme_digest(&or_circ->crypto); } -/** Keep the current inbound cell digest for the next SENDME digest. This part - * is only done by the client as the circuit came back from the Exit. */ -void -sendme_circuit_record_inbound_cell(crypt_path_t *cpath) -{ - tor_assert(cpath); - relay_crypto_record_sendme_digest(&cpath->crypto); -} - /** Return true iff the next cell for the given cell window is expected to be * a SENDME. * @@ -387,7 +379,7 @@ sendme_circuit_consider_sending(circuit_t *circ, crypt_path_t *layer_hint) log_debug(LD_CIRC,"Queuing circuit sendme."); if (layer_hint) { layer_hint->deliver_window += CIRCWINDOW_INCREMENT; - digest = relay_crypto_get_sendme_digest(&layer_hint->crypto); + digest = cpath_get_sendme_digest(layer_hint); } else { circ->deliver_window += CIRCWINDOW_INCREMENT; digest = relay_crypto_get_sendme_digest(&TO_OR_CIRCUIT(circ)->crypto); diff --git a/src/core/or/sendme.h b/src/core/or/sendme.h index 78273eb9a8..ac18bbdd31 100644 --- a/src/core/or/sendme.h +++ b/src/core/or/sendme.h @@ -36,7 +36,6 @@ int sendme_note_stream_data_packaged(edge_connection_t *conn); /* Track cell digest. */ void sendme_record_cell_digest(circuit_t *circ); -void sendme_circuit_record_inbound_cell(crypt_path_t *cpath); void sendme_circuit_record_outbound_cell(or_circuit_t *or_circ); /* Circuit level information. */ From 07e4b09b5f27326a1aa58b5b2ed3c25ceb5a6db6 Mon Sep 17 00:00:00 2001 From: David Goulet Date: Fri, 3 May 2019 13:24:06 -0400 Subject: [PATCH 0967/2557] sendme: Add FlowCtrl protover value See proposal 289 section 4.3 for more details. It describes the flow control protocol at the circuit and stream level. If there is no FlowCtrl protocol version, tor supports the unauthenticated flow control features from its supported Relay protocols. At this commit, relay will start advertising FlowCtrl=1 meaning they support authenticated SENDMEs v1. Closes #30363 Signed-off-by: David Goulet --- src/core/or/protover.c | 6 ++++-- src/core/or/protover.h | 1 + src/rust/protover/ffi.rs | 1 + src/rust/protover/protover.rs | 8 ++++++-- 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/core/or/protover.c b/src/core/or/protover.c index 1edf78ec87..e12919f0a8 100644 --- a/src/core/or/protover.c +++ b/src/core/or/protover.c @@ -53,7 +53,8 @@ static const struct { { PRT_DESC, "Desc" }, { PRT_MICRODESC, "Microdesc"}, { PRT_PADDING, "Padding"}, - { PRT_CONS, "Cons" } + { PRT_CONS, "Cons" }, + { PRT_FLOWCTRL, "FlowCtrl"}, }; #define N_PROTOCOL_NAMES ARRAY_LENGTH(PROTOCOL_NAMES) @@ -401,7 +402,8 @@ protover_get_supported_protocols(void) #endif "Microdesc=1-2 " "Relay=1-2 " - "Padding=1"; + "Padding=1 " + "FlowCtrl=1"; } /** The protocols from protover_get_supported_protocols(), as parsed into a diff --git a/src/core/or/protover.h b/src/core/or/protover.h index 567b94a168..d8e541735f 100644 --- a/src/core/or/protover.h +++ b/src/core/or/protover.h @@ -44,6 +44,7 @@ typedef enum protocol_type_t { PRT_MICRODESC = 8, PRT_CONS = 9, PRT_PADDING = 10, + PRT_FLOWCTRL = 11, } protocol_type_t; bool protover_contains_long_protocol_names(const char *s); diff --git a/src/rust/protover/ffi.rs b/src/rust/protover/ffi.rs index 066b08eddb..14170d0353 100644 --- a/src/rust/protover/ffi.rs +++ b/src/rust/protover/ffi.rs @@ -31,6 +31,7 @@ fn translate_to_rust(c_proto: uint32_t) -> Result { 8 => Ok(Protocol::Microdesc), 9 => Ok(Protocol::Cons), 10 => Ok(Protocol::Padding), + 11 => Ok(Protocol::FlowCtrl), _ => Err(ProtoverError::UnknownProtocol), } } diff --git a/src/rust/protover/protover.rs b/src/rust/protover/protover.rs index 74158d9f6d..f7d9d6d15f 100644 --- a/src/rust/protover/protover.rs +++ b/src/rust/protover/protover.rs @@ -47,6 +47,7 @@ pub enum Protocol { Microdesc, Relay, Padding, + FlowCtrl, } impl fmt::Display for Protocol { @@ -75,6 +76,7 @@ impl FromStr for Protocol { "Microdesc" => Ok(Protocol::Microdesc), "Relay" => Ok(Protocol::Relay), "Padding" => Ok(Protocol::Padding), + "FlowCtrl" => Ok(Protocol::FlowCtrl), _ => Err(ProtoverError::UnknownProtocol), } } @@ -166,7 +168,8 @@ pub(crate) fn get_supported_protocols_cstr() -> &'static CStr { LinkAuth=3 \ Microdesc=1-2 \ Relay=1-2 \ - Padding=1" + Padding=1 \ + FlowCtrl=1" ) } else { cstr!( @@ -180,7 +183,8 @@ pub(crate) fn get_supported_protocols_cstr() -> &'static CStr { LinkAuth=1,3 \ Microdesc=1-2 \ Relay=1-2 \ - Padding=1" + Padding=1 \ + FlowCtrl=1" ) } } From e9769d621769c2ee31657b6da25032d86f79b15d Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Mon, 6 May 2019 17:54:51 +0300 Subject: [PATCH 0968/2557] Hiding crypt_path_t: Add changes file. --- changes/bug30236 | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 changes/bug30236 diff --git a/changes/bug30236 b/changes/bug30236 new file mode 100644 index 0000000000..ceaa98c8f1 --- /dev/null +++ b/changes/bug30236 @@ -0,0 +1,3 @@ + o Code simplification and refactoring: + - Refactor and encapsulate parts of the codebase that manipulate + crypt_path_t objects. Resolves issue 30236. \ No newline at end of file From b394b5b2af7f38a5e66c9875d3a55be5af840933 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 1 May 2019 15:36:18 -0400 Subject: [PATCH 0969/2557] Create a relay subsystem and move the shutdown functions there --- src/app/main/shutdown.c | 9 ------- src/core/include.am | 2 ++ src/feature/relay/relay_sys.c | 46 +++++++++++++++++++++++++++++++++++ src/feature/relay/relay_sys.h | 17 +++++++++++++ 4 files changed, 65 insertions(+), 9 deletions(-) create mode 100644 src/feature/relay/relay_sys.c create mode 100644 src/feature/relay/relay_sys.h diff --git a/src/app/main/shutdown.c b/src/app/main/shutdown.c index c302ce455c..b8de0f37dc 100644 --- a/src/app/main/shutdown.c +++ b/src/app/main/shutdown.c @@ -47,17 +47,13 @@ #include "feature/nodelist/nodelist.h" #include "feature/nodelist/routerlist.h" #include "feature/nodelist/routerlist.h" -#include "feature/relay/dns.h" #include "feature/relay/ext_orport.h" -#include "feature/relay/onion_queue.h" -#include "feature/relay/routerkeys.h" #include "feature/rend/rendcache.h" #include "feature/rend/rendclient.h" #include "feature/stats/geoip_stats.h" #include "feature/stats/rephist.h" #include "lib/evloop/compat_libevent.h" #include "lib/geoip/geoip.h" -#include "src/feature/relay/router.h" void evdns_shutdown(int); @@ -127,8 +123,6 @@ tor_free_all(int postfork) rend_cache_free_all(); rend_service_authorization_free_all(); rep_hist_free_all(); - dns_free_all(); - clear_pending_onions(); circuit_free_all(); circpad_machines_free(); entry_guards_free_all(); @@ -141,7 +135,6 @@ tor_free_all(int postfork) nodelist_free_all(); microdesc_free_all(); routerparse_free_all(); - ext_orport_free_all(); control_free_all(); protover_free_all(); bridges_free_all(); @@ -155,8 +148,6 @@ tor_free_all(int postfork) if (!postfork) { config_free_all(); or_state_free_all(); - router_free_all(); - routerkeys_free_all(); policies_free_all(); } if (!postfork) { diff --git a/src/core/include.am b/src/core/include.am index 9493f79552..d477cceb35 100644 --- a/src/core/include.am +++ b/src/core/include.am @@ -137,6 +137,7 @@ LIBTOR_APP_A_SOURCES = \ src/feature/relay/dns.c \ src/feature/relay/ext_orport.c \ src/feature/relay/onion_queue.c \ + src/feature/relay/relay_sys.c \ src/feature/relay/router.c \ src/feature/relay/routerkeys.c \ src/feature/relay/routermode.c \ @@ -405,6 +406,7 @@ noinst_HEADERS += \ src/feature/relay/dns_structs.h \ src/feature/relay/ext_orport.h \ src/feature/relay/onion_queue.h \ + src/feature/relay/relay_sys.h \ src/feature/relay/router.h \ src/feature/relay/routerkeys.h \ src/feature/relay/routermode.h \ diff --git a/src/feature/relay/relay_sys.c b/src/feature/relay/relay_sys.c new file mode 100644 index 0000000000..4fdd3d3584 --- /dev/null +++ b/src/feature/relay/relay_sys.c @@ -0,0 +1,46 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file relay_sys.c + * @brief Subsystem definitions for the relay module. + **/ + +#include "orconfig.h" +#include "core/or/or.h" + +#include "feature/relay/dns.h" +#include "feature/relay/ext_orport.h" +#include "feature/relay/onion_queue.h" +#include "feature/relay/relay_sys.h" +#include "feature/relay/routerkeys.h" +#include "feature/relay/router.h" + +#include "lib/subsys/subsys.h" + +static int +subsys_relay_initialize(void) +{ + return 0; +} + +static void +subsys_relay_shutdown(void) +{ + dns_free_all(); + ext_orport_free_all(); + clear_pending_onions(); + routerkeys_free_all(); + router_free_all(); +} + +const struct subsys_fns_t sys_relay = { + .name = "relay", + .supported = true, + .level = 50, + .initialize = subsys_relay_initialize, + .shutdown = subsys_relay_shutdown, +}; diff --git a/src/feature/relay/relay_sys.h b/src/feature/relay/relay_sys.h new file mode 100644 index 0000000000..32e21d90d8 --- /dev/null +++ b/src/feature/relay/relay_sys.h @@ -0,0 +1,17 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file relay_sys.h + * @brief Header for feature/relay/relay_sys.c + **/ + +#ifndef TOR_FEATURE_RELAY_RELAY_SYS_H +#define TOR_FEATURE_RELAY_RELAY_SYS_H + +extern const struct subsys_fns_t sys_relay; + +#endif /* !defined(TOR_FEATURE_RELAY_RELAY_SYS_H) */ From 3c2648bbda53f74a4e960ad149600c3a2b12305c Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 2 May 2019 09:52:03 -0400 Subject: [PATCH 0970/2557] Move "relay" and "router" periodic callbacks out of mainloop.c (Some of these callbacks are specific to the OR module, so now it's time to have an or_sys and or_periodic.) --- changes/ticket30414 | 3 + src/app/main/shutdown.c | 6 - src/app/main/subsystem_list.c | 7 +- src/core/include.am | 6 + src/core/mainloop/mainloop.c | 288 --------------------------- src/core/mainloop/mainloop.h | 1 - src/core/or/circuitstats.c | 2 + src/core/or/or_periodic.c | 65 ++++++ src/core/or/or_periodic.h | 17 ++ src/core/or/or_sys.c | 43 ++++ src/core/or/or_sys.h | 17 ++ src/feature/relay/relay_periodic.c | 308 +++++++++++++++++++++++++++++ src/feature/relay/relay_periodic.h | 18 ++ src/feature/relay/relay_sys.c | 2 + src/feature/relay/selftest.c | 1 + 15 files changed, 488 insertions(+), 296 deletions(-) create mode 100644 changes/ticket30414 create mode 100644 src/core/or/or_periodic.c create mode 100644 src/core/or/or_periodic.h create mode 100644 src/core/or/or_sys.c create mode 100644 src/core/or/or_sys.h create mode 100644 src/feature/relay/relay_periodic.c create mode 100644 src/feature/relay/relay_periodic.h diff --git a/changes/ticket30414 b/changes/ticket30414 new file mode 100644 index 0000000000..029ed1311f --- /dev/null +++ b/changes/ticket30414 @@ -0,0 +1,3 @@ + o Code simplification and refactoring: + - Move most relay-only periodic events out of mainloop.c into the + relay subsystem. Closes ticket 30414. diff --git a/src/app/main/shutdown.c b/src/app/main/shutdown.c index b8de0f37dc..1871717ad5 100644 --- a/src/app/main/shutdown.c +++ b/src/app/main/shutdown.c @@ -25,10 +25,7 @@ #include "core/or/circuitpadding.h" #include "core/or/connection_edge.h" #include "core/or/dos.h" -#include "core/or/policies.h" -#include "core/or/protover.h" #include "core/or/scheduler.h" -#include "core/or/versions.h" #include "feature/client/addressmap.h" #include "feature/client/bridges.h" #include "feature/client/entrynodes.h" @@ -136,19 +133,16 @@ tor_free_all(int postfork) microdesc_free_all(); routerparse_free_all(); control_free_all(); - protover_free_all(); bridges_free_all(); consdiffmgr_free_all(); hs_free_all(); dos_free_all(); circuitmux_ewma_free_all(); accounting_free_all(); - protover_summary_cache_free_all(); if (!postfork) { config_free_all(); or_state_free_all(); - policies_free_all(); } if (!postfork) { #ifndef _WIN32 diff --git a/src/app/main/subsystem_list.c b/src/app/main/subsystem_list.c index 00effe01aa..f595796232 100644 --- a/src/app/main/subsystem_list.c +++ b/src/app/main/subsystem_list.c @@ -10,19 +10,21 @@ #include "core/mainloop/mainloop_sys.h" #include "core/or/ocirc_event_sys.h" +#include "core/or/or_sys.h" #include "core/or/orconn_event_sys.h" #include "feature/control/btrack_sys.h" +#include "feature/relay/relay_sys.h" #include "lib/compress/compress_sys.h" #include "lib/crypt_ops/crypto_sys.h" #include "lib/err/torerr_sys.h" #include "lib/log/log_sys.h" #include "lib/net/network_sys.h" +#include "lib/process/process_sys.h" #include "lib/process/winprocess_sys.h" #include "lib/thread/thread_sys.h" #include "lib/time/time_sys.h" #include "lib/tls/tortls_sys.h" #include "lib/wallclock/wallclock_sys.h" -#include "lib/process/process_sys.h" #include "feature/dirauth/dirauth_sys.h" @@ -49,6 +51,9 @@ const subsys_fns_t *tor_subsystems[] = { &sys_btrack, /* -30 */ &sys_mainloop, /* 5 */ + &sys_or, /* 20 */ + + &sys_relay, /* 50 */ #ifdef HAVE_MODULE_DIRAUTH &sys_dirauth, /* 70 */ diff --git a/src/core/include.am b/src/core/include.am index d477cceb35..dfbdd82685 100644 --- a/src/core/include.am +++ b/src/core/include.am @@ -45,6 +45,8 @@ LIBTOR_APP_A_SOURCES = \ src/core/or/dos.c \ src/core/or/onion.c \ src/core/or/ocirc_event.c \ + src/core/or/or_periodic.c \ + src/core/or/or_sys.c \ src/core/or/orconn_event.c \ src/core/or/policies.c \ src/core/or/protover.c \ @@ -137,6 +139,7 @@ LIBTOR_APP_A_SOURCES = \ src/feature/relay/dns.c \ src/feature/relay/ext_orport.c \ src/feature/relay/onion_queue.c \ + src/feature/relay/relay_periodic.c \ src/feature/relay/relay_sys.c \ src/feature/relay/router.c \ src/feature/relay/routerkeys.c \ @@ -261,6 +264,8 @@ noinst_HEADERS += \ src/core/or/listener_connection_st.h \ src/core/or/onion.h \ src/core/or/or.h \ + src/core/or/or_periodic.h \ + src/core/or/or_sys.h \ src/core/or/orconn_event.h \ src/core/or/orconn_event_sys.h \ src/core/or/or_circuit_st.h \ @@ -406,6 +411,7 @@ noinst_HEADERS += \ src/feature/relay/dns_structs.h \ src/feature/relay/ext_orport.h \ src/feature/relay/onion_queue.h \ + src/feature/relay/relay_periodic.h \ src/feature/relay/relay_sys.h \ src/feature/relay/router.h \ src/feature/relay/routerkeys.h \ diff --git a/src/core/mainloop/mainloop.c b/src/core/mainloop/mainloop.c index 4401f805d9..82042e8498 100644 --- a/src/core/mainloop/mainloop.c +++ b/src/core/mainloop/mainloop.c @@ -1342,28 +1342,18 @@ static int periodic_events_initialized = 0; #define CALLBACK(name) \ static int name ## _callback(time_t, const or_options_t *) CALLBACK(add_entropy); -CALLBACK(check_canonical_channels); -CALLBACK(check_descriptor); -CALLBACK(check_dns_honesty); -CALLBACK(check_ed_keys); CALLBACK(check_expired_networkstatus); -CALLBACK(check_for_reachability_bw); -CALLBACK(check_onion_keys_expiry_time); CALLBACK(clean_caches); CALLBACK(clean_consdiffmgr); -CALLBACK(expire_old_ciruits_serverside); CALLBACK(fetch_networkstatus); CALLBACK(heartbeat); CALLBACK(hs_service); CALLBACK(launch_descriptor_fetches); CALLBACK(prune_old_routers); -CALLBACK(reachability_warnings); CALLBACK(record_bridge_stats); CALLBACK(rend_cache_failure_clean); CALLBACK(reset_padding_counts); -CALLBACK(retry_dns); CALLBACK(retry_listeners); -CALLBACK(rotate_onion_key); CALLBACK(rotate_x509_certificate); CALLBACK(save_state); CALLBACK(write_stats_file); @@ -1408,20 +1398,6 @@ STATIC periodic_event_item_t mainloop_periodic_events[] = { CALLBACK(write_stats_file, NET_PARTICIPANT, FL(RUN_ON_DISABLE)), CALLBACK(prune_old_routers, NET_PARTICIPANT, FL(RUN_ON_DISABLE)), - /* Routers (bridge and relay) only. */ - CALLBACK(check_descriptor, ROUTER, FL(NEED_NET)), - CALLBACK(check_ed_keys, ROUTER, 0), - CALLBACK(check_for_reachability_bw, ROUTER, FL(NEED_NET)), - CALLBACK(check_onion_keys_expiry_time, ROUTER, 0), - CALLBACK(expire_old_ciruits_serverside, ROUTER, FL(NEED_NET)), - CALLBACK(reachability_warnings, ROUTER, FL(NEED_NET)), - CALLBACK(retry_dns, ROUTER, 0), - CALLBACK(rotate_onion_key, ROUTER, 0), - - /* Relay only. */ - CALLBACK(check_canonical_channels, RELAY, FL(NEED_NET)), - CALLBACK(check_dns_honesty, RELAY, FL(NEED_NET)), - /* Hidden Service service only. */ CALLBACK(hs_service, HS_SERVICE, FL(NEED_NET)), // XXXX break this down more @@ -1447,7 +1423,6 @@ STATIC periodic_event_item_t mainloop_periodic_events[] = { * implement particular callbacks. We keep them separate here so that we * can access them by name. We also keep them inside periodic_events[] * so that we can implement "reset all timers" in a reasonable way. */ -static periodic_event_item_t *check_descriptor_event=NULL; static periodic_event_item_t *fetch_networkstatus_event=NULL; static periodic_event_item_t *launch_descriptor_fetches_event=NULL; static periodic_event_item_t *check_dns_honesty_event=NULL; @@ -1544,7 +1519,6 @@ initialize_periodic_events(void) #define NAMED_CALLBACK(name) \ STMT_BEGIN name ## _event = periodic_events_find( #name ); STMT_END - NAMED_CALLBACK(check_descriptor); NAMED_CALLBACK(prune_old_routers); NAMED_CALLBACK(fetch_networkstatus); NAMED_CALLBACK(launch_descriptor_fetches); @@ -1556,7 +1530,6 @@ STATIC void teardown_periodic_events(void) { periodic_events_disconnect_all(); - check_descriptor_event = NULL; fetch_networkstatus_event = NULL; launch_descriptor_fetches_event = NULL; check_dns_honesty_event = NULL; @@ -1607,19 +1580,6 @@ periodic_events_on_new_options(const or_options_t *options) rescan_periodic_events(options); } -/** - * Update our schedule so that we'll check whether we need to update our - * descriptor immediately, rather than after up to CHECK_DESCRIPTOR_INTERVAL - * seconds. - */ -void -reschedule_descriptor_update_check(void) -{ - if (check_descriptor_event) { - periodic_event_reschedule(check_descriptor_event); - } -} - /** * Update our schedule so that we'll check whether we need to fetch directory * info immediately. @@ -1762,77 +1722,6 @@ second_elapsed_callback(time_t now, const or_options_t *options) return 1; } -/* Periodic callback: rotate the onion keys after the period defined by the - * "onion-key-rotation-days" consensus parameter, shut down and restart all - * cpuworkers, and update our descriptor if necessary. - */ -static int -rotate_onion_key_callback(time_t now, const or_options_t *options) -{ - if (server_mode(options)) { - int onion_key_lifetime = get_onion_key_lifetime(); - time_t rotation_time = get_onion_key_set_at()+onion_key_lifetime; - if (rotation_time > now) { - return ONION_KEY_CONSENSUS_CHECK_INTERVAL; - } - - log_info(LD_GENERAL,"Rotating onion key."); - rotate_onion_key(); - cpuworkers_rotate_keyinfo(); - if (router_rebuild_descriptor(1)<0) { - log_info(LD_CONFIG, "Couldn't rebuild router descriptor"); - } - if (advertised_server_mode() && !net_is_disabled()) - router_upload_dir_desc_to_dirservers(0); - return ONION_KEY_CONSENSUS_CHECK_INTERVAL; - } - return PERIODIC_EVENT_NO_UPDATE; -} - -/* Period callback: Check if our old onion keys are still valid after the - * period of time defined by the consensus parameter - * "onion-key-grace-period-days", otherwise expire them by setting them to - * NULL. - */ -static int -check_onion_keys_expiry_time_callback(time_t now, const or_options_t *options) -{ - if (server_mode(options)) { - int onion_key_grace_period = get_onion_key_grace_period(); - time_t expiry_time = get_onion_key_set_at()+onion_key_grace_period; - if (expiry_time > now) { - return ONION_KEY_CONSENSUS_CHECK_INTERVAL; - } - - log_info(LD_GENERAL, "Expiring old onion keys."); - expire_old_onion_keys(); - cpuworkers_rotate_keyinfo(); - return ONION_KEY_CONSENSUS_CHECK_INTERVAL; - } - - return PERIODIC_EVENT_NO_UPDATE; -} - -/* Periodic callback: Every 30 seconds, check whether it's time to make new - * Ed25519 subkeys. - */ -static int -check_ed_keys_callback(time_t now, const or_options_t *options) -{ - if (server_mode(options)) { - if (should_make_new_ed_keys(options, now)) { - int new_signing_key = load_ed_keys(options, now); - if (new_signing_key < 0 || - generate_ed_link_cert(options, now, new_signing_key > 0)) { - log_err(LD_OR, "Unable to update Ed25519 keys! Exiting."); - tor_shutdown_event_loop_and_exit(1); - } - } - return 30; - } - return PERIODIC_EVENT_NO_UPDATE; -} - /** * Periodic callback: Every {LAZY,GREEDY}_DESCRIPTOR_RETRY_INTERVAL, * see about fetching descriptors, microdescriptors, and extrainfo @@ -2053,17 +1942,6 @@ write_stats_file_callback(time_t now, const or_options_t *options) return safe_timer_diff(now, next_time_to_write_stats_files); } -#define CHANNEL_CHECK_INTERVAL (60*60) -static int -check_canonical_channels_callback(time_t now, const or_options_t *options) -{ - (void)now; - if (public_server_mode(options)) - channel_check_for_duplicates(); - - return CHANNEL_CHECK_INTERVAL; -} - static int reset_padding_counts_callback(time_t now, const or_options_t *options) { @@ -2137,19 +2015,6 @@ rend_cache_failure_clean_callback(time_t now, const or_options_t *options) return 30; } -/** - * Periodic callback: If we're a server and initializing dns failed, retry. - */ -static int -retry_dns_callback(time_t now, const or_options_t *options) -{ - (void)now; -#define RETRY_DNS_INTERVAL (10*60) - if (server_mode(options) && has_dns_init_failed()) - dns_init(); - return RETRY_DNS_INTERVAL; -} - /** * Periodic callback: prune routerlist of old information about Tor network. */ @@ -2171,71 +2036,6 @@ prune_old_routers_callback(time_t now, const or_options_t *options) return ROUTERLIST_PRUNING_INTERVAL; } -/** Periodic callback: consider rebuilding or and re-uploading our descriptor - * (if we've passed our internal checks). */ -static int -check_descriptor_callback(time_t now, const or_options_t *options) -{ -/** How often do we check whether part of our router info has changed in a - * way that would require an upload? That includes checking whether our IP - * address has changed. */ -#define CHECK_DESCRIPTOR_INTERVAL (60) - - (void)options; - - /* 2b. Once per minute, regenerate and upload the descriptor if the old - * one is inaccurate. */ - if (!net_is_disabled()) { - check_descriptor_bandwidth_changed(now); - check_descriptor_ipaddress_changed(now); - mark_my_descriptor_dirty_if_too_old(now); - consider_publishable_server(0); - } - - return CHECK_DESCRIPTOR_INTERVAL; -} - -/** - * Periodic callback: check whether we're reachable (as a relay), and - * whether our bandwidth has changed enough that we need to - * publish a new descriptor. - */ -static int -check_for_reachability_bw_callback(time_t now, const or_options_t *options) -{ - /* XXXX This whole thing was stuck in the middle of what is now - * XXXX check_descriptor_callback. I'm not sure it's right. */ - - static int dirport_reachability_count = 0; - /* also, check religiously for reachability, if it's within the first - * 20 minutes of our uptime. */ - if (server_mode(options) && - (have_completed_a_circuit() || !any_predicted_circuits(now)) && - !net_is_disabled()) { - if (get_uptime() < TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT) { - router_do_reachability_checks(1, dirport_reachability_count==0); - if (++dirport_reachability_count > 5) - dirport_reachability_count = 0; - return 1; - } else { - /* If we haven't checked for 12 hours and our bandwidth estimate is - * low, do another bandwidth test. This is especially important for - * bridges, since they might go long periods without much use. */ - const routerinfo_t *me = router_get_my_routerinfo(); - static int first_time = 1; - if (!first_time && me && - me->bandwidthcapacity < me->bandwidthrate && - me->bandwidthcapacity < 51200) { - reset_bandwidth_test(); - } - first_time = 0; -#define BANDWIDTH_RECHECK_INTERVAL (12*60*60) - return BANDWIDTH_RECHECK_INTERVAL; - } - } - return CHECK_DESCRIPTOR_INTERVAL; -} - /** * Periodic event: once a minute, (or every second if TestingTorNetwork, or * during client bootstrap), check whether we want to download any @@ -2278,93 +2078,6 @@ retry_listeners_callback(time_t now, const or_options_t *options) return PERIODIC_EVENT_NO_UPDATE; } -/** - * Periodic callback: as a server, see if we have any old unused circuits - * that should be expired */ -static int -expire_old_ciruits_serverside_callback(time_t now, const or_options_t *options) -{ - (void)options; - /* every 11 seconds, so not usually the same second as other such events */ - circuit_expire_old_circuits_serverside(now); - return 11; -} - -/** - * Callback: Send warnings if Tor doesn't find its ports reachable. - */ -static int -reachability_warnings_callback(time_t now, const or_options_t *options) -{ - (void) now; - - if (get_uptime() < TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT) { - return (int)(TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT - get_uptime()); - } - - if (server_mode(options) && - !net_is_disabled() && - have_completed_a_circuit()) { - /* every 20 minutes, check and complain if necessary */ - const routerinfo_t *me = router_get_my_routerinfo(); - if (me && !check_whether_orport_reachable(options)) { - char *address = tor_dup_ip(me->addr); - log_warn(LD_CONFIG,"Your server (%s:%d) has not managed to confirm that " - "its ORPort is reachable. Relays do not publish descriptors " - "until their ORPort and DirPort are reachable. Please check " - "your firewalls, ports, address, /etc/hosts file, etc.", - address, me->or_port); - control_event_server_status(LOG_WARN, - "REACHABILITY_FAILED ORADDRESS=%s:%d", - address, me->or_port); - tor_free(address); - } - - if (me && !check_whether_dirport_reachable(options)) { - char *address = tor_dup_ip(me->addr); - log_warn(LD_CONFIG, - "Your server (%s:%d) has not managed to confirm that its " - "DirPort is reachable. Relays do not publish descriptors " - "until their ORPort and DirPort are reachable. Please check " - "your firewalls, ports, address, /etc/hosts file, etc.", - address, me->dir_port); - control_event_server_status(LOG_WARN, - "REACHABILITY_FAILED DIRADDRESS=%s:%d", - address, me->dir_port); - tor_free(address); - } - } - - return TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT; -} - -static int dns_honesty_first_time = 1; - -/** - * Periodic event: if we're an exit, see if our DNS server is telling us - * obvious lies. - */ -static int -check_dns_honesty_callback(time_t now, const or_options_t *options) -{ - (void)now; - /* 9. and if we're an exit node, check whether our DNS is telling stories - * to us. */ - if (net_is_disabled() || - ! public_server_mode(options) || - router_my_exit_policy_is_reject_star()) - return PERIODIC_EVENT_NO_UPDATE; - - if (dns_honesty_first_time) { - /* Don't launch right when we start */ - dns_honesty_first_time = 0; - return crypto_rand_int_range(60, 180); - } - - dns_launch_correctness_checks(); - return 12*3600 + crypto_rand_int(12*3600); -} - static int heartbeat_callback_first_time = 1; /** @@ -2830,7 +2543,6 @@ tor_mainloop_free_all(void) can_complete_circuits = 0; quiet_level = 0; should_init_bridge_stats = 1; - dns_honesty_first_time = 1; heartbeat_callback_first_time = 1; current_second = 0; memset(¤t_second_last_changed, 0, diff --git a/src/core/mainloop/mainloop.h b/src/core/mainloop/mainloop.h index 3a611f81aa..cdc2bf8608 100644 --- a/src/core/mainloop/mainloop.h +++ b/src/core/mainloop/mainloop.h @@ -59,7 +59,6 @@ void directory_info_has_arrived(time_t now, int from_cache, int suppress_logs); void ip_address_changed(int at_interface); void dns_servers_relaunch_checks(void); void reset_all_main_loop_timers(void); -void reschedule_descriptor_update_check(void); void reschedule_directory_downloads(void); void reschedule_or_state_save(void); void mainloop_schedule_postloop_cleanup(void); diff --git a/src/core/or/circuitstats.c b/src/core/or/circuitstats.c index e5a3bac30b..03eea1d779 100644 --- a/src/core/or/circuitstats.c +++ b/src/core/or/circuitstats.c @@ -44,6 +44,7 @@ #include "lib/time/tvdiff.h" #include "lib/encoding/confline.h" #include "feature/dirauth/authmode.h" +#include "feature/relay/relay_periodic.h" #include "core/or/crypt_path_st.h" #include "core/or/origin_circuit_st.h" @@ -1420,6 +1421,7 @@ void circuit_build_times_network_is_live(circuit_build_times_t *cbt) { time_t now = approx_time(); + // XXXX this should use pubsub if (cbt->liveness.nonlive_timeouts > 0) { time_t time_since_live = now - cbt->liveness.network_last_live; log_notice(LD_CIRC, diff --git a/src/core/or/or_periodic.c b/src/core/or/or_periodic.c new file mode 100644 index 0000000000..93dfa8cf8b --- /dev/null +++ b/src/core/or/or_periodic.c @@ -0,0 +1,65 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file or_periodic.c + * @brief Periodic callbacks for the onion routing subsystem + **/ + +#include "orconfig.h" +#include "core/or/or.h" + +#include "core/mainloop/periodic.h" + +#include "core/or/channel.h" +#include "core/or/circuituse.h" +#include "core/or/or_periodic.h" + +#include "src/feature/relay/routermode.h" + +#define DECLARE_EVENT(name, roles, flags) \ + static periodic_event_item_t name ## _event = \ + PERIODIC_EVENT(name, \ + PERIODIC_EVENT_ROLE_##roles, \ + flags) + +#define FL(name) (PERIODIC_EVENT_FLAG_ ## name) + +#define CHANNEL_CHECK_INTERVAL (60*60) +static int +check_canonical_channels_callback(time_t now, const or_options_t *options) +{ + (void)now; + if (public_server_mode(options)) + channel_check_for_duplicates(); + + return CHANNEL_CHECK_INTERVAL; +} + +DECLARE_EVENT(check_canonical_channels, RELAY, FL(NEED_NET)); + +/** + * Periodic callback: as a server, see if we have any old unused circuits + * that should be expired */ +static int +expire_old_circuits_serverside_callback(time_t now, + const or_options_t *options) +{ + (void)options; + /* every 11 seconds, so not usually the same second as other such events */ + circuit_expire_old_circuits_serverside(now); + return 11; +} + +DECLARE_EVENT(expire_old_circuits_serverside, ROUTER, FL(NEED_NET)); + +void +or_register_periodic_events(void) +{ + // These are router-only events, but they're owned by the OR subsystem. */ + periodic_events_register(&check_canonical_channels_event); + periodic_events_register(&expire_old_circuits_serverside_event); +} diff --git a/src/core/or/or_periodic.h b/src/core/or/or_periodic.h new file mode 100644 index 0000000000..c2f47cf5ef --- /dev/null +++ b/src/core/or/or_periodic.h @@ -0,0 +1,17 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file or_periodic.h + * @brief Header for core/or/or_periodic.c + **/ + +#ifndef TOR_CORE_OR_OR_PERIODIC_H +#define TOR_CORE_OR_OR_PERIODIC_H + +void or_register_periodic_events(void); + +#endif /* !defined(TOR_CORE_OR_OR_PERIODIC_H) */ diff --git a/src/core/or/or_sys.c b/src/core/or/or_sys.c new file mode 100644 index 0000000000..6f8c81a4df --- /dev/null +++ b/src/core/or/or_sys.c @@ -0,0 +1,43 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file or_sys.c + * @brief Subsystem definitions for OR module. + **/ + +#include "orconfig.h" +#include "core/or/or.h" +#include "core/or/or_periodic.h" +#include "core/or/or_sys.h" +#include "core/or/policies.h" +#include "core/or/protover.h" +#include "core/or/versions.h" + +#include "lib/subsys/subsys.h" + +static int +subsys_or_initialize(void) +{ + or_register_periodic_events(); + return 0; +} + +static void +subsys_or_shutdown(void) +{ + protover_free_all(); + protover_summary_cache_free_all(); + policies_free_all(); +} + +const struct subsys_fns_t sys_or = { + .name = "or", + .supported = true, + .level = 20, + .initialize = subsys_or_initialize, + .shutdown = subsys_or_shutdown, +}; diff --git a/src/core/or/or_sys.h b/src/core/or/or_sys.h new file mode 100644 index 0000000000..c37ef01858 --- /dev/null +++ b/src/core/or/or_sys.h @@ -0,0 +1,17 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file or_sys.h + * @brief Header for core/or/or_sys.c + **/ + +#ifndef TOR_CORE_OR_OR_SYS_H +#define TOR_CORE_OR_OR_SYS_H + +extern const struct subsys_fns_t sys_or; + +#endif /* !defined(TOR_CORE_OR_OR_SYS_H) */ diff --git a/src/feature/relay/relay_periodic.c b/src/feature/relay/relay_periodic.c new file mode 100644 index 0000000000..8908b57415 --- /dev/null +++ b/src/feature/relay/relay_periodic.c @@ -0,0 +1,308 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file relay_periodic.c + * @brief Periodic functions for the relay subsytem + **/ + +#include "orconfig.h" +#include "core/or/or.h" + +#include "core/mainloop/periodic.h" +#include "core/mainloop/cpuworker.h" // XXXX use a pubsub event. +#include "core/mainloop/mainloop.h" +#include "core/mainloop/netstatus.h" +#include "core/or/circuituse.h" // XXXX move have_performed_bandwidth_test + +#include "feature/relay/dns.h" +#include "feature/relay/relay_periodic.h" +#include "feature/relay/router.h" +#include "feature/relay/routerkeys.h" +#include "feature/relay/routermode.h" +#include "feature/relay/selftest.h" +#include "src/feature/stats/predict_ports.h" + +#include "lib/crypt_ops/crypto_rand.h" + +#include "feature/nodelist/routerinfo_st.h" +#include "feature/control/control_events.h" + +#define DECLARE_EVENT(name, roles, flags) \ + static periodic_event_item_t name ## _event = \ + PERIODIC_EVENT(name, \ + PERIODIC_EVENT_ROLE_##roles, \ + flags) + +#define FL(name) (PERIODIC_EVENT_FLAG_##name) + +/** + * Periodic callback: If we're a server and initializing dns failed, retry. + */ +static int +retry_dns_callback(time_t now, const or_options_t *options) +{ + (void)now; +#define RETRY_DNS_INTERVAL (10*60) + if (server_mode(options) && has_dns_init_failed()) + dns_init(); + return RETRY_DNS_INTERVAL; +} + +DECLARE_EVENT(retry_dns, ROUTER, 0); + +static int dns_honesty_first_time = 1; + +/** + * Periodic event: if we're an exit, see if our DNS server is telling us + * obvious lies. + */ +static int +check_dns_honesty_callback(time_t now, const or_options_t *options) +{ + (void)now; + /* 9. and if we're an exit node, check whether our DNS is telling stories + * to us. */ + if (net_is_disabled() || + ! public_server_mode(options) || + router_my_exit_policy_is_reject_star()) + return PERIODIC_EVENT_NO_UPDATE; + + if (dns_honesty_first_time) { + /* Don't launch right when we start */ + dns_honesty_first_time = 0; + return crypto_rand_int_range(60, 180); + } + + dns_launch_correctness_checks(); + return 12*3600 + crypto_rand_int(12*3600); +} + +DECLARE_EVENT(check_dns_honesty, RELAY, FL(NEED_NET)); + +/* Periodic callback: rotate the onion keys after the period defined by the + * "onion-key-rotation-days" consensus parameter, shut down and restart all + * cpuworkers, and update our descriptor if necessary. + */ +static int +rotate_onion_key_callback(time_t now, const or_options_t *options) +{ + if (server_mode(options)) { + int onion_key_lifetime = get_onion_key_lifetime(); + time_t rotation_time = get_onion_key_set_at()+onion_key_lifetime; + if (rotation_time > now) { + return ONION_KEY_CONSENSUS_CHECK_INTERVAL; + } + + log_info(LD_GENERAL,"Rotating onion key."); + rotate_onion_key(); + cpuworkers_rotate_keyinfo(); + if (router_rebuild_descriptor(1)<0) { + log_info(LD_CONFIG, "Couldn't rebuild router descriptor"); + } + if (advertised_server_mode() && !net_is_disabled()) + router_upload_dir_desc_to_dirservers(0); + return ONION_KEY_CONSENSUS_CHECK_INTERVAL; + } + return PERIODIC_EVENT_NO_UPDATE; +} + +DECLARE_EVENT(rotate_onion_key, ROUTER, 0); + +/** Periodic callback: consider rebuilding or and re-uploading our descriptor + * (if we've passed our internal checks). */ +static int +check_descriptor_callback(time_t now, const or_options_t *options) +{ +/** How often do we check whether part of our router info has changed in a + * way that would require an upload? That includes checking whether our IP + * address has changed. */ +#define CHECK_DESCRIPTOR_INTERVAL (60) + + (void)options; + + /* 2b. Once per minute, regenerate and upload the descriptor if the old + * one is inaccurate. */ + if (!net_is_disabled()) { + check_descriptor_bandwidth_changed(now); + check_descriptor_ipaddress_changed(now); + mark_my_descriptor_dirty_if_too_old(now); + consider_publishable_server(0); + } + + return CHECK_DESCRIPTOR_INTERVAL; +} + +DECLARE_EVENT(check_descriptor, ROUTER, FL(NEED_NET)); + +static int dirport_reachability_count = 0; + +/** + * Periodic callback: check whether we're reachable (as a relay), and + * whether our bandwidth has changed enough that we need to + * publish a new descriptor. + */ +static int +check_for_reachability_bw_callback(time_t now, const or_options_t *options) +{ + /* XXXX This whole thing was stuck in the middle of what is now + * XXXX check_descriptor_callback. I'm not sure it's right. */ + + /* also, check religiously for reachability, if it's within the first + * 20 minutes of our uptime. */ + if (server_mode(options) && + (have_completed_a_circuit() || !any_predicted_circuits(now)) && + !net_is_disabled()) { + if (get_uptime() < TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT) { + router_do_reachability_checks(1, dirport_reachability_count==0); + if (++dirport_reachability_count > 5) + dirport_reachability_count = 0; + return 1; + } else { + /* If we haven't checked for 12 hours and our bandwidth estimate is + * low, do another bandwidth test. This is especially important for + * bridges, since they might go long periods without much use. */ + const routerinfo_t *me = router_get_my_routerinfo(); + static int first_time = 1; + if (!first_time && me && + me->bandwidthcapacity < me->bandwidthrate && + me->bandwidthcapacity < 51200) { + reset_bandwidth_test(); + } + first_time = 0; +#define BANDWIDTH_RECHECK_INTERVAL (12*60*60) + return BANDWIDTH_RECHECK_INTERVAL; + } + } + return CHECK_DESCRIPTOR_INTERVAL; +} + +DECLARE_EVENT(check_for_reachability_bw, ROUTER, FL(NEED_NET)); + +/** + * Callback: Send warnings if Tor doesn't find its ports reachable. + */ +static int +reachability_warnings_callback(time_t now, const or_options_t *options) +{ + (void) now; + + if (get_uptime() < TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT) { + return (int)(TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT - get_uptime()); + } + + if (server_mode(options) && + !net_is_disabled() && + have_completed_a_circuit()) { + /* every 20 minutes, check and complain if necessary */ + const routerinfo_t *me = router_get_my_routerinfo(); + if (me && !check_whether_orport_reachable(options)) { + char *address = tor_dup_ip(me->addr); + log_warn(LD_CONFIG,"Your server (%s:%d) has not managed to confirm that " + "its ORPort is reachable. Relays do not publish descriptors " + "until their ORPort and DirPort are reachable. Please check " + "your firewalls, ports, address, /etc/hosts file, etc.", + address, me->or_port); + control_event_server_status(LOG_WARN, + "REACHABILITY_FAILED ORADDRESS=%s:%d", + address, me->or_port); + tor_free(address); + } + + if (me && !check_whether_dirport_reachable(options)) { + char *address = tor_dup_ip(me->addr); + log_warn(LD_CONFIG, + "Your server (%s:%d) has not managed to confirm that its " + "DirPort is reachable. Relays do not publish descriptors " + "until their ORPort and DirPort are reachable. Please check " + "your firewalls, ports, address, /etc/hosts file, etc.", + address, me->dir_port); + control_event_server_status(LOG_WARN, + "REACHABILITY_FAILED DIRADDRESS=%s:%d", + address, me->dir_port); + tor_free(address); + } + } + + return TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT; +} + +DECLARE_EVENT(reachability_warnings, ROUTER, FL(NEED_NET)); + +/* Periodic callback: Every 30 seconds, check whether it's time to make new + * Ed25519 subkeys. + */ +static int +check_ed_keys_callback(time_t now, const or_options_t *options) +{ + if (server_mode(options)) { + if (should_make_new_ed_keys(options, now)) { + int new_signing_key = load_ed_keys(options, now); + if (new_signing_key < 0 || + generate_ed_link_cert(options, now, new_signing_key > 0)) { + log_err(LD_OR, "Unable to update Ed25519 keys! Exiting."); + tor_shutdown_event_loop_and_exit(1); + } + } + return 30; + } + return PERIODIC_EVENT_NO_UPDATE; +} + +DECLARE_EVENT(check_ed_keys, ROUTER, 0); + +/* Period callback: Check if our old onion keys are still valid after the + * period of time defined by the consensus parameter + * "onion-key-grace-period-days", otherwise expire them by setting them to + * NULL. + */ +static int +check_onion_keys_expiry_time_callback(time_t now, const or_options_t *options) +{ + if (server_mode(options)) { + int onion_key_grace_period = get_onion_key_grace_period(); + time_t expiry_time = get_onion_key_set_at()+onion_key_grace_period; + if (expiry_time > now) { + return ONION_KEY_CONSENSUS_CHECK_INTERVAL; + } + + log_info(LD_GENERAL, "Expiring old onion keys."); + expire_old_onion_keys(); + cpuworkers_rotate_keyinfo(); + return ONION_KEY_CONSENSUS_CHECK_INTERVAL; + } + + return PERIODIC_EVENT_NO_UPDATE; +} + +DECLARE_EVENT(check_onion_keys_expiry_time, ROUTER, 0); + +void +relay_register_periodic_events(void) +{ + periodic_events_register(&retry_dns_event); + periodic_events_register(&check_dns_honesty_event); + periodic_events_register(&rotate_onion_key_event); + periodic_events_register(&check_descriptor_event); + periodic_events_register(&check_for_reachability_bw_event); + periodic_events_register(&reachability_warnings_event); + periodic_events_register(&check_ed_keys_event); + periodic_events_register(&check_onion_keys_expiry_time_event); + + dns_honesty_first_time = 1; + dirport_reachability_count = 0; +} + +/** + * Update our schedule so that we'll check whether we need to update our + * descriptor immediately, rather than after up to CHECK_DESCRIPTOR_INTERVAL + * seconds. + */ +void +reschedule_descriptor_update_check(void) +{ + periodic_event_reschedule(&check_descriptor_event); +} diff --git a/src/feature/relay/relay_periodic.h b/src/feature/relay/relay_periodic.h new file mode 100644 index 0000000000..b6ea83c749 --- /dev/null +++ b/src/feature/relay/relay_periodic.h @@ -0,0 +1,18 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file relay_periodic.h + * @brief Header for feature/relay/relay_periodic.c + **/ + +#ifndef TOR_FEATURE_RELAY_RELAY_PERIODIC_H +#define TOR_FEATURE_RELAY_RELAY_PERIODIC_H + +void relay_register_periodic_events(void); +void reschedule_descriptor_update_check(void); + +#endif /* !defined(TOR_FEATURE_RELAY_RELAY_PERIODIC_H) */ diff --git a/src/feature/relay/relay_sys.c b/src/feature/relay/relay_sys.c index 4fdd3d3584..106e88b2a5 100644 --- a/src/feature/relay/relay_sys.c +++ b/src/feature/relay/relay_sys.c @@ -15,6 +15,7 @@ #include "feature/relay/dns.h" #include "feature/relay/ext_orport.h" #include "feature/relay/onion_queue.h" +#include "feature/relay/relay_periodic.h" #include "feature/relay/relay_sys.h" #include "feature/relay/routerkeys.h" #include "feature/relay/router.h" @@ -24,6 +25,7 @@ static int subsys_relay_initialize(void) { + relay_register_periodic_events(); return 0; } diff --git a/src/feature/relay/selftest.c b/src/feature/relay/selftest.c index eeddd09b63..f8b54ff45d 100644 --- a/src/feature/relay/selftest.c +++ b/src/feature/relay/selftest.c @@ -35,6 +35,7 @@ #include "feature/nodelist/routerlist.h" // but... #include "feature/nodelist/routerset.h" #include "feature/nodelist/torcert.h" +#include "feature/relay/relay_periodic.h" #include "feature/relay/router.h" #include "feature/relay/selftest.h" From 3cafdeb8c0f1dfa60a964e4fba04b54d1ad97d15 Mon Sep 17 00:00:00 2001 From: Neel Chauhan Date: Tue, 7 May 2019 11:24:53 -0400 Subject: [PATCH 0971/2557] Only call tor_addr_parse() in circuit_is_acceptable() when needed --- changes/bug22210 | 7 +++++++ scripts/maint/practracker/exceptions.txt | 4 ++-- src/core/or/circuituse.c | 17 ++++++++++------- 3 files changed, 19 insertions(+), 9 deletions(-) create mode 100644 changes/bug22210 diff --git a/changes/bug22210 b/changes/bug22210 new file mode 100644 index 0000000000..d7a00fd72c --- /dev/null +++ b/changes/bug22210 @@ -0,0 +1,7 @@ + o Minor bugfixes (onion services, performance): + - If we are building circuits to onion services, in circuit_is_acceptable() + we only call tor_addr_parse() in places where we use the returned + family and address values from this function. Previously, we called + tor_addr_parse() in circuit_is_acceptable() even if it wasn't used. + This change will improve performance when building circuits. Fixes + bug 22210; bugfix on 0.2.8.12. Patch by Neel Chauhan diff --git a/scripts/maint/practracker/exceptions.txt b/scripts/maint/practracker/exceptions.txt index 21fe9ec351..8623b00a26 100644 --- a/scripts/maint/practracker/exceptions.txt +++ b/scripts/maint/practracker/exceptions.txt @@ -92,8 +92,8 @@ problem function-size /src/core/or/circuitlist.c:circuits_handle_oom() 117 problem function-size /src/core/or/circuitmux.c:circuitmux_set_policy() 110 problem function-size /src/core/or/circuitmux.c:circuitmux_attach_circuit() 114 problem function-size /src/core/or/circuitstats.c:circuit_build_times_parse_state() 124 -problem file-size /src/core/or/circuituse.c 3152 -problem function-size /src/core/or/circuituse.c:circuit_is_acceptable() 129 +problem file-size /src/core/or/circuituse.c 3155 +problem function-size /src/core/or/circuituse.c:circuit_is_acceptable() 133 problem function-size /src/core/or/circuituse.c:circuit_expire_building() 394 problem function-size /src/core/or/circuituse.c:circuit_log_ancient_one_hop_circuits() 126 problem function-size /src/core/or/circuituse.c:circuit_build_failed() 149 diff --git a/src/core/or/circuituse.c b/src/core/or/circuituse.c index 2f27cf996f..226295425e 100644 --- a/src/core/or/circuituse.c +++ b/src/core/or/circuituse.c @@ -178,7 +178,6 @@ circuit_is_acceptable(const origin_circuit_t *origin_circ, purpose == CIRCUIT_PURPOSE_S_HSDIR_POST || purpose == CIRCUIT_PURPOSE_C_HSDIR_GET) { tor_addr_t addr; - const int family = tor_addr_parse(&addr, conn->socks_request->address); if (!exitnode && !build_state->onehop_tunnel) { log_debug(LD_CIRC,"Not considering circuit with unknown router."); return 0; /* this circuit is screwed and doesn't know it yet, @@ -199,6 +198,8 @@ circuit_is_acceptable(const origin_circuit_t *origin_circ, return 0; /* this is a circuit to somewhere else */ if (tor_digest_is_zero(digest)) { /* we don't know the digest; have to compare addr:port */ + const int family = tor_addr_parse(&addr, + conn->socks_request->address); if (family < 0 || !tor_addr_eq(&build_state->chosen_exit->addr, &addr) || build_state->chosen_exit->port != conn->socks_request->port) @@ -211,12 +212,14 @@ circuit_is_acceptable(const origin_circuit_t *origin_circ, return 0; } } - if (origin_circ->prepend_policy && family != -1) { - int r = compare_tor_addr_to_addr_policy(&addr, - conn->socks_request->port, - origin_circ->prepend_policy); - if (r == ADDR_POLICY_REJECTED) - return 0; + if (origin_circ->prepend_policy) { + if (tor_addr_parse(&addr, conn->socks_request->address) != -1) { + int r = compare_tor_addr_to_addr_policy(&addr, + conn->socks_request->port, + origin_circ->prepend_policy); + if (r == ADDR_POLICY_REJECTED) + return 0; + } } if (exitnode && !connection_ap_can_use_exit(conn, exitnode)) { /* can't exit from this router */ From 6f1527f88871d16c4edf83f56ed5cdde1f6d2b82 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Sat, 11 May 2019 18:58:14 +0300 Subject: [PATCH 0972/2557] pre-push.git-hook: Check remote name to see if it's upstream --- scripts/git/pre-push.git-hook | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/git/pre-push.git-hook b/scripts/git/pre-push.git-hook index 238541d9f0..00e048029e 100755 --- a/scripts/git/pre-push.git-hook +++ b/scripts/git/pre-push.git-hook @@ -16,6 +16,7 @@ echo "Running pre-push hook" z40=0000000000000000000000000000000000000000 +upstream_name=${TOR_UPSTREAM_REMOTE_NAME:-"upstream"} workdir=$(git rev-parse --show-toplevel) if [ -x "$workdir/.git/hooks/pre-commit" ]; then @@ -33,8 +34,9 @@ fi remote="$1" remote_loc="$2" -if [[ "$remote_loc" != *github.com/torproject/tor.git ]] && - [[ "$remote_loc" != *torproject.org/tor.git ]]; then +remote_name=$(git remote --verbose | grep "$2" | awk '{print $1}' | head -n 1) + +if [[ "$remote_name" != "$upstream_name" ]]; then echo "Not pushing to upstream - refraining from further checks" exit 0 fi From 0a4e68e4e24849a8d95dfa9ca0eb773b2cda6ebf Mon Sep 17 00:00:00 2001 From: rl1987 Date: Sat, 11 May 2019 19:03:46 +0300 Subject: [PATCH 0973/2557] Add changes file --- changes/bug30286 | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 changes/bug30286 diff --git a/changes/bug30286 b/changes/bug30286 new file mode 100644 index 0000000000..f2fc67a484 --- /dev/null +++ b/changes/bug30286 @@ -0,0 +1,4 @@ + o Minor bugfixes (developer tooling): + - Fix pre-push hook to refrain from rejecting fixup and squash commits + when pushing to non-upstream git remote. Fixes bug 30286; bugfix on + 0.4.0.1-alpha. From def96ce83858b214ebc01797e41e4f0419f9d104 Mon Sep 17 00:00:00 2001 From: David Goulet Date: Mon, 13 May 2019 10:00:39 -0400 Subject: [PATCH 0974/2557] sendme: Fix coverity CID 1444999 The code flow in theory can end up with a layer_hint to be NULL but in practice it should never happen because with an origin circuit, we must have the layer_hint. Just in case, BUG() on it if we ever end up in this situation and recover by closing the circuit. Fixes #30467. Signed-off-by: David Goulet --- src/core/or/sendme.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/core/or/sendme.c b/src/core/or/sendme.c index 46fdc3ca1c..e7c65d99e2 100644 --- a/src/core/or/sendme.c +++ b/src/core/or/sendme.c @@ -412,6 +412,11 @@ sendme_process_circuit_level(crypt_path_t *layer_hint, /* If we are the origin of the circuit, we are the Client so we use the * layer hint (the Exit hop) for the package window tracking. */ if (CIRCUIT_IS_ORIGIN(circ)) { + /* If we are the origin of the circuit, it is impossible to not have a + * cpath. Just in case, bug on it and close the circuit. */ + if (BUG(layer_hint == NULL)) { + return -END_CIRC_REASON_TORPROTOCOL; + } if ((layer_hint->package_window + CIRCWINDOW_INCREMENT) > CIRCWINDOW_START_MAX) { static struct ratelim_t exit_warn_ratelim = RATELIM_INIT(600); From f4064d6ce214b4b79017280a6c9db9b3f945ece1 Mon Sep 17 00:00:00 2001 From: Mike Perry Date: Wed, 17 Apr 2019 05:51:39 +0000 Subject: [PATCH 0975/2557] Bug 28693: Provide Torrc option to disable circuit padding. --- doc/tor.1.txt | 8 +++++++ src/app/config/config.c | 5 +++++ src/app/config/or_options_st.h | 5 +++++ src/core/or/circuitpadding.c | 41 +++++++++++++++++++++++++--------- 4 files changed, 49 insertions(+), 10 deletions(-) diff --git a/doc/tor.1.txt b/doc/tor.1.txt index f992172405..6c125e3741 100644 --- a/doc/tor.1.txt +++ b/doc/tor.1.txt @@ -955,6 +955,14 @@ The following options are useful only for clients (that is, if this option. This option should be offered via the UI to mobile users for use where bandwidth may be expensive. (Default: 0) +[[CircuitPadding]] **CircuitPadding** **0**|**1**:: + If set to 0, Tor will not pad client circuits with additional cover + traffic. Only clients may set this option. This option should be offered + via the UI to mobile users for use where bandwidth may be expensive. If + set to 1, padding will be negotiated as per the consensus and relay + support (unlike ConnectionPadding, CircuitPadding cannot be force-enabled). + (Default: 1) + [[ExcludeNodes]] **ExcludeNodes** __node__,__node__,__...__:: A list of identity fingerprints, country codes, and address patterns of nodes to avoid when building a circuit. Country codes are diff --git a/src/app/config/config.c b/src/app/config/config.c index 46dc15b069..7ad970625a 100644 --- a/src/app/config/config.c +++ b/src/app/config/config.c @@ -596,6 +596,7 @@ static config_var_t option_vars_[] = { V(ReducedConnectionPadding, BOOL, "0"), V(ConnectionPadding, AUTOBOOL, "auto"), V(RefuseUnknownExits, AUTOBOOL, "auto"), + V(CircuitPadding, BOOL, "1"), V(RejectPlaintextPorts, CSV, ""), V(RelayBandwidthBurst, MEMUNIT, "0"), V(RelayBandwidthRate, MEMUNIT, "0"), @@ -3741,6 +3742,10 @@ options_validate(or_options_t *old_options, or_options_t *options, REJECT("Relays cannot set ReducedConnectionPadding. "); } + if (server_mode(options) && options->CircuitPadding == 0) { + REJECT("Relays cannot set CircuitPadding to 0. "); + } + if (options->BridgeDistribution) { if (!options->BridgeRelay) { REJECT("You set BridgeDistribution, but you didn't set BridgeRelay!"); diff --git a/src/app/config/or_options_st.h b/src/app/config/or_options_st.h index bd707fd193..0fdeb94b4f 100644 --- a/src/app/config/or_options_st.h +++ b/src/app/config/or_options_st.h @@ -248,6 +248,11 @@ struct or_options_t { * pad to the server regardless of server support. */ int ConnectionPadding; + /** Boolean: if true, then circuit padding will be negotiated by client + * and server, subject to consenus limits (default). If 0, it will be fully + * disabled. */ + int CircuitPadding; + /** To what authority types do we publish our descriptor? Choices are * "v1", "v2", "v3", "bridge", or "". */ struct smartlist_t *PublishServerDescriptor; diff --git a/src/core/or/circuitpadding.c b/src/core/or/circuitpadding.c index bf74ecc3ff..dcd8f645c4 100644 --- a/src/core/or/circuitpadding.c +++ b/src/core/or/circuitpadding.c @@ -1099,6 +1099,24 @@ circpad_new_consensus_params(const networkstatus_t *ns) CIRCWINDOW_START_MAX, 0, 50*CIRCWINDOW_START_MAX); } +/** + * Return true if padding is allowed by torrc and consensus. + */ +STATIC bool +circpad_is_padding_allowed(void) +{ + /* If padding has been disabled in the consensus, don't send any more + * padding. Technically the machine should be shut down when the next + * machine condition check happens, but machine checks only happen on + * certain circuit events, and if padding is disabled due to some + * network overload or DoS condition, we really want to stop ASAP. */ + if (circpad_padding_disabled || !get_options()->CircuitPadding) { + return 0; + } + + return 1; +} + /** * Check this machine against its padding limits, as well as global * consensus limits. @@ -1117,15 +1135,6 @@ circpad_machine_reached_padding_limit(circpad_machine_runtime_t *mi) { const circpad_machine_spec_t *machine = CIRCPAD_GET_MACHINE(mi); - /* If padding has been disabled in the consensus, don't send any more - * padding. Technically the machine should be shut down when the next - * machine condition check happens, but machine checks only happen on - * certain circuit events, and if padding is disabled due to some - * network overload or DoS condition, we really want to stop ASAP. */ - if (circpad_padding_disabled) { - return 1; - } - /* If machine_padding_pct is non-zero, and we've sent more * than the allowed count of padding cells, then check our * percent limits for this machine. */ @@ -1176,6 +1185,18 @@ circpad_machine_schedule_padding,(circpad_machine_runtime_t *mi)) struct timeval timeout; tor_assert(mi); + /* Don't schedule padding if it is disabled */ + if (!circpad_is_padding_allowed()) { + static ratelim_t padding_lim = RATELIM_INIT(600); + log_fn_ratelim(&padding_lim,LOG_INFO,LD_CIRC, + "Padding has been disabled, but machine still on circuit %"PRIu64 + ", %d", + mi->on_circ->n_chan ? mi->on_circ->n_chan->global_identifier : 0, + mi->on_circ->n_circ_id); + + return CIRCPAD_STATE_UNCHANGED; + } + /* Don't schedule padding if we are currently in dormant mode. */ if (!is_participating_on_network()) { log_info(LD_CIRC, "Not scheduling padding because we are dormant."); @@ -1638,7 +1659,7 @@ circpad_machine_conditions_met(origin_circuit_t *circ, { /* If padding is disabled, no machines should match/apply. This has * the effect of shutting down all machines, and not adding any more. */ - if (circpad_padding_disabled) + if (circpad_padding_disabled || !get_options()->CircuitPadding) return 0; if (!(circpad_circ_purpose_to_mask(TO_CIRCUIT(circ)->purpose) From 621ea2315b3f53a9ef4ace9f3f6cb2f03a241042 Mon Sep 17 00:00:00 2001 From: Mike Perry Date: Wed, 17 Apr 2019 06:09:06 +0000 Subject: [PATCH 0976/2557] Bug 29203: Provide ReducedCircuitPadding torrc and consensus params --- doc/tor.1.txt | 6 ++++++ src/app/config/config.c | 5 +++++ src/app/config/or_options_st.h | 6 ++++++ src/core/or/circuitpadding.c | 13 +++++++++++++ src/core/or/circuitpadding.h | 11 +++++++++++ 5 files changed, 41 insertions(+) diff --git a/doc/tor.1.txt b/doc/tor.1.txt index 6c125e3741..13a85f995d 100644 --- a/doc/tor.1.txt +++ b/doc/tor.1.txt @@ -963,6 +963,12 @@ The following options are useful only for clients (that is, if support (unlike ConnectionPadding, CircuitPadding cannot be force-enabled). (Default: 1) +[[ReducedCircuitPadding]] **ReducedCircuitPadding** **0**|**1**:: + If set to 1, Tor will only use circuit padding algorithms that have low + overhead. Only clients may set this option. This option should be offered + via the UI to mobile users for use where bandwidth may be expensive. + (Default: 0) + [[ExcludeNodes]] **ExcludeNodes** __node__,__node__,__...__:: A list of identity fingerprints, country codes, and address patterns of nodes to avoid when building a circuit. Country codes are diff --git a/src/app/config/config.c b/src/app/config/config.c index 7ad970625a..1c7cb1d577 100644 --- a/src/app/config/config.c +++ b/src/app/config/config.c @@ -597,6 +597,7 @@ static config_var_t option_vars_[] = { V(ConnectionPadding, AUTOBOOL, "auto"), V(RefuseUnknownExits, AUTOBOOL, "auto"), V(CircuitPadding, BOOL, "1"), + V(ReducedCircuitPadding, BOOL, "0"), V(RejectPlaintextPorts, CSV, ""), V(RelayBandwidthBurst, MEMUNIT, "0"), V(RelayBandwidthRate, MEMUNIT, "0"), @@ -3746,6 +3747,10 @@ options_validate(or_options_t *old_options, or_options_t *options, REJECT("Relays cannot set CircuitPadding to 0. "); } + if (server_mode(options) && options->ReducedCircuitPadding == 1) { + REJECT("Relays cannot set ReducedCircuitPadding. "); + } + if (options->BridgeDistribution) { if (!options->BridgeRelay) { REJECT("You set BridgeDistribution, but you didn't set BridgeRelay!"); diff --git a/src/app/config/or_options_st.h b/src/app/config/or_options_st.h index 0fdeb94b4f..4e03bec7fa 100644 --- a/src/app/config/or_options_st.h +++ b/src/app/config/or_options_st.h @@ -253,6 +253,12 @@ struct or_options_t { * disabled. */ int CircuitPadding; + /** Boolean: if true, then this client will only use circuit padding + * algorithms that are known to use a low amount of overhead. If false, + * we will use all available circuit padding algorithms. + */ + int ReducedCircuitPadding; + /** To what authority types do we publish our descriptor? Choices are * "v1", "v2", "v3", "bridge", or "". */ struct smartlist_t *PublishServerDescriptor; diff --git a/src/core/or/circuitpadding.c b/src/core/or/circuitpadding.c index dcd8f645c4..8d2749906b 100644 --- a/src/core/or/circuitpadding.c +++ b/src/core/or/circuitpadding.c @@ -82,6 +82,7 @@ static double circpad_distribution_sample(circpad_distribution_t dist); /** Cached consensus params */ static uint8_t circpad_padding_disabled; +static uint8_t circpad_padding_reduced; static uint8_t circpad_global_max_padding_percent; static uint16_t circpad_global_allowed_cells; static uint16_t circpad_max_circ_queued_cells; @@ -1086,6 +1087,10 @@ circpad_new_consensus_params(const networkstatus_t *ns) networkstatus_get_param(ns, "circpad_padding_disabled", 0, 0, 1); + circpad_padding_reduced = + networkstatus_get_param(ns, "circpad_padding_reduced", + 0, 0, 1); + circpad_global_allowed_cells = networkstatus_get_param(ns, "circpad_global_allowed_cells", 0, 0, UINT16_MAX-1); @@ -1662,6 +1667,14 @@ circpad_machine_conditions_met(origin_circuit_t *circ, if (circpad_padding_disabled || !get_options()->CircuitPadding) return 0; + /* If the consensus or our torrc has selected reduced connection padding, + * then only allow this machine if it is flagged as acceptable under + * reduced padding conditions */ + if (circpad_padding_reduced || get_options()->ReducedCircuitPadding) { + if (!machine->conditions.reduced_padding_ok) + return 0; + } + if (!(circpad_circ_purpose_to_mask(TO_CIRCUIT(circ)->purpose) & machine->conditions.purpose_mask)) return 0; diff --git a/src/core/or/circuitpadding.h b/src/core/or/circuitpadding.h index bc2522c210..f00369eb0a 100644 --- a/src/core/or/circuitpadding.h +++ b/src/core/or/circuitpadding.h @@ -152,6 +152,17 @@ typedef struct circpad_machine_conditions_t { /** Only apply the machine *if* vanguards are enabled */ unsigned requires_vanguards : 1; + /** + * This machine is ok to use if reduced padding is set in consensus + * or torrc. This machine will still be applied even if reduced padding + * is not set; this flag only acts to exclude machines that don't have + * it set when reduced padding is requested. Therefore, reduced padding + * machines should appear at the lowest priority in the padding machine + * lists (aka first in the list), so that non-reduced padding machines + * for the same purpose are given a chance to apply when reduced padding + * is not requested. */ + unsigned reduced_padding_ok : 1; + /** Only apply the machine *if* the circuit's state matches any of * the bits set in this bitmask. */ circpad_circuit_state_t state_mask; From 42eb02a32730099e0320c7ee8b9fb0035d948bb4 Mon Sep 17 00:00:00 2001 From: Mike Perry Date: Wed, 17 Apr 2019 23:21:54 +0000 Subject: [PATCH 0977/2557] Tests for bugs 28683, 30173, and 29203. --- src/core/or/circuitpadding.c | 1 + src/test/test_circuitpadding.c | 157 ++++++++++++++++++++++++++++++++- src/test/test_options.c | 1 + 3 files changed, 157 insertions(+), 2 deletions(-) diff --git a/src/core/or/circuitpadding.c b/src/core/or/circuitpadding.c index 8d2749906b..7094e48d9d 100644 --- a/src/core/or/circuitpadding.c +++ b/src/core/or/circuitpadding.c @@ -2220,6 +2220,7 @@ circpad_circ_client_machine_init(void) circ_client_machine->conditions.state_mask = CIRCPAD_CIRC_BUILDING|CIRCPAD_CIRC_OPENED|CIRCPAD_CIRC_HAS_RELAY_EARLY; circ_client_machine->conditions.purpose_mask = CIRCPAD_PURPOSE_ALL; + circ_client_machine->conditions.reduced_padding_ok = 1; circ_client_machine->target_hopnum = 2; circ_client_machine->is_origin_side = 1; diff --git a/src/test/test_circuitpadding.c b/src/test/test_circuitpadding.c index 3289c866cf..dd84c4affd 100644 --- a/src/test/test_circuitpadding.c +++ b/src/test/test_circuitpadding.c @@ -1683,8 +1683,11 @@ static void helper_create_conditional_machines(void) { circpad_machine_spec_t *add = helper_create_conditional_machine(); - origin_padding_machines = smartlist_new(); - relay_padding_machines = smartlist_new(); + + if (!origin_padding_machines) + origin_padding_machines = smartlist_new(); + if (!relay_padding_machines) + relay_padding_machines = smartlist_new(); add->machine_num = 2; add->is_origin_side = 1; @@ -2396,6 +2399,155 @@ test_circuitpadding_global_rate_limiting(void *arg) smartlist_free(vote1.net_params); } +/* Test reduced and disabled padding */ +static void +test_circuitpadding_reduce_disable(void *arg) +{ + (void) arg; + int64_t actual_mocked_monotime_start; + + MOCK(circuitmux_attach_circuit, circuitmux_attach_circuit_mock); + + nodes_init(); + dummy_channel.cmux = circuitmux_alloc(); + relay_side = (circuit_t *)new_fake_orcirc(&dummy_channel, + &dummy_channel); + client_side = (circuit_t *)origin_circuit_new(); + relay_side->purpose = CIRCUIT_PURPOSE_OR; + client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL; + + circpad_machines_init(); + helper_create_conditional_machines(); + + monotime_init(); + monotime_enable_test_mocking(); + actual_mocked_monotime_start = MONOTIME_MOCK_START; + monotime_set_mock_time_nsec(actual_mocked_monotime_start); + monotime_coarse_set_mock_time_nsec(actual_mocked_monotime_start); + curr_mocked_time = actual_mocked_monotime_start; + timers_initialize(); + + /* This is needed so that we are not considered to be dormant */ + note_user_activity(20); + + MOCK(circuit_package_relay_cell, + circuit_package_relay_cell_mock); + MOCK(node_get_by_id, + node_get_by_id_mock); + + /* Simulate extend. This should result in the original machine getting + * added, since the circuit is not built */ + simulate_single_hop_extend(client_side, relay_side, 1); + simulate_single_hop_extend(client_side, relay_side, 1); + + /* Verify that machine #2 is added */ + tt_int_op(client_side->padding_machine[0]->machine_num, OP_EQ, 2); + tt_int_op(relay_side->padding_machine[0]->machine_num, OP_EQ, 2); + + /* Deliver a padding cell to the client, to trigger burst state */ + circpad_cell_event_padding_sent(client_side); + + /* This should have trigger length shutdown condition on client.. */ + tt_ptr_op(client_side->padding_info[0], OP_EQ, NULL); + tt_ptr_op(client_side->padding_machine[0], OP_EQ, NULL); + + /* Verify machine is gone from both sides */ + tt_ptr_op(relay_side->padding_info[0], OP_EQ, NULL); + tt_ptr_op(relay_side->padding_machine[0], OP_EQ, NULL); + + /* Now test the reduced padding machine by setting up the consensus */ + networkstatus_t vote1; + vote1.net_params = smartlist_new(); + smartlist_split_string(vote1.net_params, + "circpad_padding_reduced=1", NULL, 0, 0); + + /* Register reduced padding machine with the padding subsystem */ + circpad_new_consensus_params(&vote1); + + simulate_single_hop_extend(client_side, relay_side, 1); + + /* Verify that machine #0 is added */ + tt_int_op(client_side->padding_machine[0]->machine_num, OP_EQ, 0); + tt_int_op(relay_side->padding_machine[0]->machine_num, OP_EQ, 0); + + tt_int_op( + circpad_machine_reached_padding_limit(client_side->padding_info[0]), + OP_EQ, 0); + tt_int_op( + circpad_machine_reached_padding_limit(relay_side->padding_info[0]), + OP_EQ, 0); + + /* Test that machines get torn down when padding is disabled */ + SMARTLIST_FOREACH(vote1.net_params, char *, cp, tor_free(cp)); + smartlist_free(vote1.net_params); + vote1.net_params = smartlist_new(); + smartlist_split_string(vote1.net_params, + "circpad_padding_disabled=1", NULL, 0, 0); + + /* Register reduced padding machine with the padding subsystem */ + circpad_new_consensus_params(&vote1); + + tt_int_op( + circpad_machine_schedule_padding(client_side->padding_info[0]), + OP_EQ, CIRCPAD_STATE_UNCHANGED); + tt_int_op( + circpad_machine_schedule_padding(relay_side->padding_info[0]), + OP_EQ, CIRCPAD_STATE_UNCHANGED); + + /* Signal that circuit is built: this event causes us to re-evaluate + * machine conditions (which don't apply because padding is disabled). */ + circpad_machine_event_circ_built(TO_ORIGIN_CIRCUIT(client_side)); + + tt_ptr_op(client_side->padding_info[0], OP_EQ, NULL); + tt_ptr_op(client_side->padding_machine[0], OP_EQ, NULL); + tt_ptr_op(relay_side->padding_info[0], OP_EQ, NULL); + tt_ptr_op(relay_side->padding_machine[0], OP_EQ, NULL); + + SMARTLIST_FOREACH(vote1.net_params, char *, cp, tor_free(cp)); + smartlist_free(vote1.net_params); + vote1.net_params = NULL; + circpad_new_consensus_params(&vote1); + + get_options_mutable()->ReducedCircuitPadding = 1; + + simulate_single_hop_extend(client_side, relay_side, 1); + + /* Verify that machine #0 is added */ + tt_int_op(client_side->padding_machine[0]->machine_num, OP_EQ, 0); + tt_int_op(relay_side->padding_machine[0]->machine_num, OP_EQ, 0); + + tt_int_op( + circpad_machine_reached_padding_limit(client_side->padding_info[0]), + OP_EQ, 0); + tt_int_op( + circpad_machine_reached_padding_limit(relay_side->padding_info[0]), + OP_EQ, 0); + + get_options_mutable()->CircuitPadding = 0; + + tt_int_op( + circpad_machine_schedule_padding(client_side->padding_info[0]), + OP_EQ, CIRCPAD_STATE_UNCHANGED); + tt_int_op( + circpad_machine_schedule_padding(relay_side->padding_info[0]), + OP_EQ, CIRCPAD_STATE_UNCHANGED); + + /* Signal that circuit is built: this event causes us to re-evaluate + * machine conditions (which don't apply because padding is disabled). */ + + circpad_machine_event_circ_built(TO_ORIGIN_CIRCUIT(client_side)); + + tt_ptr_op(client_side->padding_info[0], OP_EQ, NULL); + tt_ptr_op(client_side->padding_machine[0], OP_EQ, NULL); + tt_ptr_op(relay_side->padding_info[0], OP_EQ, NULL); + tt_ptr_op(relay_side->padding_machine[0], OP_EQ, NULL); + + done: + free_fake_orcirc(relay_side); + circuitmux_detach_all_circuits(dummy_channel.cmux, NULL); + circuitmux_free(dummy_channel.cmux); +} + #define TEST_CIRCUITPADDING(name, flags) \ { #name, test_##name, (flags), NULL, NULL } @@ -2410,6 +2562,7 @@ struct testcase_t circuitpadding_tests[] = { TEST_CIRCUITPADDING(circuitpadding_sample_distribution, TT_FORK), TEST_CIRCUITPADDING(circuitpadding_machine_rate_limiting, TT_FORK), TEST_CIRCUITPADDING(circuitpadding_global_rate_limiting, TT_FORK), + TEST_CIRCUITPADDING(circuitpadding_reduce_disable, TT_FORK), TEST_CIRCUITPADDING(circuitpadding_token_removal_lower, TT_FORK), TEST_CIRCUITPADDING(circuitpadding_token_removal_higher, TT_FORK), TEST_CIRCUITPADDING(circuitpadding_closest_token_removal, TT_FORK), diff --git a/src/test/test_options.c b/src/test/test_options.c index f12e6b6763..396be6b18d 100644 --- a/src/test/test_options.c +++ b/src/test/test_options.c @@ -430,6 +430,7 @@ get_options_test_data(const char *conf) // Being kinda lame and just fixing the immedate breakage for now.. result->opt->ConnectionPadding = -1; // default must be "auto" result->opt->DormantClientTimeout = 1800; // must be over 600. + result->opt->CircuitPadding = 1; // default must be "1" rv = config_get_lines(conf, &cl, 1); tt_int_op(rv, OP_EQ, 0); From 0d275ec592761390c5ee05b2acdf5ae193c2a0ca Mon Sep 17 00:00:00 2001 From: Mike Perry Date: Wed, 17 Apr 2019 23:37:22 +0000 Subject: [PATCH 0978/2557] Changes file for bugs28693+30173+29203. --- changes/bugs28693+30173+29203 | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 changes/bugs28693+30173+29203 diff --git a/changes/bugs28693+30173+29203 b/changes/bugs28693+30173+29203 new file mode 100644 index 0000000000..9faa6279bf --- /dev/null +++ b/changes/bugs28693+30173+29203 @@ -0,0 +1,12 @@ + o Minor bugfixes (circuit padding): + - Add a torrc option to disable circuit padding. Fixes bug 28693; bugfix + on 0.4.0.1-alpha. + o Minor bugfixes (circuit padding): + - Provide consensus parameter to fully disable circuit padding, to be used + in emergency network overload situations. Fixes bug 30173; bugfix on + 0.4.0.1-alpha. + o Minor bugfixes (circuit padding): + - Allow circuit padding machines to specify that they do not contribute + much overhead, and provide consensus flags and torrc options to force + clients to only use low overhead machines. Fixes bug 29203; bugfix on + 0.4.0.1-alpha. From 507df74b319f37462d1540495f04b9b0838cb3a3 Mon Sep 17 00:00:00 2001 From: Mike Perry Date: Thu, 18 Apr 2019 18:14:31 +0000 Subject: [PATCH 0979/2557] The practracker beatings will continue until the functions get smaller. --- scripts/maint/practracker/exceptions.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/maint/practracker/exceptions.txt b/scripts/maint/practracker/exceptions.txt index 7d03bf27d6..d51e3edb53 100644 --- a/scripts/maint/practracker/exceptions.txt +++ b/scripts/maint/practracker/exceptions.txt @@ -29,12 +29,12 @@ # # Remember: It is better to fix the problem than to add a new exception! -problem file-size /src/app/config/config.c 8491 +problem file-size /src/app/config/config.c 8501 problem include-count /src/app/config/config.c 86 problem function-size /src/app/config/config.c:options_act_reversible() 296 problem function-size /src/app/config/config.c:options_act() 588 problem function-size /src/app/config/config.c:resolve_my_address() 192 -problem function-size /src/app/config/config.c:options_validate() 1207 +problem function-size /src/app/config/config.c:options_validate() 1215 problem function-size /src/app/config/config.c:options_init_from_torrc() 202 problem function-size /src/app/config/config.c:options_init_from_string() 173 problem function-size /src/app/config/config.c:options_init_logs() 146 @@ -187,7 +187,7 @@ problem function-size /src/feature/dirclient/dirclient.c:handle_response_fetch_c problem function-size /src/feature/dircommon/consdiff.c:gen_ed_diff() 204 problem function-size /src/feature/dircommon/consdiff.c:apply_ed_diff() 159 problem function-size /src/feature/dirparse/authcert_parse.c:authority_cert_parse_from_string() 182 -problem function-size /src/feature/dirparse/microdesc_parse.c:microdescs_parse_from_string() 154 +problem function-size /src/feature/dirparse/microdesc_parse.c:microdescs_parse_from_string() 169 problem function-size /src/feature/dirparse/ns_parse.c:routerstatus_parse_entry_from_string() 286 problem function-size /src/feature/dirparse/ns_parse.c:networkstatus_verify_bw_weights() 389 problem function-size /src/feature/dirparse/ns_parse.c:networkstatus_parse_vote_from_string() 638 From 5d950f3edd7f9c46f0149327c20c01a5f1cc40e2 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 13 May 2019 14:34:16 -0400 Subject: [PATCH 0980/2557] Fix a compilation warning: function does not have to be STATIC. --- src/core/or/circuitpadding.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/or/circuitpadding.c b/src/core/or/circuitpadding.c index f21cf113cc..58e8e053c7 100644 --- a/src/core/or/circuitpadding.c +++ b/src/core/or/circuitpadding.c @@ -1107,7 +1107,7 @@ circpad_new_consensus_params(const networkstatus_t *ns) /** * Return true if padding is allowed by torrc and consensus. */ -STATIC bool +static bool circpad_is_padding_allowed(void) { /* If padding has been disabled in the consensus, don't send any more From dd537ba35fbb22efb8c6ed0d6b623269531d7900 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 14 May 2019 19:22:35 -0400 Subject: [PATCH 0981/2557] Update practracker for 30452 --- scripts/maint/practracker/exceptions.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/maint/practracker/exceptions.txt b/scripts/maint/practracker/exceptions.txt index 24452b6158..2e272dc016 100644 --- a/scripts/maint/practracker/exceptions.txt +++ b/scripts/maint/practracker/exceptions.txt @@ -29,13 +29,13 @@ # # Remember: It is better to fix the problem than to add a new exception! -problem file-size /src/app/config/config.c 8510 +problem file-size /src/app/config/config.c 8520 problem include-count /src/app/config/config.c 88 problem function-size /src/app/config/config.c:options_act_reversible() 296 problem function-size /src/app/config/config.c:options_act() 588 problem function-size /src/app/config/config.c:resolve_my_address() 192 problem function-size /src/app/config/config.c:options_validate() 1220 -problem function-size /src/app/config/config.c:options_init_from_torrc() 202 +problem function-size /src/app/config/config.c:options_init_from_torrc() 210 problem function-size /src/app/config/config.c:options_init_from_string() 173 problem function-size /src/app/config/config.c:options_init_logs() 146 problem function-size /src/app/config/config.c:parse_bridge_line() 104 From b9f50a2d77caa119d02ce42356cd0b417ef22c65 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 14 May 2019 19:56:20 -0400 Subject: [PATCH 0982/2557] update practracker for tor_init --- scripts/maint/practracker/exceptions.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/maint/practracker/exceptions.txt b/scripts/maint/practracker/exceptions.txt index 2e272dc016..8124d36f46 100644 --- a/scripts/maint/practracker/exceptions.txt +++ b/scripts/maint/practracker/exceptions.txt @@ -49,7 +49,7 @@ problem function-size /src/app/config/confparse.c:config_assign_value() 205 problem function-size /src/app/config/confparse.c:config_get_assigned_option() 129 problem include-count /src/app/main/main.c 67 problem function-size /src/app/main/main.c:dumpstats() 102 -problem function-size /src/app/main/main.c:tor_init() 136 +problem function-size /src/app/main/main.c:tor_init() 137 problem function-size /src/app/main/main.c:sandbox_init_filter() 291 problem function-size /src/app/main/main.c:run_tor_main_loop() 105 problem function-size /src/app/main/ntmain.c:nt_service_install() 125 From 010779176bb959c8106a95806ede2c80b4397f60 Mon Sep 17 00:00:00 2001 From: Mike Perry Date: Sat, 11 May 2019 02:51:14 +0000 Subject: [PATCH 0983/2557] Bug 29085: Refactor non-padding accounting out of token removal. This commit moves the padding state limit checks and the padding rate limit checks out of the token removal codepath, and causes all three functions to get called from a single circpad_machine_count_nonpadding_sent() function. It does not change functionality. --- src/core/or/circuitpadding.c | 110 +++++++++++++++++++++++++---------- src/core/or/circuitpadding.h | 5 +- 2 files changed, 81 insertions(+), 34 deletions(-) diff --git a/src/core/or/circuitpadding.c b/src/core/or/circuitpadding.c index 58e8e053c7..1386bd22c8 100644 --- a/src/core/or/circuitpadding.c +++ b/src/core/or/circuitpadding.c @@ -80,6 +80,9 @@ static void circpad_setup_machine_on_circ(circuit_t *on_circ, const circpad_machine_spec_t *machine); static double circpad_distribution_sample(circpad_distribution_t dist); +static inline void circpad_machine_update_state_length_for_nonpadding( + circpad_machine_runtime_t *mi); + /** Cached consensus params */ static uint8_t circpad_padding_disabled; static uint8_t circpad_padding_reduced; @@ -828,22 +831,15 @@ check_machine_token_supply(circpad_machine_runtime_t *mi) } /** - * Remove a token from the bin corresponding to the delta since - * last packet. If that bin is empty, choose a token based on - * the specified removal strategy in the state machine. + * Count a nonpadding packet as being sent. * - * This function also updates and checks rate limit and state - * limit counters. - * - * Returns 1 if we transition states, 0 otherwise. + * This function updates our overhead accounting variables, as well + * as decrements the state limit packet counter, if the latter was + * flagged as applying to non-padding as well. */ -STATIC circpad_decision_t -circpad_machine_remove_token(circpad_machine_runtime_t *mi) +static inline void +circpad_machine_count_nonpadding_sent(circpad_machine_runtime_t *mi) { - const circpad_state_t *state = NULL; - circpad_time_t current_time; - circpad_delay_t target_bin_usec; - /* Update non-padding counts for rate limiting: We scale at UINT16_MAX * because we only use this for a percentile limit of 2 sig figs, and * space is scare in the machineinfo struct. */ @@ -853,12 +849,67 @@ circpad_machine_remove_token(circpad_machine_runtime_t *mi) mi->nonpadding_sent /= 2; } + /* Update any state packet length limits that apply */ + circpad_machine_update_state_length_for_nonpadding(mi); + + /* Remove a token from the histrogram, if applicable */ + circpad_machine_remove_token(mi); +} + +/** + * Decrement the state length counter for a non-padding packet. + * + * Only updates the state length if we're using that feature, we + * have a state, and the machine wants to count non-padding packets + * towards the state length. + */ +static inline void +circpad_machine_update_state_length_for_nonpadding( + circpad_machine_runtime_t *mi) +{ + const circpad_state_t *state = NULL; + + if (mi->state_length != CIRCPAD_STATE_LENGTH_INFINITE) + return; + + state = circpad_machine_current_state(mi); + + /* If we are not in a padding state (like start or end), we're done */ + if (!state) + return; + + /* If we're enforcing a state length on non-padding packets, + * decrement it */ + if (state->length_includes_nonpadding && + mi->state_length > 0) { + mi->state_length--; + } +} + +/** + * When a non-padding packet arrives, remove a token from the bin + * corresponding to the delta since last sent packet. If that bin + * is empty, choose a token based on the specified removal strategy + * in the state machine. + */ +STATIC void +circpad_machine_remove_token(circpad_machine_runtime_t *mi) +{ + const circpad_state_t *state = NULL; + circpad_time_t current_time; + circpad_delay_t target_bin_usec; + /* Dont remove any tokens if there was no padding scheduled */ if (!mi->padding_scheduled_at_usec) { - return CIRCPAD_STATE_UNCHANGED; + return; } state = circpad_machine_current_state(mi); + + /* Don't remove any tokens if we're not doing token removal */ + if (!state || state->token_removal == CIRCPAD_TOKEN_REMOVAL_NONE) + return; + current_time = monotime_absolute_usec(); /* If we have scheduled padding some time in the future, we want to see what @@ -877,20 +928,10 @@ circpad_machine_remove_token(circpad_machine_runtime_t *mi) /* If we are not in a padding state (like start or end), we're done */ if (!state) - return CIRCPAD_STATE_UNCHANGED; - - /* If we're enforcing a state length on non-padding packets, - * decrement it */ - if (mi->state_length != CIRCPAD_STATE_LENGTH_INFINITE && - state->length_includes_nonpadding && - mi->state_length > 0) { - mi->state_length--; - } + return; /* Perform the specified token removal strategy */ switch (state->token_removal) { - case CIRCPAD_TOKEN_REMOVAL_NONE: - break; case CIRCPAD_TOKEN_REMOVAL_CLOSEST_USEC: circpad_machine_remove_closest_token(mi, target_bin_usec, 1); break; @@ -906,10 +947,13 @@ circpad_machine_remove_token(circpad_machine_runtime_t *mi) case CIRCPAD_TOKEN_REMOVAL_EXACT: circpad_machine_remove_exact(mi, target_bin_usec); break; + case CIRCPAD_TOKEN_REMOVAL_NONE: + default: + tor_assert_nonfatal_unreached(); + log_warn(LD_BUG, "Circpad: Unknown token removal strategy %d", + state->token_removal); + break; } - - /* Check our token and state length limits */ - return check_machine_token_supply(mi); } /** @@ -1541,9 +1585,13 @@ circpad_cell_event_nonpadding_sent(circuit_t *on_circ) /* First, update any RTT estimate */ circpad_estimate_circ_rtt_on_send(on_circ, on_circ->padding_info[i]); - /* Remove a token: this is the idea of adaptive padding, since we have an - * ideal distribution that we want our distribution to look like. */ - if (!circpad_machine_remove_token(on_circ->padding_info[i])) { + /* Then, do accounting */ + circpad_machine_count_nonpadding_sent(on_circ->padding_info[i]); + + /* Check to see if we've run out of tokens for this state already, + * and if not, check for other state transitions */ + if (check_machine_token_supply(on_circ->padding_info[i]) + == CIRCPAD_STATE_UNCHANGED) { /* If removing a token did not cause a transition, check if * non-padding sent event should */ circpad_machine_spec_transition(on_circ->padding_info[i], diff --git a/src/core/or/circuitpadding.h b/src/core/or/circuitpadding.h index f00369eb0a..7d0f8dacfa 100644 --- a/src/core/or/circuitpadding.h +++ b/src/core/or/circuitpadding.h @@ -712,9 +712,6 @@ circpad_machine_sample_delay(circpad_machine_runtime_t *mi); STATIC bool circpad_machine_reached_padding_limit(circpad_machine_runtime_t *mi); -STATIC -circpad_decision_t circpad_machine_remove_token(circpad_machine_runtime_t *mi); - STATIC circpad_delay_t circpad_histogram_bin_to_usec(const circpad_machine_runtime_t *mi, circpad_hist_index_t bin); @@ -722,6 +719,8 @@ circpad_histogram_bin_to_usec(const circpad_machine_runtime_t *mi, STATIC const circpad_state_t * circpad_machine_current_state(const circpad_machine_runtime_t *mi); +STATIC void circpad_machine_remove_token(circpad_machine_runtime_t *mi); + STATIC circpad_hist_index_t circpad_histogram_usec_to_bin( const circpad_machine_runtime_t *mi, circpad_delay_t us); From 1c46790e0d3d6c722a0f3e74f24d60d03650e7d8 Mon Sep 17 00:00:00 2001 From: Mike Perry Date: Sat, 11 May 2019 03:26:53 +0000 Subject: [PATCH 0984/2557] Bug 29085: Refactor padding sent accounting out of callback. This commit moves code that updates the state length and padding limit counts out from the callback to its own function, for clarity. It does not change functionality. --- src/core/or/circuitpadding.c | 73 ++++++++++++++++++++++-------------- 1 file changed, 45 insertions(+), 28 deletions(-) diff --git a/src/core/or/circuitpadding.c b/src/core/or/circuitpadding.c index 1386bd22c8..d6a79004f5 100644 --- a/src/core/or/circuitpadding.c +++ b/src/core/or/circuitpadding.c @@ -830,6 +830,50 @@ check_machine_token_supply(circpad_machine_runtime_t *mi) return CIRCPAD_STATE_UNCHANGED; } +/** + * Count that a padding packet was sent. + * + * This updates our state length count, our machine rate limit counts, + * and if token removal is used, decrements the histogram. + */ +static inline void +circpad_machine_count_padding_sent(circpad_machine_runtime_t *mi) +{ + /* If we have a valid state length bound, consider it */ + if (mi->state_length != CIRCPAD_STATE_LENGTH_INFINITE && + !BUG(mi->state_length <= 0)) { + mi->state_length--; + } + + /* + * Update non-padding counts for rate limiting: We scale at UINT16_MAX + * because we only use this for a percentile limit of 2 sig figs, and + * space is scare in the machineinfo struct. + */ + mi->padding_sent++; + if (mi->padding_sent == UINT16_MAX) { + mi->padding_sent /= 2; + mi->nonpadding_sent /= 2; + } + + circpad_global_padding_sent++; + + /* If we have a mutable histogram, reduce the token count from + * the chosen padding bin (this assumes we always send padding + * when we intended to). */ + if (mi->histogram && mi->histogram_len) { + /* Ensure that we have a token removal strategy set */ + const circpad_state_t *state = circpad_machine_current_state(mi); + tor_assert_nonfatal(state->token_removal != CIRCPAD_TOKEN_REMOVAL_NONE); + + /* Basic sanity check on the histogram before removing anything */ + if (!BUG(mi->chosen_bin >= mi->histogram_len) && + !BUG(mi->histogram[mi->chosen_bin] == 0)) { + mi->histogram[mi->chosen_bin]--; + } + } +} + /** * Count a nonpadding packet as being sent. * @@ -1023,34 +1067,7 @@ circpad_send_padding_cell_for_callback(circpad_machine_runtime_t *mi) return CIRCPAD_STATE_CHANGED; } - /* If it's a histogram, reduce the token count */ - if (mi->histogram && mi->histogram_len) { - /* Basic sanity check on the histogram before removing anything */ - if (BUG(mi->chosen_bin >= mi->histogram_len) || - BUG(mi->histogram[mi->chosen_bin] == 0)) { - return CIRCPAD_STATE_CHANGED; - } - - mi->histogram[mi->chosen_bin]--; - } - - /* If we have a valid state length bound, consider it */ - if (mi->state_length != CIRCPAD_STATE_LENGTH_INFINITE && - !BUG(mi->state_length <= 0)) { - mi->state_length--; - } - - /* - * Update non-padding counts for rate limiting: We scale at UINT16_MAX - * because we only use this for a percentile limit of 2 sig figs, and - * space is scare in the machineinfo struct. - */ - mi->padding_sent++; - if (mi->padding_sent == UINT16_MAX) { - mi->padding_sent /= 2; - mi->nonpadding_sent /= 2; - } - circpad_global_padding_sent++; + circpad_machine_count_padding_sent(mi); if (CIRCUIT_IS_ORIGIN(mi->on_circ)) { circpad_send_command_to_hop(TO_ORIGIN_CIRCUIT(mi->on_circ), From 57e5e940d39754ec5b3ca1c7055ff7ca9a47c238 Mon Sep 17 00:00:00 2001 From: Mike Perry Date: Tue, 7 May 2019 23:37:33 +0000 Subject: [PATCH 0985/2557] Bug 29085: Minor unit test updates for refactoring. Deliver nonpadding events instead of calling token removal functions. --- src/test/test_circuitpadding.c | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/src/test/test_circuitpadding.c b/src/test/test_circuitpadding.c index db175fecee..b7dbf022aa 100644 --- a/src/test/test_circuitpadding.c +++ b/src/test/test_circuitpadding.c @@ -586,12 +586,12 @@ test_circuitpadding_token_removal_higher(void *arg) tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 2); mi->padding_scheduled_at_usec = current_time - 57; - circpad_machine_remove_token(mi); + circpad_cell_event_nonpadding_sent((circuit_t*)client_side); tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 1); mi->padding_scheduled_at_usec = current_time - 57; - circpad_machine_remove_token(mi); + circpad_cell_event_nonpadding_sent((circuit_t*)client_side); /* Test that we cleaned out this bin. Don't do this in the case of the last bin since the tokens will get refilled */ @@ -610,7 +610,7 @@ test_circuitpadding_token_removal_higher(void *arg) CIRCPAD_STATE_BURST); circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_edges[0] = 100; mi->padding_scheduled_at_usec = current_time; - circpad_machine_remove_token(mi); + circpad_cell_event_nonpadding_sent((circuit_t*)client_side); tt_int_op(mi->histogram[0], OP_EQ, 1); done: @@ -683,12 +683,12 @@ test_circuitpadding_token_removal_lower(void *arg) tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 2); mi->padding_scheduled_at_usec = current_time - 57; - circpad_machine_remove_token(mi); + circpad_cell_event_nonpadding_sent((circuit_t*)client_side); tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 1); mi->padding_scheduled_at_usec = current_time - 57; - circpad_machine_remove_token(mi); + circpad_cell_event_nonpadding_sent((circuit_t*)client_side); /* Test that we cleaned out this bin. Don't do this in the case of the last bin since the tokens will get refilled */ @@ -708,7 +708,7 @@ test_circuitpadding_token_removal_lower(void *arg) circ_client_machine.states[CIRCPAD_STATE_BURST]. histogram_edges[BIG_HISTOGRAM_LEN-2] = 100; mi->padding_scheduled_at_usec = current_time - 29202; - circpad_machine_remove_token(mi); + circpad_cell_event_nonpadding_sent((circuit_t*)client_side); tt_int_op(mi->histogram[BIG_HISTOGRAM_LEN-2], OP_EQ, 1); done: @@ -780,12 +780,12 @@ test_circuitpadding_closest_token_removal(void *arg) tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 2); mi->padding_scheduled_at_usec = current_time - 57; - circpad_machine_remove_token(mi); + circpad_cell_event_nonpadding_sent((circuit_t*)client_side); tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 1); mi->padding_scheduled_at_usec = current_time - 57; - circpad_machine_remove_token(mi); + circpad_cell_event_nonpadding_sent((circuit_t*)client_side); /* Test that we cleaned out this bin. Don't do this in the case of the last bin since the tokens will get refilled */ @@ -807,14 +807,14 @@ test_circuitpadding_closest_token_removal(void *arg) circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_edges[2] = 120; mi->padding_scheduled_at_usec = current_time - 102; mi->histogram[0] = 0; - circpad_machine_remove_token(mi); + circpad_cell_event_nonpadding_sent((circuit_t*)client_side); tt_int_op(mi->histogram[1], OP_EQ, 1); /* Test above the highest bin, for coverage */ tt_int_op(client_side->padding_info[0]->current_state, OP_EQ, CIRCPAD_STATE_BURST); mi->padding_scheduled_at_usec = current_time - 29202; - circpad_machine_remove_token(mi); + circpad_cell_event_nonpadding_sent((circuit_t*)client_side); tt_int_op(mi->histogram[BIG_HISTOGRAM_LEN-2], OP_EQ, 1); done: @@ -889,12 +889,12 @@ test_circuitpadding_closest_token_removal_usec(void *arg) tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 2); mi->padding_scheduled_at_usec = current_time - 57; - circpad_machine_remove_token(mi); + circpad_cell_event_nonpadding_sent((circuit_t*)client_side); tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 1); mi->padding_scheduled_at_usec = current_time - 57; - circpad_machine_remove_token(mi); + circpad_cell_event_nonpadding_sent((circuit_t*)client_side); /* Test that we cleaned out this bin. Don't do this in the case of the last bin since the tokens will get refilled */ @@ -916,7 +916,7 @@ test_circuitpadding_closest_token_removal_usec(void *arg) circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_edges[2] = 120; mi->padding_scheduled_at_usec = current_time - 102; mi->histogram[0] = 0; - circpad_machine_remove_token(mi); + circpad_cell_event_nonpadding_sent((circuit_t*)client_side); tt_int_op(mi->histogram[1], OP_EQ, 1); /* Test above the highest bin, for coverage */ @@ -925,7 +925,7 @@ test_circuitpadding_closest_token_removal_usec(void *arg) circ_client_machine.states[CIRCPAD_STATE_BURST]. histogram_edges[BIG_HISTOGRAM_LEN-2] = 100; mi->padding_scheduled_at_usec = current_time - 29202; - circpad_machine_remove_token(mi); + circpad_cell_event_nonpadding_sent((circuit_t*)client_side); tt_int_op(mi->histogram[BIG_HISTOGRAM_LEN-2], OP_EQ, 1); done: @@ -971,16 +971,16 @@ test_circuitpadding_token_removal_exact(void *arg) /* Ensure that we will clear out bin #4 with this usec */ mi->padding_scheduled_at_usec = current_time - 57; tt_int_op(mi->histogram[4], OP_EQ, 2); - circpad_machine_remove_token(mi); + circpad_cell_event_nonpadding_sent((circuit_t*)client_side); mi->padding_scheduled_at_usec = current_time - 57; tt_int_op(mi->histogram[4], OP_EQ, 1); - circpad_machine_remove_token(mi); + circpad_cell_event_nonpadding_sent((circuit_t*)client_side); tt_int_op(mi->histogram[4], OP_EQ, 0); /* Ensure that we will not remove any other tokens even tho we try to, since * this is what the exact strategy dictates */ mi->padding_scheduled_at_usec = current_time - 57; - circpad_machine_remove_token(mi); + circpad_cell_event_nonpadding_sent((circuit_t*)client_side); for (int i = 0; i < BIG_HISTOGRAM_LEN ; i++) { if (i != 4) { tt_int_op(mi->histogram[i], OP_EQ, 2); From e4feb4ad01e7f3de592adbecd7f1ffae040f51ed Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 14 May 2019 19:47:43 -0400 Subject: [PATCH 0986/2557] Give tinytest a function to say whether the current test has failed --- src/ext/tinytest.c | 6 ++++++ src/ext/tinytest.h | 3 +++ 2 files changed, 9 insertions(+) diff --git a/src/ext/tinytest.c b/src/ext/tinytest.c index 16f11e4639..239fdd0a38 100644 --- a/src/ext/tinytest.c +++ b/src/ext/tinytest.c @@ -492,6 +492,12 @@ tinytest_set_test_skipped_(void) cur_test_outcome = SKIP; } +int +tinytest_cur_test_has_failed(void) +{ + return (cur_test_outcome == FAIL); +} + char * tinytest_format_hex_(const void *val_, unsigned long len) { diff --git a/src/ext/tinytest.h b/src/ext/tinytest.h index ed07b26bc0..05c2fda052 100644 --- a/src/ext/tinytest.h +++ b/src/ext/tinytest.h @@ -72,6 +72,9 @@ struct testlist_alias_t { }; #define END_OF_ALIASES { NULL, NULL } +/** Return true iff the current test has failed. */ +int tinytest_cur_test_has_failed(void); + /** Implementation: called from a test to indicate failure, before logging. */ void tinytest_set_test_failed_(void); /** Implementation: called from a test to indicate that we're skipping. */ From 261d43cdd5f9dbd9b67303f8c8042a38a37995ca Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 14 May 2019 19:49:50 -0400 Subject: [PATCH 0987/2557] Make testing_disable_reproducible_rng() log seed on test failure This should let us simplify test_prob_distr.c and other stuff in the future. --- src/test/rng_test_helpers.c | 17 ++++++++++++++++- src/test/rng_test_helpers.h | 3 +-- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/src/test/rng_test_helpers.c b/src/test/rng_test_helpers.c index 262d380bda..c4243f5e9a 100644 --- a/src/test/rng_test_helpers.c +++ b/src/test/rng_test_helpers.c @@ -17,6 +17,7 @@ #include "core/or/or.h" #include "lib/crypt_ops/crypto_rand.h" +#include "ext/tinytest.h" #include "test/rng_test_helpers.h" @@ -54,7 +55,8 @@ static uint8_t rng_seed[16]; static crypto_xof_t *rng_xof = NULL; /** - * Print the seed for our PRNG to stdout. We use this when we're + * Print the seed for our PRNG to stdout. We use this when we're failed + * test that had a reproducible RNG set. **/ void testing_dump_reproducible_rng_seed(void) @@ -224,3 +226,16 @@ testing_disable_rng_override(void) rng_is_replaced = false; } + +/** + * As testing_disable_rng_override(), but dump the seed if the current + * test has failed. + */ +void +testing_disable_reproducible_rng(void) +{ + if (tinytest_cur_test_has_failed()) { + testing_dump_reproducible_rng_seed(); + } + testing_disable_rng_override(); +} diff --git a/src/test/rng_test_helpers.h b/src/test/rng_test_helpers.h index 907099450d..d7925148ae 100644 --- a/src/test/rng_test_helpers.h +++ b/src/test/rng_test_helpers.h @@ -14,8 +14,7 @@ void testing_prefilled_rng_reset(void); void testing_disable_rng_override(void); -#define testing_disable_reproducible_rng() \ - testing_disable_rng_override() +void testing_disable_reproducible_rng(void); #define testing_disable_deterministic_rng() \ testing_disable_rng_override() #define testing_disable_prefilled_rng() \ From 2d467544feadeefa9a484142c433170c66d32786 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 14 May 2019 19:51:16 -0400 Subject: [PATCH 0988/2557] Use new RNG and tinytest code to simplify prob_distr tests Since the reproducible RNG dumps its own seed, we don't need to do it for it. Since tinytest can tell us if the test failed, we don't need our own test_failed booleans. --- src/test/test_prob_distr.c | 29 ++++++++++------------------- 1 file changed, 10 insertions(+), 19 deletions(-) diff --git a/src/test/test_prob_distr.c b/src/test/test_prob_distr.c index 747c3d98e6..0ecbf65f41 100644 --- a/src/test/test_prob_distr.c +++ b/src/test/test_prob_distr.c @@ -1119,13 +1119,14 @@ test_psi_dist_sample(const struct dist *dist) } static void -dump_seed(void) +write_stochastic_warning(void) { - printf("\n" + if (tinytest_cur_test_has_failed()) { + printf("\n" "NOTE: This is a stochastic test, and we expect it to fail from\n" "time to time, with some low probability. If you see it fail more\n" "than one trial in 100, though, please tell us.\n\n"); - testing_dump_reproducible_rng_seed(); + } } static void @@ -1180,7 +1181,7 @@ test_stochastic_uniform(void *arg) done: if (tests_failed) { - dump_seed(); + write_stochastic_warning(); } testing_disable_reproducible_rng(); } @@ -1273,7 +1274,7 @@ test_stochastic_genpareto(void *arg) done: if (tests_failed) { - dump_seed(); + write_stochastic_warning(); } testing_disable_reproducible_rng(); } @@ -1301,7 +1302,7 @@ test_stochastic_geometric(void *arg) done: if (tests_failed) { - dump_seed(); + write_stochastic_warning(); } testing_disable_reproducible_rng(); } @@ -1328,7 +1329,7 @@ test_stochastic_logistic(void *arg) done: if (tests_failed) { - dump_seed(); + write_stochastic_warning(); } testing_disable_reproducible_rng(); } @@ -1337,7 +1338,6 @@ static void test_stochastic_log_logistic(void *arg) { bool ok = 0; - bool tests_failed = true; (void) arg; testing_enable_reproducible_rng(); @@ -1351,12 +1351,8 @@ test_stochastic_log_logistic(void *arg) ok = test_stochastic_log_logistic_impl(exp(-10), 1e-2); tt_assert(ok); - tests_failed = false; - done: - if (tests_failed) { - dump_seed(); - } + write_stochastic_warning(); testing_disable_reproducible_rng(); } @@ -1364,7 +1360,6 @@ static void test_stochastic_weibull(void *arg) { bool ok = 0; - bool tests_failed = true; (void) arg; testing_enable_reproducible_rng(); @@ -1380,12 +1375,8 @@ test_stochastic_weibull(void *arg) ok = test_stochastic_weibull_impl(10, 1); tt_assert(ok); - tests_failed = false; - done: - if (tests_failed) { - dump_seed(); - } + write_stochastic_warning(); testing_disable_reproducible_rng(); UNMOCK(crypto_rand); } From 72e9c427b81a1d31f28c8398f516fa7892200ef3 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 14 May 2019 20:05:53 -0400 Subject: [PATCH 0989/2557] Testing: allow the user to pass a seed in for reproducible-RNG tests The environment variable TOR_TEST_RNG_SEED, if provided, is a hex value for the RNG seed. --- src/test/rng_test_helpers.c | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/src/test/rng_test_helpers.c b/src/test/rng_test_helpers.c index c4243f5e9a..1ee2672302 100644 --- a/src/test/rng_test_helpers.c +++ b/src/test/rng_test_helpers.c @@ -124,9 +124,22 @@ enable_deterministic_rng_impl(const uint8_t *seed, size_t seed_len) void testing_enable_reproducible_rng(void) { - uint8_t seed[16]; - crypto_rand((char*)seed, sizeof(seed)); - enable_deterministic_rng_impl(seed, sizeof(seed)); + const char *provided_seed = getenv("TOR_TEST_RNG_SEED"); + if (provided_seed) { + size_t hexlen = strlen(provided_seed); + size_t seedlen = hexlen / 2; + uint8_t *seed = tor_malloc(hexlen / 2); + if (base16_decode((char*)seed, seedlen, provided_seed, hexlen) < 0) { + puts("Cannot decode value in TOR_TEST_RNG_SEED"); + exit(1); + } + enable_deterministic_rng_impl(seed, seedlen); + tor_free(seed); + } else { + uint8_t seed[16]; + crypto_rand((char*)seed, sizeof(seed)); + enable_deterministic_rng_impl(seed, sizeof(seed)); + } } /** From 536ba09ad1903f36fbb4a5a9e3a205f8142d8d1f Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 14 May 2019 20:12:50 -0400 Subject: [PATCH 0990/2557] Use reproducible RNG logic in circuitpadding tests. --- src/test/test_circuitpadding.c | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/src/test/test_circuitpadding.c b/src/test/test_circuitpadding.c index db175fecee..4dc284423c 100644 --- a/src/test/test_circuitpadding.c +++ b/src/test/test_circuitpadding.c @@ -34,6 +34,8 @@ #include "core/or/or_circuit_st.h" #include "core/or/origin_circuit_st.h" +#include "test/rng_test_helpers.h" + /* Start our monotime mocking at 1 second past whatever monotime_init() * thought the actual wall clock time was, for platforms with bad resolution * and weird timevalues during monotime_init() before mocking. */ @@ -300,6 +302,7 @@ test_circuitpadding_rtt(void *arg) MOCK(circuitmux_attach_circuit, circuitmux_attach_circuit_mock); MOCK(circpad_send_command_to_hop, circpad_send_command_to_hop_mock); + testing_enable_reproducible_rng(); dummy_channel.cmux = circuitmux_alloc(); relay_side = TO_CIRCUIT(new_fake_orcirc(&dummy_channel, &dummy_channel)); @@ -403,6 +406,7 @@ test_circuitpadding_rtt(void *arg) UNMOCK(circuit_package_relay_cell); UNMOCK(circuitmux_attach_circuit); tor_free(circ_client_machine.states); + testing_disable_reproducible_rng(); return; } @@ -524,6 +528,7 @@ test_circuitpadding_token_removal_higher(void *arg) /* Mock it up */ MOCK(monotime_absolute_usec, mock_monotime_absolute_usec); MOCK(circpad_machine_schedule_padding,circpad_machine_schedule_padding_mock); + testing_enable_reproducible_rng(); /* Setup test environment (time etc.) */ client_side = (circuit_t *)origin_circuit_new(); @@ -617,6 +622,7 @@ test_circuitpadding_token_removal_higher(void *arg) free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side)); monotime_disable_test_mocking(); tor_free(circ_client_machine.states); + testing_disable_reproducible_rng(); } /** Test lower token removal strategy by bin */ @@ -629,6 +635,7 @@ test_circuitpadding_token_removal_lower(void *arg) /* Mock it up */ MOCK(monotime_absolute_usec, mock_monotime_absolute_usec); MOCK(circpad_machine_schedule_padding,circpad_machine_schedule_padding_mock); + testing_enable_reproducible_rng(); /* Setup test environment (time etc.) */ client_side = (circuit_t *)origin_circuit_new(); @@ -715,6 +722,7 @@ test_circuitpadding_token_removal_lower(void *arg) free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side)); monotime_disable_test_mocking(); tor_free(circ_client_machine.states); + testing_disable_reproducible_rng(); } /** Test closest token removal strategy by bin */ @@ -727,6 +735,7 @@ test_circuitpadding_closest_token_removal(void *arg) /* Mock it up */ MOCK(monotime_absolute_usec, mock_monotime_absolute_usec); MOCK(circpad_machine_schedule_padding,circpad_machine_schedule_padding_mock); + testing_enable_reproducible_rng(); /* Setup test environment (time etc.) */ client_side = (circuit_t *)origin_circuit_new(); @@ -821,6 +830,7 @@ test_circuitpadding_closest_token_removal(void *arg) free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side)); monotime_disable_test_mocking(); tor_free(circ_client_machine.states); + testing_disable_reproducible_rng(); } /** Test closest token removal strategy with usec */ @@ -833,6 +843,7 @@ test_circuitpadding_closest_token_removal_usec(void *arg) /* Mock it up */ MOCK(monotime_absolute_usec, mock_monotime_absolute_usec); MOCK(circpad_machine_schedule_padding,circpad_machine_schedule_padding_mock); + testing_enable_reproducible_rng(); /* Setup test environment (time etc.) */ client_side = (circuit_t *)origin_circuit_new(); @@ -932,6 +943,7 @@ test_circuitpadding_closest_token_removal_usec(void *arg) free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side)); monotime_disable_test_mocking(); tor_free(circ_client_machine.states); + testing_disable_reproducible_rng(); } /** Test closest token removal strategy with usec */ @@ -944,6 +956,7 @@ test_circuitpadding_token_removal_exact(void *arg) /* Mock it up */ MOCK(monotime_absolute_usec, mock_monotime_absolute_usec); MOCK(circpad_machine_schedule_padding,circpad_machine_schedule_padding_mock); + testing_enable_reproducible_rng(); /* Setup test environment (time etc.) */ client_side = (circuit_t *)origin_circuit_new(); @@ -991,6 +1004,7 @@ test_circuitpadding_token_removal_exact(void *arg) free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side)); monotime_disable_test_mocking(); tor_free(circ_client_machine.states); + testing_disable_reproducible_rng(); } #undef BIG_HISTOGRAM_LEN @@ -1003,6 +1017,8 @@ test_circuitpadding_tokens(void *arg) int64_t actual_mocked_monotime_start; (void)arg; + testing_enable_reproducible_rng(); + /** Test plan: * * 1. Test symmetry between bin_to_usec and usec_to_bin @@ -1256,6 +1272,7 @@ test_circuitpadding_tokens(void *arg) free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side)); monotime_disable_test_mocking(); tor_free(circ_client_machine.states); + testing_disable_reproducible_rng(); } void @@ -1283,6 +1300,7 @@ test_circuitpadding_wronghop(void *arg) /* Mock this function so that our cell counting tests don't get confused by * padding that gets sent by scheduled timers. */ MOCK(circpad_machine_schedule_padding,circpad_machine_schedule_padding_mock); + testing_enable_reproducible_rng(); client_side = (circuit_t *)origin_circuit_new(); dummy_channel.cmux = circuitmux_alloc(); @@ -1456,6 +1474,7 @@ test_circuitpadding_wronghop(void *arg) UNMOCK(circuit_package_relay_cell); UNMOCK(circuitmux_attach_circuit); nodes_free(); + testing_disable_reproducible_rng(); } void @@ -1758,6 +1777,7 @@ test_circuitpadding_conditions(void *arg) int64_t actual_mocked_monotime_start; (void)arg; MOCK(circuitmux_attach_circuit, circuitmux_attach_circuit_mock); + testing_enable_reproducible_rng(); nodes_init(); dummy_channel.cmux = circuitmux_alloc(); @@ -1862,6 +1882,7 @@ test_circuitpadding_conditions(void *arg) done: /* XXX: Free everything */ + testing_disable_reproducible_rng(); return; } @@ -2191,6 +2212,7 @@ test_circuitpadding_sample_distribution(void *arg) /* mock this function so that we dont actually schedule any padding */ MOCK(circpad_machine_schedule_padding, circpad_machine_schedule_padding_mock); + testing_enable_reproducible_rng(); /* Initialize a machine with multiple probability distributions */ circpad_machines_init(); @@ -2223,6 +2245,7 @@ test_circuitpadding_sample_distribution(void *arg) done: free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side)); UNMOCK(circpad_machine_schedule_padding); + testing_disable_reproducible_rng(); } static circpad_decision_t @@ -2248,6 +2271,7 @@ test_circuitpadding_machine_rate_limiting(void *arg) * really care about padding counts */ MOCK(circpad_machine_spec_transition, circpad_machine_spec_transition_mock); MOCK(circpad_send_command_to_hop, circpad_send_command_to_hop_mock); + testing_enable_reproducible_rng(); /* Setup machine and circuits */ client_side = TO_CIRCUIT(origin_circuit_new()); @@ -2301,6 +2325,7 @@ test_circuitpadding_machine_rate_limiting(void *arg) done: free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side)); + testing_disable_reproducible_rng(); } /* Test global padding rate limits */ @@ -2320,6 +2345,7 @@ test_circuitpadding_global_rate_limiting(void *arg) MOCK(circuit_package_relay_cell, circuit_package_relay_cell_mock); MOCK(monotime_absolute_usec, mock_monotime_absolute_usec); + testing_enable_reproducible_rng(); monotime_init(); monotime_enable_test_mocking(); @@ -2399,6 +2425,7 @@ test_circuitpadding_global_rate_limiting(void *arg) circuitmux_free(dummy_channel.cmux); SMARTLIST_FOREACH(vote1.net_params, char *, cp, tor_free(cp)); smartlist_free(vote1.net_params); + testing_disable_reproducible_rng(); } /* Test reduced and disabled padding */ @@ -2409,6 +2436,7 @@ test_circuitpadding_reduce_disable(void *arg) int64_t actual_mocked_monotime_start; MOCK(circuitmux_attach_circuit, circuitmux_attach_circuit_mock); + testing_enable_reproducible_rng(); nodes_init(); dummy_channel.cmux = circuitmux_alloc(); @@ -2548,6 +2576,7 @@ test_circuitpadding_reduce_disable(void *arg) free_fake_orcirc(relay_side); circuitmux_detach_all_circuits(dummy_channel.cmux, NULL); circuitmux_free(dummy_channel.cmux); + testing_disable_reproducible_rng(); } #define TEST_CIRCUITPADDING(name, flags) \ From 669f6e34eaae0d6937825bdb345f37912123ddd6 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 14 May 2019 20:16:35 -0400 Subject: [PATCH 0991/2557] Travis: fix an rng seed for coverage builds. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index f78333611f..330388aa47 100644 --- a/.travis.yml +++ b/.travis.yml @@ -42,7 +42,7 @@ matrix: ## include creates builds with gcc, linux include: ## We include a single coverage build with the best options for coverage - - env: COVERAGE_OPTIONS="--enable-coverage" HARDENING_OPTIONS="" + - env: COVERAGE_OPTIONS="--enable-coverage" HARDENING_OPTIONS="" TOR_TEST_RNG_SEED="636f766572616765" ## We only want to check these build option combinations once ## (they shouldn't vary by compiler or OS) ## We run rust and coverage with hardening off, which seems like enough From 9e5c27bd2c278238d003b2fc1891d5de48b766d7 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 14 May 2019 20:22:30 -0400 Subject: [PATCH 0992/2557] changes file for ticket 28878 --- changes/ticket28878 | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 changes/ticket28878 diff --git a/changes/ticket28878 b/changes/ticket28878 new file mode 100644 index 0000000000..73ca47c72f --- /dev/null +++ b/changes/ticket28878 @@ -0,0 +1,11 @@ + o Minor features (testing): + - The circuitpadding tests now use a reproducible RNG implementation, + so that if a test fails, we can learn why. Part of ticket 28878. + - Tor's tests now support an environment variable, TOR_TEST_RNG_SEED, + to set the RNG seed for tests that use a reproducible RNG. + Part of ticket 28878. + + o Minor features (continuous integration): + - When running coverage builds on Travis, we now set TOR_TEST_RNG_SEED, + to avoid RNG-based coverage differences. + Part of ticket 28878. From 5c2d2b5d11b72fac01c5958f18c83df252f5313d Mon Sep 17 00:00:00 2001 From: Mike Perry Date: Sat, 11 May 2019 03:12:33 +0000 Subject: [PATCH 0993/2557] Make the relationship between mutable histograms and token removal explicit. --- src/core/or/circuitpadding.c | 65 +++++++++++++++++++++++++++--------- 1 file changed, 49 insertions(+), 16 deletions(-) diff --git a/src/core/or/circuitpadding.c b/src/core/or/circuitpadding.c index d6a79004f5..eeedbc124f 100644 --- a/src/core/or/circuitpadding.c +++ b/src/core/or/circuitpadding.c @@ -330,6 +330,39 @@ circpad_histogram_usec_to_bin(const circpad_machine_runtime_t *mi, return CIRCPAD_INFINITY_BIN(state)-1; } +/** + * Return true if the machine supports token removal. + * + * Token removal is equivalent to having a mutable histogram in the + * circpad_machine_runtime_t mutable info. So while we're at it, + * let's assert that everything is consistent between the mutable + * runtime and the readonly machine spec. + */ +static inline int +circpad_is_token_removal_supported(circpad_machine_runtime_t *mi) +{ + /* No runtime histogram == no token removal */ + if (mi->histogram == NULL) { + /* Machines that don't want token removal are trying to avoid + * potentially expensive mallocs, extra memory accesses, and/or + * potentially expensive monotime calls. Let's minimize checks + * and keep this path fast. */ + tor_assert_nonfatal(mi->histogram_len == 0); + return 0; + } else { + /* Machines that do want token removal are less sensitive to performance. + * Let's spend some time to check that our state is consistent and sane */ + const circpad_state_t *state = circpad_machine_current_state(mi); + tor_assert_nonfatal(state->token_removal != CIRCPAD_TOKEN_REMOVAL_NONE); + tor_assert_nonfatal(state->histogram_len == mi->histogram_len); + tor_assert_nonfatal(mi->histogram_len != 0); + return 1; + } + + tor_assert_nonfatal_unreached(); + return 0; +} + /** * This function frees any token bins allocated from a previous state * @@ -441,13 +474,7 @@ circpad_machine_sample_delay(circpad_machine_runtime_t *mi) mi->rtt_estimate_usec + state->dist_added_shift_usec : state->dist_added_shift_usec; return circpad_distribution_sample_iat_delay(state, iat_delay_shift); - } else if (state->token_removal != CIRCPAD_TOKEN_REMOVAL_NONE) { - /* We have a mutable histogram. Do basic sanity check and apply: */ - if (BUG(!mi->histogram) || - BUG(mi->histogram_len != state->histogram_len)) { - return CIRCPAD_DELAY_INFINITE; - } - + } else if (circpad_is_token_removal_supported(mi)) { histogram = mi->histogram; for (circpad_hist_index_t b = 0; b < state->histogram_len; b++) histogram_total_tokens += histogram[b]; @@ -812,7 +839,7 @@ check_machine_token_supply(circpad_machine_runtime_t *mi) * * We also do not count infinity bin in histogram totals. */ - if (mi->histogram_len && mi->histogram) { + if (circpad_is_token_removal_supported(mi)) { for (circpad_hist_index_t b = 0; b < CIRCPAD_INFINITY_BIN(mi); b++) histogram_total_tokens += mi->histogram[b]; @@ -861,12 +888,8 @@ circpad_machine_count_padding_sent(circpad_machine_runtime_t *mi) /* If we have a mutable histogram, reduce the token count from * the chosen padding bin (this assumes we always send padding * when we intended to). */ - if (mi->histogram && mi->histogram_len) { - /* Ensure that we have a token removal strategy set */ - const circpad_state_t *state = circpad_machine_current_state(mi); - tor_assert_nonfatal(state->token_removal != CIRCPAD_TOKEN_REMOVAL_NONE); - - /* Basic sanity check on the histogram before removing anything */ + if (circpad_is_token_removal_supported(mi)) { + /* Check array bounds and token count before removing */ if (!BUG(mi->chosen_bin >= mi->histogram_len) && !BUG(mi->histogram[mi->chosen_bin] == 0)) { mi->histogram[mi->chosen_bin]--; @@ -896,7 +919,7 @@ circpad_machine_count_nonpadding_sent(circpad_machine_runtime_t *mi) /* Update any state packet length limits that apply */ circpad_machine_update_state_length_for_nonpadding(mi); - /* Remove a token from the histrogram, if applicable */ + /* Remove a token from the histogram, if applicable */ circpad_machine_remove_token(mi); } @@ -1301,7 +1324,17 @@ circpad_machine_schedule_padding,(circpad_machine_runtime_t *mi)) /* in_usec = in microseconds */ in_usec = circpad_machine_sample_delay(mi); - mi->padding_scheduled_at_usec = monotime_absolute_usec(); + /* If we're using token removal, we need to know when the padding + * was scheduled at, so we can remove the appropriate token if + * a non-padding cell is sent before the padding timer expires. + * + * However, since monotime is unpredictably expensive, let's avoid + * using it for machines that don't need token removal. */ + if (circpad_is_token_removal_supported(mi)) { + mi->padding_scheduled_at_usec = monotime_absolute_usec(); + } else { + mi->padding_scheduled_at_usec = 1; + } log_fn(LOG_INFO,LD_CIRC,"\tPadding in %u usec", in_usec); // Don't schedule if we have infinite delay. From 14ec8b89f83ea9cc4fbc7b8bd24194f5408cfea4 Mon Sep 17 00:00:00 2001 From: Mike Perry Date: Wed, 8 May 2019 00:40:49 +0000 Subject: [PATCH 0994/2557] Bug 29085: Avoid monotime usage for rtt estimates if it is not in use. --- src/core/or/circuitpadding.c | 25 +++++++++++++++++++++---- src/test/test_circuitpadding.c | 1 + 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/src/core/or/circuitpadding.c b/src/core/or/circuitpadding.c index eeedbc124f..7674f3510c 100644 --- a/src/core/or/circuitpadding.c +++ b/src/core/or/circuitpadding.c @@ -1547,10 +1547,26 @@ circpad_estimate_circ_rtt_on_received(circuit_t *circ, ", %d) after two back to back packets. Current RTT: %d", circ->n_chan ? circ->n_chan->global_identifier : 0, circ->n_circ_id, mi->rtt_estimate_usec); - mi->stop_rtt_update = 1; + mi->stop_rtt_update = 1; + + if (!mi->rtt_estimate_usec) { + static ratelim_t rtt_lim = RATELIM_INIT(600); + log_fn_ratelim(&rtt_lim,LOG_NOTICE,LD_BUG, + "Circuit got two cells back to back before estimating RTT."); + } } } else { - mi->last_received_time_usec = monotime_absolute_usec(); + const circpad_state_t *state = circpad_machine_current_state(mi); + + /* Since monotime is unpredictably expensive, only update this field + * if rtt estimates are needed. Otherwise, stop the rtt update. */ + if (state->use_rtt_estimate) { + mi->last_received_time_usec = monotime_absolute_usec(); + } else { + /* Let's fast-path future decisions not to update rtt if the + * feature is not in use. */ + mi->stop_rtt_update = 1; + } } } @@ -1610,8 +1626,9 @@ circpad_estimate_circ_rtt_on_send(circuit_t *circ, mi->stop_rtt_update = 1; if (!mi->rtt_estimate_usec) { - log_fn(LOG_NOTICE, LD_CIRC, - "Got two cells back to back on a circuit before estimating RTT."); + static ratelim_t rtt_lim = RATELIM_INIT(600); + log_fn_ratelim(&rtt_lim,LOG_NOTICE,LD_BUG, + "Circuit sent two cells back to back before estimating RTT."); } } } diff --git a/src/test/test_circuitpadding.c b/src/test/test_circuitpadding.c index b7dbf022aa..13371df687 100644 --- a/src/test/test_circuitpadding.c +++ b/src/test/test_circuitpadding.c @@ -415,6 +415,7 @@ helper_create_basic_machine(void) circ_client_machine.states[CIRCPAD_STATE_START]. next_state[CIRCPAD_EVENT_NONPADDING_RECV] = CIRCPAD_STATE_BURST; + circ_client_machine.states[CIRCPAD_STATE_START].use_rtt_estimate = 1; circ_client_machine.states[CIRCPAD_STATE_BURST]. next_state[CIRCPAD_EVENT_PADDING_RECV] = CIRCPAD_STATE_BURST; From 5ef0d89d912c2b4cbf42510df5f609b3704ede73 Mon Sep 17 00:00:00 2001 From: Mike Perry Date: Wed, 8 May 2019 00:50:52 +0000 Subject: [PATCH 0995/2557] Bug 29085: Changes file. --- changes/bug29085 | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 changes/bug29085 diff --git a/changes/bug29085 b/changes/bug29085 new file mode 100644 index 0000000000..b17c06378f --- /dev/null +++ b/changes/bug29085 @@ -0,0 +1,4 @@ + o Code simplification and refactoring (circuit padding): + - Avoid calling monotime_absolute_usec() in circuit padding machines + that do not use token removal or circuit RTT estimation. Fixes bug + 29085; bugfix on 0.4.0.1-alpha. From aef9be6ace841e89c47c53a6c419fa9c1a223cd2 Mon Sep 17 00:00:00 2001 From: Mike Perry Date: Tue, 14 May 2019 22:25:15 +0000 Subject: [PATCH 0996/2557] Eliminate unneeded casts to circuit_t in circpad tests. --- src/test/test_circuitpadding.c | 114 ++++++++++++++++----------------- 1 file changed, 57 insertions(+), 57 deletions(-) diff --git a/src/test/test_circuitpadding.c b/src/test/test_circuitpadding.c index 13371df687..b19bbcc94d 100644 --- a/src/test/test_circuitpadding.c +++ b/src/test/test_circuitpadding.c @@ -329,12 +329,12 @@ test_circuitpadding_rtt(void *arg) relay_side->padding_info[0] = circpad_circuit_machineinfo_new(client_side,0); /* Test 1: Test measuring RTT */ - circpad_cell_event_nonpadding_received((circuit_t*)relay_side); + circpad_cell_event_nonpadding_received(relay_side); tt_u64_op(relay_side->padding_info[0]->last_received_time_usec, OP_NE, 0); timers_advance_and_run(20); - circpad_cell_event_nonpadding_sent((circuit_t*)relay_side); + circpad_cell_event_nonpadding_sent(relay_side); tt_u64_op(relay_side->padding_info[0]->last_received_time_usec, OP_EQ, 0); tt_int_op(relay_side->padding_info[0]->rtt_estimate_usec, OP_GE, 19000); @@ -345,12 +345,12 @@ test_circuitpadding_rtt(void *arg) circpad_machine_current_state( relay_side->padding_info[0])->histogram_edges[0]); - circpad_cell_event_nonpadding_received((circuit_t*)relay_side); - circpad_cell_event_nonpadding_received((circuit_t*)relay_side); + circpad_cell_event_nonpadding_received(relay_side); + circpad_cell_event_nonpadding_received(relay_side); tt_u64_op(relay_side->padding_info[0]->last_received_time_usec, OP_NE, 0); timers_advance_and_run(20); - circpad_cell_event_nonpadding_sent((circuit_t*)relay_side); - circpad_cell_event_nonpadding_sent((circuit_t*)relay_side); + circpad_cell_event_nonpadding_sent(relay_side); + circpad_cell_event_nonpadding_sent(relay_side); tt_u64_op(relay_side->padding_info[0]->last_received_time_usec, OP_EQ, 0); tt_int_op(relay_side->padding_info[0]->rtt_estimate_usec, OP_GE, 20000); @@ -365,9 +365,9 @@ test_circuitpadding_rtt(void *arg) tt_int_op(relay_side->padding_info[0]->stop_rtt_update, OP_EQ, 1); rtt_estimate = relay_side->padding_info[0]->rtt_estimate_usec; - circpad_cell_event_nonpadding_received((circuit_t*)relay_side); + circpad_cell_event_nonpadding_received(relay_side); timers_advance_and_run(4); - circpad_cell_event_nonpadding_sent((circuit_t*)relay_side); + circpad_cell_event_nonpadding_sent(relay_side); tt_int_op(relay_side->padding_info[0]->rtt_estimate_usec, OP_EQ, rtt_estimate); @@ -380,11 +380,11 @@ test_circuitpadding_rtt(void *arg) relay_side->padding_info[0])->histogram_edges[0]); /* Test 3: Make sure client side machine properly ignores RTT */ - circpad_cell_event_nonpadding_received((circuit_t*)client_side); + circpad_cell_event_nonpadding_received(client_side); tt_u64_op(client_side->padding_info[0]->last_received_time_usec, OP_EQ, 0); timers_advance_and_run(20); - circpad_cell_event_nonpadding_sent((circuit_t*)client_side); + circpad_cell_event_nonpadding_sent(client_side); tt_u64_op(client_side->padding_info[0]->last_received_time_usec, OP_EQ, 0); tt_int_op(client_side->padding_info[0]->rtt_estimate_usec, OP_EQ, 0); @@ -527,7 +527,7 @@ test_circuitpadding_token_removal_higher(void *arg) MOCK(circpad_machine_schedule_padding,circpad_machine_schedule_padding_mock); /* Setup test environment (time etc.) */ - client_side = (circuit_t *)origin_circuit_new(); + client_side = TO_CIRCUIT(origin_circuit_new()); client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL; monotime_enable_test_mocking(); @@ -538,7 +538,7 @@ test_circuitpadding_token_removal_higher(void *arg) circpad_circuit_machineinfo_new(client_side, 0); /* move the machine to the right state */ - circpad_cell_event_nonpadding_received((circuit_t*)client_side); + circpad_cell_event_nonpadding_received(client_side); tt_int_op(client_side->padding_info[0]->current_state, OP_EQ, CIRCPAD_STATE_BURST); @@ -587,12 +587,12 @@ test_circuitpadding_token_removal_higher(void *arg) tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 2); mi->padding_scheduled_at_usec = current_time - 57; - circpad_cell_event_nonpadding_sent((circuit_t*)client_side); + circpad_cell_event_nonpadding_sent(client_side); tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 1); mi->padding_scheduled_at_usec = current_time - 57; - circpad_cell_event_nonpadding_sent((circuit_t*)client_side); + circpad_cell_event_nonpadding_sent(client_side); /* Test that we cleaned out this bin. Don't do this in the case of the last bin since the tokens will get refilled */ @@ -611,7 +611,7 @@ test_circuitpadding_token_removal_higher(void *arg) CIRCPAD_STATE_BURST); circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_edges[0] = 100; mi->padding_scheduled_at_usec = current_time; - circpad_cell_event_nonpadding_sent((circuit_t*)client_side); + circpad_cell_event_nonpadding_sent(client_side); tt_int_op(mi->histogram[0], OP_EQ, 1); done: @@ -632,7 +632,7 @@ test_circuitpadding_token_removal_lower(void *arg) MOCK(circpad_machine_schedule_padding,circpad_machine_schedule_padding_mock); /* Setup test environment (time etc.) */ - client_side = (circuit_t *)origin_circuit_new(); + client_side = TO_CIRCUIT(origin_circuit_new()); client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL; monotime_enable_test_mocking(); @@ -643,7 +643,7 @@ test_circuitpadding_token_removal_lower(void *arg) circpad_circuit_machineinfo_new(client_side, 0); /* move the machine to the right state */ - circpad_cell_event_nonpadding_received((circuit_t*)client_side); + circpad_cell_event_nonpadding_received(client_side); tt_int_op(client_side->padding_info[0]->current_state, OP_EQ, CIRCPAD_STATE_BURST); @@ -684,12 +684,12 @@ test_circuitpadding_token_removal_lower(void *arg) tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 2); mi->padding_scheduled_at_usec = current_time - 57; - circpad_cell_event_nonpadding_sent((circuit_t*)client_side); + circpad_cell_event_nonpadding_sent(client_side); tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 1); mi->padding_scheduled_at_usec = current_time - 57; - circpad_cell_event_nonpadding_sent((circuit_t*)client_side); + circpad_cell_event_nonpadding_sent(client_side); /* Test that we cleaned out this bin. Don't do this in the case of the last bin since the tokens will get refilled */ @@ -709,7 +709,7 @@ test_circuitpadding_token_removal_lower(void *arg) circ_client_machine.states[CIRCPAD_STATE_BURST]. histogram_edges[BIG_HISTOGRAM_LEN-2] = 100; mi->padding_scheduled_at_usec = current_time - 29202; - circpad_cell_event_nonpadding_sent((circuit_t*)client_side); + circpad_cell_event_nonpadding_sent(client_side); tt_int_op(mi->histogram[BIG_HISTOGRAM_LEN-2], OP_EQ, 1); done: @@ -730,7 +730,7 @@ test_circuitpadding_closest_token_removal(void *arg) MOCK(circpad_machine_schedule_padding,circpad_machine_schedule_padding_mock); /* Setup test environment (time etc.) */ - client_side = (circuit_t *)origin_circuit_new(); + client_side = TO_CIRCUIT(origin_circuit_new()); client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL; monotime_enable_test_mocking(); @@ -741,7 +741,7 @@ test_circuitpadding_closest_token_removal(void *arg) circpad_circuit_machineinfo_new(client_side, 0); /* move the machine to the right state */ - circpad_cell_event_nonpadding_received((circuit_t*)client_side); + circpad_cell_event_nonpadding_received(client_side); tt_int_op(client_side->padding_info[0]->current_state, OP_EQ, CIRCPAD_STATE_BURST); @@ -781,12 +781,12 @@ test_circuitpadding_closest_token_removal(void *arg) tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 2); mi->padding_scheduled_at_usec = current_time - 57; - circpad_cell_event_nonpadding_sent((circuit_t*)client_side); + circpad_cell_event_nonpadding_sent(client_side); tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 1); mi->padding_scheduled_at_usec = current_time - 57; - circpad_cell_event_nonpadding_sent((circuit_t*)client_side); + circpad_cell_event_nonpadding_sent(client_side); /* Test that we cleaned out this bin. Don't do this in the case of the last bin since the tokens will get refilled */ @@ -808,14 +808,14 @@ test_circuitpadding_closest_token_removal(void *arg) circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_edges[2] = 120; mi->padding_scheduled_at_usec = current_time - 102; mi->histogram[0] = 0; - circpad_cell_event_nonpadding_sent((circuit_t*)client_side); + circpad_cell_event_nonpadding_sent(client_side); tt_int_op(mi->histogram[1], OP_EQ, 1); /* Test above the highest bin, for coverage */ tt_int_op(client_side->padding_info[0]->current_state, OP_EQ, CIRCPAD_STATE_BURST); mi->padding_scheduled_at_usec = current_time - 29202; - circpad_cell_event_nonpadding_sent((circuit_t*)client_side); + circpad_cell_event_nonpadding_sent(client_side); tt_int_op(mi->histogram[BIG_HISTOGRAM_LEN-2], OP_EQ, 1); done: @@ -836,7 +836,7 @@ test_circuitpadding_closest_token_removal_usec(void *arg) MOCK(circpad_machine_schedule_padding,circpad_machine_schedule_padding_mock); /* Setup test environment (time etc.) */ - client_side = (circuit_t *)origin_circuit_new(); + client_side = TO_CIRCUIT(origin_circuit_new()); client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL; monotime_enable_test_mocking(); @@ -847,7 +847,7 @@ test_circuitpadding_closest_token_removal_usec(void *arg) circpad_circuit_machineinfo_new(client_side, 0); /* move the machine to the right state */ - circpad_cell_event_nonpadding_received((circuit_t*)client_side); + circpad_cell_event_nonpadding_received(client_side); tt_int_op(client_side->padding_info[0]->current_state, OP_EQ, CIRCPAD_STATE_BURST); @@ -890,12 +890,12 @@ test_circuitpadding_closest_token_removal_usec(void *arg) tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 2); mi->padding_scheduled_at_usec = current_time - 57; - circpad_cell_event_nonpadding_sent((circuit_t*)client_side); + circpad_cell_event_nonpadding_sent(client_side); tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 1); mi->padding_scheduled_at_usec = current_time - 57; - circpad_cell_event_nonpadding_sent((circuit_t*)client_side); + circpad_cell_event_nonpadding_sent(client_side); /* Test that we cleaned out this bin. Don't do this in the case of the last bin since the tokens will get refilled */ @@ -917,7 +917,7 @@ test_circuitpadding_closest_token_removal_usec(void *arg) circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_edges[2] = 120; mi->padding_scheduled_at_usec = current_time - 102; mi->histogram[0] = 0; - circpad_cell_event_nonpadding_sent((circuit_t*)client_side); + circpad_cell_event_nonpadding_sent(client_side); tt_int_op(mi->histogram[1], OP_EQ, 1); /* Test above the highest bin, for coverage */ @@ -926,7 +926,7 @@ test_circuitpadding_closest_token_removal_usec(void *arg) circ_client_machine.states[CIRCPAD_STATE_BURST]. histogram_edges[BIG_HISTOGRAM_LEN-2] = 100; mi->padding_scheduled_at_usec = current_time - 29202; - circpad_cell_event_nonpadding_sent((circuit_t*)client_side); + circpad_cell_event_nonpadding_sent(client_side); tt_int_op(mi->histogram[BIG_HISTOGRAM_LEN-2], OP_EQ, 1); done: @@ -947,7 +947,7 @@ test_circuitpadding_token_removal_exact(void *arg) MOCK(circpad_machine_schedule_padding,circpad_machine_schedule_padding_mock); /* Setup test environment (time etc.) */ - client_side = (circuit_t *)origin_circuit_new(); + client_side = TO_CIRCUIT(origin_circuit_new()); client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL; monotime_enable_test_mocking(); @@ -958,7 +958,7 @@ test_circuitpadding_token_removal_exact(void *arg) circpad_circuit_machineinfo_new(client_side, 0); /* move the machine to the right state */ - circpad_cell_event_nonpadding_received((circuit_t*)client_side); + circpad_cell_event_nonpadding_received(client_side); tt_int_op(client_side->padding_info[0]->current_state, OP_EQ, CIRCPAD_STATE_BURST); @@ -972,16 +972,16 @@ test_circuitpadding_token_removal_exact(void *arg) /* Ensure that we will clear out bin #4 with this usec */ mi->padding_scheduled_at_usec = current_time - 57; tt_int_op(mi->histogram[4], OP_EQ, 2); - circpad_cell_event_nonpadding_sent((circuit_t*)client_side); + circpad_cell_event_nonpadding_sent(client_side); mi->padding_scheduled_at_usec = current_time - 57; tt_int_op(mi->histogram[4], OP_EQ, 1); - circpad_cell_event_nonpadding_sent((circuit_t*)client_side); + circpad_cell_event_nonpadding_sent(client_side); tt_int_op(mi->histogram[4], OP_EQ, 0); /* Ensure that we will not remove any other tokens even tho we try to, since * this is what the exact strategy dictates */ mi->padding_scheduled_at_usec = current_time - 57; - circpad_cell_event_nonpadding_sent((circuit_t*)client_side); + circpad_cell_event_nonpadding_sent(client_side); for (int i = 0; i < BIG_HISTOGRAM_LEN ; i++) { if (i != 4) { tt_int_op(mi->histogram[i], OP_EQ, 2); @@ -1047,8 +1047,8 @@ test_circuitpadding_tokens(void *arg) mi = client_side->padding_info[0]; // Pretend a non-padding cell was sent - circpad_cell_event_nonpadding_received((circuit_t*)client_side); - circpad_cell_event_nonpadding_sent((circuit_t*)client_side); + circpad_cell_event_nonpadding_received(client_side); + circpad_cell_event_nonpadding_sent(client_side); /* We have to save the infinity bin because one inf delay * could have been chosen when we transition to burst */ circpad_hist_token_t inf_bin = mi->histogram[4]; @@ -1157,11 +1157,11 @@ test_circuitpadding_tokens(void *arg) /* Drain the infinity bin and cause a refill */ while (inf_bin != 0) { tt_int_op(mi->histogram[4], OP_EQ, inf_bin); - circpad_cell_event_nonpadding_received((circuit_t*)client_side); + circpad_cell_event_nonpadding_received(client_side); inf_bin--; } - circpad_cell_event_nonpadding_sent((circuit_t*)client_side); + circpad_cell_event_nonpadding_sent(client_side); // We should have refilled here. tt_int_op(mi->histogram[4], OP_EQ, 2); @@ -1285,10 +1285,10 @@ test_circuitpadding_wronghop(void *arg) * padding that gets sent by scheduled timers. */ MOCK(circpad_machine_schedule_padding,circpad_machine_schedule_padding_mock); - client_side = (circuit_t *)origin_circuit_new(); + client_side = TO_CIRCUIT(origin_circuit_new()); dummy_channel.cmux = circuitmux_alloc(); - relay_side = (circuit_t *)new_fake_orcirc(&dummy_channel, - &dummy_channel); + relay_side = TO_CIRCUIT(new_fake_orcirc(&dummy_channel, + &dummy_channel)); orig_client = TO_ORIGIN_CIRCUIT(client_side); relay_side->purpose = CIRCUIT_PURPOSE_OR; @@ -1406,9 +1406,9 @@ test_circuitpadding_wronghop(void *arg) free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side)); free_fake_orcirc(relay_side); - client_side = (circuit_t *)origin_circuit_new(); - relay_side = (circuit_t *)new_fake_orcirc(&dummy_channel, - &dummy_channel); + client_side = TO_CIRCUIT(origin_circuit_new()); + relay_side = TO_CIRCUIT(new_fake_orcirc(&dummy_channel, + &dummy_channel)); relay_side->purpose = CIRCUIT_PURPOSE_OR; client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL; @@ -1602,10 +1602,10 @@ simulate_single_hop_extend(circuit_t *client, circuit_t *mid_relay, tor_addr_t addr; // Pretend a non-padding cell was sent - circpad_cell_event_nonpadding_sent((circuit_t*)client); + circpad_cell_event_nonpadding_sent(client); // Receive extend cell at middle - circpad_cell_event_nonpadding_received((circuit_t*)mid_relay); + circpad_cell_event_nonpadding_received(mid_relay); // Advance time a tiny bit so we can calculate an RTT curr_mocked_time += 10 * TOR_NSEC_PER_MSEC; @@ -1613,10 +1613,10 @@ simulate_single_hop_extend(circuit_t *client, circuit_t *mid_relay, monotime_set_mock_time_nsec(curr_mocked_time); // Receive extended cell at middle - circpad_cell_event_nonpadding_sent((circuit_t*)mid_relay); + circpad_cell_event_nonpadding_sent(mid_relay); // Receive extended cell at first hop - circpad_cell_event_nonpadding_received((circuit_t*)client); + circpad_cell_event_nonpadding_received(client); // Add a hop to cpath crypt_path_t *hop = tor_malloc_zero(sizeof(crypt_path_t)); @@ -1762,9 +1762,9 @@ test_circuitpadding_conditions(void *arg) nodes_init(); dummy_channel.cmux = circuitmux_alloc(); - relay_side = (circuit_t *)new_fake_orcirc(&dummy_channel, - &dummy_channel); - client_side = (circuit_t *)origin_circuit_new(); + relay_side = TO_CIRCUIT(new_fake_orcirc(&dummy_channel, + &dummy_channel)); + client_side = TO_CIRCUIT(origin_circuit_new()); relay_side->purpose = CIRCUIT_PURPOSE_OR; client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL; @@ -2218,7 +2218,7 @@ test_circuitpadding_sample_distribution(void *arg) } /* send a non-padding cell to move to the next machine state */ - circpad_cell_event_nonpadding_received((circuit_t*)client_side); + circpad_cell_event_nonpadding_received(client_side); } done: @@ -2330,12 +2330,12 @@ test_circuitpadding_global_rate_limiting(void *arg) curr_mocked_time = actual_mocked_monotime_start; timers_initialize(); - client_side = (circuit_t *)origin_circuit_new(); + client_side = TO_CIRCUIT(origin_circuit_new()); client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL; dummy_channel.cmux = circuitmux_alloc(); /* Setup machine and circuits */ - relay_side = (circuit_t *)new_fake_orcirc(&dummy_channel, &dummy_channel); + relay_side = TO_CIRCUIT(new_fake_orcirc(&dummy_channel, &dummy_channel)); relay_side->purpose = CIRCUIT_PURPOSE_OR; helper_create_basic_machine(); relay_side->padding_machine[0] = &circ_client_machine; From e8a1f241783878a242ebb153119af06631fe7cdb Mon Sep 17 00:00:00 2001 From: Mike Perry Date: Wed, 15 May 2019 03:45:03 +0000 Subject: [PATCH 0997/2557] Add test to explicitly check state lengths and token counts. Our other tests tested state lengths against padding packets, and token counts against non-padding packets. This test checks state lengths against non-padding packets (and also padding packets too), and checks token counts against padding packets (and also non-padding packets too). The next three commits are needed to make this test pass (it found 3 bugs). Yay? --- src/test/test_circuitpadding.c | 180 +++++++++++++++++++++++++++++++++ 1 file changed, 180 insertions(+) diff --git a/src/test/test_circuitpadding.c b/src/test/test_circuitpadding.c index b19bbcc94d..b2ae659a60 100644 --- a/src/test/test_circuitpadding.c +++ b/src/test/test_circuitpadding.c @@ -55,6 +55,7 @@ void test_circuitpadding_conditions(void *arg); void test_circuitpadding_serialize(void *arg); void test_circuitpadding_rtt(void *arg); void test_circuitpadding_tokens(void *arg); +void test_circuitpadding_state_length(void *arg); static void simulate_single_hop_extend(circuit_t *client, circuit_t *mid_relay, @@ -1643,6 +1644,55 @@ simulate_single_hop_extend(circuit_t *client, circuit_t *mid_relay, circpad_machine_event_circ_added_hop(TO_ORIGIN_CIRCUIT(client)); } +static circpad_machine_spec_t * +helper_create_length_machine(void) +{ + circpad_machine_spec_t *ret = + tor_malloc_zero(sizeof(circpad_machine_spec_t)); + + /* Start, burst */ + circpad_machine_states_init(ret, 2); + + ret->states[CIRCPAD_STATE_START]. + next_state[CIRCPAD_EVENT_PADDING_SENT] = CIRCPAD_STATE_BURST; + + ret->states[CIRCPAD_STATE_BURST]. + next_state[CIRCPAD_EVENT_PADDING_SENT] = CIRCPAD_STATE_BURST; + + ret->states[CIRCPAD_STATE_BURST]. + next_state[CIRCPAD_EVENT_LENGTH_COUNT] = CIRCPAD_STATE_END; + + ret->states[CIRCPAD_STATE_BURST]. + next_state[CIRCPAD_EVENT_BINS_EMPTY] = CIRCPAD_STATE_END; + + /* No token removal.. end via state_length only */ + ret->states[CIRCPAD_STATE_BURST].token_removal = + CIRCPAD_TOKEN_REMOVAL_NONE; + + /* Let's have this one end after 12 packets */ + ret->states[CIRCPAD_STATE_BURST].length_dist.type = CIRCPAD_DIST_UNIFORM; + ret->states[CIRCPAD_STATE_BURST].length_dist.param1 = 12; + ret->states[CIRCPAD_STATE_BURST].length_dist.param2 = 13; + ret->states[CIRCPAD_STATE_BURST].max_length = 12; + + ret->states[CIRCPAD_STATE_BURST].histogram_len = 4; + + ret->states[CIRCPAD_STATE_BURST].histogram_edges[0] = 0; + ret->states[CIRCPAD_STATE_BURST].histogram_edges[1] = 1; + ret->states[CIRCPAD_STATE_BURST].histogram_edges[2] = 1000000; + ret->states[CIRCPAD_STATE_BURST].histogram_edges[3] = 10000000; + + ret->states[CIRCPAD_STATE_BURST].histogram[0] = 0; + ret->states[CIRCPAD_STATE_BURST].histogram[1] = 0; + ret->states[CIRCPAD_STATE_BURST].histogram[2] = 6; + + ret->states[CIRCPAD_STATE_BURST].histogram_total_tokens = 6; + ret->states[CIRCPAD_STATE_BURST].use_rtt_estimate = 0; + ret->states[CIRCPAD_STATE_BURST].length_includes_nonpadding = 0; + + return ret; +} + static circpad_machine_spec_t * helper_create_conditional_machine(void) { @@ -1738,6 +1788,135 @@ helper_create_conditional_machines(void) register_padding_machine(add, relay_padding_machines); } +void +test_circuitpadding_state_length(void *arg) +{ + /** + * Test plan: + * * Explicitly test that with no token removal enabled, we hit + * the state length limit due to either padding, or non-padding. + * * Repeat test with an arbitrary token removal strategy, and + * verify that if we run out of tokens due to padding before we + * hit the state length, we still go to state end (all our + * token removal tests only test nonpadding token removal). + */ + int64_t actual_mocked_monotime_start; + (void)arg; + MOCK(circuitmux_attach_circuit, circuitmux_attach_circuit_mock); + MOCK(circpad_send_command_to_hop, circpad_send_command_to_hop_mock); + + nodes_init(); + dummy_channel.cmux = circuitmux_alloc(); + relay_side = TO_CIRCUIT(new_fake_orcirc(&dummy_channel, + &dummy_channel)); + client_side = TO_CIRCUIT(origin_circuit_new()); + relay_side->purpose = CIRCUIT_PURPOSE_OR; + client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL; + + monotime_init(); + monotime_enable_test_mocking(); + actual_mocked_monotime_start = MONOTIME_MOCK_START; + monotime_set_mock_time_nsec(actual_mocked_monotime_start); + monotime_coarse_set_mock_time_nsec(actual_mocked_monotime_start); + curr_mocked_time = actual_mocked_monotime_start; + + /* This is needed so that we are not considered to be dormant */ + note_user_activity(20); + + timers_initialize(); + circpad_machine_spec_t *client_machine = + helper_create_length_machine(); + + MOCK(circuit_package_relay_cell, + circuit_package_relay_cell_mock); + MOCK(node_get_by_id, + node_get_by_id_mock); + + client_side->padding_machine[0] = client_machine; + client_side->padding_info[0] = + circpad_circuit_machineinfo_new(client_side, 0); + circpad_machine_runtime_t *mi = client_side->padding_info[0]; + + circpad_cell_event_padding_sent(client_side); + tt_int_op(mi->state_length, OP_EQ, 12); + tt_ptr_op(mi->histogram, OP_EQ, NULL); + + /* Verify that non-padding does not change our state length */ + circpad_cell_event_nonpadding_sent(client_side); + tt_int_op(mi->state_length, OP_EQ, 12); + + /* verify that sending padding changes our state length */ + for (uint64_t i = mi->state_length-1; i > 0; i--) { + circpad_send_padding_cell_for_callback(mi); + tt_int_op(mi->state_length, OP_EQ, i); + } + circpad_send_padding_cell_for_callback(mi); + + tt_int_op(mi->state_length, OP_EQ, -1); + tt_int_op(mi->current_state, OP_EQ, CIRCPAD_STATE_END); + + /* Restart machine */ + mi->current_state = CIRCPAD_STATE_START; + + /* Now, count nonpadding as part of the state length */ + client_machine->states[CIRCPAD_STATE_BURST].length_includes_nonpadding = 1; + + circpad_cell_event_padding_sent(client_side); + tt_int_op(mi->state_length, OP_EQ, 12); + + /* Verify that non-padding does change our state length now */ + for (uint64_t i = mi->state_length-1; i > 0; i--) { + circpad_cell_event_nonpadding_sent(client_side); + tt_int_op(mi->state_length, OP_EQ, i); + } + + circpad_cell_event_nonpadding_sent(client_side); + tt_int_op(mi->state_length, OP_EQ, -1); + tt_int_op(mi->current_state, OP_EQ, CIRCPAD_STATE_END); + + /* Now, just test token removal when we send padding */ + client_machine->states[CIRCPAD_STATE_BURST].token_removal = + CIRCPAD_TOKEN_REMOVAL_EXACT; + + /* Restart machine */ + mi->current_state = CIRCPAD_STATE_START; + circpad_cell_event_padding_sent(client_side); + tt_int_op(mi->state_length, OP_EQ, 12); + tt_ptr_op(mi->histogram, OP_NE, NULL); + tt_int_op(mi->chosen_bin, OP_EQ, 2); + + /* verify that sending padding changes our state length and + * our histogram now */ + for (uint32_t i = mi->histogram[2]-1; i > 0; i--) { + circpad_send_padding_cell_for_callback(mi); + tt_int_op(mi->chosen_bin, OP_EQ, 2); + tt_int_op(mi->histogram[2], OP_EQ, i); + } + + tt_int_op(mi->state_length, OP_EQ, 7); + tt_int_op(mi->histogram[2], OP_EQ, 1); + + circpad_send_padding_cell_for_callback(mi); + tt_int_op(mi->current_state, OP_EQ, CIRCPAD_STATE_END); + + done: + tor_free(client_machine->states); + tor_free(client_machine); + + free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side)); + free_fake_orcirc(relay_side); + + circuitmux_detach_all_circuits(dummy_channel.cmux, NULL); + circuitmux_free(dummy_channel.cmux); + timers_shutdown(); + monotime_disable_test_mocking(); + UNMOCK(circuit_package_relay_cell); + UNMOCK(circuitmux_attach_circuit); + UNMOCK(node_get_by_id); + + return; +} + void test_circuitpadding_conditions(void *arg) { @@ -2556,6 +2735,7 @@ test_circuitpadding_reduce_disable(void *arg) struct testcase_t circuitpadding_tests[] = { TEST_CIRCUITPADDING(circuitpadding_tokens, TT_FORK), + TEST_CIRCUITPADDING(circuitpadding_state_length, TT_FORK), TEST_CIRCUITPADDING(circuitpadding_negotiation, TT_FORK), TEST_CIRCUITPADDING(circuitpadding_wronghop, TT_FORK), /** Disabled unstable test until #29298 is implemented (see #29122) */ From 148c2d5bab051eb60e802fc6569c84a875c5f663 Mon Sep 17 00:00:00 2001 From: Mike Perry Date: Wed, 15 May 2019 03:33:53 +0000 Subject: [PATCH 0998/2557] Fix two typo bugs found by new state length test. --- src/core/or/circuitpadding.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/or/circuitpadding.c b/src/core/or/circuitpadding.c index 7674f3510c..3124e74033 100644 --- a/src/core/or/circuitpadding.c +++ b/src/core/or/circuitpadding.c @@ -253,7 +253,7 @@ circpad_histogram_bin_to_usec(const circpad_machine_runtime_t *mi, /* The infinity bin has an upper bound of infinity, so make sure we return * that if they ask for it. */ - if (bin > CIRCPAD_INFINITY_BIN(mi)) { + if (bin > CIRCPAD_INFINITY_BIN(state)) { return CIRCPAD_DELAY_INFINITE; } @@ -936,7 +936,7 @@ circpad_machine_update_state_length_for_nonpadding( { const circpad_state_t *state = NULL; - if (mi->state_length != CIRCPAD_STATE_LENGTH_INFINITE) + if (mi->state_length == CIRCPAD_STATE_LENGTH_INFINITE) return; state = circpad_machine_current_state(mi); From 5638d65f792393a85703342bfb14cc1a60550c18 Mon Sep 17 00:00:00 2001 From: Mike Perry Date: Wed, 15 May 2019 03:43:57 +0000 Subject: [PATCH 0999/2557] Check the token supply when we received a padding event, too. We need to check here because otherwise we can try to schedule padding with no tokens left upon the receipt of a padding event when our bins just became empty. --- src/core/or/circuitpadding.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/core/or/circuitpadding.c b/src/core/or/circuitpadding.c index 3124e74033..cd58a191d6 100644 --- a/src/core/or/circuitpadding.c +++ b/src/core/or/circuitpadding.c @@ -1699,8 +1699,16 @@ void circpad_cell_event_padding_sent(circuit_t *on_circ) { FOR_EACH_ACTIVE_CIRCUIT_MACHINE_BEGIN(i, on_circ) { - circpad_machine_spec_transition(on_circ->padding_info[i], + /* Check to see if we've run out of tokens for this state already, + * and if not, check for other state transitions */ + if (check_machine_token_supply(on_circ->padding_info[i]) + == CIRCPAD_STATE_UNCHANGED) { + /* If removing a token did not cause a transition, check if + * non-padding sent event should */ + + circpad_machine_spec_transition(on_circ->padding_info[i], CIRCPAD_EVENT_PADDING_SENT); + } } FOR_EACH_ACTIVE_CIRCUIT_MACHINE_END; } From 5f47d582d50134d9504ec8e7a912887829904c8a Mon Sep 17 00:00:00 2001 From: Mike Perry Date: Wed, 15 May 2019 04:58:21 +0000 Subject: [PATCH 1000/2557] Practracker beatings are even more fun when they get caused mid-PR due to a github auto-rebase of a PR Because github PRs choose the most recent origin/master at the time of the PR (and for any fixups pushed to a PR later to send to CI), there are tons of conflicts and unexpected practracker issues. This means CI can suddenly fail after fixups to a branch that pass locally. Then CI fails and we have to close and re-open the PR. --- scripts/maint/practracker/exceptions.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/maint/practracker/exceptions.txt b/scripts/maint/practracker/exceptions.txt index 24452b6158..b380d507b1 100644 --- a/scripts/maint/practracker/exceptions.txt +++ b/scripts/maint/practracker/exceptions.txt @@ -289,3 +289,4 @@ problem function-size /src/tools/tor-gencert.c:parse_commandline() 111 problem function-size /src/tools/tor-resolve.c:build_socks5_resolve_request() 104 problem function-size /src/tools/tor-resolve.c:do_resolve() 175 problem function-size /src/tools/tor-resolve.c:main() 112 +problem function-size /src/core/or/circuitpadding.c:circpad_machine_schedule_padding() 107 From d44e3e57b0904259041fa19a2fe14639ce476135 Mon Sep 17 00:00:00 2001 From: Mike Perry Date: Wed, 15 May 2019 04:42:28 +0000 Subject: [PATCH 1001/2557] Bug 28780: Add purpose for keeping padding circuits open --- src/core/or/circuitlist.c | 6 ++++++ src/core/or/circuitlist.h | 28 +++++++++++++++------------- src/core/or/circuituse.c | 1 + 3 files changed, 22 insertions(+), 13 deletions(-) diff --git a/src/core/or/circuitlist.c b/src/core/or/circuitlist.c index cd2259c98d..91a02ce56f 100644 --- a/src/core/or/circuitlist.c +++ b/src/core/or/circuitlist.c @@ -823,6 +823,8 @@ circuit_purpose_to_controller_string(uint8_t purpose) return "PATH_BIAS_TESTING"; case CIRCUIT_PURPOSE_HS_VANGUARDS: return "HS_VANGUARDS"; + case CIRCUIT_PURPOSE_C_CIRCUIT_PADDING: + return "CIRCUIT_PADDING"; default: tor_snprintf(buf, sizeof(buf), "UNKNOWN_%d", (int)purpose); @@ -852,6 +854,7 @@ circuit_purpose_to_controller_hs_state_string(uint8_t purpose) case CIRCUIT_PURPOSE_CONTROLLER: case CIRCUIT_PURPOSE_PATH_BIAS_TESTING: case CIRCUIT_PURPOSE_HS_VANGUARDS: + case CIRCUIT_PURPOSE_C_CIRCUIT_PADDING: return NULL; case CIRCUIT_PURPOSE_INTRO_POINT: @@ -952,6 +955,9 @@ circuit_purpose_to_string(uint8_t purpose) case CIRCUIT_PURPOSE_HS_VANGUARDS: return "Hidden service: Pre-built vanguard circuit"; + case CIRCUIT_PURPOSE_C_CIRCUIT_PADDING: + return "Circuit kept open for padding"; + default: tor_snprintf(buf, sizeof(buf), "UNKNOWN_%d", (int)purpose); return buf; diff --git a/src/core/or/circuitlist.h b/src/core/or/circuitlist.h index a50e23716a..6f5fce4875 100644 --- a/src/core/or/circuitlist.h +++ b/src/core/or/circuitlist.h @@ -92,31 +92,33 @@ #define CIRCUIT_PURPOSE_C_HS_MAX_ 13 /** This circuit is used for build time measurement only */ #define CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT 14 -#define CIRCUIT_PURPOSE_C_MAX_ 14 +/** This circuit is being held open by circuit padding */ +#define CIRCUIT_PURPOSE_C_CIRCUIT_PADDING 15 +#define CIRCUIT_PURPOSE_C_MAX_ 15 -#define CIRCUIT_PURPOSE_S_HS_MIN_ 15 +#define CIRCUIT_PURPOSE_S_HS_MIN_ 16 /** Hidden-service-side circuit purpose: at the service, waiting for * introductions. */ -#define CIRCUIT_PURPOSE_S_ESTABLISH_INTRO 15 +#define CIRCUIT_PURPOSE_S_ESTABLISH_INTRO 16 /** Hidden-service-side circuit purpose: at the service, successfully * established intro. */ -#define CIRCUIT_PURPOSE_S_INTRO 16 +#define CIRCUIT_PURPOSE_S_INTRO 17 /** Hidden-service-side circuit purpose: at the service, connecting to rend * point. */ -#define CIRCUIT_PURPOSE_S_CONNECT_REND 17 +#define CIRCUIT_PURPOSE_S_CONNECT_REND 18 /** Hidden-service-side circuit purpose: at the service, rendezvous * established. */ -#define CIRCUIT_PURPOSE_S_REND_JOINED 18 +#define CIRCUIT_PURPOSE_S_REND_JOINED 19 /** This circuit is used for uploading hsdirs */ -#define CIRCUIT_PURPOSE_S_HSDIR_POST 19 -#define CIRCUIT_PURPOSE_S_HS_MAX_ 19 +#define CIRCUIT_PURPOSE_S_HSDIR_POST 20 +#define CIRCUIT_PURPOSE_S_HS_MAX_ 20 /** A testing circuit; not meant to be used for actual traffic. */ -#define CIRCUIT_PURPOSE_TESTING 20 +#define CIRCUIT_PURPOSE_TESTING 21 /** A controller made this circuit and Tor should not use it. */ -#define CIRCUIT_PURPOSE_CONTROLLER 21 +#define CIRCUIT_PURPOSE_CONTROLLER 22 /** This circuit is used for path bias probing only */ -#define CIRCUIT_PURPOSE_PATH_BIAS_TESTING 22 +#define CIRCUIT_PURPOSE_PATH_BIAS_TESTING 23 /** This circuit is used for vanguards/restricted paths. * @@ -124,9 +126,9 @@ * on-demand. When an HS operation needs to take place (e.g. connect to an * intro point), these circuits are then cannibalized and repurposed to the * actual needed HS purpose. */ -#define CIRCUIT_PURPOSE_HS_VANGUARDS 23 +#define CIRCUIT_PURPOSE_HS_VANGUARDS 24 -#define CIRCUIT_PURPOSE_MAX_ 23 +#define CIRCUIT_PURPOSE_MAX_ 24 /** A catch-all for unrecognized purposes. Currently we don't expect * to make or see any circuits with this purpose. */ #define CIRCUIT_PURPOSE_UNKNOWN 255 diff --git a/src/core/or/circuituse.c b/src/core/or/circuituse.c index 226295425e..bdab476837 100644 --- a/src/core/or/circuituse.c +++ b/src/core/or/circuituse.c @@ -1514,6 +1514,7 @@ circuit_expire_old_circuits_clientside(void) circ->purpose == CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT || circ->purpose == CIRCUIT_PURPOSE_S_ESTABLISH_INTRO || circ->purpose == CIRCUIT_PURPOSE_TESTING || + circ->purpose == CIRCUIT_PURPOSE_C_CIRCUIT_PADDING || (circ->purpose >= CIRCUIT_PURPOSE_C_INTRODUCING && circ->purpose <= CIRCUIT_PURPOSE_C_REND_READY_INTRO_ACKED) || circ->purpose == CIRCUIT_PURPOSE_S_CONNECT_REND) { From 662825474cb7d5fb0e713844aef5e912edd6ed8a Mon Sep 17 00:00:00 2001 From: Mike Perry Date: Thu, 18 Apr 2019 20:40:55 +0000 Subject: [PATCH 1002/2557] Bug 28780: Make use of purpose to keep padding circuits open. When a circuit is marked for close, check to see if any of our padding machines want to take ownership of it and continue padding until the machine hits the END state. For safety, we also ensure that machines that do not terminate are still closed as follows: Because padding machine timers are UINT32_MAX in size, if some sort of network event doesn't happen on a padding-only circuit within that time, we can conclude it is deadlocked and allow circuit_expire_old_circuits_clientside() to close it. If too much network activity happens, then per-machine padding limits can be used to cease padding, which will cause network cell events to cease, on the circuit, which will cause circpad to abandon the circuit as per the above time limit. --- src/core/or/circuitlist.c | 5 ++ src/core/or/circuitpadding.c | 119 ++++++++++++++++++++++++++++++++++- src/core/or/circuitpadding.h | 31 +++++++++ 3 files changed, 153 insertions(+), 2 deletions(-) diff --git a/src/core/or/circuitlist.c b/src/core/or/circuitlist.c index 91a02ce56f..72952a8a52 100644 --- a/src/core/or/circuitlist.c +++ b/src/core/or/circuitlist.c @@ -2195,6 +2195,11 @@ circuit_mark_for_close_, (circuit_t *circ, int reason, int line, tor_assert(line); tor_assert(file); + /* Check whether the circuitpadding subsystem wants to block this close */ + if (circpad_marked_circuit_for_padding(circ, reason)) { + return; + } + if (circ->marked_for_close) { log_warn(LD_BUG, "Duplicate call to circuit_mark_for_close at %s:%d" diff --git a/src/core/or/circuitpadding.c b/src/core/or/circuitpadding.c index 58e8e053c7..68140356c9 100644 --- a/src/core/or/circuitpadding.c +++ b/src/core/or/circuitpadding.c @@ -163,6 +163,116 @@ circpad_circuit_machineinfo_free_idx(circuit_t *circ, int idx) } } +/** + * Return true if circpad has decided to hold the circuit open for additional + * padding. This function is used to take and retain ownership of certain + * types of circuits that have padding machines on them, that have been passed + * to circuit_mark_for_close(). + * + * circuit_mark_for_close() calls this function to ask circpad if any padding + * machines want to keep the circuit open longer to pad. + * + * Any non-measurement circuit that was closed for a normal, non-error reason + * code may be held open for up to CIRCPAD_DELAY_INFINITE microseconds between + * network-driven cell events. + * + * After CIRCPAD_DELAY_INFINITE microseconds of silence on a circuit, this + * function will no longer hold it open (it will return 0 regardless of + * what the machines ask for, and thus circuit_expire_old_circuits_clientside() + * will close the circuit after roughly 1.25hr of idle time, maximum, + * regardless of the padding machine state. + */ +int +circpad_marked_circuit_for_padding(circuit_t *circ, int reason) +{ + /* If the circuit purpose is measurement or path bias, don't + * hold it open */ + if (circ->purpose == CIRCUIT_PURPOSE_PATH_BIAS_TESTING || + circ->purpose == CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT) { + return 0; + } + + /* If the circuit is closed for any reason other than these three valid, + * client-side close reasons, do not try to keep it open. It is probably + * damaged or unusable. Note this is OK with vanguards because + * controller-closed circuits have REASON=REQUESTED, so vanguards-closed + * circuits will not be held open (we want them to close ASAP). */ + if (!(reason == END_CIRC_REASON_NONE || + reason == END_CIRC_REASON_FINISHED || + reason == END_CIRC_REASON_IP_NOW_REDUNDANT)) { + return 0; + } + + FOR_EACH_ACTIVE_CIRCUIT_MACHINE_BEGIN(i, circ) { + circpad_machine_runtime_t *mi = circ->padding_info[i]; + if (!mi) { + continue; // No padding runtime info; check next machine + } + + const circpad_state_t *state = circpad_machine_current_state(mi); + + /* If we're in END state (NULL here), then check next machine */ + if (!state) { + continue; // check next machine + } + + /* If the machine does not want to control the circuit close itself, then + * check the next machine */ + if (!circ->padding_machine[i]->manage_circ_lifetime) { + continue; // check next machine + } + + /* If the machine has reached the END state, we can close. Check next + * machine. */ + if (mi->current_state == CIRCPAD_STATE_END) { + continue; // check next machine + } + + log_info(LD_CIRC, "Circuit %d is not marked for close because of a " + " pending padding machine.", CIRCUIT_IS_ORIGIN(circ) ? + TO_ORIGIN_CIRCUIT(circ)->global_identifier : 0); + + /* If the machine has had no network events at all within the + * last circpad_delay_t timespan, it's in some deadlock state. + * Tell circuit_mark_for_close() that we don't own it anymore. + * This will allow circuit_expire_old_circuits_clientside() to + * close it. + */ + if (circ->padding_info[i]->last_cell_time_sec + + (time_t)CIRCPAD_DELAY_MAX_SECS < approx_time()) { + log_notice(LD_BUG, "Circuit %d was not marked for close because of a " + " pending padding machine for over an hour. Circuit is a %s", + CIRCUIT_IS_ORIGIN(circ) ? + TO_ORIGIN_CIRCUIT(circ)->global_identifier : 0, + circuit_purpose_to_string(circ->purpose)); + + return 0; // abort timer reached; mark the circuit for close now + } + + /* If we weren't marked dirty yet, let's pretend we're dirty now. + * ("Dirty" means that a circuit has been used for application traffic + * by Tor.. Dirty circuits have different expiry times, and are not + * considered in counts of built circuits, etc. By claiming that we're + * dirty, the rest of Tor will make decisions as if we were actually + * used by application data. + * + * This is most important for circuit_expire_old_circuits_clientside(), + * where we want that function to expire us after the padding machine + * has shut down, but using the MaxCircuitDirtiness timer instead of + * the idle circuit timer (again, we want this because we're not + * supposed to look idle to Guard nodes that can see our lifespan). */ + if (!circ->timestamp_dirty) + circ->timestamp_dirty = approx_time(); + + /* Take ownership of the circuit */ + circuit_change_purpose(circ, CIRCUIT_PURPOSE_C_CIRCUIT_PADDING); + + return 1; + } FOR_EACH_ACTIVE_CIRCUIT_MACHINE_END; + + return 0; // No machine wanted to keep the circuit open; mark for close +} + /** Free all the machineinfos in circ that match machine_num. */ static void free_circ_machineinfos_with_machine_num(circuit_t *circ, int machine_num) @@ -197,6 +307,7 @@ circpad_circuit_machineinfo_new(circuit_t *on_circ, int machine_index) tor_malloc_zero(sizeof(circpad_machine_runtime_t)); mi->machine_index = machine_index; mi->on_circ = on_circ; + mi->last_cell_time_sec = approx_time(); return mi; } @@ -1538,7 +1649,8 @@ circpad_cell_event_nonpadding_sent(circuit_t *on_circ) /* If there are no machines then this loop should not iterate */ FOR_EACH_ACTIVE_CIRCUIT_MACHINE_BEGIN(i, on_circ) { - /* First, update any RTT estimate */ + /* First, update any timestamps */ + on_circ->padding_info[i]->last_cell_time_sec = approx_time(); circpad_estimate_circ_rtt_on_send(on_circ, on_circ->padding_info[i]); /* Remove a token: this is the idea of adaptive padding, since we have an @@ -1564,7 +1676,8 @@ void circpad_cell_event_nonpadding_received(circuit_t *on_circ) { FOR_EACH_ACTIVE_CIRCUIT_MACHINE_BEGIN(i, on_circ) { - /* First, update any RTT estimate */ + /* First, update any timestamps */ + on_circ->padding_info[i]->last_cell_time_sec = approx_time(); circpad_estimate_circ_rtt_on_received(on_circ, on_circ->padding_info[i]); circpad_machine_spec_transition(on_circ->padding_info[i], @@ -1584,6 +1697,7 @@ void circpad_cell_event_padding_sent(circuit_t *on_circ) { FOR_EACH_ACTIVE_CIRCUIT_MACHINE_BEGIN(i, on_circ) { + on_circ->padding_info[i]->last_cell_time_sec = approx_time(); circpad_machine_spec_transition(on_circ->padding_info[i], CIRCPAD_EVENT_PADDING_SENT); } FOR_EACH_ACTIVE_CIRCUIT_MACHINE_END; @@ -1602,6 +1716,7 @@ circpad_cell_event_padding_received(circuit_t *on_circ) { /* identical to padding sent */ FOR_EACH_ACTIVE_CIRCUIT_MACHINE_BEGIN(i, on_circ) { + on_circ->padding_info[i]->last_cell_time_sec = approx_time(); circpad_machine_spec_transition(on_circ->padding_info[i], CIRCPAD_EVENT_PADDING_RECV); } FOR_EACH_ACTIVE_CIRCUIT_MACHINE_END; diff --git a/src/core/or/circuitpadding.h b/src/core/or/circuitpadding.h index f00369eb0a..56ed0a3467 100644 --- a/src/core/or/circuitpadding.h +++ b/src/core/or/circuitpadding.h @@ -73,6 +73,7 @@ typedef uint64_t circpad_time_t; /** The type for timer delays, in microseconds */ typedef uint32_t circpad_delay_t; +#define CIRCPAD_DELAY_UNITS_PER_SECOND (1000*1000) /** * An infinite padding cell delay means don't schedule any padding -- @@ -85,6 +86,13 @@ typedef uint32_t circpad_delay_t; */ #define CIRCPAD_DELAY_INFINITE (UINT32_MAX) +/** + * This is the maximum delay that the circuit padding system can have, in + * seconds. + */ +#define CIRCPAD_DELAY_MAX_SECS \ + ((CIRCPAD_DELAY_INFINITE/CIRCPAD_DELAY_UNITS_PER_SECOND)+1) + /** * Macro to clarify when we're checking the infinity bin. * @@ -524,6 +532,14 @@ typedef struct circpad_machine_runtime_t { * half. */ uint16_t nonpadding_sent; + /** + * Timestamp of the most recent cell event (sent, received, padding, + * non-padding), in seconds from approx_time(). + * + * Used as an emergency break to stop holding padding circuits open. + */ + time_t last_cell_time_sec; + /** * EWMA estimate of the RTT of the circuit from this hop * to the exit end, in microseconds. */ @@ -605,6 +621,19 @@ typedef struct circpad_machine_spec_t { * 1-indexed (ie: hop #1 is guard, #2 middle, #3 exit). */ unsigned target_hopnum : 3; + /** If this flag is enabled, don't close circuits that use this machine even + * if another part of Tor wants to close this circuit. + * + * If this flag is set, the circuitpadding subsystem will close circuits the + * moment the machine transitions to the END state, and only if the circuit + * has already been asked to be closed by another part of Tor. + * + * Circuits that should have been closed but were kept open by a padding + * machine are re-purposed to CIRCUIT_PURPOSE_C_CIRCUIT_PADDING, hence + * machines should take that purpose into account if they are filtering + * circuits by purpose. */ + unsigned manage_circ_lifetime : 1; + /** This machine only kills fascists if the following conditions are met. */ circpad_machine_conditions_t conditions; @@ -630,6 +659,8 @@ typedef struct circpad_machine_spec_t { void circpad_new_consensus_params(const networkstatus_t *ns); +int circpad_marked_circuit_for_padding(circuit_t *circ, int reason); + /** * The following are event call-in points that are of interest to * the state machines. They are called during cell processing. */ From e253a117c0a2db9315ac44f09cfba5e854f4e172 Mon Sep 17 00:00:00 2001 From: Mike Perry Date: Wed, 15 May 2019 04:46:05 +0000 Subject: [PATCH 1003/2557] Bug 28780: Add tests Also test circpad expiry safeguard. --- src/core/or/circuituse.c | 4 +- src/test/test_circuitpadding.c | 188 +++++++++++++++++++++++++++++++++ 2 files changed, 190 insertions(+), 2 deletions(-) diff --git a/src/core/or/circuituse.c b/src/core/or/circuituse.c index bdab476837..485c389054 100644 --- a/src/core/or/circuituse.c +++ b/src/core/or/circuituse.c @@ -70,7 +70,7 @@ #include "core/or/origin_circuit_st.h" #include "core/or/socks_request_st.h" -static void circuit_expire_old_circuits_clientside(void); +STATIC void circuit_expire_old_circuits_clientside(void); static void circuit_increment_failure_count(void); /** Check whether the hidden service destination of the stream at @@ -1474,7 +1474,7 @@ circuit_detach_stream(circuit_t *circ, edge_connection_t *conn) /** Find each circuit that has been unused for too long, or dirty * for too long and has no streams on it: mark it for close. */ -static void +STATIC void circuit_expire_old_circuits_clientside(void) { struct timeval cutoff, now; diff --git a/src/test/test_circuitpadding.c b/src/test/test_circuitpadding.c index db175fecee..2e67159440 100644 --- a/src/test/test_circuitpadding.c +++ b/src/test/test_circuitpadding.c @@ -41,6 +41,7 @@ TOR_NSEC_PER_USEC*TOR_USEC_PER_SEC) extern smartlist_t *connection_array; +void circuit_expire_old_circuits_clientside(void); circid_t get_unique_circ_id_by_chan(channel_t *chan); void helper_create_basic_machine(void); @@ -2550,6 +2551,192 @@ test_circuitpadding_reduce_disable(void *arg) circuitmux_free(dummy_channel.cmux); } +/** Just a basic machine whose whole purpose is to reach the END state */ +static void +helper_create_ender_machine(void) +{ + /* Start, burst */ + circpad_machine_states_init(&circ_client_machine, 2); + + circ_client_machine.states[CIRCPAD_STATE_START]. + next_state[CIRCPAD_EVENT_NONPADDING_RECV] = CIRCPAD_STATE_END; + + circ_client_machine.conditions.state_mask = CIRCPAD_STATE_ALL; + circ_client_machine.conditions.purpose_mask = CIRCPAD_PURPOSE_ALL; +} + +static time_t mocked_timeofday; +/** Set timeval to a mock date and time. This is necessary + * to make tor_gettimeofday() mockable. */ +static void +mock_tor_gettimeofday(struct timeval *timeval) +{ + timeval->tv_sec = mocked_timeofday; + timeval->tv_usec = 0; +} + +/** Test manual managing of circuit lifetimes by the circuitpadding + * subsystem. In particular this test goes through all the cases of the + * circpad_marked_circuit_for_padding() function, via + * circuit_mark_for_close() as well as + * circuit_expire_old_circuits_clientside(). */ +static void +test_circuitpadding_manage_circuit_lifetime(void *arg) +{ + circpad_machine_runtime_t *mi; + + (void) arg; + + client_side = (circuit_t *)origin_circuit_new(); + client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL; + monotime_enable_test_mocking(); + MOCK(tor_gettimeofday, mock_tor_gettimeofday); + mocked_timeofday = 23; + + helper_create_ender_machine(); + + /* Enable manual circuit lifetime manage for this test */ + circ_client_machine.manage_circ_lifetime = 1; + + /* Test setup */ + client_side->padding_machine[0] = &circ_client_machine; + client_side->padding_info[0] = + circpad_circuit_machineinfo_new(client_side, 0); + mi = client_side->padding_info[0]; + + tt_int_op(mi->current_state, OP_EQ, CIRCPAD_STATE_START); + + /* Check that the circuit is not marked for close */ + tt_int_op(client_side->marked_for_close, OP_EQ, 0); + tt_int_op(client_side->purpose, OP_EQ, CIRCUIT_PURPOSE_C_GENERAL); + + /* Mark this circuit for close due to a remote reason */ + circuit_mark_for_close(client_side, + END_CIRC_REASON_FLAG_REMOTE|END_CIRC_REASON_NONE); + tt_ptr_op(client_side->padding_info[0], OP_NE, NULL); + tt_int_op(client_side->marked_for_close, OP_NE, 0); + tt_int_op(client_side->purpose, OP_EQ, CIRCUIT_PURPOSE_C_GENERAL); + client_side->marked_for_close = 0; + + /* Mark this circuit for close due to a protocol issue */ + circuit_mark_for_close(client_side, END_CIRC_REASON_TORPROTOCOL); + tt_int_op(client_side->marked_for_close, OP_NE, 0); + tt_int_op(client_side->purpose, OP_EQ, CIRCUIT_PURPOSE_C_GENERAL); + client_side->marked_for_close = 0; + + /* Mark a measurement circuit for close */ + client_side->purpose = CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT; + circuit_mark_for_close(client_side, END_CIRC_REASON_NONE); + tt_int_op(client_side->marked_for_close, OP_NE, 0); + tt_int_op(client_side->purpose, OP_EQ, CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT); + client_side->marked_for_close = 0; + + /* Mark a general circuit for close */ + client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL; + circuit_mark_for_close(client_side, END_CIRC_REASON_NONE); + + /* Check that this circuit is still not marked for close since we are + * managing the lifetime manually, but the circuit was tagged as such by the + * circpadding subsystem */ + tt_int_op(client_side->marked_for_close, OP_EQ, 0); + tt_int_op(client_side->purpose, OP_EQ, CIRCUIT_PURPOSE_C_CIRCUIT_PADDING); + + /* We just tested case (1) from the comments of + * circpad_circuit_should_be_marked_for_close() */ + + /* Transition the machine to the END state but did not delete its machine */ + tt_ptr_op(client_side->padding_info[0], OP_NE, NULL); + circpad_cell_event_nonpadding_received(client_side); + tt_int_op(mi->current_state, OP_EQ, CIRCPAD_STATE_END); + + /* We just tested case (3) from the comments of + * circpad_circuit_should_be_marked_for_close(). + * Now let's go for case (2). */ + + /* Reset the close mark */ + client_side->marked_for_close = 0; + + /* Mark this circuit for close */ + circuit_mark_for_close(client_side, 0); + + /* See that the circ got closed since we are already in END state */ + tt_int_op(client_side->marked_for_close, OP_NE, 0); + + /* We just tested case (2). Now let's see that case (4) is unreachable as + that comment claims */ + + /* First, reset all close marks and tags */ + client_side->marked_for_close = 0; + client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL; + + /* Now re-create the ender machine so that we can transition to END again */ + /* Free up some stuff first */ + circpad_circuit_free_all_machineinfos(client_side); + tor_free(circ_client_machine.states); + helper_create_ender_machine(); + + client_side->padding_machine[0] = &circ_client_machine; + client_side->padding_info[0] = + circpad_circuit_machineinfo_new(client_side, 0); + mi = client_side->padding_info[0]; + + /* Check we are in START. */ + tt_int_op(mi->current_state, OP_EQ, CIRCPAD_STATE_START); + + /* Test that we don't expire this circuit yet */ + client_side->timestamp_dirty = 0; + client_side->state = CIRCUIT_STATE_OPEN; + tor_gettimeofday(&client_side->timestamp_began); + TO_ORIGIN_CIRCUIT(client_side)->circuit_idle_timeout = 23; + mocked_timeofday += 24; + circuit_expire_old_circuits_clientside(); + circuit_expire_old_circuits_clientside(); + circuit_expire_old_circuits_clientside(); + tt_int_op(client_side->timestamp_dirty, OP_NE, 0); + tt_int_op(client_side->marked_for_close, OP_EQ, 0); + tt_int_op(client_side->purpose, OP_EQ, CIRCUIT_PURPOSE_C_CIRCUIT_PADDING); + + /* Runaway circpad test: if the machine does not transition to end, + * test that after CIRCPAD_DELAY_MAX_SECS, we get marked anyway */ + mocked_timeofday = client_side->timestamp_dirty + + get_options()->MaxCircuitDirtiness + 2; + client_side->padding_info[0]->last_cell_time_sec = + approx_time()-(CIRCPAD_DELAY_MAX_SECS+10); + circuit_expire_old_circuits_clientside(); + tt_int_op(client_side->marked_for_close, OP_NE, 0); + + /* Test back to normal: if we had activity, we won't close */ + client_side->padding_info[0]->last_cell_time_sec = approx_time(); + client_side->marked_for_close = 0; + circuit_expire_old_circuits_clientside(); + tt_int_op(client_side->marked_for_close, OP_EQ, 0); + + /* Transition to END, but before we're past the dirty timer */ + mocked_timeofday = client_side->timestamp_dirty; + circpad_cell_event_nonpadding_received(client_side); + tt_int_op(mi->current_state, OP_EQ, CIRCPAD_STATE_END); + + /* Verify that the circuit was not closed. */ + tt_int_op(client_side->marked_for_close, OP_EQ, 0); + + /* Now that we are in END state, we can be closed by expiry, but via + * the timestamp_dirty path, not the idle path. So first test not dirty + * enough. */ + mocked_timeofday = client_side->timestamp_dirty; + circuit_expire_old_circuits_clientside(); + tt_int_op(client_side->marked_for_close, OP_EQ, 0); + mocked_timeofday = client_side->timestamp_dirty + + get_options()->MaxCircuitDirtiness + 2; + circuit_expire_old_circuits_clientside(); + tt_int_op(client_side->marked_for_close, OP_NE, 0); + + done: + free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side)); + tor_free(circ_client_machine.states); + monotime_disable_test_mocking(); + UNMOCK(tor_gettimeofday); +} + #define TEST_CIRCUITPADDING(name, flags) \ { #name, test_##name, (flags), NULL, NULL } @@ -2570,5 +2757,6 @@ struct testcase_t circuitpadding_tests[] = { TEST_CIRCUITPADDING(circuitpadding_closest_token_removal, TT_FORK), TEST_CIRCUITPADDING(circuitpadding_closest_token_removal_usec, TT_FORK), TEST_CIRCUITPADDING(circuitpadding_token_removal_exact, TT_FORK), + TEST_CIRCUITPADDING(circuitpadding_manage_circuit_lifetime, TT_FORK), END_OF_TESTCASES }; From 5cc988a947e8ba10e2682274465a82ff22e7c93d Mon Sep 17 00:00:00 2001 From: Mike Perry Date: Thu, 18 Apr 2019 20:35:22 +0000 Subject: [PATCH 1004/2557] Bug 28780: Changes file --- changes/ticket28780 | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 changes/ticket28780 diff --git a/changes/ticket28780 b/changes/ticket28780 new file mode 100644 index 0000000000..d7c6693f8c --- /dev/null +++ b/changes/ticket28780 @@ -0,0 +1,3 @@ + o Minor features (circuit padding): + - Provide the ability for circuit padding machines to hold a circuit open + until they are done padding it. Closes ticket 28780. From b98bcd789e45fb6a09ef92ab15052a6c19e24887 Mon Sep 17 00:00:00 2001 From: Mike Perry Date: Wed, 8 May 2019 21:21:03 +0000 Subject: [PATCH 1005/2557] Pathbias should continue to ignore previously ignored circs. If circuit padding wants to keep a circuit open and pathbias used to ignore it, pathbias should continue to ignore it. This may catch other purpose-change related miscounts (such as timeout measurement, cannibalization, onion service circuit transitions, and vanguards). --- src/feature/client/circpathbias.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/feature/client/circpathbias.c b/src/feature/client/circpathbias.c index e6af649ba7..2be9984a64 100644 --- a/src/feature/client/circpathbias.c +++ b/src/feature/client/circpathbias.c @@ -400,12 +400,13 @@ pathbias_should_count(origin_circuit_t *circ) /* Check to see if the shouldcount result has changed due to a * unexpected purpose change that would affect our results */ if (circ->pathbias_shouldcount == PATHBIAS_SHOULDCOUNT_IGNORED) { - log_info(LD_BUG, - "Circuit %d is now being counted despite being ignored " - "in the past. Purpose is %s, path state is %s", - circ->global_identifier, - circuit_purpose_to_string(circ->base_.purpose), - pathbias_state_to_string(circ->path_state)); + log_info(LD_CIRC, + "Circuit %d is not being counted by pathbias because it was " + "ignored in the past. Purpose is %s, path state is %s", + circ->global_identifier, + circuit_purpose_to_string(circ->base_.purpose), + pathbias_state_to_string(circ->path_state)); + return 0; } circ->pathbias_shouldcount = PATHBIAS_SHOULDCOUNT_COUNTED; From 56738ff8c626b21d6e3d146c9a2c0c4d18f3884a Mon Sep 17 00:00:00 2001 From: Mike Perry Date: Wed, 8 May 2019 21:28:00 +0000 Subject: [PATCH 1006/2557] Add control port circuit ID to all pathbias bug messages. To ease debugging of miscount issues, attach vanguards with --loglevel DEBUG and obtain control port logs (or use any other control port CIRC and CIRC_MINOR event logging mechanism). --- src/feature/client/circpathbias.c | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/feature/client/circpathbias.c b/src/feature/client/circpathbias.c index 2be9984a64..3544dbb859 100644 --- a/src/feature/client/circpathbias.c +++ b/src/feature/client/circpathbias.c @@ -371,8 +371,9 @@ pathbias_should_count(origin_circuit_t *circ) !circ->build_state->onehop_tunnel) { if ((rate_msg = rate_limit_log(&count_limit, approx_time()))) { log_info(LD_BUG, - "One-hop circuit has length %d. Path state is %s. " + "One-hop circuit %d has length %d. Path state is %s. " "Circuit is a %s currently %s.%s", + circ->global_identifier, circ->build_state->desired_path_len, pathbias_state_to_string(circ->path_state), circuit_purpose_to_string(circ->base_.purpose), @@ -437,8 +438,9 @@ pathbias_count_build_attempt(origin_circuit_t *circ) if ((rate_msg = rate_limit_log(&circ_attempt_notice_limit, approx_time()))) { log_info(LD_BUG, - "Opened circuit is in strange path state %s. " + "Opened circuit %d is in strange path state %s. " "Circuit is a %s currently %s.%s", + circ->global_identifier, pathbias_state_to_string(circ->path_state), circuit_purpose_to_string(circ->base_.purpose), circuit_state_to_string(circ->base_.state), @@ -471,8 +473,9 @@ pathbias_count_build_attempt(origin_circuit_t *circ) if ((rate_msg = rate_limit_log(&circ_attempt_notice_limit, approx_time()))) { log_info(LD_BUG, - "Unopened circuit has strange path state %s. " + "Unopened circuit %d has strange path state %s. " "Circuit is a %s currently %s.%s", + circ->global_identifier, pathbias_state_to_string(circ->path_state), circuit_purpose_to_string(circ->base_.purpose), circuit_state_to_string(circ->base_.state), @@ -541,8 +544,9 @@ pathbias_count_build_success(origin_circuit_t *circ) if ((rate_msg = rate_limit_log(&success_notice_limit, approx_time()))) { log_info(LD_BUG, - "Succeeded circuit is in strange path state %s. " + "Succeeded circuit %d is in strange path state %s. " "Circuit is a %s currently %s.%s", + circ->global_identifier, pathbias_state_to_string(circ->path_state), circuit_purpose_to_string(circ->base_.purpose), circuit_state_to_string(circ->base_.state), @@ -577,8 +581,9 @@ pathbias_count_build_success(origin_circuit_t *circ) if ((rate_msg = rate_limit_log(&success_notice_limit, approx_time()))) { log_info(LD_BUG, - "Opened circuit is in strange path state %s. " + "Opened circuit %d is in strange path state %s. " "Circuit is a %s currently %s.%s", + circ->global_identifier, pathbias_state_to_string(circ->path_state), circuit_purpose_to_string(circ->base_.purpose), circuit_state_to_string(circ->base_.state), @@ -604,8 +609,9 @@ pathbias_count_use_attempt(origin_circuit_t *circ) if (circ->path_state < PATH_STATE_BUILD_SUCCEEDED) { log_notice(LD_BUG, - "Used circuit is in strange path state %s. " + "Used circuit %d is in strange path state %s. " "Circuit is a %s currently %s.", + circ->global_identifier, pathbias_state_to_string(circ->path_state), circuit_purpose_to_string(circ->base_.purpose), circuit_state_to_string(circ->base_.state)); From ff2a9809353c1166cb494dac3a14502b8ee24013 Mon Sep 17 00:00:00 2001 From: Mike Perry Date: Wed, 15 May 2019 04:47:50 +0000 Subject: [PATCH 1007/2557] The practracker beatings will continue until the files get smaller. --- scripts/maint/practracker/exceptions.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/maint/practracker/exceptions.txt b/scripts/maint/practracker/exceptions.txt index 24452b6158..aea6ef84bf 100644 --- a/scripts/maint/practracker/exceptions.txt +++ b/scripts/maint/practracker/exceptions.txt @@ -92,7 +92,7 @@ problem function-size /src/core/or/circuitlist.c:circuits_handle_oom() 117 problem function-size /src/core/or/circuitmux.c:circuitmux_set_policy() 110 problem function-size /src/core/or/circuitmux.c:circuitmux_attach_circuit() 114 problem function-size /src/core/or/circuitstats.c:circuit_build_times_parse_state() 124 -problem file-size /src/core/or/circuituse.c 3155 +problem file-size /src/core/or/circuituse.c 3156 problem function-size /src/core/or/circuituse.c:circuit_is_acceptable() 133 problem function-size /src/core/or/circuituse.c:circuit_expire_building() 394 problem function-size /src/core/or/circuituse.c:circuit_log_ancient_one_hop_circuits() 126 From d86896b29c7a5278bedd89a7150ddbe7531b365b Mon Sep 17 00:00:00 2001 From: Roger Dingledine Date: Wed, 15 May 2019 23:20:03 -0400 Subject: [PATCH 1008/2557] fix typos, whitespace, comments --- src/core/include.am | 22 +++++++++++----------- src/core/or/connection_edge.c | 4 ++-- src/core/or/relay.c | 6 +++--- src/core/or/sendme.c | 18 +++++++++--------- src/feature/dirauth/shared_random_state.c | 2 +- src/feature/hs/hs_client.c | 2 +- src/feature/hs/hs_common.c | 6 +++--- src/feature/hs/hs_service.c | 2 +- src/trunnel/include.am | 2 +- 9 files changed, 32 insertions(+), 32 deletions(-) diff --git a/src/core/include.am b/src/core/include.am index b04c5db541..f722b70481 100644 --- a/src/core/include.am +++ b/src/core/include.am @@ -47,7 +47,7 @@ LIBTOR_APP_A_SOURCES = \ src/core/or/onion.c \ src/core/or/ocirc_event.c \ src/core/or/or_periodic.c \ - src/core/or/or_sys.c \ + src/core/or/or_sys.c \ src/core/or/orconn_event.c \ src/core/or/policies.c \ src/core/or/protover.c \ @@ -140,7 +140,7 @@ LIBTOR_APP_A_SOURCES = \ src/feature/relay/dns.c \ src/feature/relay/ext_orport.c \ src/feature/relay/onion_queue.c \ - src/feature/relay/relay_periodic.c \ + src/feature/relay/relay_periodic.c \ src/feature/relay/relay_sys.c \ src/feature/relay/router.c \ src/feature/relay/routerkeys.c \ @@ -171,7 +171,7 @@ LIBTOR_APP_TESTING_A_SOURCES = $(LIBTOR_APP_A_SOURCES) # The Directory Authority module. MODULE_DIRAUTH_SOURCES = \ src/feature/dirauth/authmode.c \ - src/feature/dirauth/bridgeauth.c \ + src/feature/dirauth/bridgeauth.c \ src/feature/dirauth/bwauth.c \ src/feature/dirauth/dirauth_periodic.c \ src/feature/dirauth/dirauth_sys.c \ @@ -252,14 +252,14 @@ noinst_HEADERS += \ src/core/or/connection_edge.h \ src/core/or/connection_or.h \ src/core/or/connection_st.h \ - src/core/or/crypt_path.h \ + src/core/or/crypt_path.h \ src/core/or/cpath_build_state_st.h \ src/core/or/crypt_path_reference_st.h \ src/core/or/crypt_path_st.h \ src/core/or/destroy_cell_queue_st.h \ src/core/or/dos.h \ src/core/or/edge_connection_st.h \ - src/core/or/half_edge_st.h \ + src/core/or/half_edge_st.h \ src/core/or/entry_connection_st.h \ src/core/or/entry_port_cfg_st.h \ src/core/or/extend_info_st.h \ @@ -267,7 +267,7 @@ noinst_HEADERS += \ src/core/or/onion.h \ src/core/or/or.h \ src/core/or/or_periodic.h \ - src/core/or/or_sys.h \ + src/core/or/or_sys.h \ src/core/or/orconn_event.h \ src/core/or/orconn_event_sys.h \ src/core/or/or_circuit_st.h \ @@ -313,14 +313,14 @@ noinst_HEADERS += \ src/feature/control/control_cmd.h \ src/feature/control/control_cmd_args_st.h \ src/feature/control/control_connection_st.h \ - src/feature/control/control_events.h \ + src/feature/control/control_events.h \ src/feature/control/control_fmt.h \ src/feature/control/control_getinfo.h \ src/feature/control/control_proto.h \ src/feature/control/fmt_serverstatus.h \ src/feature/control/getinfo_geoip.h \ src/feature/dirauth/authmode.h \ - src/feature/dirauth/bridgeauth.h \ + src/feature/dirauth/bridgeauth.h \ src/feature/dirauth/bwauth.h \ src/feature/dirauth/dirauth_periodic.h \ src/feature/dirauth/dirauth_sys.h \ @@ -395,8 +395,8 @@ noinst_HEADERS += \ src/feature/nodelist/networkstatus_voter_info_st.h \ src/feature/nodelist/nickname.h \ src/feature/nodelist/node_st.h \ - src/feature/nodelist/nodefamily.h \ - src/feature/nodelist/nodefamily_st.h \ + src/feature/nodelist/nodefamily.h \ + src/feature/nodelist/nodefamily_st.h \ src/feature/nodelist/nodelist.h \ src/feature/nodelist/node_select.h \ src/feature/nodelist/routerinfo.h \ @@ -413,7 +413,7 @@ noinst_HEADERS += \ src/feature/relay/dns_structs.h \ src/feature/relay/ext_orport.h \ src/feature/relay/onion_queue.h \ - src/feature/relay/relay_periodic.h \ + src/feature/relay/relay_periodic.h \ src/feature/relay/relay_sys.h \ src/feature/relay/router.h \ src/feature/relay/routerkeys.h \ diff --git a/src/core/or/connection_edge.c b/src/core/or/connection_edge.c index 8ed4034560..c08d2a9ff5 100644 --- a/src/core/or/connection_edge.c +++ b/src/core/or/connection_edge.c @@ -2816,7 +2816,7 @@ static const char HTTP_CONNECT_IS_NOT_AN_HTTP_PROXY_MSG[] = "Content-Type: text/html; charset=iso-8859-1\r\n\r\n" "\n" "\n" - "This is an HTTP CONNECT tunnel, not an full HTTP Proxy\n" + "This is an HTTP CONNECT tunnel, not a full HTTP Proxy\n" "\n" "\n" "

This is an HTTP CONNECT tunnel, not an HTTP proxy.

\n" @@ -2826,7 +2826,7 @@ static const char HTTP_CONNECT_IS_NOT_AN_HTTP_PROXY_MSG[] = "

\n" "This is not correct: This port is configured as a CONNECT tunnel, not\n" "an HTTP proxy. Please configure your client accordingly. You can also\n" - "use HTTPS, then the client should automatically use HTTP CONNECT." + "use HTTPS; then the client should automatically use HTTP CONNECT." "

\n" "

\n" "See " diff --git a/src/core/or/relay.c b/src/core/or/relay.c index 8b3a1be18e..066ef19dbf 100644 --- a/src/core/or/relay.c +++ b/src/core/or/relay.c @@ -543,7 +543,7 @@ relay_command_to_string(uint8_t command) STATIC size_t get_pad_cell_offset(size_t data_len) { - /* This is never suppose to happen but in case it does, stop right away + /* This is never supposed to happen but in case it does, stop right away * because if tor is tricked somehow into not adding random bytes to the * payload with this function returning 0 for a bad data_len, the entire * authenticated SENDME design can be bypassed leading to bad denial of @@ -577,7 +577,7 @@ pad_cell_payload(uint8_t *cell_payload, size_t data_len) } /* Remember here that the cell_payload is the length of the header and - * payload size so we offset it using the full lenght of the cell. */ + * payload size so we offset it using the full length of the cell. */ pad_len = CELL_PAYLOAD_SIZE - pad_offset; crypto_fast_rng_getbytes(get_thread_fast_rng(), cell_payload + pad_offset, pad_len); @@ -1621,7 +1621,7 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, } } - /* Tell circpad that we've recieved a recognized cell */ + /* Tell circpad that we've received a recognized cell */ circpad_deliver_recognized_relay_cell_events(circ, rh.command, layer_hint); /* either conn is NULL, in which case we've got a control cell, or else diff --git a/src/core/or/sendme.c b/src/core/or/sendme.c index e7c65d99e2..0743c48c32 100644 --- a/src/core/or/sendme.c +++ b/src/core/or/sendme.c @@ -157,7 +157,7 @@ cell_version_is_valid(uint8_t cell_version) * * This is the main critical function to make sure we can continue to * send/recv cells on a circuit. If the SENDME is invalid, the circuit should - * be mark for close. */ + * be marked for close by the caller. */ STATIC bool sendme_is_valid(const circuit_t *circ, const uint8_t *cell_payload, size_t cell_payload_len) @@ -396,11 +396,11 @@ sendme_circuit_consider_sending(circuit_t *circ, crypt_path_t *layer_hint) * the length of the SENDME cell payload (excluding the header). The * cell_payload is the payload. * - * Return 0 on success that is the SENDME is valid and the package window has - * been updated properly. + * Return 0 on success (the SENDME is valid and the package window has + * been updated properly). * - * On error, a negative value is returned which indicate that the circuit must - * be closed using the value as the reason for it. */ + * On error, a negative value is returned, which indicates that the + * circuit must be closed using the value as the reason for it. */ int sendme_process_circuit_level(crypt_path_t *layer_hint, circuit_t *circ, const uint8_t *cell_payload, @@ -462,11 +462,11 @@ sendme_process_circuit_level(crypt_path_t *layer_hint, * edge connection (stream) that the circuit circ is associated with. The * cell_body_len is the length of the payload (excluding the header). * - * Return 0 on success that is the SENDME is valid and the package window has - * been updated properly. + * Return 0 on success (the SENDME is valid and the package window has + * been updated properly). * - * On error, a negative value is returned which indicate that the circuit must - * be closed using the value as the reason for it. */ + * On error, a negative value is returned, which indicates that the + * circuit must be closed using the value as the reason for it. */ int sendme_process_stream_level(edge_connection_t *conn, circuit_t *circ, uint16_t cell_body_len) diff --git a/src/feature/dirauth/shared_random_state.c b/src/feature/dirauth/shared_random_state.c index 9a045b283d..b669e3836e 100644 --- a/src/feature/dirauth/shared_random_state.c +++ b/src/feature/dirauth/shared_random_state.c @@ -937,7 +937,7 @@ state_query_del_all_(sr_state_object_t obj_type) } DIGESTMAP_FOREACH_END; break; } - /* The following object are _NOT_ suppose to be removed. */ + /* The following objects are _NOT_ supposed to be removed. */ case SR_STATE_OBJ_CURSRV: case SR_STATE_OBJ_PREVSRV: case SR_STATE_OBJ_PHASE: diff --git a/src/feature/hs/hs_client.c b/src/feature/hs/hs_client.c index 7aec6d80bb..ad5bf15563 100644 --- a/src/feature/hs/hs_client.c +++ b/src/feature/hs/hs_client.c @@ -697,7 +697,7 @@ setup_intro_circ_auth_key(origin_circuit_t *circ) } /* Reaching this point means we didn't find any intro point for this circuit - * which is not suppose to happen. */ + * which is not supposed to happen. */ tor_assert_nonfatal_unreached(); end: diff --git a/src/feature/hs/hs_common.c b/src/feature/hs/hs_common.c index 597409a1e9..a5747fe170 100644 --- a/src/feature/hs/hs_common.c +++ b/src/feature/hs/hs_common.c @@ -941,7 +941,7 @@ hs_parse_address(const char *address, ed25519_public_key_t *key_out, return -1; } -/* Validate a given onion address. The length, the base32 decoding and +/* Validate a given onion address. The length, the base32 decoding, and * checksum are validated. Return 1 if valid else 0. */ int hs_address_is_valid(const char *address) @@ -956,7 +956,7 @@ hs_address_is_valid(const char *address) goto invalid; } - /* Get the checksum it's suppose to be and compare it with what we have + /* Get the checksum it's supposed to be and compare it with what we have * encoded in the address. */ build_hs_checksum(&service_pubkey, version, target_checksum); if (tor_memcmp(checksum, target_checksum, sizeof(checksum))) { @@ -984,7 +984,7 @@ hs_address_is_valid(const char *address) * The returned address is base32 encoded and put in addr_out. The caller MUST * make sure the addr_out is at least HS_SERVICE_ADDR_LEN_BASE32 + 1 long. * - * Format is as follow: + * Format is as follows: * base32(PUBKEY || CHECKSUM || VERSION) * CHECKSUM = H(".onion checksum" || PUBKEY || VERSION) * */ diff --git a/src/feature/hs/hs_service.c b/src/feature/hs/hs_service.c index 76e55e4f83..8a4f1efb16 100644 --- a/src/feature/hs/hs_service.c +++ b/src/feature/hs/hs_service.c @@ -1231,7 +1231,7 @@ load_client_keys(hs_service_t *service) client_key_str = read_file_to_str(client_key_file_path, 0, NULL); /* If we cannot read the file, continue with the next file. */ - if (!client_key_str) { + if (!client_key_str) { log_warn(LD_REND, "Client authorization file %s can't be read. " "Corrupted or verify permission? Ignoring.", client_key_file_path); diff --git a/src/trunnel/include.am b/src/trunnel/include.am index 82e7a66959..ce15570b15 100644 --- a/src/trunnel/include.am +++ b/src/trunnel/include.am @@ -11,7 +11,7 @@ TRUNNELINPUTS = \ src/trunnel/link_handshake.trunnel \ src/trunnel/pwbox.trunnel \ src/trunnel/channelpadding_negotiation.trunnel \ - src/trunnel/sendme.trunnel \ + src/trunnel/sendme.trunnel \ src/trunnel/socks5.trunnel \ src/trunnel/circpad_negotiation.trunnel From a014e01b686cbe84bcfc9907d0a98ac71be91e3e Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Wed, 20 Mar 2019 17:42:56 +0200 Subject: [PATCH 1009/2557] Behave correctly when state->max_length is zero. --- src/core/or/circuitpadding.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/core/or/circuitpadding.c b/src/core/or/circuitpadding.c index ddf28ea624..61e222cbb9 100644 --- a/src/core/or/circuitpadding.c +++ b/src/core/or/circuitpadding.c @@ -525,7 +525,10 @@ circpad_choose_state_length(circpad_machine_runtime_t *mi) length = circpad_distribution_sample(state->length_dist); length = MAX(0, length); length += state->start_length; - length = MIN(length, state->max_length); + + if (state->max_length) { + length = MIN(length, state->max_length); + } mi->state_length = clamp_double_to_int64(length); } From 39c52d14a619c9944321f404c74c9129a5e664cb Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Wed, 20 Mar 2019 17:41:10 +0200 Subject: [PATCH 1010/2557] Make register_padding_machine part of the public API. We are gonna use this function to register our new machine. --- src/core/or/circuitpadding.c | 14 ++++++++------ src/core/or/circuitpadding.h | 5 ++--- src/test/test_circuitpadding.c | 8 ++++---- 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/src/core/or/circuitpadding.c b/src/core/or/circuitpadding.c index 61e222cbb9..48d0d4a95a 100644 --- a/src/core/or/circuitpadding.c +++ b/src/core/or/circuitpadding.c @@ -2371,7 +2371,6 @@ circpad_setup_machine_on_circ(circuit_t *on_circ, on_circ->padding_machine[machine->machine_index] = machine; } -#ifdef TOR_UNIT_TESTS /** Validate a single state of a padding machine */ static bool padding_machine_state_is_valid(const circpad_state_t *state) @@ -2435,9 +2434,9 @@ padding_machine_is_valid(const circpad_machine_spec_t *machine) /* Validate and register machine into machine_list. If * machine_list is NULL, then just validate. */ -STATIC void -register_padding_machine(circpad_machine_spec_t *machine, - smartlist_t *machine_list) +void +circpad_register_padding_machine(circpad_machine_spec_t *machine, + smartlist_t *machine_list) { if (!padding_machine_is_valid(machine)) { log_warn(LD_GENERAL, "Machine #%u is invalid. Ignoring.", @@ -2450,6 +2449,7 @@ register_padding_machine(circpad_machine_spec_t *machine, } } +#ifdef TOR_UNIT_TESTS /* These padding machines are only used for tests pending #28634. */ static void circpad_circ_client_machine_init(void) @@ -2502,7 +2502,8 @@ circpad_circ_client_machine_init(void) circ_client_machine->states[CIRCPAD_STATE_BURST].histogram_total_tokens = 5; circ_client_machine->machine_num = smartlist_len(origin_padding_machines); - register_padding_machine(circ_client_machine, origin_padding_machines); + circpad_register_padding_machine(circ_client_machine, + origin_padding_machines); } static void @@ -2602,7 +2603,8 @@ circpad_circ_responder_machine_init(void) CIRCPAD_TOKEN_REMOVAL_CLOSEST_USEC; circ_responder_machine->machine_num = smartlist_len(relay_padding_machines); - register_padding_machine(circ_responder_machine, relay_padding_machines); + circpad_register_padding_machine(circ_responder_machine, + relay_padding_machines); } #endif diff --git a/src/core/or/circuitpadding.h b/src/core/or/circuitpadding.h index 277a78001e..059896ddf0 100644 --- a/src/core/or/circuitpadding.h +++ b/src/core/or/circuitpadding.h @@ -698,6 +698,8 @@ circpad_machine_event_circ_has_no_relay_early(struct origin_circuit_t *circ); void circpad_machines_init(void); void circpad_machines_free(void); +void circpad_register_padding_machine(circpad_machine_spec_t *machine, + smartlist_t *machine_list); void circpad_machine_states_init(circpad_machine_spec_t *machine, circpad_statenum_t num_states); @@ -781,9 +783,6 @@ histogram_get_bin_upper_bound(const circpad_machine_runtime_t *mi, extern smartlist_t *origin_padding_machines; extern smartlist_t *relay_padding_machines; -STATIC void -register_padding_machine(circpad_machine_spec_t *machine, - smartlist_t *machine_list); #endif #endif diff --git a/src/test/test_circuitpadding.c b/src/test/test_circuitpadding.c index bd6697922e..c5aad0f5d9 100644 --- a/src/test/test_circuitpadding.c +++ b/src/test/test_circuitpadding.c @@ -1759,7 +1759,7 @@ helper_create_conditional_machines(void) add->conditions.state_mask = CIRCPAD_CIRC_BUILDING| CIRCPAD_CIRC_NO_STREAMS|CIRCPAD_CIRC_HAS_RELAY_EARLY; add->conditions.purpose_mask = CIRCPAD_PURPOSE_ALL; - register_padding_machine(add, origin_padding_machines); + circpad_register_padding_machine(add, origin_padding_machines); add = helper_create_conditional_machine(); add->machine_num = 3; @@ -1778,15 +1778,15 @@ helper_create_conditional_machines(void) add->conditions.state_mask = CIRCPAD_CIRC_OPENED| CIRCPAD_CIRC_STREAMS|CIRCPAD_CIRC_HAS_NO_RELAY_EARLY; add->conditions.purpose_mask = CIRCPAD_PURPOSE_ALL; - register_padding_machine(add, origin_padding_machines); + circpad_register_padding_machine(add, origin_padding_machines); add = helper_create_conditional_machine(); add->machine_num = 2; - register_padding_machine(add, relay_padding_machines); + circpad_register_padding_machine(add, relay_padding_machines); add = helper_create_conditional_machine(); add->machine_num = 3; - register_padding_machine(add, relay_padding_machines); + circpad_register_padding_machine(add, relay_padding_machines); } void From 5791bc9d768df8897bc471dba4def75c812cfcf2 Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Fri, 12 Apr 2019 23:42:46 +0300 Subject: [PATCH 1011/2557] Generate non-padding circpad events for PADDING_NEGOTIATE(D). As part of our machines, we need to know when a PADDING_NEGOATIATE(D) cell gets sent out, so we add an event for this. --- src/core/or/circuitpadding.c | 25 +++++++------------------ 1 file changed, 7 insertions(+), 18 deletions(-) diff --git a/src/core/or/circuitpadding.c b/src/core/or/circuitpadding.c index 48d0d4a95a..1a46296d18 100644 --- a/src/core/or/circuitpadding.c +++ b/src/core/or/circuitpadding.c @@ -1737,13 +1737,12 @@ circpad_estimate_circ_rtt_on_send(circuit_t *circ, * to back. Stop estimating RTT in this case. Note that we only * stop RTT update if the circuit is opened, to allow for RTT estimates * of var cells during circ setup. */ - mi->stop_rtt_update = 1; - - if (!mi->rtt_estimate_usec) { + if (!mi->rtt_estimate_usec && !mi->stop_rtt_update) { static ratelim_t rtt_lim = RATELIM_INIT(600); log_fn_ratelim(&rtt_lim,LOG_NOTICE,LD_BUG, "Circuit sent two cells back to back before estimating RTT."); } + mi->stop_rtt_update = 1; } } @@ -2256,13 +2255,6 @@ circpad_deliver_recognized_relay_cell_events(circuit_t *circ, uint8_t relay_command, crypt_path_t *layer_hint) { - /* Padding negotiate cells are ignored by the state machines - * for simplicity. */ - if (relay_command == RELAY_COMMAND_PADDING_NEGOTIATE || - relay_command == RELAY_COMMAND_PADDING_NEGOTIATED) { - return; - } - if (relay_command == RELAY_COMMAND_DROP) { rep_hist_padding_count_read(PADDING_TYPE_DROP); @@ -2299,16 +2291,12 @@ void circpad_deliver_sent_relay_cell_events(circuit_t *circ, uint8_t relay_command) { - /* Padding negotiate cells are ignored by the state machines - * for simplicity. */ - if (relay_command == RELAY_COMMAND_PADDING_NEGOTIATE || - relay_command == RELAY_COMMAND_PADDING_NEGOTIATED) { - return; - } - /* RELAY_COMMAND_DROP is the multi-hop (aka circuit-level) padding cell in * tor. (CELL_PADDING is a channel-level padding cell, which is not relayed - * or processed here) */ + * or processed here). + * + * We do generate events for PADDING_NEGOTIATE and PADDING_NEGOTIATED cells. + */ if (relay_command == RELAY_COMMAND_DROP) { /* Optimization: The event for RELAY_COMMAND_DROP is sent directly * from circpad_send_padding_cell_for_callback(). This is to avoid @@ -2823,6 +2811,7 @@ circpad_handle_padding_negotiate(circuit_t *circ, cell_t *cell) const circpad_machine_spec_t *, m) { if (m->machine_num == negotiate->machine_type) { circpad_setup_machine_on_circ(circ, m); + circpad_cell_event_nonpadding_received(circ); goto done; } } SMARTLIST_FOREACH_END(m); From 69a277f635d9f5d6e40d1f1e85110621b5c49169 Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Wed, 17 Apr 2019 14:52:51 +0300 Subject: [PATCH 1012/2557] Introduce circpad free_all() function. --- src/app/main/shutdown.c | 1 + src/core/or/circuitpadding.c | 37 ++++++++++++++++++++++++++++++++++++ src/core/or/circuitpadding.h | 6 ++++++ 3 files changed, 44 insertions(+) diff --git a/src/app/main/shutdown.c b/src/app/main/shutdown.c index 1871717ad5..e4dcaa1324 100644 --- a/src/app/main/shutdown.c +++ b/src/app/main/shutdown.c @@ -139,6 +139,7 @@ tor_free_all(int postfork) dos_free_all(); circuitmux_ewma_free_all(); accounting_free_all(); + circpad_free_all(); if (!postfork) { config_free_all(); diff --git a/src/core/or/circuitpadding.c b/src/core/or/circuitpadding.c index 1a46296d18..c0caacf581 100644 --- a/src/core/or/circuitpadding.c +++ b/src/core/or/circuitpadding.c @@ -37,6 +37,13 @@ * When a padding machine reaches the END state, it gets wiped from the circuit * so that other padding machines can take over if needed (see * circpad_machine_spec_transitioned_to_end()). + * + **************************** + * General notes: + * + * All used machines should be heap allocated and placed into + * origin_padding_machines/relay_padding_machines so that they get correctly + * cleaned up by the circpad_free_all() function. **/ #define CIRCUITPADDING_PRIVATE @@ -2885,6 +2892,36 @@ circpad_handle_padding_negotiated(circuit_t *circ, cell_t *cell, return 0; } +/** Free memory allocated by this machine spec. */ +STATIC void +machine_spec_free_(circpad_machine_spec_t *m) +{ + if (!m) return; + + tor_free(m->states); + tor_free(m); +} + +/** Free all memory allocated by the circuitpadding subsystem. */ +void +circpad_free_all(void) +{ + if (origin_padding_machines) { + SMARTLIST_FOREACH_BEGIN(origin_padding_machines, + circpad_machine_spec_t *, m) { + machine_spec_free(m); + } SMARTLIST_FOREACH_END(m); + smartlist_free(origin_padding_machines); + } + if (relay_padding_machines) { + SMARTLIST_FOREACH_BEGIN(relay_padding_machines, + circpad_machine_spec_t *, m) { + machine_spec_free(m); + } SMARTLIST_FOREACH_END(m); + smartlist_free(relay_padding_machines); + } +} + /* Serialization */ // TODO: Should we use keyword=value here? Are there helpers for that? #if 0 diff --git a/src/core/or/circuitpadding.h b/src/core/or/circuitpadding.h index 059896ddf0..c7c5f6c690 100644 --- a/src/core/or/circuitpadding.h +++ b/src/core/or/circuitpadding.h @@ -738,7 +738,13 @@ circpad_machine_spec_transition, (circpad_machine_runtime_t *mi, circpad_decision_t circpad_send_padding_cell_for_callback( circpad_machine_runtime_t *mi); +void circpad_free_all(void); + #ifdef CIRCUITPADDING_PRIVATE +STATIC void machine_spec_free_(circpad_machine_spec_t *m); +#define machine_spec_free(chan) \ + FREE_AND_NULL(circpad_machine_spec_t,machine_spec_free_, (m)) + STATIC circpad_delay_t circpad_machine_sample_delay(circpad_machine_runtime_t *mi); From 9b582edddb3a56d0eb2049820c6e14bfead7b143 Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Wed, 17 Apr 2019 13:23:23 +0300 Subject: [PATCH 1013/2557] Correctly handle machines out of tokens that have not closed yet. Perhaps the machine on the other side is still not done. --- src/core/or/circuitpadding.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/core/or/circuitpadding.c b/src/core/or/circuitpadding.c index c0caacf581..e02cfb7bce 100644 --- a/src/core/or/circuitpadding.c +++ b/src/core/or/circuitpadding.c @@ -605,6 +605,11 @@ circpad_machine_sample_delay(circpad_machine_runtime_t *mi) histogram_total_tokens = state->histogram_total_tokens; } + /* If we are out of tokens, don't schedule padding. */ + if (!histogram_total_tokens) { + return CIRCPAD_DELAY_INFINITE; + } + bin_choice = crypto_fast_rng_get_uint64(get_thread_fast_rng(), histogram_total_tokens); From ac895fa40552b78744eddfb648dadde23834e553 Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Thu, 16 May 2019 14:02:16 +0300 Subject: [PATCH 1014/2557] Add client-side onion service circuit hiding machines. --- src/core/include.am | 2 + src/core/or/circuitpadding.c | 18 +- src/core/or/circuitpadding.h | 5 + src/core/or/circuitpadding_machines.c | 418 ++++++++++++++++++++++++++ src/core/or/circuitpadding_machines.h | 35 +++ src/test/test_circuitpadding.c | 188 ++++++++++++ 6 files changed, 662 insertions(+), 4 deletions(-) create mode 100644 src/core/or/circuitpadding_machines.c create mode 100644 src/core/or/circuitpadding_machines.h diff --git a/src/core/include.am b/src/core/include.am index f722b70481..66ce2c9650 100644 --- a/src/core/include.am +++ b/src/core/include.am @@ -37,6 +37,7 @@ LIBTOR_APP_A_SOURCES = \ src/core/or/circuitmux.c \ src/core/or/circuitmux_ewma.c \ src/core/or/circuitpadding.c \ + src/core/or/circuitpadding_machines.c \ src/core/or/circuitstats.c \ src/core/or/circuituse.c \ src/core/or/crypt_path.c \ @@ -247,6 +248,7 @@ noinst_HEADERS += \ src/core/or/circuitmux_ewma.h \ src/core/or/circuitstats.h \ src/core/or/circuitpadding.h \ + src/core/or/circuitpadding_machines.h \ src/core/or/circuituse.h \ src/core/or/command.h \ src/core/or/connection_edge.h \ diff --git a/src/core/or/circuitpadding.c b/src/core/or/circuitpadding.c index e02cfb7bce..6c48bbe6f5 100644 --- a/src/core/or/circuitpadding.c +++ b/src/core/or/circuitpadding.c @@ -53,6 +53,7 @@ #include "lib/math/prob_distr.h" #include "core/or/or.h" #include "core/or/circuitpadding.h" +#include "core/or/circuitpadding_machines.h" #include "core/or/circuitlist.h" #include "core/or/circuituse.h" #include "core/mainloop/netstatus.h" @@ -79,8 +80,6 @@ #include "app/config/config.h" -static inline circpad_purpose_mask_t circpad_circ_purpose_to_mask(uint8_t - circ_purpose); static inline circpad_circuit_state_t circpad_circuit_state( origin_circuit_t *circ); static void circpad_setup_machine_on_circ(circuit_t *on_circ, @@ -492,7 +491,10 @@ circpad_machine_setup_tokens(circpad_machine_runtime_t *mi) const circpad_state_t *state = circpad_machine_current_state(mi); /* If this state doesn't exist, or doesn't have token removal, - * free any previous state's histogram, and bail */ + * free any previous state's runtime histogram, and bail. + * + * If we don't have a token removal strategy, we also don't need a runtime + * histogram and we rely on the immutable one in machine_spec_t. */ if (!state || state->token_removal == CIRCPAD_TOKEN_REMOVAL_NONE) { if (mi->histogram) { tor_free(mi->histogram); @@ -1990,7 +1992,6 @@ circpad_circuit_state(origin_circuit_t *circ) * Convert a normal circuit purpose into a bitmask that we can * use for determining matching circuits. */ -static inline circpad_purpose_mask_t circpad_circ_purpose_to_mask(uint8_t circ_purpose) { @@ -2623,6 +2624,14 @@ circpad_machines_init(void) origin_padding_machines = smartlist_new(); relay_padding_machines = smartlist_new(); + /* Register machines for hiding client-side intro circuits */ + circpad_machine_client_hide_intro_circuits(origin_padding_machines); + circpad_machine_relay_hide_intro_circuits(relay_padding_machines); + + /* Register machines for hiding client-side rendezvous circuits */ + circpad_machine_client_hide_rend_circuits(origin_padding_machines); + circpad_machine_relay_hide_rend_circuits(relay_padding_machines); + // TODO: Parse machines from consensus and torrc #ifdef TOR_UNIT_TESTS circpad_circ_client_machine_init(); @@ -2877,6 +2886,7 @@ circpad_handle_padding_negotiated(circuit_t *circ, cell_t *cell, } if (negotiated->command == CIRCPAD_COMMAND_STOP) { + log_info(LD_CIRC, "Received STOP command on PADDING_NEGOTIATED"); /* There may not be a padding_info here if we shut down the * machine in circpad_shutdown_old_machines(). Or, if * circpad_add_matching_matchines() added a new machine, diff --git a/src/core/or/circuitpadding.h b/src/core/or/circuitpadding.h index c7c5f6c690..37b4603f0e 100644 --- a/src/core/or/circuitpadding.h +++ b/src/core/or/circuitpadding.h @@ -603,6 +603,9 @@ typedef uint8_t circpad_machine_num_t; /** Global state machine structure from the consensus */ typedef struct circpad_machine_spec_t { + /* Just a user-friendly machine name for logs */ + const char *name; + /** Global machine number */ circpad_machine_num_t machine_num; @@ -728,6 +731,8 @@ bool circpad_padding_negotiated(struct circuit_t *circ, uint8_t command, uint8_t response); +circpad_purpose_mask_t circpad_circ_purpose_to_mask(uint8_t circ_purpose); + MOCK_DECL(circpad_decision_t, circpad_machine_schedule_padding,(circpad_machine_runtime_t *)); diff --git a/src/core/or/circuitpadding_machines.c b/src/core/or/circuitpadding_machines.c new file mode 100644 index 0000000000..4d348f9593 --- /dev/null +++ b/src/core/or/circuitpadding_machines.c @@ -0,0 +1,418 @@ +/* Copyright (c) 2019 The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file circuitpadding_machines.c + * \brief Circuit padding state machines + * + * \detail + * + * Introduce circuit padding machines that will be used by Tor circuits, as + * specified by proposal 302 "Hiding onion service clients using padding". + * + * Right now this file introduces two machines that aim to hide the client-side + * of onion service circuits against naive classifiers like the ones from the + * "Circuit Fingerprinting Attacks: Passive Deanonymization of Tor Hidden + * Services" paper from USENIX. By naive classifiers we mean classifiers that + * use basic features like "circuit construction circuits" and "incoming and + * outgoing cell counts" and "duration of activity". + * + * In particular, these machines aim to be lightweight and protect against + * these basic classifiers. They don't aim to protect against more advanced + * attacks that use deep learning or even correlate various circuit + * construction events together. Machines that fool such advanced classifiers + * are also possible, but they can't be so lightweight and might require more + * WTF-PAD features. So for now we opt for the following two machines: + * + * Client-side introduction circuit hiding machine: + * + * This machine hides client-side introduction circuits by making their + * circuit consruction sequence look like normal general circuits that + * download directory information. Furthermore, the circuits are kept open + * until all the padding has been sent, since intro circuits are usually + * very short lived and this act as a distinguisher. For more info see + * circpad_machine_client_hide_intro_circuits() and the sec. + * + * Client-side rendezvous circuit hiding machine: + * + * This machine hides client-side rendezvous circuits by making their + * circuit construction sequence look like normal general circuits. For more + * details see circpad_machine_client_hide_rend_circuits() and the spec. + * + * TODO: These are simple machines that carefully manipulate the cells of the + * initial circuit setup procedure to make them look like general + * circuits. In the future, more states can be baked into their state machine + * to do more advanced obfuscation. + **/ + +#define CIRCUITPADDING_MACHINES_PRIVATE + +#include "core/or/or.h" +#include "feature/nodelist/networkstatus.h" + +#include "lib/crypt_ops/crypto_rand.h" + +#include "core/or/circuitlist.h" + +#include "core/or/circuitpadding_machines.h" +#include "core/or/circuitpadding.h" + +/* Setup the simple state machine we use for all HS padding machines */ +static void +setup_state_machine_for_hiding_intro_circuits(circpad_machine_spec_t *machine) +{ + /* Two states: START, OBFUSCATE_CIRC_SETUP (and END) */ + circpad_machine_states_init(machine, 2); + + /* For the relay-side machine, we want to transition + * START -> OBFUSCATE_CIRC_SETUP upon first non-padding + * cell sent (PADDING_NEGOTIATED in this case). + * + * For the origin-side machine, we transition to OBFUSCATE_CIRC_SETUP after + * sending PADDING_NEGOTIATE, and we stay there (without sending any padding) + * until we receive a STOP from the other side. */ + machine->states[CIRCPAD_STATE_START]. + next_state[CIRCPAD_EVENT_NONPADDING_SENT] = + CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP; + + /* For the relay-side, we want to transition from OBFUSCATE_CIRC_SETUP to END + * state when the length finishes. + * + * For the origin-side, we don't care because the relay-side machine is gonna + * END us. */ + machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP]. + next_state[CIRCPAD_EVENT_LENGTH_COUNT] = CIRCPAD_STATE_END; + + /* Now let's define the OBF -> OBF transitions that maintain our padding + * flow: + * + * For the relay-side machine, we want to keep on sending padding bytes even + * when nothing else happens on this circuit. */ + machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP]. + next_state[CIRCPAD_EVENT_PADDING_SENT] = + CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP; + /* For the relay-side machine, we need this transition so that we re-enter + the state, after PADDING_NEGOTIATED is sent. Otherwise, the remove token + function will disable the timer, and nothing will restart it since there + is no other motion on an intro circuit. */ + machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP]. + next_state[CIRCPAD_EVENT_NONPADDING_SENT] = + CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP; +} + +/* Setup the OBFUSCATE_CIRC_SETUP state of the machine that hides client-side + * intro circuits. */ +static void +setup_obf_state_for_hiding_intro_circuits(circpad_state_t *obf_state, + bool is_client) +{ + /* Token removal strategy for OBFUSCATE_CIRC_SETUP state. We pick the + * simplest one since we rely on the state length sampling and not the + * tokens. */ + obf_state->token_removal = CIRCPAD_TOKEN_REMOVAL_NONE; + + /* Figure out the length of the OBFUSCATE_CIRC_SETUP state so that it's + * randomized. We will set the histogram such that we don't send any padding + * from the origin-side, so just tune these parameteres for the + * relay-side. */ + obf_state->length_dist.type = CIRCPAD_DIST_UNIFORM; + obf_state->length_dist.param1 = INTRO_MACHINE_MINIMUM_PADDING; + obf_state->length_dist.param2 = INTRO_MACHINE_MAXIMUM_PADDING; + + /* Configure histogram */ + obf_state->histogram_len = 2; + if (is_client) { + /* For the origin-side machine we don't want to send any padding, so setup + * infinite delays. */ + obf_state->histogram_edges[0] = CIRCPAD_DELAY_INFINITE-1; + obf_state->histogram_edges[1] = CIRCPAD_DELAY_INFINITE; + /* zero tokens */ + obf_state->histogram[0] = 0; + } else { + /* For the relay-side machine we want to batch padding instantly to pretend + * its an incoming directory download. So set the histogram edges tight: + * (1, 10ms, infinity). */ + obf_state->histogram_edges[0] = 1000; + obf_state->histogram_edges[1] = 10000; + /* padding is controlled by state length, so this is just a high value */ + obf_state->histogram[0] = 1000; + } + + /* just one bin, so setup the total tokens */ + obf_state->histogram_total_tokens = obf_state->histogram[0]; +} + +/** Create a client-side padding machine that aims to hide IP circuits. In + * particular, it keeps intro circuits alive until a bunch of fake traffic has + * been pushed through. + */ +void +circpad_machine_client_hide_intro_circuits(smartlist_t *machines_sl) +{ + circpad_machine_spec_t *client_machine + = tor_malloc_zero(sizeof(circpad_machine_spec_t)); + + client_machine->name = "client_ip_circ"; + + client_machine->conditions.state_mask = CIRCPAD_CIRC_OPENED; + client_machine->target_hopnum = 2; + + /* This is a client machine */ + client_machine->is_origin_side = 1; + + /* We only want to pad introduction circuits, and we want to start padding + * only after the INTRODUCE1 cell has been sent, so set the purposes + * appropriately. + * + * In particular we want introduction circuits to blend as much as possible + * with general circuits. Most general circuits have the following initial + * relay cell sequence (outgoing cells marked in [brackets]): + * + * [EXTEND2] -> EXTENDED2 -> [EXTEND2] -> EXTENDED2 -> [BEGIN] -> CONNECTED + * -> [DATA] -> [DATA] -> DATA -> DATA...(inbound data cells continue) + * + * Whereas normal introduction circuits usually look like: + * + * [EXTEND2] -> EXTENDED2 -> [EXTEND2] -> EXTENDED2 -> [EXTEND2] -> EXTENDED2 + * -> [INTRO1] -> INTRODUCE_ACK + * + * This means that up to the sixth cell (first line of each sequence above), + * both general and intro circuits have identical cell sequences. After that + * we want to mimic the second line sequence of + * -> [DATA] -> [DATA] -> DATA -> DATA...(inbound data cells continue) + * + * We achieve this by starting padding INTRODUCE1 has been sent. With padding + * negotiation cells, in the common case of the second line looks like: + * -> [INTRO1] -> [PADDING_NEGOTIATE] -> PADDING_NEGOTIATED -> INTRO_ACK + * + * Then, the middle node will send between INTRO_MACHINE_MINIMUM_PADDING and + * INTRO_MACHINE_MAXIMUM_PADDING cells, to match the "...(inbound data cells + * continue)" portion of the trace (aka the rest of an HTTPS response body). + */ + client_machine->conditions.purpose_mask = + circpad_circ_purpose_to_mask(CIRCUIT_PURPOSE_C_INTRODUCE_ACK_WAIT)| + circpad_circ_purpose_to_mask(CIRCUIT_PURPOSE_C_INTRODUCE_ACKED)| + circpad_circ_purpose_to_mask(CIRCUIT_PURPOSE_C_CIRCUIT_PADDING); + + /* Keep the circuit alive even after the introduction has been finished, + * otherwise the short-term lifetime of the circuit will blow our cover */ + client_machine->manage_circ_lifetime = 1; + + /* Set padding machine limits to help guard against excessive padding */ + client_machine->allowed_padding_count = INTRO_MACHINE_MAXIMUM_PADDING; + client_machine->max_padding_percent = 1; + + /* Setup states and histograms */ + setup_state_machine_for_hiding_intro_circuits(client_machine); + setup_obf_state_for_hiding_intro_circuits( + &client_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP], + true); + + /* Register the machine */ + client_machine->machine_num = smartlist_len(machines_sl); + circpad_register_padding_machine(client_machine, machines_sl); + + log_warn(LD_GENERAL, + "Registered client intro point hiding padding machine (%u)", + client_machine->machine_num); +} + +/** Create a relay-side padding machine that aims to hide IP circuits. See + * comments on the function above for more details on the workings of the + * machine. */ +void +circpad_machine_relay_hide_intro_circuits(smartlist_t *machines_sl) +{ + circpad_machine_spec_t *relay_machine + = tor_malloc_zero(sizeof(circpad_machine_spec_t)); + + relay_machine->name = "relay_ip_circ"; + + relay_machine->conditions.state_mask = CIRCPAD_CIRC_OPENED; + relay_machine->target_hopnum = 2; + + /* This is a relay-side machine */ + relay_machine->is_origin_side = 0; + + /* We want to negotiate END from this side after all our padding is done, so + * that the origin-side machine goes into END state, and eventually closes + * the circuit. */ + relay_machine->should_negotiate_end = 1; + + /* Set padding machine limits to help guard against excessive padding */ + relay_machine->allowed_padding_count = INTRO_MACHINE_MAXIMUM_PADDING; + relay_machine->max_padding_percent = 1; + + /* Setup states and histograms */ + setup_state_machine_for_hiding_intro_circuits(relay_machine); + setup_obf_state_for_hiding_intro_circuits( + &relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP], + false); + + /* Register the machine */ + relay_machine->machine_num = smartlist_len(machines_sl); + circpad_register_padding_machine(relay_machine, machines_sl); + + log_warn(LD_GENERAL, + "Registered relay intro circuit hiding padding machine (%u)", + relay_machine->machine_num); +} + +/************************** Rendezvous-circuit machine ***********************/ + +/* Setup the obf state of the machine that hides client-side rend + * circuits. */ +static void +setup_obf_state_for_hiding_rend_circuits(circpad_state_t *obf_state) +{ + /* Don't use a token removal strategy since we don't want to use monotime + functions. We want this machine to be light. */ + obf_state->token_removal = CIRCPAD_TOKEN_REMOVAL_NONE; + + /* Instead, to control the volume of padding (we just want to send a single + * padding cell) we will use a static state length. We just want one token, + * since we want to make the following pattern: + * [PADDING_NEGOTIATE] -> [DROP] -> PADDING_NEGOTIATED -> DROP */ + obf_state->length_dist.type = CIRCPAD_DIST_UNIFORM; + obf_state->length_dist.param1 = 1; + obf_state->length_dist.param2 = 2; + + /* Histogram is: (0 msecs, 50 msecs, infinity). We want this to be fast so + * that the incoming PADDING_NEGOTIATED cell always arrives after the + * outgoing [DROP]. */ + obf_state->histogram_len = 2; + obf_state->histogram_edges[0] = 0; + obf_state->histogram_edges[1] = 1000; + + /* dummy amount of tokens. they don't matter. we rely on state length. */ + obf_state->histogram[0] = 1; + obf_state->histogram_total_tokens = 1; +} + +/* Setup the simple state machine we use for all HS padding machines */ +static void +setup_state_machine_for_hiding_rend_circuits(circpad_machine_spec_t *machine) +{ + /* Two states: START, OBFUSCATE_CIRC_SETUP (and END) */ + circpad_machine_states_init(machine, 2); + + /* START -> OBFUSCATE_CIRC_SETUP transition upon sending the first + * non-padding cell (which is PADDING_NEGOTIATE) */ + machine->states[CIRCPAD_STATE_START]. + next_state[CIRCPAD_EVENT_NONPADDING_SENT] = + CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP; + + /* OBFUSCATE_CIRC_SETUP -> END transition when we finish all the tokens */ + machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP]. + next_state[CIRCPAD_EVENT_PADDING_RECV] = CIRCPAD_STATE_END; + machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP]. + next_state[CIRCPAD_EVENT_LENGTH_COUNT] = CIRCPAD_STATE_END; +} + +/** Create a client-side padding machine that aims to hide rendezvous + * circuits.*/ +void +circpad_machine_client_hide_rend_circuits(smartlist_t *machines_sl) +{ + circpad_machine_spec_t *client_machine + = tor_malloc_zero(sizeof(circpad_machine_spec_t)); + + client_machine->name = "client_rp_circ"; + + /* Only pad after the circuit has been built and pad to the middle */ + client_machine->conditions.state_mask = CIRCPAD_CIRC_OPENED; + client_machine->target_hopnum = 2; + + /* This is a client machine */ + client_machine->is_origin_side = 1; + + /* We only want to pad rendezvous circuits, and we want to start padding only + * after the rendezvous circuit has been established. + * + * Following a similar argument as for intro circuits, we are aiming for + * padded rendezvous circuits to blend in with the initial cell sequence of + * general circuits which usually look like this: + * + * [EXTEND2] -> EXTENDED2 -> [EXTEND2] -> EXTENDED2 -> [BEGIN] -> CONNECTED + * -> [DATA] -> [DATA] -> DATA -> DATA...(incoming cells continue) + * + * Whereas normal rendezvous circuits usually look like: + * + * [EXTEND2] -> EXTENDED2 -> [EXTEND2] -> EXTENDED2 -> [EST_REND] -> REND_EST + * -> REND2 -> [BEGIN] + * + * This means that up to the sixth cell (in the first line), both general and + * rend circuits have identical cell sequences. + * + * After that we want to mimic a [DATA] -> [DATA] -> DATA -> DATA sequence. + * + * With padding negotiation right after the REND_ESTABLISHED, the sequence + * becomes: + * + * [EXTEND2] -> EXTENDED2 -> [EXTEND2] -> EXTENDED2 -> [EST_REND] -> REND_EST + * -> [PADDING_NEGOTIATE] -> [DROP] -> PADDING_NEGOTIATED -> DROP... + * + * After which normal application DATA cells continue on the circuit. + * + * Hence this way we make rendezvous circuits look like general circuits up + * till the end of the circuit setup. */ + client_machine->conditions.purpose_mask = + circpad_circ_purpose_to_mask(CIRCUIT_PURPOSE_C_REND_JOINED)| + circpad_circ_purpose_to_mask(CIRCUIT_PURPOSE_C_REND_READY)| + circpad_circ_purpose_to_mask(CIRCUIT_PURPOSE_C_REND_READY_INTRO_ACKED); + + /* Set padding machine limits to help guard against excessive padding */ + client_machine->allowed_padding_count = 1; + client_machine->max_padding_percent = 1; + + /* Setup states and histograms */ + setup_state_machine_for_hiding_rend_circuits(client_machine); + setup_obf_state_for_hiding_rend_circuits( + &client_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP]); + + /* Register the machine */ + client_machine->machine_num = smartlist_len(machines_sl); + circpad_register_padding_machine(client_machine, machines_sl); + + log_warn(LD_GENERAL, + "Registered client rendezvous circuit hiding padding machine (%u)", + client_machine->machine_num); +} + +/** Create a relay-side padding machine that aims to hide IP circuits. + * + * This is meant to follow the client-side machine. + */ +void +circpad_machine_relay_hide_rend_circuits(smartlist_t *machines_sl) +{ + circpad_machine_spec_t *relay_machine + = tor_malloc_zero(sizeof(circpad_machine_spec_t)); + + relay_machine->name = "relay_rp_circ"; + + /* Only pad after the circuit has been built and pad to the middle */ + relay_machine->conditions.min_hops = 2; + relay_machine->conditions.state_mask = CIRCPAD_CIRC_OPENED; + relay_machine->target_hopnum = 2; + + /* This is a relay-side machine */ + relay_machine->is_origin_side = 0; + + /* Set padding machine limits to help guard against excessive padding */ + relay_machine->allowed_padding_count = 1; + relay_machine->max_padding_percent = 1; + + /* Setup states and histograms */ + setup_state_machine_for_hiding_rend_circuits(relay_machine); + setup_obf_state_for_hiding_rend_circuits( + &relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP]); + + /* Register the machine */ + relay_machine->machine_num = smartlist_len(machines_sl); + circpad_register_padding_machine(relay_machine, machines_sl); + + log_warn(LD_GENERAL, + "Registered relay rendezvous circuit hiding padding machine (%u)", + relay_machine->machine_num); +} diff --git a/src/core/or/circuitpadding_machines.h b/src/core/or/circuitpadding_machines.h new file mode 100644 index 0000000000..c44a70f2cc --- /dev/null +++ b/src/core/or/circuitpadding_machines.h @@ -0,0 +1,35 @@ +/* Copyright (c) 2018 The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file circuitpadding_machines.h + * \brief Header file for circuitpadding_machines.c. + **/ + +#ifndef TOR_CIRCUITPADDING_MACHINES_H +#define TOR_CIRCUITPADDING_MACHINES_H + +void circpad_machine_relay_hide_intro_circuits(smartlist_t *machines_sl); +void circpad_machine_client_hide_intro_circuits(smartlist_t *machines_sl); +void circpad_machine_relay_hide_rend_circuits(smartlist_t *machines_sl); +void circpad_machine_client_hide_rend_circuits(smartlist_t *machines_sl); + +#ifdef CIRCUITPADDING_MACHINES_PRIVATE + +/** State of the padding machines that actually sends padding */ +#define CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP CIRCPAD_STATE_BURST + +/** Constants defining the amount of padding that a machine will send to hide + * HS circuits. The actual value is sampled uniformly random between the + * min/max values. + */ + +/** Minimum number of relay-side padding cells to be sent by this machine */ +#define INTRO_MACHINE_MINIMUM_PADDING 7 +/** Maximum number of relay-side padding cells to be sent by this machine. + * The actual value will be sampled between the min and max.*/ +#define INTRO_MACHINE_MAXIMUM_PADDING 10 + +#endif + +#endif diff --git a/src/test/test_circuitpadding.c b/src/test/test_circuitpadding.c index c5aad0f5d9..7835b438f8 100644 --- a/src/test/test_circuitpadding.c +++ b/src/test/test_circuitpadding.c @@ -415,6 +415,8 @@ helper_create_basic_machine(void) /* Start, burst */ circpad_machine_states_init(&circ_client_machine, 2); + circ_client_machine.name = "basic"; + circ_client_machine.states[CIRCPAD_STATE_START]. next_state[CIRCPAD_EVENT_NONPADDING_RECV] = CIRCPAD_STATE_BURST; circ_client_machine.states[CIRCPAD_STATE_START].use_rtt_estimate = 1; @@ -2917,6 +2919,192 @@ test_circuitpadding_manage_circuit_lifetime(void *arg) UNMOCK(tor_gettimeofday); } +/** Helper for the test_circuitpadding_hs_machines test: + * + * - Create a client and relay circuit. + * - Setup right circuit purpose and attach a machine to the client circuit. + * - Verify that state transitions work as intended and state length gets + * enforced. + * + * This function is able to do this test both for intro and rend circuits + * depending on the value of test_intro_circs. + */ +static void +helper_test_hs_machines(bool test_intro_circs) +{ + /* Setup the circuits */ + origin_circuit_t *origin_client_side = origin_circuit_new(); + client_side = TO_CIRCUIT(origin_client_side); + client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL; + + dummy_channel.cmux = circuitmux_alloc(); + relay_side = TO_CIRCUIT(new_fake_orcirc(&dummy_channel, &dummy_channel)); + relay_side->purpose = CIRCUIT_PURPOSE_OR; + + /* extend the client circ to two hops */ + simulate_single_hop_extend(client_side, relay_side, 1); + simulate_single_hop_extend(client_side, relay_side, 1); + + /* machines only apply on opened circuits */ + origin_client_side->has_opened = 1; + + /************************************/ + + /* Attaching the client machine now won't work here because of a wrong + * purpose */ + tt_assert(!client_side->padding_machine[0]); + circpad_add_matching_machines(origin_client_side, origin_padding_machines); + tt_assert(!client_side->padding_machine[0]); + + /* Change the purpose, see the machine getting attached */ + client_side->purpose = test_intro_circs ? + CIRCUIT_PURPOSE_C_INTRODUCE_ACK_WAIT : CIRCUIT_PURPOSE_C_REND_JOINED; + circpad_add_matching_machines(origin_client_side, origin_padding_machines); + tt_ptr_op(client_side->padding_info[0], OP_NE, NULL); + tt_ptr_op(client_side->padding_machine[0], OP_NE, NULL); + + tt_ptr_op(relay_side->padding_info[0], OP_NE, NULL); + tt_ptr_op(relay_side->padding_machine[0], OP_NE, NULL); + + /* Verify that the right machine is attached */ + tt_str_op(client_side->padding_machine[0]->name, OP_EQ, + test_intro_circs ? "client_ip_circ" : "client_rp_circ"); + tt_str_op(relay_side->padding_machine[0]->name, OP_EQ, + test_intro_circs ? "relay_ip_circ": "relay_rp_circ"); + + /***********************************/ + + /* Intro machines are at START state, but rend machines have already skipped + * to OBFUSCATE_CIRC_SETUP because of the sent PADDING_NEGOTIATE. */ + tt_int_op(client_side->padding_info[0]->current_state, OP_EQ, + CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP); + tt_int_op(relay_side->padding_info[0]->current_state, OP_EQ, + CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP); + + /*Send non-padding to move the machines from START to OBFUSCATE_CIRC_SETUP */ + circpad_cell_event_nonpadding_received(client_side); + circpad_cell_event_nonpadding_received(relay_side); + tt_int_op(client_side->padding_info[0]->current_state, OP_EQ, + CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP); + tt_int_op(relay_side->padding_info[0]->current_state, OP_EQ, + CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP); + + /* For rendezvous circuit machines we can stop early since are simpler than + * the intro circuit machines. */ + if (!test_intro_circs) { + tt_int_op(client_side->padding_info[0]->histogram[0], OP_EQ, 1); + goto done; + } + + /* Check that the state lengths have been sampled and are within range */ + circpad_machine_runtime_t *client_machine_runtime = + client_side->padding_info[0]; + circpad_machine_runtime_t *relay_machine_runtime = + relay_side->padding_info[0]; + if (test_intro_circs) { + tt_int_op(client_machine_runtime->state_length, OP_GE, + INTRO_MACHINE_MINIMUM_PADDING); + tt_int_op(client_machine_runtime->state_length, OP_LT, + INTRO_MACHINE_MAXIMUM_PADDING); + tt_int_op(relay_machine_runtime->state_length, OP_GE, + INTRO_MACHINE_MINIMUM_PADDING); + tt_int_op(relay_machine_runtime->state_length, OP_LT, + INTRO_MACHINE_MAXIMUM_PADDING); + } else { + tt_int_op(client_machine_runtime->state_length, OP_EQ, 1); + tt_int_op(relay_machine_runtime->state_length, OP_EQ, 1); + } + + /* Send state_length worth of padding and see that the state goes to END */ + int i; + for (i = (int) client_machine_runtime->state_length ; i > 0 ; i--) { + circpad_send_padding_cell_for_callback(client_machine_runtime); + } + /* See that the machine has been teared down after all the length has been + * exhausted. */ + tt_int_op(client_side->padding_info[0]->current_state, OP_EQ, + CIRCPAD_STATE_END); + + done: + free_fake_orcirc(relay_side); + circuitmux_detach_all_circuits(dummy_channel.cmux, NULL); + circuitmux_free(dummy_channel.cmux); + free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side)); +} + +/** Test that the HS circuit padding machines work as intended. */ +static void +test_circuitpadding_hs_machines(void *arg) +{ + (void)arg; + + /* Test logic: + * + * 1) Register the HS machines, which aim to hide the presense of + * onion service traffic on the client-side + * + * 2) Call helper_test_hs_machines() to perform tests for the intro circuit + * machines and for the rend circuit machines. + */ + + MOCK(circuitmux_attach_circuit, circuitmux_attach_circuit_mock); + MOCK(circuit_package_relay_cell, circuit_package_relay_cell_mock); + MOCK(circuit_get_nth_node, circuit_get_nth_node_mock); + MOCK(circpad_machine_schedule_padding,circpad_machine_schedule_padding_mock); + + origin_padding_machines = smartlist_new(); + relay_padding_machines = smartlist_new(); + + nodes_init(); + + monotime_init(); + monotime_enable_test_mocking(); + monotime_set_mock_time_nsec(1*TOR_NSEC_PER_USEC); + monotime_coarse_set_mock_time_nsec(1*TOR_NSEC_PER_USEC); + curr_mocked_time = 1*TOR_NSEC_PER_USEC; + + timers_initialize(); + + /* This is needed so that we are not considered to be dormant */ + note_user_activity(20); + + /************************************/ + + /* Register the HS machines */ + circpad_machine_client_hide_intro_circuits(origin_padding_machines); + circpad_machine_client_hide_rend_circuits(origin_padding_machines); + circpad_machine_relay_hide_intro_circuits(relay_padding_machines); + circpad_machine_relay_hide_rend_circuits(relay_padding_machines); + + /***********************************/ + + /* Do the tests for the intro circuit machines */ + helper_test_hs_machines(true); + /* Do the tests for the rend circuit machines */ + helper_test_hs_machines(false); + + timers_shutdown(); + monotime_disable_test_mocking(); + + SMARTLIST_FOREACH_BEGIN(origin_padding_machines, + circpad_machine_spec_t *, m) { + machine_spec_free(m); + } SMARTLIST_FOREACH_END(m); + + SMARTLIST_FOREACH_BEGIN(relay_padding_machines, + circpad_machine_spec_t *, m) { + machine_spec_free(m); + } SMARTLIST_FOREACH_END(m); + + smartlist_free(origin_padding_machines); + smartlist_free(relay_padding_machines); + + UNMOCK(circuitmux_attach_circuit); + UNMOCK(circuit_package_relay_cell); + UNMOCK(circuit_get_nth_node); + UNMOCK(circpad_machine_schedule_padding); +} + #define TEST_CIRCUITPADDING(name, flags) \ { #name, test_##name, (flags), NULL, NULL } From 953dc601d9369db54ad48a21768a7852f18a617b Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Thu, 28 Mar 2019 15:39:08 +0200 Subject: [PATCH 1015/2557] Add unittests for the new machines. --- src/core/or/circuitpadding.c | 28 +++++++++++++++------------- src/core/or/circuitpadding.h | 7 +++++++ src/test/test_circuitpadding.c | 28 +++++++++++++++++----------- 3 files changed, 39 insertions(+), 24 deletions(-) diff --git a/src/core/or/circuitpadding.c b/src/core/or/circuitpadding.c index 6c48bbe6f5..72909bf723 100644 --- a/src/core/or/circuitpadding.c +++ b/src/core/or/circuitpadding.c @@ -2034,7 +2034,8 @@ circpad_shutdown_old_machines(origin_circuit_t *on_circ) } /** - * Negotiate new machines that would apply to this circuit. + * Negotiate new machines that would apply to this circuit, given the machines + * inside machines_sl. * * This function checks to see if we have any free machine indexes, * and for each free machine index, it initializes the most recently @@ -2042,14 +2043,15 @@ circpad_shutdown_old_machines(origin_circuit_t *on_circ) * index and circuit conditions, and negotiates it with the appropriate * middle relay. */ -static void -circpad_add_matching_machines(origin_circuit_t *on_circ) +STATIC void +circpad_add_matching_machines(origin_circuit_t *on_circ, + smartlist_t *machines_sl) { circuit_t *circ = TO_CIRCUIT(on_circ); #ifdef TOR_UNIT_TESTS /* Tests don't have to init our padding machines */ - if (!origin_padding_machines) + if (!machines_sl) return; #endif @@ -2066,7 +2068,7 @@ circpad_add_matching_machines(origin_circuit_t *on_circ) /* We have a free machine index. Check the origin padding * machines in reverse order, so that more recently added * machines take priority over older ones. */ - SMARTLIST_FOREACH_REVERSE_BEGIN(origin_padding_machines, + SMARTLIST_FOREACH_REVERSE_BEGIN(machines_sl, circpad_machine_spec_t *, machine) { /* Machine definitions have a specific target machine index. @@ -2117,7 +2119,7 @@ circpad_machine_event_circ_added_hop(origin_circuit_t *on_circ) { /* Since our padding conditions do not specify a max_hops, * all we can do is add machines here */ - circpad_add_matching_machines(on_circ); + circpad_add_matching_machines(on_circ, origin_padding_machines); } /** @@ -2130,7 +2132,7 @@ void circpad_machine_event_circ_built(origin_circuit_t *circ) { circpad_shutdown_old_machines(circ); - circpad_add_matching_machines(circ); + circpad_add_matching_machines(circ, origin_padding_machines); } /** @@ -2143,7 +2145,7 @@ void circpad_machine_event_circ_purpose_changed(origin_circuit_t *circ) { circpad_shutdown_old_machines(circ); - circpad_add_matching_machines(circ); + circpad_add_matching_machines(circ, origin_padding_machines); } /** @@ -2157,7 +2159,7 @@ void circpad_machine_event_circ_has_no_relay_early(origin_circuit_t *circ) { circpad_shutdown_old_machines(circ); - circpad_add_matching_machines(circ); + circpad_add_matching_machines(circ, origin_padding_machines); } /** @@ -2172,7 +2174,7 @@ void circpad_machine_event_circ_has_streams(origin_circuit_t *circ) { circpad_shutdown_old_machines(circ); - circpad_add_matching_machines(circ); + circpad_add_matching_machines(circ, origin_padding_machines); } /** @@ -2187,7 +2189,7 @@ void circpad_machine_event_circ_has_no_streams(origin_circuit_t *circ) { circpad_shutdown_old_machines(circ); - circpad_add_matching_machines(circ); + circpad_add_matching_machines(circ, origin_padding_machines); } /** @@ -2682,8 +2684,8 @@ circpad_node_supports_padding(const node_t *node) * Returns node_t from the consensus for that hop, if it is opened. * Otherwise returns NULL. */ -static const node_t * -circuit_get_nth_node(origin_circuit_t *circ, int hop) +MOCK_IMPL(STATIC const node_t *, +circuit_get_nth_node,(origin_circuit_t *circ, int hop)) { crypt_path_t *iter = circuit_get_cpath_hop(circ, hop); diff --git a/src/core/or/circuitpadding.h b/src/core/or/circuitpadding.h index 37b4603f0e..0dc66246d9 100644 --- a/src/core/or/circuitpadding.h +++ b/src/core/or/circuitpadding.h @@ -786,10 +786,17 @@ circpad_send_command_to_hop,(struct origin_circuit_t *circ, uint8_t hopnum, uint8_t relay_command, const uint8_t *payload, ssize_t payload_len)); +MOCK_DECL(STATIC const node_t *, +circuit_get_nth_node,(origin_circuit_t *circ, int hop)); + STATIC circpad_delay_t histogram_get_bin_upper_bound(const circpad_machine_runtime_t *mi, circpad_hist_index_t bin); +STATIC void +circpad_add_matching_machines(origin_circuit_t *on_circ, + smartlist_t *machines_sl); + #ifdef TOR_UNIT_TESTS extern smartlist_t *origin_padding_machines; extern smartlist_t *relay_padding_machines; diff --git a/src/test/test_circuitpadding.c b/src/test/test_circuitpadding.c index 7835b438f8..68d1de91ba 100644 --- a/src/test/test_circuitpadding.c +++ b/src/test/test_circuitpadding.c @@ -1,6 +1,7 @@ #define TOR_CHANNEL_INTERNAL_ #define TOR_TIMERS_PRIVATE #define CIRCUITPADDING_PRIVATE +#define CIRCUITPADDING_MACHINES_PRIVATE #define NETWORKSTATUS_PRIVATE #define CRYPT_PATH_PRIVATE @@ -19,6 +20,7 @@ #include "core/or/circuitlist.h" #include "core/or/circuitbuild.h" #include "core/or/circuitpadding.h" +#include "core/or/circuitpadding_machines.h" #include "core/mainloop/netstatus.h" #include "core/crypto/relay_crypto.h" #include "core/or/protover.h" @@ -112,6 +114,15 @@ node_get_by_id_mock(const char *identity_digest) return NULL; } +static const node_t * +circuit_get_nth_node_mock(origin_circuit_t *circ, int hop) +{ + (void) circ; + (void) hop; + + return &padding_node; +} + static or_circuit_t * new_fake_orcirc(channel_t *nchan, channel_t *pchan) { @@ -2652,8 +2663,8 @@ test_circuitpadding_reduce_disable(void *arg) simulate_single_hop_extend(client_side, relay_side, 1); /* Verify that machine #0 is added */ - tt_int_op(client_side->padding_machine[0]->machine_num, OP_EQ, 0); - tt_int_op(relay_side->padding_machine[0]->machine_num, OP_EQ, 0); + tt_int_op(client_side->padding_machine[0]->machine_num, OP_EQ, 2); + tt_int_op(relay_side->padding_machine[0]->machine_num, OP_EQ, 2); tt_int_op( circpad_machine_reached_padding_limit(client_side->padding_info[0]), @@ -2698,8 +2709,8 @@ test_circuitpadding_reduce_disable(void *arg) simulate_single_hop_extend(client_side, relay_side, 1); /* Verify that machine #0 is added */ - tt_int_op(client_side->padding_machine[0]->machine_num, OP_EQ, 0); - tt_int_op(relay_side->padding_machine[0]->machine_num, OP_EQ, 0); + tt_int_op(client_side->padding_machine[0]->machine_num, OP_EQ, 2); + tt_int_op(relay_side->padding_machine[0]->machine_num, OP_EQ, 2); tt_int_op( circpad_machine_reached_padding_limit(client_side->padding_info[0]), @@ -2989,18 +3000,12 @@ helper_test_hs_machines(bool test_intro_circs) tt_int_op(relay_side->padding_info[0]->current_state, OP_EQ, CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP); - /* For rendezvous circuit machines we can stop early since are simpler than - * the intro circuit machines. */ - if (!test_intro_circs) { - tt_int_op(client_side->padding_info[0]->histogram[0], OP_EQ, 1); - goto done; - } - /* Check that the state lengths have been sampled and are within range */ circpad_machine_runtime_t *client_machine_runtime = client_side->padding_info[0]; circpad_machine_runtime_t *relay_machine_runtime = relay_side->padding_info[0]; + if (test_intro_circs) { tt_int_op(client_machine_runtime->state_length, OP_GE, INTRO_MACHINE_MINIMUM_PADDING); @@ -3127,5 +3132,6 @@ struct testcase_t circuitpadding_tests[] = { TEST_CIRCUITPADDING(circuitpadding_closest_token_removal_usec, TT_FORK), TEST_CIRCUITPADDING(circuitpadding_token_removal_exact, TT_FORK), TEST_CIRCUITPADDING(circuitpadding_manage_circuit_lifetime, TT_FORK), + TEST_CIRCUITPADDING(circuitpadding_hs_machines, TT_FORK), END_OF_TESTCASES }; From 42ea3a416e649eddf4bf9a0dee88a8b4fdbbef20 Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Thu, 28 Mar 2019 15:38:33 +0200 Subject: [PATCH 1016/2557] Improve logging around the circpad module.. - Add some more useful logs for future debugging. - Stop usage of circpad_state_to_string(). It's innacurate. - Reduce severity and fix up log domain of some logging messages. --- src/core/or/circuitpadding.c | 70 +++++++++++---------------- src/core/or/circuitpadding_machines.c | 8 +-- 2 files changed, 32 insertions(+), 46 deletions(-) diff --git a/src/core/or/circuitpadding.c b/src/core/or/circuitpadding.c index 72909bf723..b2315d822f 100644 --- a/src/core/or/circuitpadding.c +++ b/src/core/or/circuitpadding.c @@ -131,34 +131,6 @@ STATIC smartlist_t *relay_padding_machines = NULL; continue; #define FOR_EACH_ACTIVE_CIRCUIT_MACHINE_END } STMT_END ; -/** - * Return a human-readable description for a circuit padding state. - */ -static const char * -circpad_state_to_string(circpad_statenum_t state) -{ - const char *descr; - - switch (state) { - case CIRCPAD_STATE_START: - descr = "START"; - break; - case CIRCPAD_STATE_BURST: - descr = "BURST"; - break; - case CIRCPAD_STATE_GAP: - descr = "GAP"; - break; - case CIRCPAD_STATE_END: - descr = "END"; - break; - default: - descr = "CUSTOM"; // XXX: Just return # in static char buf? - } - - return descr; -} - /** * Free the machineinfo at an index */ @@ -540,6 +512,8 @@ circpad_choose_state_length(circpad_machine_runtime_t *mi) } mi->state_length = clamp_double_to_int64(length); + + log_info(LD_CIRC, "State length sampled to %"PRIu64".", mi->state_length); } /** @@ -914,7 +888,7 @@ circpad_machine_remove_closest_token(circpad_machine_runtime_t *mi, bin_to_remove = lower; } mi->histogram[bin_to_remove]--; - log_debug(LD_GENERAL, "Removing token from bin %d", bin_to_remove); + log_debug(LD_CIRC, "Removing token from bin %d", bin_to_remove); return; } else { if (current - lower > higher - current) { @@ -1224,14 +1198,16 @@ circpad_send_padding_cell_for_callback(circpad_machine_runtime_t *mi) circpad_send_command_to_hop(TO_ORIGIN_CIRCUIT(mi->on_circ), CIRCPAD_GET_MACHINE(mi)->target_hopnum, RELAY_COMMAND_DROP, NULL, 0); - log_fn(LOG_INFO,LD_CIRC, "Callback: Sending padding to origin circuit %u.", - TO_ORIGIN_CIRCUIT(mi->on_circ)->global_identifier); + log_info(LD_CIRC, "Callback: Sending padding to origin circuit %u" + " (%d) [length: %"PRIu64"]", + TO_ORIGIN_CIRCUIT(mi->on_circ)->global_identifier, + mi->on_circ->purpose, mi->state_length); } else { // If we're a non-origin circ, we can just send from here as if we're the // edge. if (TO_OR_CIRCUIT(circ)->p_chan_cells.n <= circpad_max_circ_queued_cells) { - log_fn(LOG_INFO,LD_CIRC, - "Callback: Sending padding to non-origin circuit."); + log_info(LD_CIRC, "Callback: Sending padding to circuit (%d)" + " [length: %"PRIu64"]", mi->on_circ->purpose, mi->state_length); relay_send_command_from_edge(0, mi->on_circ, RELAY_COMMAND_DROP, NULL, 0, NULL); rep_hist_padding_count_write(PADDING_TYPE_DROP); @@ -1599,9 +1575,8 @@ circpad_machine_spec_transition,(circpad_machine_runtime_t *mi, * a transition to itself. All non-specified events are ignored. */ log_fn(LOG_INFO, LD_CIRC, - "Circpad machine %d transitioning from %s to %s", - mi->machine_index, circpad_state_to_string(mi->current_state), - circpad_state_to_string(s)); + "Circpad machine %d transitioning from %u to %u", + mi->machine_index, mi->current_state, s); /* If this is not the same state, switch and init tokens, * otherwise just reschedule padding. */ @@ -2096,6 +2071,7 @@ circpad_add_matching_machines(origin_circuit_t *on_circ, if (circpad_negotiate_padding(on_circ, machine->machine_num, machine->target_hopnum, CIRCPAD_COMMAND_START) < 0) { + log_info(LD_CIRC, "Padding not negotiated. Cleaning machine"); circpad_circuit_machineinfo_free_idx(circ, i); circ->padding_machine[i] = NULL; on_circ->padding_negotiation_failed = 1; @@ -2369,6 +2345,16 @@ circpad_setup_machine_on_circ(circuit_t *on_circ, == NULL); tor_assert_nonfatal(on_circ->padding_info[machine->machine_index] == NULL); + /* Log message */ + if (CIRCUIT_IS_ORIGIN(on_circ)) { + log_info(LD_CIRC, "Registering machine %s to origin circ %u (%d)", + machine->name, + TO_ORIGIN_CIRCUIT(on_circ)->global_identifier, on_circ->purpose); + } else { + log_info(LD_CIRC, "Registering machine %s to non-origin circ (%d)", + machine->name, on_circ->purpose); + } + on_circ->padding_info[machine->machine_index] = circpad_circuit_machineinfo_new(on_circ, machine->machine_index); on_circ->padding_machine[machine->machine_index] = machine; @@ -2389,7 +2375,7 @@ padding_machine_state_is_valid(const circpad_state_t *state) /* We need at least two bins in a histogram */ if (state->histogram_len < 2) { - log_warn(LD_GENERAL, "You can't have a histogram with less than 2 bins"); + log_warn(LD_CIRC, "You can't have a histogram with less than 2 bins"); return false; } @@ -2399,7 +2385,7 @@ padding_machine_state_is_valid(const circpad_state_t *state) /* Check that histogram edges are strictly increasing. Ignore the first * edge since it can be zero. */ if (prev_bin_edge >= state->histogram_edges[b] && b > 0) { - log_warn(LD_GENERAL, "Histogram edges are not increasing [%u/%u]", + log_warn(LD_CIRC, "Histogram edges are not increasing [%u/%u]", prev_bin_edge, state->histogram_edges[b]); return false; } @@ -2411,7 +2397,7 @@ padding_machine_state_is_valid(const circpad_state_t *state) } /* Verify that the total number of tokens is correct */ if (tokens_count != state->histogram_total_tokens) { - log_warn(LD_GENERAL, "Histogram token count is wrong [%u/%u]", + log_warn(LD_CIRC, "Histogram token count is wrong [%u/%u]", tokens_count, state->histogram_total_tokens); return false; } @@ -2442,7 +2428,7 @@ circpad_register_padding_machine(circpad_machine_spec_t *machine, smartlist_t *machine_list) { if (!padding_machine_is_valid(machine)) { - log_warn(LD_GENERAL, "Machine #%u is invalid. Ignoring.", + log_warn(LD_CIRC, "Machine #%u is invalid. Ignoring.", machine->machine_num); return; } @@ -2748,8 +2734,8 @@ circpad_negotiate_padding(origin_circuit_t *circ, &type)) < 0) return -1; - log_fn(LOG_INFO,LD_CIRC, "Negotiating padding on circuit %u", - circ->global_identifier); + log_fn(LOG_INFO,LD_CIRC, "Negotiating padding on circuit %u (%d)", + circ->global_identifier, TO_CIRCUIT(circ)->purpose); return circpad_send_command_to_hop(circ, target_hopnum, RELAY_COMMAND_PADDING_NEGOTIATE, diff --git a/src/core/or/circuitpadding_machines.c b/src/core/or/circuitpadding_machines.c index 4d348f9593..c096822899 100644 --- a/src/core/or/circuitpadding_machines.c +++ b/src/core/or/circuitpadding_machines.c @@ -212,7 +212,7 @@ circpad_machine_client_hide_intro_circuits(smartlist_t *machines_sl) client_machine->machine_num = smartlist_len(machines_sl); circpad_register_padding_machine(client_machine, machines_sl); - log_warn(LD_GENERAL, + log_info(LD_CIRC, "Registered client intro point hiding padding machine (%u)", client_machine->machine_num); } @@ -253,7 +253,7 @@ circpad_machine_relay_hide_intro_circuits(smartlist_t *machines_sl) relay_machine->machine_num = smartlist_len(machines_sl); circpad_register_padding_machine(relay_machine, machines_sl); - log_warn(LD_GENERAL, + log_info(LD_CIRC, "Registered relay intro circuit hiding padding machine (%u)", relay_machine->machine_num); } @@ -374,7 +374,7 @@ circpad_machine_client_hide_rend_circuits(smartlist_t *machines_sl) client_machine->machine_num = smartlist_len(machines_sl); circpad_register_padding_machine(client_machine, machines_sl); - log_warn(LD_GENERAL, + log_info(LD_CIRC, "Registered client rendezvous circuit hiding padding machine (%u)", client_machine->machine_num); } @@ -412,7 +412,7 @@ circpad_machine_relay_hide_rend_circuits(smartlist_t *machines_sl) relay_machine->machine_num = smartlist_len(machines_sl); circpad_register_padding_machine(relay_machine, machines_sl); - log_warn(LD_GENERAL, + log_info(LD_CIRC, "Registered relay rendezvous circuit hiding padding machine (%u)", relay_machine->machine_num); } From 58cb98af32e8436eccf9536255b8158271f1c03d Mon Sep 17 00:00:00 2001 From: "Iain R. Learmonth" Date: Thu, 16 May 2019 12:54:31 +0100 Subject: [PATCH 1017/2557] Prop 301: No longer vote on RecommendedPackages This is the first half of implementing proposal 301. The RecommendedPackages torrc option is marked as obsolete and the test cases for the option removed. Additionally, the code relating to generating and formatting package lines in votes is removed. These lines may still appear in votes from other directory authorities running earlier versions of the code and so consensuses may still contain package lines. A new consensus method will be needed to stop including package lines in consensuses. Fixes: #28465 --- changes/ticket29738 | 6 ++++++ doc/tor.1.txt | 6 ------ src/app/config/config.c | 9 +-------- src/app/config/or_options_st.h | 1 - src/feature/dirauth/dirvote.c | 26 -------------------------- src/test/test_options.c | 24 ------------------------ 6 files changed, 7 insertions(+), 65 deletions(-) create mode 100644 changes/ticket29738 diff --git a/changes/ticket29738 b/changes/ticket29738 new file mode 100644 index 0000000000..9217cc9a5f --- /dev/null +++ b/changes/ticket29738 @@ -0,0 +1,6 @@ + o Minor features (recommended packages): + - No longer include recommended packages in votes as detailed in proposal + 301. The RecommendedPackages torrc option is deprecated and will no + longer have any effect. "package" lines will still be considered when + computing consensuses for consensus methods that include them. Fixes + ticket 29738. diff --git a/doc/tor.1.txt b/doc/tor.1.txt index 4bd365c774..064259b15f 100644 --- a/doc/tor.1.txt +++ b/doc/tor.1.txt @@ -2720,12 +2720,6 @@ on the public Tor network. multiple times: the values from multiple lines are spliced together. When this is set then **VersioningAuthoritativeDirectory** should be set too. -[[RecommendedPackages]] **RecommendedPackages** __PACKAGENAME__ __VERSION__ __URL__ __DIGESTTYPE__**=**__DIGEST__ :: - Adds "package" line to the directory authority's vote. This information - is used to vote on the correct URL and digest for the released versions - of different Tor-related packages, so that the consensus can certify - them. This line may appear any number of times. - [[RecommendedClientVersions]] **RecommendedClientVersions** __STRING__:: STRING is a comma-separated list of Tor versions currently believed to be safe for clients to use. This information is included in version 2 diff --git a/src/app/config/config.c b/src/app/config/config.c index d03305627b..4382a44723 100644 --- a/src/app/config/config.c +++ b/src/app/config/config.c @@ -593,7 +593,7 @@ static config_var_t option_vars_[] = { V(RecommendedVersions, LINELIST, NULL), V(RecommendedClientVersions, LINELIST, NULL), V(RecommendedServerVersions, LINELIST, NULL), - V(RecommendedPackages, LINELIST, NULL), + OBSOLETE("RecommendedPackages"), V(ReducedConnectionPadding, BOOL, "0"), V(ConnectionPadding, AUTOBOOL, "auto"), V(RefuseUnknownExits, AUTOBOOL, "auto"), @@ -3521,13 +3521,6 @@ options_validate(or_options_t *old_options, or_options_t *options, "features to be broken in unpredictable ways."); } - for (cl = options->RecommendedPackages; cl; cl = cl->next) { - if (! validate_recommended_package_line(cl->value)) { - log_warn(LD_CONFIG, "Invalid RecommendedPackage line %s will be ignored", - escaped(cl->value)); - } - } - if (options->AuthoritativeDir) { if (!options->ContactInfo && !options->TestingTorNetwork) REJECT("Authoritative directory servers must set ContactInfo"); diff --git a/src/app/config/or_options_st.h b/src/app/config/or_options_st.h index 4e03bec7fa..7e79834f8c 100644 --- a/src/app/config/or_options_st.h +++ b/src/app/config/or_options_st.h @@ -121,7 +121,6 @@ struct or_options_t { struct config_line_t *RecommendedVersions; struct config_line_t *RecommendedClientVersions; struct config_line_t *RecommendedServerVersions; - struct config_line_t *RecommendedPackages; /** Whether dirservers allow router descriptors with private IPs. */ int DirAllowPrivateAddresses; /** Whether routers accept EXTEND cells to routers with private IPs. */ diff --git a/src/feature/dirauth/dirvote.c b/src/feature/dirauth/dirvote.c index b841ab240f..0084fea1e3 100644 --- a/src/feature/dirauth/dirvote.c +++ b/src/feature/dirauth/dirvote.c @@ -220,7 +220,6 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key, networkstatus_t *v3_ns) { smartlist_t *chunks = smartlist_new(); - char *packages = NULL; char fingerprint[FINGERPRINT_LEN+1]; char digest[DIGEST_LEN]; uint32_t addr; @@ -246,19 +245,6 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key, v3_ns->server_versions); protocols_lines = format_protocols_lines_for_vote(v3_ns); - if (v3_ns->package_lines) { - smartlist_t *tmp = smartlist_new(); - SMARTLIST_FOREACH(v3_ns->package_lines, const char *, p, - if (validate_recommended_package_line(p)) - smartlist_add_asprintf(tmp, "package %s\n", p)); - smartlist_sort_strings(tmp); - packages = smartlist_join_strings(tmp, "", 0, NULL); - SMARTLIST_FOREACH(tmp, char *, cp, tor_free(cp)); - smartlist_free(tmp); - } else { - packages = tor_strdup(""); - } - /* Get shared random commitments/reveals line(s). */ shared_random_vote_str = sr_get_string_for_vote(); @@ -344,7 +330,6 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key, "voting-delay %d %d\n" "%s%s" /* versions */ "%s" /* protocols */ - "%s" /* packages */ "known-flags %s\n" "flag-thresholds %s\n" "params %s\n" @@ -361,7 +346,6 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key, client_versions_line, server_versions_line, protocols_lines, - packages, flags, flag_thresholds, params, @@ -461,7 +445,6 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key, tor_free(client_versions_line); tor_free(server_versions_line); tor_free(protocols_lines); - tor_free(packages); SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp)); smartlist_free(chunks); @@ -4669,15 +4652,6 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key, tor_assert_nonfatal(protover_all_supported( v3_out->recommended_client_protocols, NULL)); - v3_out->package_lines = smartlist_new(); - { - config_line_t *cl; - for (cl = get_options()->RecommendedPackages; cl; cl = cl->next) { - if (validate_recommended_package_line(cl->value)) - smartlist_add_strdup(v3_out->package_lines, cl->value); - } - } - v3_out->known_flags = smartlist_new(); smartlist_split_string(v3_out->known_flags, DIRVOTE_UNIVERSAL_FLAGS, diff --git a/src/test/test_options.c b/src/test/test_options.c index 396be6b18d..d693fe0568 100644 --- a/src/test/test_options.c +++ b/src/test/test_options.c @@ -1342,29 +1342,6 @@ test_options_validate__token_bucket(void *ignored) tor_free(msg); } -static void -test_options_validate__recommended_packages(void *ignored) -{ - (void)ignored; - int ret; - char *msg; - setup_capture_of_logs(LOG_WARN); - options_test_data_t *tdata = get_options_test_data( - "RecommendedPackages foo 1.2 http://foo.com sha1=123123123123\n" - "RecommendedPackages invalid-package-line\n"); - - ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg); - tt_int_op(ret, OP_EQ, -1); - expect_no_log_msg("Invalid RecommendedPackage line " - "invalid-package-line will be ignored\n"); - - done: - escaped(NULL); // This will free the leaking memory from the previous escaped - teardown_capture_of_logs(); - free_options_test_data(tdata); - tor_free(msg); -} - static void test_options_validate__fetch_dir(void *ignored) { @@ -4200,7 +4177,6 @@ struct testcase_t options_tests[] = { LOCAL_VALIDATE_TEST(exclude_nodes), LOCAL_VALIDATE_TEST(node_families), LOCAL_VALIDATE_TEST(token_bucket), - LOCAL_VALIDATE_TEST(recommended_packages), LOCAL_VALIDATE_TEST(fetch_dir), LOCAL_VALIDATE_TEST(conn_limit), LOCAL_VALIDATE_TEST(paths_needed), From 1bf451cffba5da84166dda48ec957e0b9cf45bee Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 16 May 2019 15:03:54 -0400 Subject: [PATCH 1018/2557] rng_test_helpers: add a needless lock/unlock pair to please coverity Fix for CID 1444908 --- src/test/rng_test_helpers.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/test/rng_test_helpers.c b/src/test/rng_test_helpers.c index 262d380bda..04b7138471 100644 --- a/src/test/rng_test_helpers.c +++ b/src/test/rng_test_helpers.c @@ -183,11 +183,14 @@ void testing_enable_prefilled_rng(const void *buffer, size_t buflen) { tor_assert(buflen > 0); + tor_assert(!rng_mutex); rng_mutex = tor_mutex_new(); + tor_mutex_acquire(rng_mutex); prefilled_rng_buffer = tor_memdup(buffer, buflen); prefilled_rng_buflen = buflen; prefilled_rng_idx = 0; + tor_mutex_release(rng_mutex); MOCK(crypto_rand, crypto_rand_prefilled); MOCK(crypto_strongest_rand_, mock_crypto_strongest_rand); From f237fed7466e84e6b6399fe5c7cb80bd59e3e2d8 Mon Sep 17 00:00:00 2001 From: Mike Perry Date: Thu, 16 May 2019 18:41:21 +0000 Subject: [PATCH 1019/2557] Refactor intro machines, stage 1/2: Move state transition code. This just moves the state transition directives into the proper client/relay side functions. It also allows us to remove some dead-code from the client side (since the client doesn't send padding). --- src/core/or/circuitpadding_machines.c | 91 +++++++++++++-------------- 1 file changed, 44 insertions(+), 47 deletions(-) diff --git a/src/core/or/circuitpadding_machines.c b/src/core/or/circuitpadding_machines.c index c096822899..4256a7be4a 100644 --- a/src/core/or/circuitpadding_machines.c +++ b/src/core/or/circuitpadding_machines.c @@ -57,49 +57,6 @@ #include "core/or/circuitpadding_machines.h" #include "core/or/circuitpadding.h" -/* Setup the simple state machine we use for all HS padding machines */ -static void -setup_state_machine_for_hiding_intro_circuits(circpad_machine_spec_t *machine) -{ - /* Two states: START, OBFUSCATE_CIRC_SETUP (and END) */ - circpad_machine_states_init(machine, 2); - - /* For the relay-side machine, we want to transition - * START -> OBFUSCATE_CIRC_SETUP upon first non-padding - * cell sent (PADDING_NEGOTIATED in this case). - * - * For the origin-side machine, we transition to OBFUSCATE_CIRC_SETUP after - * sending PADDING_NEGOTIATE, and we stay there (without sending any padding) - * until we receive a STOP from the other side. */ - machine->states[CIRCPAD_STATE_START]. - next_state[CIRCPAD_EVENT_NONPADDING_SENT] = - CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP; - - /* For the relay-side, we want to transition from OBFUSCATE_CIRC_SETUP to END - * state when the length finishes. - * - * For the origin-side, we don't care because the relay-side machine is gonna - * END us. */ - machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP]. - next_state[CIRCPAD_EVENT_LENGTH_COUNT] = CIRCPAD_STATE_END; - - /* Now let's define the OBF -> OBF transitions that maintain our padding - * flow: - * - * For the relay-side machine, we want to keep on sending padding bytes even - * when nothing else happens on this circuit. */ - machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP]. - next_state[CIRCPAD_EVENT_PADDING_SENT] = - CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP; - /* For the relay-side machine, we need this transition so that we re-enter - the state, after PADDING_NEGOTIATED is sent. Otherwise, the remove token - function will disable the timer, and nothing will restart it since there - is no other motion on an intro circuit. */ - machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP]. - next_state[CIRCPAD_EVENT_NONPADDING_SENT] = - CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP; -} - /* Setup the OBFUSCATE_CIRC_SETUP state of the machine that hides client-side * intro circuits. */ static void @@ -202,8 +159,19 @@ circpad_machine_client_hide_intro_circuits(smartlist_t *machines_sl) client_machine->allowed_padding_count = INTRO_MACHINE_MAXIMUM_PADDING; client_machine->max_padding_percent = 1; - /* Setup states and histograms */ - setup_state_machine_for_hiding_intro_circuits(client_machine); + /* Two states: START, OBFUSCATE_CIRC_SETUP (and END) */ + circpad_machine_states_init(client_machine, 2); + + /* For the origin-side machine, we transition to OBFUSCATE_CIRC_SETUP after + * sending PADDING_NEGOTIATE, and we stay there (without sending any padding) + * until we receive a STOP from the other side. */ + client_machine->states[CIRCPAD_STATE_START]. + next_state[CIRCPAD_EVENT_NONPADDING_SENT] = + CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP; + + /* origin-side machine has no event reactions while in + * CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP, so no more state transitions here). */ + setup_obf_state_for_hiding_intro_circuits( &client_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP], true); @@ -243,8 +211,37 @@ circpad_machine_relay_hide_intro_circuits(smartlist_t *machines_sl) relay_machine->allowed_padding_count = INTRO_MACHINE_MAXIMUM_PADDING; relay_machine->max_padding_percent = 1; - /* Setup states and histograms */ - setup_state_machine_for_hiding_intro_circuits(relay_machine); + /* Two states: START, OBFUSCATE_CIRC_SETUP (and END) */ + circpad_machine_states_init(relay_machine, 2); + + /* For the relay-side machine, we want to transition + * START -> OBFUSCATE_CIRC_SETUP upon first non-padding + * cell sent (PADDING_NEGOTIATED in this case). */ + relay_machine->states[CIRCPAD_STATE_START]. + next_state[CIRCPAD_EVENT_NONPADDING_SENT] = + CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP; + + /* For the relay-side, we want to transition from OBFUSCATE_CIRC_SETUP to END + * state when the length finishes. */ + relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP]. + next_state[CIRCPAD_EVENT_LENGTH_COUNT] = CIRCPAD_STATE_END; + + /* Now let's define the OBF -> OBF transitions that maintain our padding + * flow: + * + * For the relay-side machine, we want to keep on sending padding bytes even + * when nothing else happens on this circuit. */ + relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP]. + next_state[CIRCPAD_EVENT_PADDING_SENT] = + CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP; + /* For the relay-side machine, we need this transition so that we re-enter + the state, after PADDING_NEGOTIATED is sent. Otherwise, the remove token + function will disable the timer, and nothing will restart it since there + is no other motion on an intro circuit. */ + relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP]. + next_state[CIRCPAD_EVENT_NONPADDING_SENT] = + CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP; + setup_obf_state_for_hiding_intro_circuits( &relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP], false); From 0a9685b3a7437e8851f8cb65fea3d0a16b7833a7 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 16 May 2019 15:21:18 -0400 Subject: [PATCH 1020/2557] hs tests: explicitly free 'service' variable. This should fix about 15 CID issues, where coverity can't tell that hs_free_all() frees the service we allocated. --- src/test/test_hs_common.c | 4 ++++ src/test/test_hs_service.c | 48 +++++++++++++++++++++++++++++++++----- 2 files changed, 46 insertions(+), 6 deletions(-) diff --git a/src/test/test_hs_common.c b/src/test/test_hs_common.c index bb41f1f870..abded6021e 100644 --- a/src/test/test_hs_common.c +++ b/src/test/test_hs_common.c @@ -603,6 +603,10 @@ test_desc_reupload_logic(void *arg) SMARTLIST_FOREACH(ns->routerstatus_list, routerstatus_t *, rs, routerstatus_free(rs)); smartlist_clear(ns->routerstatus_list); + if (service) { + remove_service(get_hs_service_map(), service); + hs_service_free(service); + } networkstatus_vote_free(ns); cleanup_nodelist(); hs_free_all(); diff --git a/src/test/test_hs_service.c b/src/test/test_hs_service.c index c4a8583696..a303f10411 100644 --- a/src/test/test_hs_service.c +++ b/src/test/test_hs_service.c @@ -878,6 +878,10 @@ test_helper_functions(void *arg) done: /* This will free the service and all objects associated to it. */ + if (service) { + remove_service(get_hs_service_map(), service); + hs_service_free(service); + } hs_service_free_all(); UNMOCK(node_get_by_id); } @@ -887,7 +891,7 @@ static void test_intro_circuit_opened(void *arg) { int flags = CIRCLAUNCH_NEED_UPTIME | CIRCLAUNCH_IS_INTERNAL; - hs_service_t *service; + hs_service_t *service = NULL; origin_circuit_t *circ = NULL; (void) arg; @@ -935,6 +939,10 @@ test_intro_circuit_opened(void *arg) done: circuit_free_(TO_CIRCUIT(circ)); + if (service) { + remove_service(get_hs_service_map(), service); + hs_service_free(service); + } hs_free_all(); UNMOCK(circuit_mark_for_close_); UNMOCK(relay_send_command_from_edge_); @@ -949,7 +957,7 @@ test_intro_established(void *arg) int flags = CIRCLAUNCH_NEED_UPTIME | CIRCLAUNCH_IS_INTERNAL; uint8_t payload[RELAY_PAYLOAD_SIZE] = {0}; origin_circuit_t *circ = NULL; - hs_service_t *service; + hs_service_t *service = NULL; hs_service_intro_point_t *ip = NULL; (void) arg; @@ -1010,6 +1018,10 @@ test_intro_established(void *arg) done: if (circ) circuit_free_(TO_CIRCUIT(circ)); + if (service) { + remove_service(get_hs_service_map(), service); + hs_service_free(service); + } hs_free_all(); UNMOCK(circuit_mark_for_close_); } @@ -1021,7 +1033,7 @@ test_rdv_circuit_opened(void *arg) { int flags = CIRCLAUNCH_NEED_UPTIME | CIRCLAUNCH_IS_INTERNAL; origin_circuit_t *circ = NULL; - hs_service_t *service; + hs_service_t *service = NULL; (void) arg; @@ -1052,6 +1064,10 @@ test_rdv_circuit_opened(void *arg) done: circuit_free_(TO_CIRCUIT(circ)); + if (service) { + remove_service(get_hs_service_map(), service); + hs_service_free(service); + } hs_free_all(); UNMOCK(circuit_mark_for_close_); UNMOCK(relay_send_command_from_edge_); @@ -1139,6 +1155,10 @@ test_closing_intro_circs(void *arg) circuit_free_(TO_CIRCUIT(intro_circ)); } /* Frees the service object. */ + if (service) { + remove_service(get_hs_service_map(), service); + hs_service_free(service); + } hs_free_all(); UNMOCK(assert_circuit_ok); } @@ -1151,7 +1171,7 @@ test_introduce2(void *arg) int flags = CIRCLAUNCH_NEED_UPTIME | CIRCLAUNCH_IS_INTERNAL; uint8_t payload[RELAY_PAYLOAD_SIZE] = {0}; origin_circuit_t *circ = NULL; - hs_service_t *service; + hs_service_t *service = NULL; hs_service_intro_point_t *ip = NULL; (void) arg; @@ -1218,6 +1238,10 @@ test_introduce2(void *arg) dummy_state = NULL; if (circ) circuit_free_(TO_CIRCUIT(circ)); + if (service) { + remove_service(get_hs_service_map(), service); + hs_service_free(service); + } hs_free_all(); UNMOCK(circuit_mark_for_close_); } @@ -1302,6 +1326,10 @@ test_service_event(void *arg) done: hs_circuitmap_remove_circuit(TO_CIRCUIT(circ)); circuit_free_(TO_CIRCUIT(circ)); + if (service) { + remove_service(get_hs_service_map(), service); + hs_service_free(service); + } hs_free_all(); UNMOCK(circuit_mark_for_close_); } @@ -1312,7 +1340,7 @@ test_rotate_descriptors(void *arg) { int ret; time_t next_rotation_time, now; - hs_service_t *service; + hs_service_t *service = NULL; hs_service_descriptor_t *desc_next; (void) arg; @@ -1404,6 +1432,10 @@ test_rotate_descriptors(void *arg) tt_assert(service->desc_next); done: + if (service) { + remove_service(get_hs_service_map(), service); + hs_service_free(service); + } hs_free_all(); UNMOCK(get_or_state); UNMOCK(circuit_mark_for_close_); @@ -1417,7 +1449,7 @@ test_build_update_descriptors(void *arg) { int ret; node_t *node; - hs_service_t *service; + hs_service_t *service = NULL; hs_service_intro_point_t *ip_cur, *ip_next; routerinfo_t ri; @@ -1634,6 +1666,10 @@ test_build_update_descriptors(void *arg) tt_u64_op(service->desc_next->next_upload_time, OP_EQ, 0); done: + if (service) { + remove_service(get_hs_service_map(), service); + hs_service_free(service); + } hs_free_all(); nodelist_free_all(); } From bbb974234c67d8d7dc95a760101ca61a9e88d647 Mon Sep 17 00:00:00 2001 From: Mike Perry Date: Thu, 16 May 2019 19:17:31 +0000 Subject: [PATCH 1021/2557] Refactor intro machines, stage 2/2: Move histogram code. The client side had garbage histograms and deadcode here, too. That code has been removed. The tests have also been updated to properly test the intro circ by sending padding from the relay side to the client, and verifying that both shut down when padding was up. (The tests previously erroneously tested only the client side of intro circs, which actually were supposed to be doing nothing). --- src/core/or/circuitpadding_machines.c | 96 +++++++++++++-------------- src/test/test_circuitpadding.c | 37 +++++++---- 2 files changed, 72 insertions(+), 61 deletions(-) diff --git a/src/core/or/circuitpadding_machines.c b/src/core/or/circuitpadding_machines.c index 4256a7be4a..be02a597dc 100644 --- a/src/core/or/circuitpadding_machines.c +++ b/src/core/or/circuitpadding_machines.c @@ -57,48 +57,6 @@ #include "core/or/circuitpadding_machines.h" #include "core/or/circuitpadding.h" -/* Setup the OBFUSCATE_CIRC_SETUP state of the machine that hides client-side - * intro circuits. */ -static void -setup_obf_state_for_hiding_intro_circuits(circpad_state_t *obf_state, - bool is_client) -{ - /* Token removal strategy for OBFUSCATE_CIRC_SETUP state. We pick the - * simplest one since we rely on the state length sampling and not the - * tokens. */ - obf_state->token_removal = CIRCPAD_TOKEN_REMOVAL_NONE; - - /* Figure out the length of the OBFUSCATE_CIRC_SETUP state so that it's - * randomized. We will set the histogram such that we don't send any padding - * from the origin-side, so just tune these parameteres for the - * relay-side. */ - obf_state->length_dist.type = CIRCPAD_DIST_UNIFORM; - obf_state->length_dist.param1 = INTRO_MACHINE_MINIMUM_PADDING; - obf_state->length_dist.param2 = INTRO_MACHINE_MAXIMUM_PADDING; - - /* Configure histogram */ - obf_state->histogram_len = 2; - if (is_client) { - /* For the origin-side machine we don't want to send any padding, so setup - * infinite delays. */ - obf_state->histogram_edges[0] = CIRCPAD_DELAY_INFINITE-1; - obf_state->histogram_edges[1] = CIRCPAD_DELAY_INFINITE; - /* zero tokens */ - obf_state->histogram[0] = 0; - } else { - /* For the relay-side machine we want to batch padding instantly to pretend - * its an incoming directory download. So set the histogram edges tight: - * (1, 10ms, infinity). */ - obf_state->histogram_edges[0] = 1000; - obf_state->histogram_edges[1] = 10000; - /* padding is controlled by state length, so this is just a high value */ - obf_state->histogram[0] = 1000; - } - - /* just one bin, so setup the total tokens */ - obf_state->histogram_total_tokens = obf_state->histogram[0]; -} - /** Create a client-side padding machine that aims to hide IP circuits. In * particular, it keeps intro circuits alive until a bunch of fake traffic has * been pushed through. @@ -170,11 +128,11 @@ circpad_machine_client_hide_intro_circuits(smartlist_t *machines_sl) CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP; /* origin-side machine has no event reactions while in - * CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP, so no more state transitions here). */ + * CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP, so no more state transitions here. */ - setup_obf_state_for_hiding_intro_circuits( - &client_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP], - true); + /* The client side should never send padding, so it does not need + * to specify token removal, or a histogram definition or state lengths. + * That is all controlled by the middle node. */ /* Register the machine */ client_machine->machine_num = smartlist_len(machines_sl); @@ -242,9 +200,49 @@ circpad_machine_relay_hide_intro_circuits(smartlist_t *machines_sl) next_state[CIRCPAD_EVENT_NONPADDING_SENT] = CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP; - setup_obf_state_for_hiding_intro_circuits( - &relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP], - false); + /* Token removal strategy for OBFUSCATE_CIRC_SETUP state: Don't + * remove any tokens. + * + * We rely on the state length sampling and not token removal, to avoid + * the mallocs required to copy the histograms for token removal, + * and to avoid monotime calls needed to determine histogram + * bins for token removal. */ + relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP]. + token_removal = CIRCPAD_TOKEN_REMOVAL_NONE; + + /* Figure out the length of the OBFUSCATE_CIRC_SETUP state so that it's + * randomized. The relay side will send between INTRO_MACHINE_MINIMUM_PADDING + * and INTRO_MACHINE_MAXIMUM_PADDING padding cells towards the client. */ + relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP]. + length_dist.type = CIRCPAD_DIST_UNIFORM; + relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP]. + length_dist.param1 = INTRO_MACHINE_MINIMUM_PADDING; + relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP]. + length_dist.param2 = INTRO_MACHINE_MAXIMUM_PADDING; + + /* Configure histogram */ + relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP]. + histogram_len = 2; + + /* For the relay-side machine we want to batch padding instantly to pretend + * its an incoming directory download. So set the histogram edges tight: + * (1, 10ms, infinity). */ + relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP]. + histogram_edges[0] = 1000; + relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP]. + histogram_edges[1] = 10000; + + /* We put all our tokens in bin 0, which means we want 100% probability + * for choosing a inter-packet delay of between 1000 and 10000 microseconds + * (1 to 10ms). Since we only have 1 bin, it doesn't matter how many tokens + * there are, 1000 out of 1000 is 100% */ + relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP]. + histogram[0] = 1000; + + /* just one bin, so setup the total tokens */ + relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP]. + histogram_total_tokens = + relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].histogram[0]; /* Register the machine */ relay_machine->machine_num = smartlist_len(machines_sl); diff --git a/src/test/test_circuitpadding.c b/src/test/test_circuitpadding.c index 68d1de91ba..ca65438430 100644 --- a/src/test/test_circuitpadding.c +++ b/src/test/test_circuitpadding.c @@ -3007,10 +3007,10 @@ helper_test_hs_machines(bool test_intro_circs) relay_side->padding_info[0]; if (test_intro_circs) { - tt_int_op(client_machine_runtime->state_length, OP_GE, - INTRO_MACHINE_MINIMUM_PADDING); - tt_int_op(client_machine_runtime->state_length, OP_LT, - INTRO_MACHINE_MAXIMUM_PADDING); + /* on the client side, we don't send any padding so + * state length is not set */ + tt_int_op(client_machine_runtime->state_length, OP_EQ, -1); + /* relay side has state limits. check them */ tt_int_op(relay_machine_runtime->state_length, OP_GE, INTRO_MACHINE_MINIMUM_PADDING); tt_int_op(relay_machine_runtime->state_length, OP_LT, @@ -3020,15 +3020,28 @@ helper_test_hs_machines(bool test_intro_circs) tt_int_op(relay_machine_runtime->state_length, OP_EQ, 1); } - /* Send state_length worth of padding and see that the state goes to END */ - int i; - for (i = (int) client_machine_runtime->state_length ; i > 0 ; i--) { - circpad_send_padding_cell_for_callback(client_machine_runtime); + if (test_intro_circs) { + int i; + /* Send state_length worth of padding from the relay and see that the + * client state goes to END */ + for (i = (int) relay_machine_runtime->state_length ; i > 0 ; i--) { + circpad_send_padding_cell_for_callback(relay_machine_runtime); + } + /* See that the machine has been teared down after all the length has been + * exhausted (the padding info should now be null on both sides) */ + tt_ptr_op(relay_side->padding_info[0], OP_EQ, NULL); + tt_ptr_op(client_side->padding_info[0], OP_EQ, NULL); + } else { + int i; + /* Send state_length worth of padding and see that the state goes to END */ + for (i = (int) client_machine_runtime->state_length ; i > 0 ; i--) { + circpad_send_padding_cell_for_callback(client_machine_runtime); + } + /* See that the machine has been teared down after all the length has been + * exhausted. */ + tt_int_op(client_side->padding_info[0]->current_state, OP_EQ, + CIRCPAD_STATE_END); } - /* See that the machine has been teared down after all the length has been - * exhausted. */ - tt_int_op(client_side->padding_info[0]->current_state, OP_EQ, - CIRCPAD_STATE_END); done: free_fake_orcirc(relay_side); From d5db40a0145457f6a1ee24ac1310511eaf62b959 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 16 May 2019 15:34:28 -0400 Subject: [PATCH 1022/2557] test_channel_listener: free 'chan' explicitly This should fix CID 1437442, where coverity can't tell that channel_free_all() frees the fake channel we allocated. --- src/test/test_channel.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/test/test_channel.c b/src/test/test_channel.c index 4c2bbc86b4..6a6bc9d810 100644 --- a/src/test/test_channel.c +++ b/src/test/test_channel.c @@ -1540,6 +1540,10 @@ test_channel_listener(void *arg) channel_listener_dump_statistics(chan, LOG_INFO); done: + if (chan) { + channel_listener_unregister(chan); + tor_free(chan); + } channel_free_all(); } @@ -1566,4 +1570,3 @@ struct testcase_t channel_tests[] = { NULL, NULL }, END_OF_TESTCASES }; - From 6246f4539ed317e05b773b0e373f1aa369d3759d Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 16 May 2019 15:38:08 -0400 Subject: [PATCH 1023/2557] changes file for coverity fixes in tests (30150) --- changes/ticket30150 | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 changes/ticket30150 diff --git a/changes/ticket30150 b/changes/ticket30150 new file mode 100644 index 0000000000..70808fa5db --- /dev/null +++ b/changes/ticket30150 @@ -0,0 +1,4 @@ + o Minor bugfixes (static analysis): + - Fix several spurious Coverity warnings about the unit tests, to lower our + chances of missing any real warnings in the future. Fixes bug 30150; + bugfix on 0.3.5.1-alpha and various other Tor versions. From 0cba53c6edcc8c9c67a83a836e562552adf95e2d Mon Sep 17 00:00:00 2001 From: Mike Perry Date: Thu, 16 May 2019 19:30:42 +0000 Subject: [PATCH 1024/2557] Refactor rend machines, stage 1/2: Move state transition code. --- src/core/or/circuitpadding_machines.c | 56 +++++++++++++++------------ 1 file changed, 32 insertions(+), 24 deletions(-) diff --git a/src/core/or/circuitpadding_machines.c b/src/core/or/circuitpadding_machines.c index be02a597dc..5350e6e86a 100644 --- a/src/core/or/circuitpadding_machines.c +++ b/src/core/or/circuitpadding_machines.c @@ -284,26 +284,6 @@ setup_obf_state_for_hiding_rend_circuits(circpad_state_t *obf_state) obf_state->histogram_total_tokens = 1; } -/* Setup the simple state machine we use for all HS padding machines */ -static void -setup_state_machine_for_hiding_rend_circuits(circpad_machine_spec_t *machine) -{ - /* Two states: START, OBFUSCATE_CIRC_SETUP (and END) */ - circpad_machine_states_init(machine, 2); - - /* START -> OBFUSCATE_CIRC_SETUP transition upon sending the first - * non-padding cell (which is PADDING_NEGOTIATE) */ - machine->states[CIRCPAD_STATE_START]. - next_state[CIRCPAD_EVENT_NONPADDING_SENT] = - CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP; - - /* OBFUSCATE_CIRC_SETUP -> END transition when we finish all the tokens */ - machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP]. - next_state[CIRCPAD_EVENT_PADDING_RECV] = CIRCPAD_STATE_END; - machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP]. - next_state[CIRCPAD_EVENT_LENGTH_COUNT] = CIRCPAD_STATE_END; -} - /** Create a client-side padding machine that aims to hide rendezvous * circuits.*/ void @@ -360,8 +340,22 @@ circpad_machine_client_hide_rend_circuits(smartlist_t *machines_sl) client_machine->allowed_padding_count = 1; client_machine->max_padding_percent = 1; - /* Setup states and histograms */ - setup_state_machine_for_hiding_rend_circuits(client_machine); + /* Two states: START, OBFUSCATE_CIRC_SETUP (and END) */ + circpad_machine_states_init(client_machine, 2); + + /* START -> OBFUSCATE_CIRC_SETUP transition upon sending the first + * non-padding cell (which is PADDING_NEGOTIATE) */ + client_machine->states[CIRCPAD_STATE_START]. + next_state[CIRCPAD_EVENT_NONPADDING_SENT] = + CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP; + + /* OBFUSCATE_CIRC_SETUP -> END transition when we send our first + * padding packet and/or hit the state length (the state length is 1). */ + client_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP]. + next_state[CIRCPAD_EVENT_PADDING_RECV] = CIRCPAD_STATE_END; + client_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP]. + next_state[CIRCPAD_EVENT_LENGTH_COUNT] = CIRCPAD_STATE_END; + setup_obf_state_for_hiding_rend_circuits( &client_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP]); @@ -398,8 +392,22 @@ circpad_machine_relay_hide_rend_circuits(smartlist_t *machines_sl) relay_machine->allowed_padding_count = 1; relay_machine->max_padding_percent = 1; - /* Setup states and histograms */ - setup_state_machine_for_hiding_rend_circuits(relay_machine); + /* Two states: START, OBFUSCATE_CIRC_SETUP (and END) */ + circpad_machine_states_init(relay_machine, 2); + + /* START -> OBFUSCATE_CIRC_SETUP transition upon sending the first + * non-padding cell (which is PADDING_NEGOTIATED) */ + relay_machine->states[CIRCPAD_STATE_START]. + next_state[CIRCPAD_EVENT_NONPADDING_SENT] = + CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP; + + /* OBFUSCATE_CIRC_SETUP -> END transition when we send our first + * padding packet and/or hit the state length (the state length is 1). */ + relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP]. + next_state[CIRCPAD_EVENT_PADDING_RECV] = CIRCPAD_STATE_END; + relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP]. + next_state[CIRCPAD_EVENT_LENGTH_COUNT] = CIRCPAD_STATE_END; + setup_obf_state_for_hiding_rend_circuits( &relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP]); From 857c54ca038baf0881ce6859dbdb2c3446f54c50 Mon Sep 17 00:00:00 2001 From: Mike Perry Date: Thu, 16 May 2019 19:42:45 +0000 Subject: [PATCH 1025/2557] Refactor rend machines, stage 2/2: Move histogram code. Comment clarifications now that the code is seperated. It's the same code, but its doing this for different reasons on each side. --- src/core/or/circuitpadding_machines.c | 103 +++++++++++++++++--------- 1 file changed, 70 insertions(+), 33 deletions(-) diff --git a/src/core/or/circuitpadding_machines.c b/src/core/or/circuitpadding_machines.c index 5350e6e86a..75d2614aca 100644 --- a/src/core/or/circuitpadding_machines.c +++ b/src/core/or/circuitpadding_machines.c @@ -255,35 +255,6 @@ circpad_machine_relay_hide_intro_circuits(smartlist_t *machines_sl) /************************** Rendezvous-circuit machine ***********************/ -/* Setup the obf state of the machine that hides client-side rend - * circuits. */ -static void -setup_obf_state_for_hiding_rend_circuits(circpad_state_t *obf_state) -{ - /* Don't use a token removal strategy since we don't want to use monotime - functions. We want this machine to be light. */ - obf_state->token_removal = CIRCPAD_TOKEN_REMOVAL_NONE; - - /* Instead, to control the volume of padding (we just want to send a single - * padding cell) we will use a static state length. We just want one token, - * since we want to make the following pattern: - * [PADDING_NEGOTIATE] -> [DROP] -> PADDING_NEGOTIATED -> DROP */ - obf_state->length_dist.type = CIRCPAD_DIST_UNIFORM; - obf_state->length_dist.param1 = 1; - obf_state->length_dist.param2 = 2; - - /* Histogram is: (0 msecs, 50 msecs, infinity). We want this to be fast so - * that the incoming PADDING_NEGOTIATED cell always arrives after the - * outgoing [DROP]. */ - obf_state->histogram_len = 2; - obf_state->histogram_edges[0] = 0; - obf_state->histogram_edges[1] = 1000; - - /* dummy amount of tokens. they don't matter. we rely on state length. */ - obf_state->histogram[0] = 1; - obf_state->histogram_total_tokens = 1; -} - /** Create a client-side padding machine that aims to hide rendezvous * circuits.*/ void @@ -356,8 +327,41 @@ circpad_machine_client_hide_rend_circuits(smartlist_t *machines_sl) client_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP]. next_state[CIRCPAD_EVENT_LENGTH_COUNT] = CIRCPAD_STATE_END; - setup_obf_state_for_hiding_rend_circuits( - &client_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP]); + /* Don't use a token removal strategy since we don't want to use monotime + * functions and we want to avoid mallocing histogram copies. We want + * this machine to be light. */ + client_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP]. + token_removal = CIRCPAD_TOKEN_REMOVAL_NONE; + + /* Instead, to control the volume of padding (we just want to send a single + * padding cell) we will use a static state length. We just want one token, + * since we want to make the following pattern: + * [PADDING_NEGOTIATE] -> [DROP] -> PADDING_NEGOTIATED -> DROP */ + client_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP]. + length_dist.type = CIRCPAD_DIST_UNIFORM; + client_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP]. + length_dist.param1 = 1; + client_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP]. + length_dist.param2 = 2; // rand(1,2) is always 1 + + /* Histogram is: (0 msecs, 1 msec, infinity). We want this to be fast so + * that we send our outgoing [DROP] before the PADDING_NEGOTIATED comes + * back from the relay side. */ + client_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP]. + histogram_len = 2; + client_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP]. + histogram_edges[0] = 0; + client_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP]. + histogram_edges[1] = 1000; + + /* We want a 100% probability of choosing an inter-packet delay of + * between 0 and 1ms. Since we don't use token removal, + * the number of tokens does not matter. (And also, state_length + * governs how many packets we send). */ + client_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP]. + histogram[0] = 1; + client_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP]. + histogram_total_tokens = 1; /* Register the machine */ client_machine->machine_num = smartlist_len(machines_sl); @@ -408,8 +412,41 @@ circpad_machine_relay_hide_rend_circuits(smartlist_t *machines_sl) relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP]. next_state[CIRCPAD_EVENT_LENGTH_COUNT] = CIRCPAD_STATE_END; - setup_obf_state_for_hiding_rend_circuits( - &relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP]); + /* Don't use a token removal strategy since we don't want to use monotime + * functions and we want to avoid mallocing histogram copies. We want + * this machine to be light. */ + relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP]. + token_removal = CIRCPAD_TOKEN_REMOVAL_NONE; + + /* Instead, to control the volume of padding (we just want to send a single + * padding cell) we will use a static state length. We just want one token, + * since we want to make the following pattern: + * [PADDING_NEGOTIATE] -> [DROP] -> PADDING_NEGOTIATED -> DROP */ + relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP]. + length_dist.type = CIRCPAD_DIST_UNIFORM; + relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP]. + length_dist.param1 = 1; + relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP]. + length_dist.param2 = 2; // rand(1,2) is always 1 + + /* Histogram is: (0 msecs, 1 msec, infinity). We want this to be fast so + * that the outgoing DROP cell is sent immediately after the + * PADDING_NEGOTIATED. */ + relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP]. + histogram_len = 2; + relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP]. + histogram_edges[0] = 0; + relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP]. + histogram_edges[1] = 1000; + + /* We want a 100% probability of choosing an inter-packet delay of + * between 0 and 1ms. Since we don't use token removal, + * the number of tokens does not matter. (And also, state_length + * governs how many packets we send). */ + relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP]. + histogram[0] = 1; + relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP]. + histogram_total_tokens = 1; /* Register the machine */ relay_machine->machine_num = smartlist_len(machines_sl); From 1a79bedd97bb67931b87c0ec7e53ceceb77ea754 Mon Sep 17 00:00:00 2001 From: Mike Perry Date: Thu, 16 May 2019 20:25:25 +0000 Subject: [PATCH 1026/2557] Changes file. --- changes/ticket28634 | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 changes/ticket28634 diff --git a/changes/ticket28634 b/changes/ticket28634 new file mode 100644 index 0000000000..7ba05e5c55 --- /dev/null +++ b/changes/ticket28634 @@ -0,0 +1,10 @@ + o Major features (Circuit padding): + - Onion service clients will now add padding cells to the initial portions + of their INTRODUCE and RENDEZVOUS circuits, to make those circuits' + traffic patterns look more like general purpose Exit traffic. The + overhead for this is 2 extra cells in each direction for RENDEZVOUS + circuits, and 1 extra upstream cell and 10 downstream cells for INTRODUCE + circuits. This will only be enabled if the circuit's middle node supports + this feature, too. (Clients may specify fixed middle nodes with the MiddleNodes + torrc directive, and may force-disable this feature with the CircuitPadding + torrc directive). Closes ticket 28634. From 84274000d8a12a62ced9ef95d1bf355e7ab6e5c6 Mon Sep 17 00:00:00 2001 From: Mike Perry Date: Thu, 16 May 2019 20:29:09 +0000 Subject: [PATCH 1027/2557] Yes, these functions really do have to be this long. --- scripts/maint/practracker/exceptions.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/maint/practracker/exceptions.txt b/scripts/maint/practracker/exceptions.txt index 4cee5453f0..896180156b 100644 --- a/scripts/maint/practracker/exceptions.txt +++ b/scripts/maint/practracker/exceptions.txt @@ -290,3 +290,5 @@ problem function-size /src/tools/tor-resolve.c:build_socks5_resolve_request() 10 problem function-size /src/tools/tor-resolve.c:do_resolve() 175 problem function-size /src/tools/tor-resolve.c:main() 112 problem function-size /src/core/or/circuitpadding.c:circpad_machine_schedule_padding() 107 +problem function-size /src/core/or/circuitpadding_machines.c:circpad_machine_relay_hide_intro_circuits() 104 +problem function-size /src/core/or/circuitpadding_machines.c:circpad_machine_client_hide_rend_circuits() 112 From aa4f2f739737edd8d1570474ee459d5b98713aba Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 17 May 2019 10:12:07 -0400 Subject: [PATCH 1028/2557] Initial changelog draft for 0.4.1.1-alpha (mostly automated) --- ChangeLog | 432 ++++++++++++++++++++++++++++++++++ changes/bug17357 | 7 - changes/bug22210 | 7 - changes/bug22781 | 4 - changes/bug23576 | 7 - changes/bug23588 | 5 - changes/bug24338 | 4 - changes/bug24490 | 5 - changes/bug28269 | 7 - changes/bug28636 | 8 - changes/bug29018 | 5 - changes/bug29061 | 4 - changes/bug29063 | 2 - changes/bug29085 | 4 - changes/bug29204 | 4 - changes/bug29221 | 5 - changes/bug29231 | 4 - changes/bug29243 | 3 - changes/bug29298 | 6 - changes/bug29613 | 5 - changes/bug29640 | 4 - changes/bug29805 | 3 - changes/bug29823 | 5 - changes/bug29926 | 2 - changes/bug29939 | 4 - changes/bug30002 | 2 - changes/bug30109 | 3 - changes/bug30148 | 4 - changes/bug30151 | 5 - changes/bug30189 | 4 - changes/bug30190 | 3 - changes/bug30236 | 3 - changes/bug30309 | 3 - changes/bug30452 | 3 - changes/bug30475 | 4 - changes/bugs28693+30173+29203 | 12 - changes/coverity_falsepos | 4 - changes/feature29532 | 4 - changes/geoip-2019-05-13 | 4 - changes/pubsub | 5 - changes/ticket25110 | 4 - changes/ticket25417 | 4 - changes/ticket25614 | 3 - changes/ticket26069 | 2 - changes/ticket26288 | 6 - changes/ticket27251 | 4 - changes/ticket27821 | 3 - changes/ticket28634 | 10 - changes/ticket28780 | 3 - changes/ticket28816 | 4 - changes/ticket28837 | 4 - changes/ticket28913 | 4 - changes/ticket29059 | 3 - changes/ticket29060 | 2 - changes/ticket29062 | 3 - changes/ticket29064 | 2 - changes/ticket29065 | 3 - changes/ticket29067 | 3 - changes/ticket29068 | 2 - changes/ticket29070 | 2 - changes/ticket29071 | 3 - changes/ticket29108 | 5 - changes/ticket29391 | 3 - changes/ticket29434 | 3 - changes/ticket29436 | 4 - changes/ticket29536 | 9 - changes/ticket29537 | 3 - changes/ticket29542 | 7 - changes/ticket29553 | 5 - changes/ticket29588 | 4 - changes/ticket29635 | 3 - changes/ticket29660 | 5 - changes/ticket29662 | 5 - changes/ticket29732 | 5 - changes/ticket29756 | 3 - changes/ticket29894 | 4 - changes/ticket29913 | 4 - changes/ticket29984 | 5 - changes/ticket30007 | 3 - changes/ticket30033 | 4 - changes/ticket30051 | 5 - changes/ticket30075 | 3 - changes/ticket30076 | 2 - changes/ticket30077 | 2 - changes/ticket30078 | 3 - changes/ticket30079 | 3 - changes/ticket30091 | 4 - changes/ticket30114 | 3 - changes/ticket30149 | 3 - changes/ticket30176 | 4 - changes/ticket30213 | 3 - changes/ticket30234 | 2 - changes/ticket30261 | 4 - changes/ticket30293 | 5 - changes/ticket30307 | 4 - changes/ticket30308 | 5 - changes/ticket30345 | 3 - changes/ticket30414 | 3 - 98 files changed, 432 insertions(+), 396 deletions(-) delete mode 100644 changes/bug17357 delete mode 100644 changes/bug22210 delete mode 100644 changes/bug22781 delete mode 100644 changes/bug23576 delete mode 100644 changes/bug23588 delete mode 100644 changes/bug24338 delete mode 100644 changes/bug24490 delete mode 100644 changes/bug28269 delete mode 100644 changes/bug28636 delete mode 100644 changes/bug29018 delete mode 100644 changes/bug29061 delete mode 100644 changes/bug29063 delete mode 100644 changes/bug29085 delete mode 100644 changes/bug29204 delete mode 100644 changes/bug29221 delete mode 100644 changes/bug29231 delete mode 100644 changes/bug29243 delete mode 100644 changes/bug29298 delete mode 100644 changes/bug29613 delete mode 100644 changes/bug29640 delete mode 100644 changes/bug29805 delete mode 100644 changes/bug29823 delete mode 100644 changes/bug29926 delete mode 100644 changes/bug29939 delete mode 100644 changes/bug30002 delete mode 100644 changes/bug30109 delete mode 100644 changes/bug30148 delete mode 100644 changes/bug30151 delete mode 100644 changes/bug30189 delete mode 100644 changes/bug30190 delete mode 100644 changes/bug30236 delete mode 100644 changes/bug30309 delete mode 100644 changes/bug30452 delete mode 100644 changes/bug30475 delete mode 100644 changes/bugs28693+30173+29203 delete mode 100644 changes/coverity_falsepos delete mode 100644 changes/feature29532 delete mode 100644 changes/geoip-2019-05-13 delete mode 100644 changes/pubsub delete mode 100644 changes/ticket25110 delete mode 100644 changes/ticket25417 delete mode 100644 changes/ticket25614 delete mode 100644 changes/ticket26069 delete mode 100644 changes/ticket26288 delete mode 100644 changes/ticket27251 delete mode 100644 changes/ticket27821 delete mode 100644 changes/ticket28634 delete mode 100644 changes/ticket28780 delete mode 100644 changes/ticket28816 delete mode 100644 changes/ticket28837 delete mode 100644 changes/ticket28913 delete mode 100644 changes/ticket29059 delete mode 100644 changes/ticket29060 delete mode 100644 changes/ticket29062 delete mode 100644 changes/ticket29064 delete mode 100644 changes/ticket29065 delete mode 100644 changes/ticket29067 delete mode 100644 changes/ticket29068 delete mode 100644 changes/ticket29070 delete mode 100644 changes/ticket29071 delete mode 100644 changes/ticket29108 delete mode 100644 changes/ticket29391 delete mode 100644 changes/ticket29434 delete mode 100644 changes/ticket29436 delete mode 100644 changes/ticket29536 delete mode 100644 changes/ticket29537 delete mode 100644 changes/ticket29542 delete mode 100644 changes/ticket29553 delete mode 100644 changes/ticket29588 delete mode 100644 changes/ticket29635 delete mode 100644 changes/ticket29660 delete mode 100644 changes/ticket29662 delete mode 100644 changes/ticket29732 delete mode 100644 changes/ticket29756 delete mode 100644 changes/ticket29894 delete mode 100644 changes/ticket29913 delete mode 100644 changes/ticket29984 delete mode 100644 changes/ticket30007 delete mode 100644 changes/ticket30033 delete mode 100644 changes/ticket30051 delete mode 100644 changes/ticket30075 delete mode 100644 changes/ticket30076 delete mode 100644 changes/ticket30077 delete mode 100644 changes/ticket30078 delete mode 100644 changes/ticket30079 delete mode 100644 changes/ticket30091 delete mode 100644 changes/ticket30114 delete mode 100644 changes/ticket30149 delete mode 100644 changes/ticket30176 delete mode 100644 changes/ticket30213 delete mode 100644 changes/ticket30234 delete mode 100644 changes/ticket30261 delete mode 100644 changes/ticket30293 delete mode 100644 changes/ticket30307 delete mode 100644 changes/ticket30308 delete mode 100644 changes/ticket30345 delete mode 100644 changes/ticket30414 diff --git a/ChangeLog b/ChangeLog index a69a7253b0..827c4c3131 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,435 @@ +Changes in version 0.4.1.1-alpha - 2019-05-?? + This is the first alpha in the 0.4.1.x series. It introduces + lightweight circuit padding to make some onion-service circuits harder + to distinguish, includes a new "authenticated SENDME" feature to make + certain denial-of-service attacks more difficult, and improves + performance in several areas. + + o Major features (Circuit padding): + - Onion service clients will now add padding cells to the initial + portions of their INTRODUCE and RENDEZVOUS circuits, to make those + circuits' traffic patterns look more like general purpose Exit + traffic. The overhead for this is 2 extra cells in each direction + for RENDEZVOUS circuits, and 1 extra upstream cell and 10 + downstream cells for INTRODUCE circuits. This will only be enabled + if the circuit's middle node supports this feature, too. (Clients + may specify fixed middle nodes with the MiddleNodes torrc + directive, and may force-disable this feature with the + CircuitPadding torrc directive). Closes ticket 28634. + + o Major features (code organization): + - Tor now includes a generic publish-subscribe message-passing + subsystem that we can use to organize intermodule dependencies. We + hope to use this to reduce dependencies between modules that don't + need to be related, and to generally simplify our codebase. Closes + ticket 28226. + + o Major features (controller protocol): + - Controller commands are now parsed using a generalized parsing + subsystem. Previously, each controller command was responsible for + parsing its own input. Closes ticket 30091. + + o Major features (flow control): + - Implement authenticated SENDMEs detailed in proposal 289. A SENDME + cell now includes the digest of the last cell received so once the + end point receives the SENDME, it can confirm the other side's + knowledge of the previous cells that were sent. This behavior is + controlled by two new consensus parameters, see proposal for more + details. Fixes ticket 26288. + + o Major features (performance): + - Update our node selection algorithm to exclude nodes in linear + time. Previously, the algorithm was quadratic, which could slow + down heavily used onion services. Closes ticket 30307. + + o Minor feature (circuit padding): + - We now use a fast RNG when scheduling circuit padding. Part of + ticket 28636. + + o Minor feature (maintenance scripts): + - Add to scripts/maint/ helper maintainer scripts used for git + maintenance. Closes ticket 29391. + + o Minor features (circuit padding): + - Allow the padding machine designer to pick the edges of their + histogram instead of trying to compute them automatically using an + exponential formula. Resolves some undefined behavior in the case + of small histograms and allows greater flexibility on machine + design. Closes ticket 29298; bugfix on 0.4.0.1-alpha. + - Provide the ability for circuit padding machines to hold a circuit + open until they are done padding it. Closes ticket 28780. + + o Minor features (compile-time modules): + - Add a --list-modules command to print a list of which compile-time + modules are enabled. Closes ticket 30452. + + o Minor features (continuous integration): + - Remove sudo configuration lines from .travis.yml as they are no + longer needed with current Travis build environment. Resolves + issue 30213. + + o Minor features (controller): + - Add onion service version 3 support to HSFETCH. Previously, only + version 2 onion services were supported. Closes ticket 25417. + Patch by Neel Chauhan + + o Minor features (debugging): + - Introduce tor_assertf() and tor_assertf_nonfatal() to enable + logging of additional information during assert failure. Now we + can use format strings to include pieces of information that are + relevant for trouble shooting. Resolves ticket 29662. + + o Minor features (defense in depth): + - In smartlist_remove_keeporder(), set any pointers that become + unused to NULL, in case a bug causes them to be used later. Closes + ticket 30176. Patch from Tobias Stoeckmann. + - Tor now uses a fast cryptographically strong PRNG even for + decisions that we do not believe are security-sensitive. + Previously, for performance reasons, we had used a trivially + predictable linear congruential generator algorithm for certain + load-balancing and statistical sampling decisions. Now we use our + fast RNG in those cases. Closes ticket 29542. + + o Minor features (developer tooling): + - Call practracker from pre-push and pre-commit git hooks to let a + developer know if they made any code style violations in their + last commit. This should help preventing code style violations + appearing upstream. Closes ticket 30051. + - Call pre-commit git hook from pre-push hook to make sure we're + running documentation and code style checks before pushing to + remote git repository. Implements feature 30033. + - Modify git pre-push hook script to disallow pushing branches other + than master, release-* and maint-* to origin remote. Implements + feature 29532. + + o Minor features (developer tools): + - Add a script to check that each header has a well-formed and + unique guard marco. Closes ticket 29756. + - Introduce a post-merge git hook script to check if we're pulling + in any changes to our git workspace management scripts from + upstream. Resolves issue 29588. + + o Minor features (development tools): + - Tor's test scripts now check for files and functions that seem too + long and complicated. Existing overlong functions and files are + accepted for now, but should eventually be refactored. Closes + ticket 29221. + + o Minor features (geoip): + - Update geoip and geoip6 to the May 13 2019 Maxmind GeoLite2 + Country database. Closes ticket 30522. + + o Minor features (git scripts): + - In git-pull-all.sh, also fetch the latest tor-github pull + requests. Implements ticket 30114. + + o Minor features (HTTP tunnel): + - Return an informative web page when the HTTPTunnelPort is used as + an HTTP proxy. Closes ticket 27821, patch by "eighthave". + + o Minor features (IPv6, v3 onion services): + - Make v3 onion services put IPv6 addresses in service descriptors. + Before this change, service descriptors only contained IPv4 + addressesd. Implements 26992. + + o Minor features (modularity): + - The --disable-module-dirauth compile-time option now disables even + more dirauth-only code. Closes ticket 30345. + + o Minor features (performance): + - Use OpenSSL's implementations of SHA3 when available (in OpenSSL + 1.1.1 and later), since they tend to be faster than tiny-keccak. + Closes ticket 28837. + + o Minor features (performance, RNG): + - Tor now constructs a fast secure pseudorandom number generator for + each thread, to use for cases where performance is critical. This + PRNG is based on AES-CTR, using a buffering construction similar + to libottery and the (newer) OpenBSD arc4random() code. It + outperforms OpenSSL 1.1.1a's CSPRNG by roughly a factor of 100 for + small outputs. Although we believe it to be cryptographically + strong, we are only using it when necessary for reasonable + performance. Implements tickets 29023 and 29536. + + o Minor features (testing): + - Tor's unit test code now contains a standard set of functions to + replace the PRNG with a deterministic or reproducible version for + testing. Previously, various tests implemented this in various + ways. Implements ticket 29732. + - We now have a script, cov-test-determinism.sh, to identify places + where our unit test coverage has become nondeterministic. Closes + ticket 29436. + + o Minor bugfixes (bridge authority): + - We set bridges as running when we dump the bridge status to a + file. Previously, we set bridges as running in a GETINFO + controller, but these shouldn't modify vital data structures. + Fixes bug 24490; bugfix on 0.2.0.13-alpha. Patch by Neel Chauhan + + o Minor bugfixes (Channel padding statistics): + - Channel padding write totals and padding-enabled totals are now + counted properly in relay extrainfo descriptors. Fixes bug 29231; + bugfix on 0.3.1.1-alpha + + o Minor bugfixes (circuit padding): + - Add a torrc option to disable circuit padding. Fixes bug 28693; + bugfix on 0.4.0.1-alpha. + - Allow circuit padding machines to specify that they do not + contribute much overhead, and provide consensus flags and torrc + options to force clients to only use low overhead machines. Fixes + bug 29203; bugfix on 0.4.0.1-alpha. + - Provide consensus parameter to fully disable circuit padding, to + be used in emergency network overload situations. Fixes bug 30173; + bugfix on 0.4.0.1-alpha. + - The circuit padding subsystem does not schedule padding if dormant + mode is enabled. Fixes bug 28636; bugfix on 0.4.0.1-alpha. + + o Minor bugfixes (circuitpadding): + - Inspect circuit-level cell queue before sending padding, to avoid + sending padding while too much data is queued. Fixes bug 29204; + bugfix on 0.4.0.1-alpha. + + o Minor bugfixes (compilation, unusual configuration): + - Avoid failures when building with ALL_BUGS_ARE_FAILED due to + missing declarations of abort(), and prevent other such failures + in the future. Fixes bug 30189; bugfix on 0.3.4.1-alpha. + + o Minor bugfixes (controller protocol): + - Teach the controller parser to correctly distinguish an object + preceded by an argument list from one without. Previously, it + couldn't distinguish an argument list from the first line of a + multiline object. Fixes bug 29984; bugfix on 0.2.3.8-alpha. + + o Minor bugfixes (developer tools): + - Update our pre-commit.git-hook script to work correctly on older + Tor branches and release branches without any changes files, and + to actually exit when something fails. Fixes bug 29553; bugfix + on 0.4.0.2-alpha. + + o Minor bugfixes (dirauth, ipv6): + - If we are a durauth with IPv6 and are marking relays as running, + mark ourselves as reachable on IPv6. Fixes bug 24338; bugfix on + 0.4.0.2-alpha. Patch by Neel Chauhan + + o Minor bugfixes (documentation): + - Improve the documentation for MapAddress .exit. Fixes bug 30109; + bugfix on 0.1.0.1-rc. + - Improve the monotonic time module and function documentation. + Explain what "monotonic" actually means, and document some results + that have surprised people. Fixes bug 29640; bugfix + on 0.2.9.1-alpha. + + o Minor bugfixes (documentation, manpage): + - Use proper formatting when providing an example on quoting options + that contain whitespace. Fixes bug 29635; bugfix on 0.2.3.18-rc. + + o Minor bugfixes (lib): + + o Minor bugfixes (logging): + - Do not log a warning for OpenSSL versions that should be + compatible. Fixes bug 30190; bugfix on 0.2.4.2-alpha + + o Minor bugfixes (logging, configuration): + - Warn operators when MyFamily option is set but ContactInfo is + missing, as the latter should be set too. Fixes bug 25110; bugfix + on 0.3.3.1-alpha. + + o Minor bugfixes (memory leak): + - Avoid a minor memory leak that could occur on relays when creating + a keys directory failed. Fixes bug 30148; bugfix on 0.3.3.1-alpha. + + o Minor bugfixes (onion services): + - Avoid a GCC 9.1.1 warning (and possible crash depending on libc + implemenation) when failing to load an onion service client + authorization file. Fixes bug 30475; bugfix on 0.3.5.1-alpha. + - If we are launching repeated HSFETCH queries and are rate-limited, + we introduce a new controller response QUERY_RATE_LIMITED instead + of QUERY_NO_HSDIR, while keeping the latter for when onion service + directories are missing a descriptor. Previously, we returned + QUERY_NO_HSDIR for both cases. Fixes bug 28269; bugfix on + 0.3.1.1-alpha. Patch by Neel Chauhan + - If we are relaunching a circuit to a rendevous service in + rend_service_relaunch_rendezvous() and + hs_service_requires_uptime_circ() is true, the + CIRCLAUNCH_NEED_UPTIME flag is added to the circuit. Previously, + we only set this flag when we received a INTRODUCE2 cell in + rend_service_receive_introduction(). Fixes bug 17357; bugfix on + 0.4.0.2-alpha. Patch by Neel Chauhan + + o Minor bugfixes (onion services, performance): + - If we are building circuits to onion services, in + circuit_is_acceptable() we only call tor_addr_parse() in places + where we use the returned family and address values from this + function. Previously, we called tor_addr_parse() in + circuit_is_acceptable() even if it wasn't used. This change will + improve performance when building circuits. Fixes bug 22210; + bugfix on 0.2.8.12. Patch by Neel Chauhan + + o Minor bugfixes (performance): + - When checking a node for bridge status, use a fast check to make + sure that its identity is set. Previously, we used a constant-time + check, which is not necessary when verifying a BUG() condition that + causes a stack trace. Fixes bug 30308; bugfix on 0.3.5.1-alpha. + + o Minor bugfixes (pluggable transports): + - Tor now sets TOR_PT_EXIT_ON_STDIN_CLOSE=1 for client transports as + well as servers. Fixes bug 25614; bugfix on 0.2.7.1-alpha. + + o Minor bugfixes (probability distributions): + - Refactor and improve parts of the probability distribution code + that made Coverity complain. Fixes bug 29805; bugfix + on 0.4.0.1-alpha. + + o Minor bugfixes (python): + - Stop assuming that /usr/bin/python3 exists. For scripts that work + with python2, use /usr/bin/python. Otherwise, use /usr/bin/env + python3. Fixes bug 29913; bugfix on 0.2.5.3-alpha. + + o Minor bugfixes (relay): + - If we are are a relay and have IPv6Exit to 1 while ExitRelay is + auto, we act as if ExitRelay is 1. Previously, we ignored IPv6Exit + if ExitRelay was 0 or auto. Fixes bug 29613; bugfix on + 0.3.5.1-alpha. Patch by Neel Chauhan. + + o Minor bugfixes (stats): + - When ExtraInfoStatistics is 0, stop including bandwidth usage + statistics, GeoIPFile hashes, ServerTransportPlugin lines, and + bridge statistics by country in extra-info documents. Fixes bug + 29018; bugfix on 0.2.4.1-alpha. + + o Minor bugfixes (testing): + - Call setrlimit() to disable core dumps in test_bt_cl.c instead of + using `ulimit -c` in test_bt.sh, which violates POSIX shell + compatibility. Fixes bug 29061; bugfix on 0.3.5.1-alpha. + + o Minor bugfixes (testing, v3 onion services): + - Fix some incorrect code in the v3 onion service unit tests. Fixes + bug 29243; bugfix on 0.3.2.1-alpha. + + o Minor bugfixes (tor-resolve): + - Fix a memory leak in tor-resolve that could happen if Tor gave it + a malformed SOCKS response. (Memory leaks in tor-resolve don't + actually matter, but it's good to fix them anyway.) Fixes bug + 30151; bugfix on 0.4.0.1-alpha. + + o Minor bugfixes (unit tests): + - In the "routerkeys/*" tests, check the return values of mkdir() + for possible failures. Fixes bug 29939; bugfix on 0.2.7.2-alpha. + Found by Coverity as CID 1444254. + - Split test_utils_general() to several smaller test functions in + test_utils_general(). This makes it easier to perform resource + deallocation on assert failure and fixes Coverity warnings CID + 1444117 and CID 1444118. Fixes bug 29823; bugfix on 0.2.9.1-alpha. + + o Minor bugfixes (v3 onion services): + - Stop ignoring IPv6 link specifiers sent to v3 onion services. v3 + onion service IPv6 support is still incomplete, see 23493 for + details. Fixes bug 23588; bugfix on 0.3.2.1-alpha. Patch by + Neel Chauhan. + + o Code simplification and refactoring: + - Abstract out the low-level formatting of replies on the control + port. Implements ticket 30007. + - Add several assertions in an attempt to fix some Coverity + warnings. Closes ticket 30149. + - Introduce a connection_dir_buf_add() helper function that checks + for compress_state of dir_connection_t and automatically writes a + string to directory connection with or without compression. + Resolves issue 28816. + - Make the base32_decode() API return the number of bytes written, + for consistency with base64_decode(). Closes ticket 28913. + - Move most relay-only periodic events out of mainloop.c into the + relay subsystem. Closes ticket 30414. + - Refactor and encapsulate parts of the codebase that manipulate + crypt_path_t objects. Resolves issue 30236. + - Refactor several places in our code that coverity incorrectly + believed that we might have memory leaks, so that we can analyze + our software more easily. Closes ticket 30147. + - Remove redundant return values in crypto_format, and the + associated return value checks elsewhere in the code. Make the + implementations in crypto_format consistent, and remove redundant + code. Resolves ticket 29660. + - Rename tor_mem_is_zero() to fast_mem_is_zero(), to emphasize that + it is not a constant-time function. Closes ticket 30309. + - Replace hs_desc_link_specifier_t with link_specifier_t, and remove + all hs_desc_link_specifier_t-specific code. Fixes bug 22781; + bugfix on 0.3.2.1-alpha. + - Simplify v3 onion service link specifier handling code. Fixes bug + 23576; bugfix on 0.3.2.1-alpha. + - Split crypto_digest.c into three parts: 1) general code that does + not depend on either NSS or OpenSSL (stays in crypto_digest.c); 2) + code that depends on NSS API (moved to crypto_digest_nss.c); 3) + code that depends on OpenSSL API (moved to + crypto_digest_openssl.c). Resolves ticket 29108. + - Split up the control.c file into several submodules, in + preparation for distributing its current responsibilities + throughout the codebase. Closes ticket 29894. + - Start move responsibility for knowing about periodic events to the + appropriate subsystems, so that the mainloop doesn't need to know + all the periodic events in the rest of the codebase. Implements + tickets 30293 and 30294. + + o Documentation: + - Document how to find git commits and tags for bug fixes in + CodingStandards.md. And update some changes file documentation. + Closes ticket 30261. + + o Removed features: + - Remove linux-tor-prio.sh script from contrib/operator-tools + directory. Resolves issue 29434. + - Remove obsolete OpenSUSE initscript. Resolves issue 30076. + - Remove the obsolete script at contrib/dist/tor.sh.in. Resolves + issue 30075. + + o Testing: + - Check that representative subsets of values of `int` and `unsigned + int` can be represented by `void *`. Resolves issue 29537. + + o Code simplification and refactoring (circuit padding): + - Avoid calling monotime_absolute_usec() in circuit padding machines + that do not use token removal or circuit RTT estimation. Fixes bug + 29085; bugfix on 0.4.0.1-alpha. + + o Code simplification and refactoring (shell scripts): + - Cleanup autogen.sh to silence shellcheck warnings. Closes + ticket 26069. + - Cleanup test_keygen.sh to silence all shellcheck warnings. Closes + ticket 29062. + - Cleanup test_switch_id.sh to silence shellcheck warnings. Closes + ticket 29065. + - Fix issues shellcheck found in test_rebind.sh. Resolves + issue 29063. + - Fix shellcheck warning SC2006 in src/test/fuzz/minimize.sh. + Resolves issue 30079. + - Fix shellcheck warning in test_rust.sh. Fixes issue 29064. + - Fix shellcheck warning in torify script. Resolves issue 29070. + - Fix shellcheck warnings in asciidoc-helper.sh. Resolves + issue 29926. + - Fix shellcheck warnings in fuzz_multi.sh. Resolves issue 30077. + - Fix shellcheck warnings in fuzz_static_testcases.sh. Resolves + ticket 29059. + - Fix shellcheck warnings in nagios-check-tor-authority-cert script. + Resolves issue 29071. + - Fix shellcheck warnings in src/test/fuzz/fixup_filenames.sh. + Resolves issue 30078. + - Fix shellcheck warnings in test-network.sh. Resolves issue 29060. + - Fix shellcheck warnings in test_key_expiration.sh. Resolves + issue 30002. + - Fix shellcheck warnings in zero_length_keys.sh. Resolves + issue 29068. + - Fix test_workqueue_*.sh scripts to silence shellcheck SC2086 + warnings. Fixes issue 29067. + + o Testing (chutney): + - In "make test-network-all", test IPv6-only v3 single onion + services, using the chutney network single-onion-v23-ipv6-md. This + test will not pass until 23588 has been merged. Closes + ticket 27251. + + o Testing (continuous integration): + - In Travis, show stem's tor log after failure. Closes ticket 30234. + + Changes in version 0.4.0.5 - 2019-05-02 This is the first stable release in the 0.4.0.x series. It contains improvements for power management and bootstrap reporting, as well as diff --git a/changes/bug17357 b/changes/bug17357 deleted file mode 100644 index 1188b65fd7..0000000000 --- a/changes/bug17357 +++ /dev/null @@ -1,7 +0,0 @@ - o Minor bugfixes (onion services): - - If we are relaunching a circuit to a rendevous service in - rend_service_relaunch_rendezvous() and hs_service_requires_uptime_circ() - is true, the CIRCLAUNCH_NEED_UPTIME flag is added to the circuit. - Previously, we only set this flag when we received a INTRODUCE2 - cell in rend_service_receive_introduction(). Fixes bug 17357; - bugfix on 0.4.0.2-alpha. Patch by Neel Chauhan diff --git a/changes/bug22210 b/changes/bug22210 deleted file mode 100644 index d7a00fd72c..0000000000 --- a/changes/bug22210 +++ /dev/null @@ -1,7 +0,0 @@ - o Minor bugfixes (onion services, performance): - - If we are building circuits to onion services, in circuit_is_acceptable() - we only call tor_addr_parse() in places where we use the returned - family and address values from this function. Previously, we called - tor_addr_parse() in circuit_is_acceptable() even if it wasn't used. - This change will improve performance when building circuits. Fixes - bug 22210; bugfix on 0.2.8.12. Patch by Neel Chauhan diff --git a/changes/bug22781 b/changes/bug22781 deleted file mode 100644 index 5606dfa5e2..0000000000 --- a/changes/bug22781 +++ /dev/null @@ -1,4 +0,0 @@ - o Code simplification and refactoring: - - Replace hs_desc_link_specifier_t with link_specifier_t, - and remove all hs_desc_link_specifier_t-specific code. - Fixes bug 22781; bugfix on 0.3.2.1-alpha. diff --git a/changes/bug23576 b/changes/bug23576 deleted file mode 100644 index edcae02e5e..0000000000 --- a/changes/bug23576 +++ /dev/null @@ -1,7 +0,0 @@ - o Minor features (IPv6, v3 onion services): - - Make v3 onion services put IPv6 addresses in service - descriptors. Before this change, service descriptors only - contained IPv4 addressesd. Implements 26992. - o Code simplification and refactoring: - - Simplify v3 onion service link specifier handling code. - Fixes bug 23576; bugfix on 0.3.2.1-alpha. diff --git a/changes/bug23588 b/changes/bug23588 deleted file mode 100644 index 86064ab313..0000000000 --- a/changes/bug23588 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor bugfixes (v3 onion services): - - Stop ignoring IPv6 link specifiers sent to v3 onion services. - v3 onion service IPv6 support is still incomplete, see 23493 for - details. Fixes bug 23588; bugfix on 0.3.2.1-alpha. - Patch by Neel Chauhan. diff --git a/changes/bug24338 b/changes/bug24338 deleted file mode 100644 index 75984b6329..0000000000 --- a/changes/bug24338 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (dirauth, ipv6): - - If we are a durauth with IPv6 and are marking relays as running, mark - ourselves as reachable on IPv6. Fixes bug 24338; bugfix on 0.4.0.2-alpha. - Patch by Neel Chauhan diff --git a/changes/bug24490 b/changes/bug24490 deleted file mode 100644 index cf9281c878..0000000000 --- a/changes/bug24490 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor bugfixes (bridge authority): - - We set bridges as running when we dump the bridge status to a file. - Previously, we set bridges as running in a GETINFO controller, but - these shouldn't modify vital data structures. Fixes bug 24490; - bugfix on 0.2.0.13-alpha. Patch by Neel Chauhan diff --git a/changes/bug28269 b/changes/bug28269 deleted file mode 100644 index bdfe9e1aae..0000000000 --- a/changes/bug28269 +++ /dev/null @@ -1,7 +0,0 @@ - o Minor bugfixes (onion services): - - If we are launching repeated HSFETCH queries and are rate-limited, - we introduce a new controller response QUERY_RATE_LIMITED instead - of QUERY_NO_HSDIR, while keeping the latter for when onion service - directories are missing a descriptor. Previously, we returned - QUERY_NO_HSDIR for both cases. Fixes bug 28269; bugfix on - 0.3.1.1-alpha. Patch by Neel Chauhan diff --git a/changes/bug28636 b/changes/bug28636 deleted file mode 100644 index 240655cbea..0000000000 --- a/changes/bug28636 +++ /dev/null @@ -1,8 +0,0 @@ - o Minor bugfixes (circuit padding): - - The circuit padding subsystem does not schedule padding if dormant mode - is enabled. Fixes bug 28636; bugfix on 0.4.0.1-alpha. - - o Minor feature (circuit padding): - - We now use a fast RNG when scheduling circuit padding. Part of ticket - 28636. - diff --git a/changes/bug29018 b/changes/bug29018 deleted file mode 100644 index b006ae36a7..0000000000 --- a/changes/bug29018 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor bugfixes (stats): - - When ExtraInfoStatistics is 0, stop including bandwidth usage statistics, - GeoIPFile hashes, ServerTransportPlugin lines, and bridge statistics - by country in extra-info documents. Fixes bug 29018; - bugfix on 0.2.4.1-alpha. diff --git a/changes/bug29061 b/changes/bug29061 deleted file mode 100644 index 58fc4f22e9..0000000000 --- a/changes/bug29061 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (testing): - - Call setrlimit() to disable core dumps in test_bt_cl.c instead of - using `ulimit -c` in test_bt.sh, which violates POSIX shell - compatibility. Fixes bug 29061; bugfix on 0.3.5.1-alpha. diff --git a/changes/bug29063 b/changes/bug29063 deleted file mode 100644 index 8cbcbebc6e..0000000000 --- a/changes/bug29063 +++ /dev/null @@ -1,2 +0,0 @@ - o Code simplification and refactoring (shell scripts): - - Fix issues shellcheck found in test_rebind.sh. Resolves issue 29063. diff --git a/changes/bug29085 b/changes/bug29085 deleted file mode 100644 index b17c06378f..0000000000 --- a/changes/bug29085 +++ /dev/null @@ -1,4 +0,0 @@ - o Code simplification and refactoring (circuit padding): - - Avoid calling monotime_absolute_usec() in circuit padding machines - that do not use token removal or circuit RTT estimation. Fixes bug - 29085; bugfix on 0.4.0.1-alpha. diff --git a/changes/bug29204 b/changes/bug29204 deleted file mode 100644 index ec2cf67b2f..0000000000 --- a/changes/bug29204 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (circuitpadding): - - Inspect circuit-level cell queue before sending padding, to avoid - sending padding while too much data is queued. Fixes bug 29204; - bugfix on 0.4.0.1-alpha. diff --git a/changes/bug29221 b/changes/bug29221 deleted file mode 100644 index fbe08aa9a0..0000000000 --- a/changes/bug29221 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor features (development tools): - - Tor's test scripts now check for files and functions that seem - too long and complicated. Existing overlong functions and files are - accepted for now, but should eventually be refactored. Closes - ticket 29221. diff --git a/changes/bug29231 b/changes/bug29231 deleted file mode 100644 index bcc19e1b48..0000000000 --- a/changes/bug29231 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (Channel padding statistics): - - Channel padding write totals and padding-enabled totals are now - counted properly in relay extrainfo descriptors. Fixes bug 29231; - bugfix on 0.3.1.1-alpha diff --git a/changes/bug29243 b/changes/bug29243 deleted file mode 100644 index b5694f7568..0000000000 --- a/changes/bug29243 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor bugfixes (testing, v3 onion services): - - Fix some incorrect code in the v3 onion service unit tests. - Fixes bug 29243; bugfix on 0.3.2.1-alpha. diff --git a/changes/bug29298 b/changes/bug29298 deleted file mode 100644 index 6e447b62dd..0000000000 --- a/changes/bug29298 +++ /dev/null @@ -1,6 +0,0 @@ - o Minor features (circuit padding): - - Allow the padding machine designer to pick the edges of their histogram - instead of trying to compute them automatically using an exponential - formula. Resolves some undefined behavior in the case of small histograms - and allows greater flexibility on machine design. Closes ticket 29298; - bugfix on 0.4.0.1-alpha. \ No newline at end of file diff --git a/changes/bug29613 b/changes/bug29613 deleted file mode 100644 index e966973255..0000000000 --- a/changes/bug29613 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor bugfixes (relay): - - If we are are a relay and have IPv6Exit to 1 while ExitRelay is - auto, we act as if ExitRelay is 1. Previously, we ignored IPv6Exit - if ExitRelay was 0 or auto. Fixes bug 29613; bugfix on 0.3.5.1-alpha. - Patch by Neel Chauhan. diff --git a/changes/bug29640 b/changes/bug29640 deleted file mode 100644 index 81adeae32a..0000000000 --- a/changes/bug29640 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (documentation): - - Improve the monotonic time module and function documentation. Explain - what "monotonic" actually means, and document some results that have - surprised people. Fixes bug 29640; bugfix on 0.2.9.1-alpha. diff --git a/changes/bug29805 b/changes/bug29805 deleted file mode 100644 index 00c846e9af..0000000000 --- a/changes/bug29805 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor bugfixes (probability distributions): - - Refactor and improve parts of the probability distribution code that made - Coverity complain. Fixes bug 29805; bugfix on 0.4.0.1-alpha. \ No newline at end of file diff --git a/changes/bug29823 b/changes/bug29823 deleted file mode 100644 index d856cf1fef..0000000000 --- a/changes/bug29823 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor bugfixes (unit tests): - - Split test_utils_general() to several smaller test functions in - test_utils_general(). This makes it easier to perform resource - deallocation on assert failure and fixes Coverity warnings CID 1444117 - and CID 1444118. Fixes bug 29823; bugfix on 0.2.9.1-alpha. diff --git a/changes/bug29926 b/changes/bug29926 deleted file mode 100644 index ab1417c603..0000000000 --- a/changes/bug29926 +++ /dev/null @@ -1,2 +0,0 @@ - o Code simplification and refactoring (shell scripts): - - Fix shellcheck warnings in asciidoc-helper.sh. Resolves issue 29926. diff --git a/changes/bug29939 b/changes/bug29939 deleted file mode 100644 index 0e9b46c075..0000000000 --- a/changes/bug29939 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (unit tests): - - In the "routerkeys/*" tests, check the return values of mkdir() for - possible failures. Fixes bug 29939; bugfix on 0.2.7.2-alpha. Found by - Coverity as CID 1444254. diff --git a/changes/bug30002 b/changes/bug30002 deleted file mode 100644 index da61c9e4b2..0000000000 --- a/changes/bug30002 +++ /dev/null @@ -1,2 +0,0 @@ - o Code simplification and refactoring (shell scripts): - - Fix shellcheck warnings in test_key_expiration.sh. Resolves issue 30002. diff --git a/changes/bug30109 b/changes/bug30109 deleted file mode 100644 index b25aa803bb..0000000000 --- a/changes/bug30109 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor bugfixes (documentation): - - Improve the documentation for MapAddress .exit. - Fixes bug 30109; bugfix on 0.1.0.1-rc. diff --git a/changes/bug30148 b/changes/bug30148 deleted file mode 100644 index 7d0257e3fe..0000000000 --- a/changes/bug30148 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (memory leak): - - Avoid a minor memory leak that could occur on relays when - creating a keys directory failed. Fixes bug 30148; bugfix on - 0.3.3.1-alpha. diff --git a/changes/bug30151 b/changes/bug30151 deleted file mode 100644 index 8ac9a320a0..0000000000 --- a/changes/bug30151 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor bugfixes (tor-resolve): - - Fix a memory leak in tor-resolve that could happen if Tor gave it a - malformed SOCKS response. (Memory leaks in tor-resolve don't actually - matter, but it's good to fix them anyway.) Fixes bug 30151; bugfix on - 0.4.0.1-alpha. diff --git a/changes/bug30189 b/changes/bug30189 deleted file mode 100644 index f8c932a5f9..0000000000 --- a/changes/bug30189 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (compilation, unusual configuration): - - Avoid failures when building with ALL_BUGS_ARE_FAILED due to - missing declarations of abort(), and prevent other such failures - in the future. Fixes bug 30189; bugfix on 0.3.4.1-alpha. diff --git a/changes/bug30190 b/changes/bug30190 deleted file mode 100644 index e2352c3b9c..0000000000 --- a/changes/bug30190 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor bugfixes (lib): - do not log a warning for OpenSSL versions that should be compatible - Fixes bug 30190; bugfix on 0.2.4.2-alpha diff --git a/changes/bug30236 b/changes/bug30236 deleted file mode 100644 index ceaa98c8f1..0000000000 --- a/changes/bug30236 +++ /dev/null @@ -1,3 +0,0 @@ - o Code simplification and refactoring: - - Refactor and encapsulate parts of the codebase that manipulate - crypt_path_t objects. Resolves issue 30236. \ No newline at end of file diff --git a/changes/bug30309 b/changes/bug30309 deleted file mode 100644 index 6cbbe8d156..0000000000 --- a/changes/bug30309 +++ /dev/null @@ -1,3 +0,0 @@ - o Code simplification and refactoring: - - Rename tor_mem_is_zero() to fast_mem_is_zero(), to emphasize that - it is not a constant-time function. Closes ticket 30309. diff --git a/changes/bug30452 b/changes/bug30452 deleted file mode 100644 index 2bb401d87d..0000000000 --- a/changes/bug30452 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor features (compile-time modules): - - Add a --list-modules command to print a list of which compile-time - modules are enabled. Closes ticket 30452. diff --git a/changes/bug30475 b/changes/bug30475 deleted file mode 100644 index 839597b885..0000000000 --- a/changes/bug30475 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (): - - Avoid a GCC 9.1.1 warning (and possible crash depending on libc - implemenation) when failing to load a hidden service client authorization - file. Fixes bug 30475; bugfix on 0.3.5.1-alpha. diff --git a/changes/bugs28693+30173+29203 b/changes/bugs28693+30173+29203 deleted file mode 100644 index 9faa6279bf..0000000000 --- a/changes/bugs28693+30173+29203 +++ /dev/null @@ -1,12 +0,0 @@ - o Minor bugfixes (circuit padding): - - Add a torrc option to disable circuit padding. Fixes bug 28693; bugfix - on 0.4.0.1-alpha. - o Minor bugfixes (circuit padding): - - Provide consensus parameter to fully disable circuit padding, to be used - in emergency network overload situations. Fixes bug 30173; bugfix on - 0.4.0.1-alpha. - o Minor bugfixes (circuit padding): - - Allow circuit padding machines to specify that they do not contribute - much overhead, and provide consensus flags and torrc options to force - clients to only use low overhead machines. Fixes bug 29203; bugfix on - 0.4.0.1-alpha. diff --git a/changes/coverity_falsepos b/changes/coverity_falsepos deleted file mode 100644 index 9fbb01a0c1..0000000000 --- a/changes/coverity_falsepos +++ /dev/null @@ -1,4 +0,0 @@ - o Code simplification and refactoring: - - Refactor several places in our code that coverity incorrectly believed - that we might have memory leaks, so that we can analyze our software - more easily. Closes ticket 30147. diff --git a/changes/feature29532 b/changes/feature29532 deleted file mode 100644 index 4d95e6bca8..0000000000 --- a/changes/feature29532 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor features (developer tooling): - - Modify git pre-push hook script to disallow pushing branches other than - master, release-* and maint-* to origin remote. Implements feature - 29532. diff --git a/changes/geoip-2019-05-13 b/changes/geoip-2019-05-13 deleted file mode 100644 index 0a2fa18971..0000000000 --- a/changes/geoip-2019-05-13 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor features (geoip): - - Update geoip and geoip6 to the May 13 2019 Maxmind GeoLite2 - Country database. Closes ticket 30522. - diff --git a/changes/pubsub b/changes/pubsub deleted file mode 100644 index f67b36b988..0000000000 --- a/changes/pubsub +++ /dev/null @@ -1,5 +0,0 @@ - o Major features (code organization): - - Tor now includes a generic publish-subscribe message-passing subsystem - that we can use to organize intermodule dependencies. We hope to use - this to reduce dependencies between modules that don't need to be - related, and to generally simplify our codebase. Closes ticket 28226. diff --git a/changes/ticket25110 b/changes/ticket25110 deleted file mode 100644 index 298e33287f..0000000000 --- a/changes/ticket25110 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (logging, configuration): - - Warn operators when MyFamily option is set but ContactInfo - is missing, as the latter should be set too. - Fixes bug 25110; bugfix on 0.3.3.1-alpha. diff --git a/changes/ticket25417 b/changes/ticket25417 deleted file mode 100644 index 41f2acc988..0000000000 --- a/changes/ticket25417 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor features (controller): - - Add onion service version 3 support to HSFETCH. Previously, only - version 2 onion services were supported. Closes ticket 25417. - Patch by Neel Chauhan diff --git a/changes/ticket25614 b/changes/ticket25614 deleted file mode 100644 index 82988eeace..0000000000 --- a/changes/ticket25614 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor bugfixes (pluggable transports): - - Tor now sets TOR_PT_EXIT_ON_STDIN_CLOSE=1 for client transports as - well as servers. Fixes bug 25614; bugfix on 0.2.7.1-alpha. diff --git a/changes/ticket26069 b/changes/ticket26069 deleted file mode 100644 index caed9be348..0000000000 --- a/changes/ticket26069 +++ /dev/null @@ -1,2 +0,0 @@ - o Code simplification and refactoring (shell scripts): - - Cleanup autogen.sh to silence shellcheck warnings. Closes ticket 26069. diff --git a/changes/ticket26288 b/changes/ticket26288 deleted file mode 100644 index 59bb856dd2..0000000000 --- a/changes/ticket26288 +++ /dev/null @@ -1,6 +0,0 @@ - o Major features (flow control): - - Implement authenticated SENDMEs detailed in proposal 289. A SENDME cell - now includes the digest of the last cell received so once the end point - receives the SENDME, it can confirm the other side's knowledge of the - previous cells that were sent. This behavior is controlled by two new - consensus parameters, see proposal for more details. Fixes ticket 26288. diff --git a/changes/ticket27251 b/changes/ticket27251 deleted file mode 100644 index 7ce296e8da..0000000000 --- a/changes/ticket27251 +++ /dev/null @@ -1,4 +0,0 @@ - o Testing (chutney): - - In "make test-network-all", test IPv6-only v3 single onion services, - using the chutney network single-onion-v23-ipv6-md. This test will - not pass until 23588 has been merged. Closes ticket 27251. diff --git a/changes/ticket27821 b/changes/ticket27821 deleted file mode 100644 index 158f308fbf..0000000000 --- a/changes/ticket27821 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor features (HTTP tunnel): - - Return an informative web page when the HTTPTunnelPort is used as an - HTTP proxy. Closes ticket 27821, patch by "eighthave". diff --git a/changes/ticket28634 b/changes/ticket28634 deleted file mode 100644 index 7ba05e5c55..0000000000 --- a/changes/ticket28634 +++ /dev/null @@ -1,10 +0,0 @@ - o Major features (Circuit padding): - - Onion service clients will now add padding cells to the initial portions - of their INTRODUCE and RENDEZVOUS circuits, to make those circuits' - traffic patterns look more like general purpose Exit traffic. The - overhead for this is 2 extra cells in each direction for RENDEZVOUS - circuits, and 1 extra upstream cell and 10 downstream cells for INTRODUCE - circuits. This will only be enabled if the circuit's middle node supports - this feature, too. (Clients may specify fixed middle nodes with the MiddleNodes - torrc directive, and may force-disable this feature with the CircuitPadding - torrc directive). Closes ticket 28634. diff --git a/changes/ticket28780 b/changes/ticket28780 deleted file mode 100644 index d7c6693f8c..0000000000 --- a/changes/ticket28780 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor features (circuit padding): - - Provide the ability for circuit padding machines to hold a circuit open - until they are done padding it. Closes ticket 28780. diff --git a/changes/ticket28816 b/changes/ticket28816 deleted file mode 100644 index 02878ccfdc..0000000000 --- a/changes/ticket28816 +++ /dev/null @@ -1,4 +0,0 @@ - o Code simplification and refactoring: - - Introduce a connection_dir_buf_add() helper function that checks for - compress_state of dir_connection_t and automatically writes a string to - directory connection with or without compression. Resolves issue 28816. diff --git a/changes/ticket28837 b/changes/ticket28837 deleted file mode 100644 index 3bc8f12597..0000000000 --- a/changes/ticket28837 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor features (performance): - - Use OpenSSL's implementations of SHA3 when available (in OpenSSL 1.1.1 - and later), since they tend to be faster than tiny-keccak. Closes - ticket 28837. diff --git a/changes/ticket28913 b/changes/ticket28913 deleted file mode 100644 index e09847464d..0000000000 --- a/changes/ticket28913 +++ /dev/null @@ -1,4 +0,0 @@ - o Code simplification and refactoring: - - Make the base32_decode() API return the number of bytes written, - for consistency with base64_decode(). - Closes ticket 28913. diff --git a/changes/ticket29059 b/changes/ticket29059 deleted file mode 100644 index d47d0e2a3b..0000000000 --- a/changes/ticket29059 +++ /dev/null @@ -1,3 +0,0 @@ - o Code simplification and refactoring (shell scripts): - - Fix shellcheck warnings in fuzz_static_testcases.sh. Resolves ticket - 29059. diff --git a/changes/ticket29060 b/changes/ticket29060 deleted file mode 100644 index 380cc8eb11..0000000000 --- a/changes/ticket29060 +++ /dev/null @@ -1,2 +0,0 @@ - o Code simplification and refactoring (shell scripts): - - Fix shellcheck warnings in test-network.sh. Resolves issue 29060. diff --git a/changes/ticket29062 b/changes/ticket29062 deleted file mode 100644 index de05c621f1..0000000000 --- a/changes/ticket29062 +++ /dev/null @@ -1,3 +0,0 @@ - o Code simplification and refactoring (shell scripts): - - Cleanup test_keygen.sh to silence all shellcheck warnings. Closes - ticket 29062. diff --git a/changes/ticket29064 b/changes/ticket29064 deleted file mode 100644 index 616b8aa77e..0000000000 --- a/changes/ticket29064 +++ /dev/null @@ -1,2 +0,0 @@ - o Code simplification and refactoring (shell scripts): - - Fix shellcheck warning in test_rust.sh. Fixes issue 29064. diff --git a/changes/ticket29065 b/changes/ticket29065 deleted file mode 100644 index edf00ac99c..0000000000 --- a/changes/ticket29065 +++ /dev/null @@ -1,3 +0,0 @@ - o Code simplification and refactoring (shell scripts): - - Cleanup test_switch_id.sh to silence shellcheck warnings. Closes - ticket 29065. diff --git a/changes/ticket29067 b/changes/ticket29067 deleted file mode 100644 index a660648775..0000000000 --- a/changes/ticket29067 +++ /dev/null @@ -1,3 +0,0 @@ - o Code simplification and refactoring (shell scripts): - - Fix test_workqueue_*.sh scripts to silence shellcheck SC2086 - warnings. Fixes issue 29067. diff --git a/changes/ticket29068 b/changes/ticket29068 deleted file mode 100644 index 77ef304f1d..0000000000 --- a/changes/ticket29068 +++ /dev/null @@ -1,2 +0,0 @@ - o Code simplification and refactoring (shell scripts): - - Fix shellcheck warnings in zero_length_keys.sh. Resolves issue 29068. diff --git a/changes/ticket29070 b/changes/ticket29070 deleted file mode 100644 index 2716915359..0000000000 --- a/changes/ticket29070 +++ /dev/null @@ -1,2 +0,0 @@ - o Code simplification and refactoring (shell scripts): - - Fix shellcheck warning in torify script. Resolves issue 29070. diff --git a/changes/ticket29071 b/changes/ticket29071 deleted file mode 100644 index 0997a8d22f..0000000000 --- a/changes/ticket29071 +++ /dev/null @@ -1,3 +0,0 @@ - o Code simplification and refactoring (shell scripts): - - Fix shellcheck warnings in nagios-check-tor-authority-cert script. - Resolves issue 29071. diff --git a/changes/ticket29108 b/changes/ticket29108 deleted file mode 100644 index 7adb08ecb1..0000000000 --- a/changes/ticket29108 +++ /dev/null @@ -1,5 +0,0 @@ - o Code simplification and refactoring: - - Split crypto_digest.c into three parts: 1) general code that does not - depend on either NSS or OpenSSL (stays in crypto_digest.c); 2) code that - depends on NSS API (moved to crypto_digest_nss.c); 3) code that depends - on OpenSSL API (moved to crypto_digest_openssl.c). Resolves ticket 29108. diff --git a/changes/ticket29391 b/changes/ticket29391 deleted file mode 100644 index f00fa61c47..0000000000 --- a/changes/ticket29391 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor feature (maintenance scripts): - - Add to scripts/maint/ helper maintainer scripts used for git maintenance. - Closes ticket 29391. diff --git a/changes/ticket29434 b/changes/ticket29434 deleted file mode 100644 index 8037044f0b..0000000000 --- a/changes/ticket29434 +++ /dev/null @@ -1,3 +0,0 @@ - o Removed features: - - Remove linux-tor-prio.sh script from contrib/operator-tools directory. - Resolves issue 29434. diff --git a/changes/ticket29436 b/changes/ticket29436 deleted file mode 100644 index 025be619e5..0000000000 --- a/changes/ticket29436 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor features (testing): - - We now have a script, cov-test-determinism.sh, to identify places - where our unit test coverage has become nondeterministic. - Closes ticket 29436. diff --git a/changes/ticket29536 b/changes/ticket29536 deleted file mode 100644 index a5ae26b701..0000000000 --- a/changes/ticket29536 +++ /dev/null @@ -1,9 +0,0 @@ - o Minor features (performance, RNG): - - Tor now constructs a fast secure pseudorandom number generator for - each thread, to use for cases where performance is critical. This PRNG - is based on AES-CTR, using a buffering construction similar to - libottery and the (newer) OpenBSD arc4random() code. It outperforms - OpenSSL 1.1.1a's CSPRNG by roughly a factor of 100 for small outputs. - Although we believe it to be cryptographically strong, we are only - using it when necessary for reasonable performance. Implements tickets - 29023 and 29536. diff --git a/changes/ticket29537 b/changes/ticket29537 deleted file mode 100644 index afe2308205..0000000000 --- a/changes/ticket29537 +++ /dev/null @@ -1,3 +0,0 @@ - o Testing: - - Check that representative subsets of values of `int` and `unsigned int` - can be represented by `void *`. Resolves issue 29537. diff --git a/changes/ticket29542 b/changes/ticket29542 deleted file mode 100644 index 465a8e31bc..0000000000 --- a/changes/ticket29542 +++ /dev/null @@ -1,7 +0,0 @@ - o Minor features (defense in depth): - - Tor now uses a fast cryptographically strong PRNG even for decisions - that we do not believe are security-sensitive. Previously, for - performance reasons, we had used a trivially predictable linear - congruential generator algorithm for certain load-balancing and - statistical sampling decisions. Now we use our fast RNG in those cases. - Closes ticket 29542. diff --git a/changes/ticket29553 b/changes/ticket29553 deleted file mode 100644 index af441b92b0..0000000000 --- a/changes/ticket29553 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor bugfixes (developer tools): - - Update our pre-commit.git-hook script to work correctly on older Tor - branches and release branches without any changes files, - and to actually exit when something fails. Fixes bug 29553; bugfix on - 0.4.0.2-alpha. diff --git a/changes/ticket29588 b/changes/ticket29588 deleted file mode 100644 index c81bccb00d..0000000000 --- a/changes/ticket29588 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor features (developer tools): - - Introduce a post-merge git hook script to check if we're pulling in any - changes to our git workspace management scripts from upstream. Resolves - issue 29588. diff --git a/changes/ticket29635 b/changes/ticket29635 deleted file mode 100644 index cbadbf648a..0000000000 --- a/changes/ticket29635 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor bugfixes (documentation, manpage): - - Use proper formatting when providing an example on quoting options that - contain whitespace. Fixes bug 29635; bugfix on 0.2.3.18-rc. diff --git a/changes/ticket29660 b/changes/ticket29660 deleted file mode 100644 index 84b8059106..0000000000 --- a/changes/ticket29660 +++ /dev/null @@ -1,5 +0,0 @@ - o Code simplification and refactoring: - - Remove redundant return values in crypto_format, and the associated - return value checks elsewhere in the code. Make the implementations in - crypto_format consistent, and remove redundant code. - Resolves ticket 29660. diff --git a/changes/ticket29662 b/changes/ticket29662 deleted file mode 100644 index 872df9ad82..0000000000 --- a/changes/ticket29662 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor features (debugging): - - Introduce tor_assertf() and tor_assertf_nonfatal() to enable logging of - additional information during assert failure. Now we can use format - strings to include pieces of information that are relevant for trouble - shooting. Resolves ticket 29662. diff --git a/changes/ticket29732 b/changes/ticket29732 deleted file mode 100644 index bb72361c48..0000000000 --- a/changes/ticket29732 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor features (testing): - - Tor's unit test code now contains a standard set of functions to - replace the PRNG with a deterministic or reproducible version for - testing. Previously, various tests implemented this in various ways. - Implements ticket 29732. diff --git a/changes/ticket29756 b/changes/ticket29756 deleted file mode 100644 index 79995b4995..0000000000 --- a/changes/ticket29756 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor features (developer tools): - - Add a script to check that each header has a well-formed and unique - guard marco. Closes ticket 29756. diff --git a/changes/ticket29894 b/changes/ticket29894 deleted file mode 100644 index 6392598ec6..0000000000 --- a/changes/ticket29894 +++ /dev/null @@ -1,4 +0,0 @@ - o Code simplification and refactoring: - - Split up the control.c file into several submodules, in preparation - for distributing its current responsibilities throughout the codebase. - Closes ticket 29894. diff --git a/changes/ticket29913 b/changes/ticket29913 deleted file mode 100644 index a713b0ccef..0000000000 --- a/changes/ticket29913 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (python): - - Stop assuming that /usr/bin/python3 exists. For scripts that work with - python2, use /usr/bin/python. Otherwise, use /usr/bin/env python3. - Fixes bug 29913; bugfix on 0.2.5.3-alpha. diff --git a/changes/ticket29984 b/changes/ticket29984 deleted file mode 100644 index 8631dff27b..0000000000 --- a/changes/ticket29984 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor bugfixes (controller protocol): - - Teach the controller parser to correctly distinguish an object - preceded by an argument list from one without. Previously, it - couldn't distinguish an argument list from the first line of a - multiline object. Fixes bug 29984; bugfix on 0.2.3.8-alpha. diff --git a/changes/ticket30007 b/changes/ticket30007 deleted file mode 100644 index e87f6b956f..0000000000 --- a/changes/ticket30007 +++ /dev/null @@ -1,3 +0,0 @@ - o Code simplification and refactoring: - - Abstract out the low-level formatting of replies on the control - port. Implements ticket 30007. diff --git a/changes/ticket30033 b/changes/ticket30033 deleted file mode 100644 index 3f66d049c8..0000000000 --- a/changes/ticket30033 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor features (developer tooling): - - Call pre-commit git hook from pre-push hook to make sure we're - running documentation and code style checks before pushing to remote - git repository. Implements feature 30033. diff --git a/changes/ticket30051 b/changes/ticket30051 deleted file mode 100644 index 87b6d7611f..0000000000 --- a/changes/ticket30051 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor features (developer tooling): - - Call practracker from pre-push and pre-commit git hooks to let a - developer know if they made any code style violations in their last - commit. This should help preventing code style violations appearing - upstream. Closes ticket 30051. diff --git a/changes/ticket30075 b/changes/ticket30075 deleted file mode 100644 index 288abd7674..0000000000 --- a/changes/ticket30075 +++ /dev/null @@ -1,3 +0,0 @@ - o Removed features: - - Remove the obsolete script at contrib/dist/tor.sh.in. Resolves issue - 30075. diff --git a/changes/ticket30076 b/changes/ticket30076 deleted file mode 100644 index 1334bc4603..0000000000 --- a/changes/ticket30076 +++ /dev/null @@ -1,2 +0,0 @@ - o Removed features: - - Remove obsolete OpenSUSE initscript. Resolves issue 30076. diff --git a/changes/ticket30077 b/changes/ticket30077 deleted file mode 100644 index 9be014730e..0000000000 --- a/changes/ticket30077 +++ /dev/null @@ -1,2 +0,0 @@ - o Code simplification and refactoring (shell scripts): - - Fix shellcheck warnings in fuzz_multi.sh. Resolves issue 30077. diff --git a/changes/ticket30078 b/changes/ticket30078 deleted file mode 100644 index 5ab5abdbfd..0000000000 --- a/changes/ticket30078 +++ /dev/null @@ -1,3 +0,0 @@ - o Code simplification and refactoring (shell scripts): - - Fix shellcheck warnings in src/test/fuzz/fixup_filenames.sh. Resolves - issue 30078. diff --git a/changes/ticket30079 b/changes/ticket30079 deleted file mode 100644 index 56b88e7f53..0000000000 --- a/changes/ticket30079 +++ /dev/null @@ -1,3 +0,0 @@ - o Code simplification and refactoring (shell scripts): - - Fix shellcheck warning SC2006 in src/test/fuzz/minimize.sh. Resolves - issue 30079. diff --git a/changes/ticket30091 b/changes/ticket30091 deleted file mode 100644 index 968ea01f4a..0000000000 --- a/changes/ticket30091 +++ /dev/null @@ -1,4 +0,0 @@ - o Major features (controller protocol): - - Controller commands are now parsed using a generalized parsing - subsystem. Previously, each controller command was responsible for - parsing its own input. Closes ticket 30091. diff --git a/changes/ticket30114 b/changes/ticket30114 deleted file mode 100644 index a80f7f4dcf..0000000000 --- a/changes/ticket30114 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor features (git scripts): - - In git-pull-all.sh, also fetch the latest tor-github pull requests. - Implements ticket 30114. diff --git a/changes/ticket30149 b/changes/ticket30149 deleted file mode 100644 index a21687ac2f..0000000000 --- a/changes/ticket30149 +++ /dev/null @@ -1,3 +0,0 @@ - o Code simplification and refactoring: - - Add several assertions in an attempt to fix some Coverity warnings. - Closes ticket 30149. diff --git a/changes/ticket30176 b/changes/ticket30176 deleted file mode 100644 index da23760ce5..0000000000 --- a/changes/ticket30176 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor features (defense in depth): - - In smartlist_remove_keeporder(), set any pointers that become - unused to NULL, in case a bug causes them to be used later. Closes - ticket 30176. Patch from Tobias Stoeckmann. diff --git a/changes/ticket30213 b/changes/ticket30213 deleted file mode 100644 index acb7614807..0000000000 --- a/changes/ticket30213 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor features (continuous integration): - - Remove sudo configuration lines from .travis.yml as they are no longer - needed with current Travis build environment. Resolves issue 30213. diff --git a/changes/ticket30234 b/changes/ticket30234 deleted file mode 100644 index 5a0076bad2..0000000000 --- a/changes/ticket30234 +++ /dev/null @@ -1,2 +0,0 @@ - o Testing (continuous integration): - - In Travis, show stem's tor log after failure. Closes ticket 30234. diff --git a/changes/ticket30261 b/changes/ticket30261 deleted file mode 100644 index e4a2643c88..0000000000 --- a/changes/ticket30261 +++ /dev/null @@ -1,4 +0,0 @@ - o Documentation: - - Document how to find git commits and tags for bug fixes in - CodingStandards.md. And update some changes file documentation. - Closes ticket 30261. diff --git a/changes/ticket30293 b/changes/ticket30293 deleted file mode 100644 index c74b6cd346..0000000000 --- a/changes/ticket30293 +++ /dev/null @@ -1,5 +0,0 @@ - o Code simplification and refactoring: - - Start move responsibility for knowing about periodic events to the - appropriate subsystems, so that the mainloop doesn't need to know all - the periodic events in the rest of the codebase. Implements tickets - 30293 and 30294. diff --git a/changes/ticket30307 b/changes/ticket30307 deleted file mode 100644 index abcacb6085..0000000000 --- a/changes/ticket30307 +++ /dev/null @@ -1,4 +0,0 @@ - o Major features (performance): - - Update our node selection algorithm to exclude nodes in linear time. - Previously, the algorithm was quadratic, which could slow down heavily - used onion services. Closes ticket 30307. diff --git a/changes/ticket30308 b/changes/ticket30308 deleted file mode 100644 index b78e6b3e9f..0000000000 --- a/changes/ticket30308 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor bugfixes (performance): - - When checking a node for bridge status, use a fast check to make sure - that its identity is set. Previously, we used a constant-time check, - which is not necessary when verifying a BUG() condition that causes - a stack trace. Fixes bug 30308; bugfix on 0.3.5.1-alpha. diff --git a/changes/ticket30345 b/changes/ticket30345 deleted file mode 100644 index 639db8d7ee..0000000000 --- a/changes/ticket30345 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor features (modularity): - - The --disable-module-dirauth compile-time option now disables - even more dirauth-only code. Closes ticket 30345. diff --git a/changes/ticket30414 b/changes/ticket30414 deleted file mode 100644 index 029ed1311f..0000000000 --- a/changes/ticket30414 +++ /dev/null @@ -1,3 +0,0 @@ - o Code simplification and refactoring: - - Move most relay-only periodic events out of mainloop.c into the - relay subsystem. Closes ticket 30414. From 49acbfad23df415977fb7a196c6aae2d7406b489 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Mon, 6 May 2019 17:21:11 +0300 Subject: [PATCH 1029/2557] Don't forget to use the mutex in testing_enable_prefilled_rng() --- src/test/rng_test_helpers.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/test/rng_test_helpers.c b/src/test/rng_test_helpers.c index 262d380bda..d268cb64b7 100644 --- a/src/test/rng_test_helpers.c +++ b/src/test/rng_test_helpers.c @@ -185,10 +185,14 @@ testing_enable_prefilled_rng(const void *buffer, size_t buflen) tor_assert(buflen > 0); rng_mutex = tor_mutex_new(); + tor_mutex_acquire(rng_mutex); + prefilled_rng_buffer = tor_memdup(buffer, buflen); prefilled_rng_buflen = buflen; prefilled_rng_idx = 0; + tor_mutex_release(rng_mutex); + MOCK(crypto_rand, crypto_rand_prefilled); MOCK(crypto_strongest_rand_, mock_crypto_strongest_rand); } From 2845607f9706dc00aa799fc609d692e8d8c4786f Mon Sep 17 00:00:00 2001 From: rl1987 Date: Fri, 17 May 2019 19:44:45 +0300 Subject: [PATCH 1030/2557] In microdesc_cache_reload(), set journal length to length of string we read Hopefully this will fix CID 1444769. --- src/feature/nodelist/microdesc.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/feature/nodelist/microdesc.c b/src/feature/nodelist/microdesc.c index db2149754a..89ac0a2f83 100644 --- a/src/feature/nodelist/microdesc.c +++ b/src/feature/nodelist/microdesc.c @@ -536,8 +536,8 @@ microdesc_cache_reload(microdesc_cache_t *cache) journal_content = read_file_to_str(cache->journal_fname, RFTS_IGNORE_MISSING, &st); if (journal_content) { - cache->journal_len = (size_t) st.st_size; - warn_if_nul_found(journal_content, cache->journal_len, 0, + cache->journal_len = strlen(journal_content); + warn_if_nul_found(journal_content, (size_t)st.st_size, 0, "reading microdesc journal"); added = microdescs_add_to_cache(cache, journal_content, journal_content+st.st_size, From 2f31c8146ff12aa96a33cce47bf4b5f5eeac8a06 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 20 May 2019 11:52:45 -0400 Subject: [PATCH 1031/2557] rectify_include_paths: warn instead of aborting on duplicate headers We have two sendme.h files at the moment; we should fix that, but not in this branch. --- scripts/maint/add_c_file.py | 251 +++++++++++++++++++++++++ scripts/maint/rectify_include_paths.py | 13 +- 2 files changed, 262 insertions(+), 2 deletions(-) create mode 100755 scripts/maint/add_c_file.py diff --git a/scripts/maint/add_c_file.py b/scripts/maint/add_c_file.py new file mode 100755 index 0000000000..499415974f --- /dev/null +++ b/scripts/maint/add_c_file.py @@ -0,0 +1,251 @@ +#!/usr/bin/env python3 + +""" + Add a C file with matching header to the Tor codebase. Creates + both files from templates, and adds them to the right include.am file. + + Example usage: + + % add_c_file.py ./src/feature/dirauth/ocelot.c +""" + +import os +import re +import time + +def topdir_file(name): + """Strip opening "src" from a filename""" + if name.startswith("src/"): + name = name[4:] + return name + +def guard_macro(name): + """Return the guard macro that should be used for the header file 'name'. + """ + td = topdir_file(name).replace(".", "_").replace("/", "_").upper() + return "TOR_{}".format(td) + +def makeext(name, new_extension): + """Replace the extension for the file called 'name' with 'new_extension'. + """ + base = os.path.splitext(name)[0] + return base + "." + new_extension + +def instantiate_template(template, output_fname): + """ + Fill in a template with string using the fields that should be used + for 'output_fname'. + """ + names = { + # The relative location of the header file. + 'header_path' : makeext(topdir_file(output_fname), "h"), + # The relative location of the C file file. + 'c_file_path' : makeext(topdir_file(output_fname), "c"), + # The truncated name of the file. + 'short_name' : os.path.basename(output_fname), + # The current year, for the copyright notice + 'this_year' : time.localtime().tm_year, + # An appropriate guard macro, for the header. + 'guard_macro' : guard_macro(output_fname), + } + + return template.format(**names) + +HEADER_TEMPLATE = """\ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-{this_year}, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file {short_name} + * @brief Header for {c_file_path} + **/ + +#ifndef {guard_macro} +#define {guard_macro} + +#endif /* !defined({guard_macro}) */ +""" + +C_FILE_TEMPLATE = """\ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-{this_year}, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file {short_name} + * @brief DOCDOC + **/ + +#include "orconfig.h" +#include "{header_path}" +""" + +class AutomakeChunk: + """ + Represents part of an automake file. If it is decorated with + an ADD_C_FILE comment, it has a "kind" based on what to add to it. + Otherwise, it only has a bunch of lines in it. + """ + pat = re.compile(r'# ADD_C_FILE: INSERT (\S*) HERE', re.I) + + def __init__(self): + self.lines = [] + self.kind = "" + + def addLine(self, line): + """ + Insert a line into this chunk while parsing the automake file. + """ + m = self.pat.match(line) + if m: + if self.lines: + raise ValueError("control line not preceded by a blank line") + self.kind = m.group(1) + + self.lines.append(line) + if line.strip() == "": + return True + + return False + + def insertMember(self, member): + """ + Add a new member to this chunk. Try to insert it in alphabetical + order with matching indentation, but don't freak out too much if the + source isn't consistent. + + Assumes that this chunk is of the form: + FOOBAR = \ + X \ + Y \ + Z + """ + self.prespace = "\t" + self.postspace = "\t\t" + for lineno, line in enumerate(self.lines): + m = re.match(r'(\s+)(\S+)(\s+)\\', line) + if not m: + continue + prespace, fname, postspace = m.groups() + if fname > member: + self.insert_before(lineno, member, prespace, postspace) + return + self.insert_at_end(member) + + def insert_before(self, lineno, member, prespace, postspace): + self.lines.insert(lineno, + "{}{}{}\\\n".format(prespace, member, postspace)) + + def insert_at_end(self, member, prespace, postspace): + lastline = self.lines[-1] + self.lines[-1] += '{}\\\n'.format(postspace) + self.lines.append("{}{}\n".format(prespace, member)) + + def dump(self, f): + """Write all the lines in this chunk to the file 'f'.""" + for line in self.lines: + f.write(line) + if not line.endswith("\n"): + f.write("\n") + +class ParsedAutomake: + """A sort-of-parsed automake file, with identified chunks into which + headers and c files can be inserted. + """ + def __init__(self): + self.chunks = [] + self.by_type = {} + + def addChunk(self, chunk): + """Add a newly parsed AutomakeChunk to this file.""" + self.chunks.append(chunk) + self.by_type[chunk.kind.lower()] = chunk + + def add_file(self, fname, kind): + """Insert a file of kind 'kind' to the appropriate section of this + file. Return True if we added it. + """ + if kind.lower() in self.by_type: + self.by_type[kind.lower()].insertMember(fname) + return True + else: + return False + + def dump(self, f): + """Write this file into a file 'f'.""" + for chunk in self.chunks: + chunk.dump(f) + +def get_include_am_location(fname): + """Find the right include.am file for introducing a new file. Return None + if we can't guess one. + + Note that this function is imperfect because our include.am layout is + not (yet) consistent. + """ + td = topdir_file(fname) + m = re.match(r'^lib/([a-z0-9_]*)/', td) + if m: + return "src/lib/{}/include.am".format(m.group(1)) + + if re.match(r'^(core|feature|app)/', td): + return "src/core/include.am" + + if re.match(r'^test/', td): + return "src/test/include.am" + + return None + +def run(fn): + """ + Create a new C file and H file corresponding to the filename "fn", and + add them to include.am. + """ + + cf = makeext(fn, "c") + hf = makeext(fn, "h") + + if os.path.exists(cf): + print("{} already exists".format(cf)) + return 1 + if os.path.exists(hf): + print("{} already exists".format(hf)) + return 1 + + with open(cf, 'w') as f: + f.write(instantiate_template(C_FILE_TEMPLATE, cf)) + + with open(hf, 'w') as f: + f.write(instantiate_template(HEADER_TEMPLATE, hf)) + + iam = get_include_am_location(cf) + if iam is None or not os.path.exists(iam): + print("Made files successfully but couldn't identify include.am for {}" + .format(cf)) + return 1 + + amfile = ParsedAutomake() + cur_chunk = AutomakeChunk() + with open(iam) as f: + for line in f: + if cur_chunk.addLine(line): + amfile.addChunk(cur_chunk) + cur_chunk = AutomakeChunk() + amfile.addChunk(cur_chunk) + + amfile.add_file(cf, "sources") + amfile.add_file(hf, "headers") + + with open(iam+".tmp", 'w') as f: + amfile.dump(f) + + os.rename(iam+".tmp", iam) + +if __name__ == '__main__': + import sys + sys.exit(run(sys.argv[1])) diff --git a/scripts/maint/rectify_include_paths.py b/scripts/maint/rectify_include_paths.py index d1a2328ca6..1140e8cd22 100755 --- a/scripts/maint/rectify_include_paths.py +++ b/scripts/maint/rectify_include_paths.py @@ -3,6 +3,10 @@ import os import os.path import re +import sys + +def warn(msg): + sys.stderr.write("WARNING: %s\n"%msg) # Find all the include files, map them to their real names. @@ -11,6 +15,8 @@ def exclude(paths, dirnames): if p in dirnames: dirnames.remove(p) +DUPLICATE = object() + def get_include_map(): includes = { } @@ -19,7 +25,10 @@ def get_include_map(): for fname in fnames: if fname.endswith(".h"): - assert fname not in includes + if fname in includes: + warn("Multiple headers named %s"%fname) + includes[fname] = DUPLICATE + continue include = os.path.join(dirpath, fname) assert include.startswith("src/") includes[fname] = include[4:] @@ -37,7 +46,7 @@ def fix_includes(inp, out, mapping): if m: include,hdr,rest = m.groups() basehdr = get_base_header_name(hdr) - if basehdr in mapping: + if basehdr in mapping and mapping[basehdr] is not DUPLICATE: out.write('{}{}{}\n'.format(include,mapping[basehdr],rest)) continue From 66eb0a5a32e78e541f2b350fc814c35f17150859 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 20 May 2019 12:02:20 -0400 Subject: [PATCH 1032/2557] updateCopyright: look at the current year. --- scripts/maint/updateCopyright.pl | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/maint/updateCopyright.pl b/scripts/maint/updateCopyright.pl index 36894b1baf..6800032f87 100755 --- a/scripts/maint/updateCopyright.pl +++ b/scripts/maint/updateCopyright.pl @@ -1,7 +1,9 @@ #!/usr/bin/perl -i -w -p -$NEWYEAR=2019; +@now = gmtime(); -s/Copyright(.*) (201[^9]), The Tor Project/Copyright$1 $2-${NEWYEAR}, The Tor Project/; +$NEWYEAR=$now[5]+1900; + +s/Copyright([^-]*) (20[^-]*), The Tor Project/Copyright$1 $2-${NEWYEAR}, The Tor Project/; s/Copyright(.*)-(20..), The Tor Project/Copyright$1-${NEWYEAR}, The Tor Project/; From e2d3d444969abe2a2b802f4f42512b71a5a45468 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 20 May 2019 12:08:30 -0400 Subject: [PATCH 1033/2557] Add a new "autostyle" make target to run all of our reformatting Closes ticket 30539. --- Makefile.am | 31 ++++++++++++++++++++++++++----- changes/ticket30539 | 4 ++++ 2 files changed, 30 insertions(+), 5 deletions(-) create mode 100644 changes/ticket30539 diff --git a/Makefile.am b/Makefile.am index 827cf3dc9b..7a0d40d6a5 100644 --- a/Makefile.am +++ b/Makefile.am @@ -334,11 +334,8 @@ coverage-html-full: all lcov --remove "$(HTML_COVER_DIR)/lcov.tmp" --rc lcov_branch_coverage=1 'test/*' 'ext/tinytest*' '/usr/*' --output-file "$(HTML_COVER_DIR)/lcov.info" genhtml --branch-coverage -o "$(HTML_COVER_DIR)" "$(HTML_COVER_DIR)/lcov.info" -# Avoid strlcpy.c, strlcat.c, aes.c, OpenBSD_malloc_Linux.c, sha256.c, -# tinytest*.[ch] -check-spaces: -if USE_PERL - $(PERL) $(top_srcdir)/scripts/maint/checkSpace.pl -C \ +# For scripts: avoid src/ext and src/trunnel. +OWNED_TOR_C_FILES=\ $(top_srcdir)/src/lib/*/*.[ch] \ $(top_srcdir)/src/core/*/*.[ch] \ $(top_srcdir)/src/feature/*/*.[ch] \ @@ -346,6 +343,11 @@ if USE_PERL $(top_srcdir)/src/test/*.[ch] \ $(top_srcdir)/src/test/*/*.[ch] \ $(top_srcdir)/src/tools/*.[ch] + +check-spaces: +if USE_PERL + $(PERL) $(top_srcdir)/scripts/maint/checkSpace.pl -C \ + $(OWNED_TOR_C_FILES) endif check-includes: @@ -456,6 +458,25 @@ version: (cd "$(top_srcdir)" && git rev-parse --short=16 HEAD); \ fi +.PHONY: autostyle-ifdefs +autostyle-ifdefs: + $(PYTHON) scripts/maint/annotate_ifdef_directives $(OWNED_TOR_C_FILES) + +.PHONY: autostyle-ifdefs +autostyle-operators: + $(PERL) scripts/coccinelle/test-operator-cleanup $(OWNED_TOR_C_FILES) + +.PHONY: rectify-includes +rectify-includes: + $(PYTHON) scripts/maint/rectify_include_paths.py + +.PHONY: update-copyright +update-copyright: + $(PERL) scripts/maint/updateCopyright.pl $(OWNED_TOR_C_FILES) + +.PHONY: autostyle +autostyle: update-versions rustfmt autostyle-ifdefs rectify-includes + mostlyclean-local: rm -f $(top_builddir)/src/*/*.gc{da,no} $(top_builddir)/src/*/*/*.gc{da,no} rm -rf $(HTML_COVER_DIR) diff --git a/changes/ticket30539 b/changes/ticket30539 new file mode 100644 index 0000000000..8d105eacb9 --- /dev/null +++ b/changes/ticket30539 @@ -0,0 +1,4 @@ + o Minor features (maintenance): + - Add a new "make autostyle" target that developers can use to + apply all automatic Tor style and consistency conversions to the + codebase. Closes ticket 30539. From fb91e1c6efed6e5526170df3650fac65b701524b Mon Sep 17 00:00:00 2001 From: rl1987 Date: Tue, 21 May 2019 14:56:53 +0300 Subject: [PATCH 1034/2557] Remove torctl.in from contrib/dist --- changes/ticket30550 | 2 + configure.ac | 1 - contrib/dist/torctl.in | 195 ----------------------------------------- contrib/include.am | 1 - 4 files changed, 2 insertions(+), 197 deletions(-) create mode 100644 changes/ticket30550 delete mode 100644 contrib/dist/torctl.in diff --git a/changes/ticket30550 b/changes/ticket30550 new file mode 100644 index 0000000000..f356c4048e --- /dev/null +++ b/changes/ticket30550 @@ -0,0 +1,2 @@ + o Removed features: + - Remove torctl.in from contrib/dist directory. Resolves ticket 30550. diff --git a/configure.ac b/configure.ac index e65e960d7f..3c4ce438e0 100644 --- a/configure.ac +++ b/configure.ac @@ -2458,7 +2458,6 @@ AC_CONFIG_FILES([ Makefile config.rust contrib/operator-tools/tor.logrotate - contrib/dist/torctl contrib/dist/tor.service src/config/torrc.sample src/config/torrc.minimal diff --git a/contrib/dist/torctl.in b/contrib/dist/torctl.in deleted file mode 100644 index 4cc137da46..0000000000 --- a/contrib/dist/torctl.in +++ /dev/null @@ -1,195 +0,0 @@ -#!/bin/sh -# -# TOR control script designed to allow an easy command line interface -# to controlling The Onion Router -# -# The exit codes returned are: -# 0 - operation completed successfully. For "status", tor running. -# 1 - For "status", tor not running. -# 2 - Command not supported -# 3 - Could not be started or reloaded -# 4 - Could not be stopped -# 5 - -# 6 - -# 7 - -# 8 - -# -# When multiple arguments are given, only the error from the _last_ -# one is reported. -# -# -# |||||||||||||||||||| START CONFIGURATION SECTION |||||||||||||||||||| -# -------------------- -------------------- -# Name of the executable -EXEC=tor -# -# the path to your binary, including options if necessary -TORBIN="@BINDIR@/$EXEC" -# -# the path to the configuration file -TORCONF="@CONFDIR@/torrc" -# -# the path to your PID file -PIDFILE="@LOCALSTATEDIR@/run/tor/tor.pid" -# -# The path to the log file -LOGFILE="@LOCALSTATEDIR@/log/tor/tor.log" -# -# The path to the datadirectory -TORDATA="@LOCALSTATEDIR@/lib/tor" -# -TORARGS="--pidfile $PIDFILE --log \"notice file $LOGFILE\" --runasdaemon 1" -TORARGS="$TORARGS --datadirectory $TORDATA" - -# If user name is set in the environment, then use it; -# otherwise run as the invoking user (or whatever user the config -# file says)... unless the invoking user is root. The idea here is to -# let an unprivileged user run tor for her own use using this script, -# while still providing for it to be used as a system daemon. -if [ "x`id -u`" = "x0" ]; then - TORUSER=@TORUSER@ -fi - -if [ "x$TORUSER" != "x" ]; then - TORARGS="$TORARGS --user $TORUSER" -fi - -# We no longer wrap the Tor daemon startup in an su when running as -# root, because it's too painful to make the use of su portable. -# Just let the daemon set the UID and GID. -START="$TORBIN -f $TORCONF $TORARGS" - -# -# -------------------- -------------------- -# |||||||||||||||||||| END CONFIGURATION SECTION |||||||||||||||||||| - -ERROR=0 -ARGV="$@" -if [ "x$ARGV" = "x" ] ; then - ARGS="help" -fi - -checkIfRunning ( ) { - # check for pidfile - PID=unknown - if [ -f $PIDFILE ] ; then - PID=`/bin/cat $PIDFILE` - if [ "x$PID" != "x" ] ; then - if kill -0 $PID 2>/dev/null ; then - STATUS="$EXEC (pid $PID) running" - RUNNING=1 - else - STATUS="PID file ($PIDFILE) present, but $EXEC ($PID) not running" - RUNNING=0 - fi - else - STATUS="$EXEC (pid $PID?) not running" - RUNNING=0 - fi - else - STATUS="$EXEC apparently not running (no pid file)" - RUNNING=0 - fi - return -} - -for ARG in $@ $ARGS -do - checkIfRunning - - case $ARG in - start) - if [ $RUNNING -eq 1 ]; then - echo "$0 $ARG: $EXEC (pid $PID) already running" - continue - fi - if eval "$START" ; then - echo "$0 $ARG: $EXEC started" - # Make sure it stayed up! - /bin/sleep 1 - checkIfRunning - if [ $RUNNING -eq 0 ]; then - echo "$0 $ARG: $EXEC (pid $PID) quit unexpectedly" - fi - else - echo "$0 $ARG: $EXEC could not be started" - ERROR=3 - fi - ;; - stop) - if [ $RUNNING -eq 0 ]; then - echo "$0 $ARG: $STATUS" - continue - fi - if kill -15 $PID ; then - echo "$0 $ARG: $EXEC stopped" - else - /bin/sleep 1 - if kill -9 $PID ; then - echo "$0 $ARG: $EXEC stopped" - else - echo "$0 $ARG: $EXEC could not be stopped" - ERROR=4 - fi - fi - # Make sure it really died! - /bin/sleep 1 - checkIfRunning - if [ $RUNNING -eq 1 ]; then - echo "$0 $ARG: $EXEC (pid $PID) unexpectedly still running" - ERROR=4 - fi - ;; - restart) - $0 stop start - ;; - reload) - if [ $RUNNING -eq 0 ]; then - echo "$0 $ARG: $STATUS" - continue - fi - if kill -1 $PID; then - /bin/sleep 1 - echo "$EXEC (PID $PID) reloaded" - else - echo "Can't reload $EXEC" - ERROR=3 - fi - ;; - status) - echo $STATUS - if [ $RUNNING -eq 1 ]; then - ERROR=0 - else - ERROR=1 - fi - ;; - log) - cat $LOGFILE - ;; - help) - echo "usage: $0 (start|stop|restart|status|help)" - /bin/cat < Date: Tue, 21 May 2019 19:18:58 -0400 Subject: [PATCH 1035/2557] light movement and editing on changelog --- ChangeLog | 121 ++++++++++++++++++------------------------------------ 1 file changed, 39 insertions(+), 82 deletions(-) diff --git a/ChangeLog b/ChangeLog index 827c4c3131..4335a28c62 100644 --- a/ChangeLog +++ b/ChangeLog @@ -5,7 +5,9 @@ Changes in version 0.4.1.1-alpha - 2019-05-?? certain denial-of-service attacks more difficult, and improves performance in several areas. - o Major features (Circuit padding): + o Code simplification and refactoring (circuit padding): + + o Major features (circuit padding): - Onion service clients will now add padding cells to the initial portions of their INTRODUCE and RENDEZVOUS circuits, to make those circuits' traffic patterns look more like general purpose Exit @@ -42,15 +44,19 @@ Changes in version 0.4.1.1-alpha - 2019-05-?? time. Previously, the algorithm was quadratic, which could slow down heavily used onion services. Closes ticket 30307. - o Minor feature (circuit padding): - - We now use a fast RNG when scheduling circuit padding. Part of - ticket 28636. - - o Minor feature (maintenance scripts): - - Add to scripts/maint/ helper maintainer scripts used for git - maintenance. Closes ticket 29391. + o Major features (performance, RNG): + - Tor now constructs a fast secure pseudorandom number generator for + each thread, to use when performance is critical. This PRNG is + based on AES-CTR, using a buffering construction similar to + libottery and the (newer) OpenBSD arc4random() code. It + outperforms OpenSSL 1.1.1a's CSPRNG by roughly a factor of 100 for + small outputs. Although we believe it to be cryptographically + strong, we are only using it when necessary for reasonable + performance. Implements tickets 29023 and 29536. o Minor features (circuit padding): + - We now use a fast RNG when scheduling circuit padding. Part of + ticket 28636. - Allow the padding machine designer to pick the edges of their histogram instead of trying to compute them automatically using an exponential formula. Resolves some undefined behavior in the case @@ -90,39 +96,24 @@ Changes in version 0.4.1.1-alpha - 2019-05-?? load-balancing and statistical sampling decisions. Now we use our fast RNG in those cases. Closes ticket 29542. - o Minor features (developer tooling): - - Call practracker from pre-push and pre-commit git hooks to let a - developer know if they made any code style violations in their - last commit. This should help preventing code style violations - appearing upstream. Closes ticket 30051. - - Call pre-commit git hook from pre-push hook to make sure we're - running documentation and code style checks before pushing to - remote git repository. Implements feature 30033. - - Modify git pre-push hook script to disallow pushing branches other - than master, release-* and maint-* to origin remote. Implements - feature 29532. - o Minor features (developer tools): - - Add a script to check that each header has a well-formed and - unique guard marco. Closes ticket 29756. - - Introduce a post-merge git hook script to check if we're pulling - in any changes to our git workspace management scripts from - upstream. Resolves issue 29588. - - o Minor features (development tools): - Tor's test scripts now check for files and functions that seem too long and complicated. Existing overlong functions and files are accepted for now, but should eventually be refactored. Closes ticket 29221. + - Add to scripts/maint/ helper maintainer scripts used for git + maintenance. Closes ticket 29391. + - Call practracker from pre-push and pre-commit git hooks to let a + developer know if they made any code style violations in their + last commit. This should help preventing code style violations + appearing upstream. Closes ticket 30051. + - Add a script to check that each header has a well-formed and + unique guard marco. Closes ticket 29756. o Minor features (geoip): - Update geoip and geoip6 to the May 13 2019 Maxmind GeoLite2 Country database. Closes ticket 30522. - o Minor features (git scripts): - - In git-pull-all.sh, also fetch the latest tor-github pull - requests. Implements ticket 30114. - o Minor features (HTTP tunnel): - Return an informative web page when the HTTPTunnelPort is used as an HTTP proxy. Closes ticket 27821, patch by "eighthave". @@ -141,16 +132,6 @@ Changes in version 0.4.1.1-alpha - 2019-05-?? 1.1.1 and later), since they tend to be faster than tiny-keccak. Closes ticket 28837. - o Minor features (performance, RNG): - - Tor now constructs a fast secure pseudorandom number generator for - each thread, to use for cases where performance is critical. This - PRNG is based on AES-CTR, using a buffering construction similar - to libottery and the (newer) OpenBSD arc4random() code. It - outperforms OpenSSL 1.1.1a's CSPRNG by roughly a factor of 100 for - small outputs. Although we believe it to be cryptographically - strong, we are only using it when necessary for reasonable - performance. Implements tickets 29023 and 29536. - o Minor features (testing): - Tor's unit test code now contains a standard set of functions to replace the PRNG with a deterministic or reproducible version for @@ -159,6 +140,8 @@ Changes in version 0.4.1.1-alpha - 2019-05-?? - We now have a script, cov-test-determinism.sh, to identify places where our unit test coverage has become nondeterministic. Closes ticket 29436. + - Check that representative subsets of values of `int` and `unsigned + int` can be represented by `void *`. Resolves issue 29537. o Minor bugfixes (bridge authority): - We set bridges as running when we dump the bridge status to a @@ -166,7 +149,7 @@ Changes in version 0.4.1.1-alpha - 2019-05-?? controller, but these shouldn't modify vital data structures. Fixes bug 24490; bugfix on 0.2.0.13-alpha. Patch by Neel Chauhan - o Minor bugfixes (Channel padding statistics): + o Minor bugfixes (channel padding statistics): - Channel padding write totals and padding-enabled totals are now counted properly in relay extrainfo descriptors. Fixes bug 29231; bugfix on 0.3.1.1-alpha @@ -183,13 +166,11 @@ Changes in version 0.4.1.1-alpha - 2019-05-?? bugfix on 0.4.0.1-alpha. - The circuit padding subsystem does not schedule padding if dormant mode is enabled. Fixes bug 28636; bugfix on 0.4.0.1-alpha. - - o Minor bugfixes (circuitpadding): - Inspect circuit-level cell queue before sending padding, to avoid sending padding while too much data is queued. Fixes bug 29204; bugfix on 0.4.0.1-alpha. - o Minor bugfixes (compilation, unusual configuration): + o Minor bugfixes (compilation, unusual configurations): - Avoid failures when building with ALL_BUGS_ARE_FAILED due to missing declarations of abort(), and prevent other such failures in the future. Fixes bug 30189; bugfix on 0.3.4.1-alpha. @@ -200,16 +181,10 @@ Changes in version 0.4.1.1-alpha - 2019-05-?? couldn't distinguish an argument list from the first line of a multiline object. Fixes bug 29984; bugfix on 0.2.3.8-alpha. - o Minor bugfixes (developer tools): - - Update our pre-commit.git-hook script to work correctly on older - Tor branches and release branches without any changes files, and - to actually exit when something fails. Fixes bug 29553; bugfix - on 0.4.0.2-alpha. - - o Minor bugfixes (dirauth, ipv6): - - If we are a durauth with IPv6 and are marking relays as running, - mark ourselves as reachable on IPv6. Fixes bug 24338; bugfix on - 0.4.0.2-alpha. Patch by Neel Chauhan + o Minor bugfixes (directory authority, ipv6): + - If we are a directory authity with IPv6 and are marking relays as + running, mark ourselves as reachable on IPv6. Fixes bug 24338; + bugfix on 0.4.0.2-alpha. Patch by Neel Chauhan o Minor bugfixes (documentation): - Improve the documentation for MapAddress .exit. Fixes bug 30109; @@ -218,18 +193,12 @@ Changes in version 0.4.1.1-alpha - 2019-05-?? Explain what "monotonic" actually means, and document some results that have surprised people. Fixes bug 29640; bugfix on 0.2.9.1-alpha. - - o Minor bugfixes (documentation, manpage): - Use proper formatting when providing an example on quoting options that contain whitespace. Fixes bug 29635; bugfix on 0.2.3.18-rc. - o Minor bugfixes (lib): - o Minor bugfixes (logging): - Do not log a warning for OpenSSL versions that should be compatible. Fixes bug 30190; bugfix on 0.2.4.2-alpha - - o Minor bugfixes (logging, configuration): - Warn operators when MyFamily option is set but ContactInfo is missing, as the latter should be set too. Fixes bug 25110; bugfix on 0.3.3.1-alpha. @@ -255,6 +224,10 @@ Changes in version 0.4.1.1-alpha - 2019-05-?? we only set this flag when we received a INTRODUCE2 cell in rend_service_receive_introduction(). Fixes bug 17357; bugfix on 0.4.0.2-alpha. Patch by Neel Chauhan + - Stop ignoring IPv6 link specifiers sent to v3 onion services. v3 + onion service IPv6 support is still incomplete, see 23493 for + details. Fixes bug 23588; bugfix on 0.3.2.1-alpha. Patch by + Neel Chauhan. o Minor bugfixes (onion services, performance): - If we are building circuits to onion services, in @@ -301,18 +274,8 @@ Changes in version 0.4.1.1-alpha - 2019-05-?? - Call setrlimit() to disable core dumps in test_bt_cl.c instead of using `ulimit -c` in test_bt.sh, which violates POSIX shell compatibility. Fixes bug 29061; bugfix on 0.3.5.1-alpha. - - o Minor bugfixes (testing, v3 onion services): - Fix some incorrect code in the v3 onion service unit tests. Fixes bug 29243; bugfix on 0.3.2.1-alpha. - - o Minor bugfixes (tor-resolve): - - Fix a memory leak in tor-resolve that could happen if Tor gave it - a malformed SOCKS response. (Memory leaks in tor-resolve don't - actually matter, but it's good to fix them anyway.) Fixes bug - 30151; bugfix on 0.4.0.1-alpha. - - o Minor bugfixes (unit tests): - In the "routerkeys/*" tests, check the return values of mkdir() for possible failures. Fixes bug 29939; bugfix on 0.2.7.2-alpha. Found by Coverity as CID 1444254. @@ -321,11 +284,11 @@ Changes in version 0.4.1.1-alpha - 2019-05-?? deallocation on assert failure and fixes Coverity warnings CID 1444117 and CID 1444118. Fixes bug 29823; bugfix on 0.2.9.1-alpha. - o Minor bugfixes (v3 onion services): - - Stop ignoring IPv6 link specifiers sent to v3 onion services. v3 - onion service IPv6 support is still incomplete, see 23493 for - details. Fixes bug 23588; bugfix on 0.3.2.1-alpha. Patch by - Neel Chauhan. + o Minor bugfixes (tor-resolve): + - Fix a memory leak in tor-resolve that could happen if Tor gave it + a malformed SOCKS response. (Memory leaks in tor-resolve don't + actually matter, but it's good to fix them anyway.) Fixes bug + 30151; bugfix on 0.4.0.1-alpha. o Code simplification and refactoring: - Abstract out the low-level formatting of replies on the control @@ -380,12 +343,6 @@ Changes in version 0.4.1.1-alpha - 2019-05-?? - Remove obsolete OpenSUSE initscript. Resolves issue 30076. - Remove the obsolete script at contrib/dist/tor.sh.in. Resolves issue 30075. - - o Testing: - - Check that representative subsets of values of `int` and `unsigned - int` can be represented by `void *`. Resolves issue 29537. - - o Code simplification and refactoring (circuit padding): - Avoid calling monotime_absolute_usec() in circuit padding machines that do not use token removal or circuit RTT estimation. Fixes bug 29085; bugfix on 0.4.0.1-alpha. From 21b051d1cbf5d60ccbeaf24925c4ff446d493523 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 22 May 2019 11:06:37 -0400 Subject: [PATCH 1036/2557] Edit changelog entries for clarity and conciseness --- ChangeLog | 331 +++++++++++++++++++++++++----------------------------- 1 file changed, 151 insertions(+), 180 deletions(-) diff --git a/ChangeLog b/ChangeLog index 4335a28c62..b3562bf6a4 100644 --- a/ChangeLog +++ b/ChangeLog @@ -5,19 +5,16 @@ Changes in version 0.4.1.1-alpha - 2019-05-?? certain denial-of-service attacks more difficult, and improves performance in several areas. - o Code simplification and refactoring (circuit padding): - o Major features (circuit padding): - - Onion service clients will now add padding cells to the initial - portions of their INTRODUCE and RENDEZVOUS circuits, to make those - circuits' traffic patterns look more like general purpose Exit - traffic. The overhead for this is 2 extra cells in each direction - for RENDEZVOUS circuits, and 1 extra upstream cell and 10 - downstream cells for INTRODUCE circuits. This will only be enabled - if the circuit's middle node supports this feature, too. (Clients - may specify fixed middle nodes with the MiddleNodes torrc - directive, and may force-disable this feature with the - CircuitPadding torrc directive). Closes ticket 28634. + - Onion service clients now add padding cells at the start of their + INTRODUCE and RENDEZVOUS circuits, to make those circuits' traffic + look more like general purpose Exit traffic. The overhead for this + is 2 extra cells in each direction for RENDEZVOUS circuits, and 1 + extra upstream cell and 10 downstream cells for INTRODUCE + circuits. This feature is only enabled when also supported by the + circuit's middle node. (Clients may specify fixed middle nodes + with the MiddleNodes option, and may force-disable this feature + with the CircuitPadding torrc.) Closes ticket 28634. o Major features (code organization): - Tor now includes a generic publish-subscribe message-passing @@ -29,20 +26,22 @@ Changes in version 0.4.1.1-alpha - 2019-05-?? o Major features (controller protocol): - Controller commands are now parsed using a generalized parsing subsystem. Previously, each controller command was responsible for - parsing its own input. Closes ticket 30091. + parsing its own input, which led to strange inconsistencies. + Closes ticket 30091. o Major features (flow control): - - Implement authenticated SENDMEs detailed in proposal 289. A SENDME - cell now includes the digest of the last cell received so once the - end point receives the SENDME, it can confirm the other side's - knowledge of the previous cells that were sent. This behavior is - controlled by two new consensus parameters, see proposal for more - details. Fixes ticket 26288. + - Implement authenticated SENDMEs as detailed in proposal 289. A + SENDME cell now includes the digest of the traffic that it + acknowledges, so that once end point receives the SENDME, it can + confirm the other side's knowledge of the previous cells that were + sent, and prevent certain types of denial-of-service attacks. This + behavior is controlled by two new consensus parameters: see the + proposal for more details. Fixes ticket 26288. o Major features (performance): - - Update our node selection algorithm to exclude nodes in linear - time. Previously, the algorithm was quadratic, which could slow - down heavily used onion services. Closes ticket 30307. + - Our node selection algorithm now excludes nodes in linear time. + Previously, the algorithm was quadratic, which could slow down + heavily used onion services. Closes ticket 30307. o Major features (performance, RNG): - Tor now constructs a fast secure pseudorandom number generator for @@ -51,64 +50,64 @@ Changes in version 0.4.1.1-alpha - 2019-05-?? libottery and the (newer) OpenBSD arc4random() code. It outperforms OpenSSL 1.1.1a's CSPRNG by roughly a factor of 100 for small outputs. Although we believe it to be cryptographically - strong, we are only using it when necessary for reasonable - performance. Implements tickets 29023 and 29536. + strong, we are only using it when necessary for performance. + Implements tickets 29023 and 29536. o Minor features (circuit padding): - - We now use a fast RNG when scheduling circuit padding. Part of + - We now use a fast PRNG when scheduling circuit padding. Part of ticket 28636. - Allow the padding machine designer to pick the edges of their histogram instead of trying to compute them automatically using an exponential formula. Resolves some undefined behavior in the case of small histograms and allows greater flexibility on machine design. Closes ticket 29298; bugfix on 0.4.0.1-alpha. - - Provide the ability for circuit padding machines to hold a circuit - open until they are done padding it. Closes ticket 28780. + - Allow circuit padding machines to hold a circuit open until they + are done padding it. Closes ticket 28780. o Minor features (compile-time modules): - - Add a --list-modules command to print a list of which compile-time - modules are enabled. Closes ticket 30452. + - Add a "--list-modules" command to print a list of which compile- + time modules are enabled. Closes ticket 30452. o Minor features (continuous integration): - Remove sudo configuration lines from .travis.yml as they are no longer needed with current Travis build environment. Resolves issue 30213. + - In Travis, show stem's tor log after failure. Closes ticket 30234. o Minor features (controller): - - Add onion service version 3 support to HSFETCH. Previously, only - version 2 onion services were supported. Closes ticket 25417. - Patch by Neel Chauhan + - Add onion service version 3 support to the HSFETCH command. + Previously, only version 2 onion services were supported. Closes + ticket 25417. Patch by Neel Chauhan o Minor features (debugging): - Introduce tor_assertf() and tor_assertf_nonfatal() to enable logging of additional information during assert failure. Now we - can use format strings to include pieces of information that are - relevant for trouble shooting. Resolves ticket 29662. + can use format strings to include information for trouble + shooting. Resolves ticket 29662. o Minor features (defense in depth): - - In smartlist_remove_keeporder(), set any pointers that become - unused to NULL, in case a bug causes them to be used later. Closes - ticket 30176. Patch from Tobias Stoeckmann. - - Tor now uses a fast cryptographically strong PRNG even for - decisions that we do not believe are security-sensitive. - Previously, for performance reasons, we had used a trivially - predictable linear congruential generator algorithm for certain - load-balancing and statistical sampling decisions. Now we use our - fast RNG in those cases. Closes ticket 29542. + - In smartlist_remove_keeporder(), set unused pointers to NULL, in + case a bug causes them to be used later. Closes ticket 30176. + Patch from Tobias Stoeckmann. + - Tor now uses a cryptographically strong PRNG even for decisions + that we do not believe are security-sensitive. Previously, for + performance reasons, we had used a trivially predictable linear + congruential generator algorithm for certain load-balancing and + statistical sampling decisions. Now we use our fast RNG in those + cases. Closes ticket 29542. o Minor features (developer tools): - - Tor's test scripts now check for files and functions that seem too - long and complicated. Existing overlong functions and files are - accepted for now, but should eventually be refactored. Closes - ticket 29221. - - Add to scripts/maint/ helper maintainer scripts used for git - maintenance. Closes ticket 29391. - - Call practracker from pre-push and pre-commit git hooks to let a - developer know if they made any code style violations in their - last commit. This should help preventing code style violations - appearing upstream. Closes ticket 30051. + - Tor's "practracker" test script nows check for files and functions + that seem too long and complicated. Existing overlong functions + and files are accepted for now, but should eventually be + refactored. Closes ticket 29221. + - Add some scripts used for git maintenance to scripts/git. Closes + ticket 29391. + - Call practracker from pre-push and pre-commit git hooks to let + developers know if they made any code style violations. Closes + ticket 30051. - Add a script to check that each header has a well-formed and - unique guard marco. Closes ticket 29756. + unique guard macro. Closes ticket 29756. o Minor features (geoip): - Update geoip and geoip6 to the May 13 2019 Maxmind GeoLite2 @@ -124,8 +123,8 @@ Changes in version 0.4.1.1-alpha - 2019-05-?? addressesd. Implements 26992. o Minor features (modularity): - - The --disable-module-dirauth compile-time option now disables even - more dirauth-only code. Closes ticket 30345. + - The "--disable-module-dirauth" compile-time option now disables + even more dirauth-only code. Closes ticket 30345. o Minor features (performance): - Use OpenSSL's implementations of SHA3 when available (in OpenSSL @@ -133,10 +132,10 @@ Changes in version 0.4.1.1-alpha - 2019-05-?? Closes ticket 28837. o Minor features (testing): - - Tor's unit test code now contains a standard set of functions to - replace the PRNG with a deterministic or reproducible version for - testing. Previously, various tests implemented this in various - ways. Implements ticket 29732. + - Tor's unit test code now contains helper functions to replace the + PRNG with a deterministic or reproducible version for testing. + Previously, various tests did implemented this in various ways. + Implements ticket 29732. - We now have a script, cov-test-determinism.sh, to identify places where our unit test coverage has become nondeterministic. Closes ticket 29436. @@ -144,10 +143,11 @@ Changes in version 0.4.1.1-alpha - 2019-05-?? int` can be represented by `void *`. Resolves issue 29537. o Minor bugfixes (bridge authority): - - We set bridges as running when we dump the bridge status to a - file. Previously, we set bridges as running in a GETINFO - controller, but these shouldn't modify vital data structures. - Fixes bug 24490; bugfix on 0.2.0.13-alpha. Patch by Neel Chauhan + - Bridge authorities now set bridges as running or non-running when + about to dump their status to a file. Previously, they set bridges + as running in response to a GETINFO command, but those shouldn't + modify data structures. Fixes bug 24490; bugfix on 0.2.0.13-alpha. + Patch by Neel Chauhan o Minor bugfixes (channel padding statistics): - Channel padding write totals and padding-enabled totals are now @@ -155,94 +155,93 @@ Changes in version 0.4.1.1-alpha - 2019-05-?? bugfix on 0.3.1.1-alpha o Minor bugfixes (circuit padding): - - Add a torrc option to disable circuit padding. Fixes bug 28693; - bugfix on 0.4.0.1-alpha. + - Add a "CircuitPadding" torrc option to disable circuit padding. + Fixes bug 28693; bugfix on 0.4.0.1-alpha. - Allow circuit padding machines to specify that they do not contribute much overhead, and provide consensus flags and torrc - options to force clients to only use low overhead machines. Fixes - bug 29203; bugfix on 0.4.0.1-alpha. - - Provide consensus parameter to fully disable circuit padding, to + options to force clients to only use these low overhead machines. + Fixes bug 29203; bugfix on 0.4.0.1-alpha. + - Provide a consensus parameter to fully disable circuit padding, to be used in emergency network overload situations. Fixes bug 30173; bugfix on 0.4.0.1-alpha. - - The circuit padding subsystem does not schedule padding if dormant - mode is enabled. Fixes bug 28636; bugfix on 0.4.0.1-alpha. - - Inspect circuit-level cell queue before sending padding, to avoid - sending padding while too much data is queued. Fixes bug 29204; - bugfix on 0.4.0.1-alpha. + - The circuit padding subsystem will no longer schedule padding if + dormant mode is enabled. Fixes bug 28636; bugfix on 0.4.0.1-alpha. + - Inspect a circuit-level cell queue before sending padding, to + avoid sending padding while too much data is already queued. Fixes + bug 29204; bugfix on 0.4.0.1-alpha. + - Avoid calling monotime_absolute_usec() in circuit padding machines + that do not use token removal or circuit RTT estimation. Fixes bug + 29085; bugfix on 0.4.0.1-alpha. o Minor bugfixes (compilation, unusual configurations): - - Avoid failures when building with ALL_BUGS_ARE_FAILED due to - missing declarations of abort(), and prevent other such failures - in the future. Fixes bug 30189; bugfix on 0.3.4.1-alpha. + - Avoid failures when building with the ALL_BUGS_ARE_FATAL option + due to missing declarations of abort(), and prevent other such + failures in the future. Fixes bug 30189; bugfix on 0.3.4.1-alpha. o Minor bugfixes (controller protocol): - - Teach the controller parser to correctly distinguish an object - preceded by an argument list from one without. Previously, it - couldn't distinguish an argument list from the first line of a - multiline object. Fixes bug 29984; bugfix on 0.2.3.8-alpha. + - Teach the controller parser to distinguish an object preceded by + an argument list from one without. Previously, it couldn't + distinguish an argument list from the first line of a multiline + object. Fixes bug 29984; bugfix on 0.2.3.8-alpha. o Minor bugfixes (directory authority, ipv6): - - If we are a directory authity with IPv6 and are marking relays as - running, mark ourselves as reachable on IPv6. Fixes bug 24338; + - If we are a directory authority with IPv6 and are marking relays + as running, mark ourselves as reachable on IPv6. Fixes bug 24338; bugfix on 0.4.0.2-alpha. Patch by Neel Chauhan o Minor bugfixes (documentation): - - Improve the documentation for MapAddress .exit. Fixes bug 30109; - bugfix on 0.1.0.1-rc. - - Improve the monotonic time module and function documentation. - Explain what "monotonic" actually means, and document some results + - Improve the documentation for using MapAddress with ".exit". Fixes + bug 30109; bugfix on 0.1.0.1-rc. + - Improve the monotonic time module and function documentation to + explain what "monotonic" actually means, and document some results that have surprised people. Fixes bug 29640; bugfix on 0.2.9.1-alpha. - Use proper formatting when providing an example on quoting options that contain whitespace. Fixes bug 29635; bugfix on 0.2.3.18-rc. o Minor bugfixes (logging): - - Do not log a warning for OpenSSL versions that should be - compatible. Fixes bug 30190; bugfix on 0.2.4.2-alpha - - Warn operators when MyFamily option is set but ContactInfo is + - Do not log a warning when running with an OpenSSL version that + that should be compatible with the one we were built with. + Previously, we would warn whenever the version was different. + Fixes bug 30190; bugfix on 0.2.4.2-alpha + - Warn operators when the MyFamily option is set but ContactInfo is missing, as the latter should be set too. Fixes bug 25110; bugfix on 0.3.3.1-alpha. o Minor bugfixes (memory leak): - - Avoid a minor memory leak that could occur on relays when creating - a keys directory failed. Fixes bug 30148; bugfix on 0.3.3.1-alpha. + - Avoid a minor memory leak that could occur on relays when failing + to create a "keys" directory. Fixes bug 30148; bugfix + on 0.3.3.1-alpha. o Minor bugfixes (onion services): - Avoid a GCC 9.1.1 warning (and possible crash depending on libc implemenation) when failing to load an onion service client authorization file. Fixes bug 30475; bugfix on 0.3.5.1-alpha. - - If we are launching repeated HSFETCH queries and are rate-limited, - we introduce a new controller response QUERY_RATE_LIMITED instead - of QUERY_NO_HSDIR, while keeping the latter for when onion service - directories are missing a descriptor. Previously, we returned - QUERY_NO_HSDIR for both cases. Fixes bug 28269; bugfix on - 0.3.1.1-alpha. Patch by Neel Chauhan - - If we are relaunching a circuit to a rendevous service in - rend_service_relaunch_rendezvous() and - hs_service_requires_uptime_circ() is true, the - CIRCLAUNCH_NEED_UPTIME flag is added to the circuit. Previously, - we only set this flag when we received a INTRODUCE2 cell in - rend_service_receive_introduction(). Fixes bug 17357; bugfix on - 0.4.0.2-alpha. Patch by Neel Chauhan - - Stop ignoring IPv6 link specifiers sent to v3 onion services. v3 - onion service IPv6 support is still incomplete, see 23493 for - details. Fixes bug 23588; bugfix on 0.3.2.1-alpha. Patch by - Neel Chauhan. + - When refusing to launch a controller's HSFETCH request because of + rate-limiting, respond to the controller with a new response, + "QUERY_RATE_LIMITED". Previously, we would log QUERY_NO_HSDIR for + this case. Fixes bug 28269; bugfix on 0.3.1.1-alpha. Patch by + Neel Chauhan + - When relaunching a circuit to a rendevous service, mark the + circuit as needing high-uptime routers as appropriate. Fixes bug + 17357; bugfix on 0.4.0.2-alpha. Patch by Neel Chauhan + - Stop ignoring IPv6 link specifiers sent to v3 onion services. + (IPv6 support for v3 onion services is still incomplete: see + ticket 23493 for details.) Fixes bug 23588; bugfix on + 0.3.2.1-alpha. Patch by Neel Chauhan. o Minor bugfixes (onion services, performance): - - If we are building circuits to onion services, in - circuit_is_acceptable() we only call tor_addr_parse() in places - where we use the returned family and address values from this - function. Previously, we called tor_addr_parse() in - circuit_is_acceptable() even if it wasn't used. This change will - improve performance when building circuits. Fixes bug 22210; - bugfix on 0.2.8.12. Patch by Neel Chauhan + - When building circuits to onion services, call tor_addr_parse() + less often. Previously, we called tor_addr_parse() in + circuit_is_acceptable() even if its output it wasn't used. This + change should improve performance when building circuits. Fixes + bug 22210; bugfix on 0.2.8.12. Patch by Neel Chauhan o Minor bugfixes (performance): - - When checking a node for bridge status, use a fast check to make + - When checking whether a node is a bridge, use a fast check to make sure that its identity is set. Previously, we used a constant-time - check, which is not necessary when verifying a BUG() condition that - causes a stack trace. Fixes bug 30308; bugfix on 0.3.5.1-alpha. + check, which is not necessary in this case. Fixes bug 30308; + bugfix on 0.3.5.1-alpha. o Minor bugfixes (pluggable transports): - Tor now sets TOR_PT_EXIT_ON_STDIN_CLOSE=1 for client transports as @@ -271,18 +270,18 @@ Changes in version 0.4.1.1-alpha - 2019-05-?? 29018; bugfix on 0.2.4.1-alpha. o Minor bugfixes (testing): - - Call setrlimit() to disable core dumps in test_bt_cl.c instead of - using `ulimit -c` in test_bt.sh, which violates POSIX shell + - Call setrlimit() to disable core dumps in test_bt_cl.c. Previously + we used `ulimit -c` in test_bt.sh, which violates POSIX shell compatibility. Fixes bug 29061; bugfix on 0.3.5.1-alpha. - Fix some incorrect code in the v3 onion service unit tests. Fixes bug 29243; bugfix on 0.3.2.1-alpha. - In the "routerkeys/*" tests, check the return values of mkdir() for possible failures. Fixes bug 29939; bugfix on 0.2.7.2-alpha. Found by Coverity as CID 1444254. - - Split test_utils_general() to several smaller test functions in - test_utils_general(). This makes it easier to perform resource - deallocation on assert failure and fixes Coverity warnings CID - 1444117 and CID 1444118. Fixes bug 29823; bugfix on 0.2.9.1-alpha. + - Split test_utils_general() into several smaller test functions. + This makes it easier to perform resource deallocation on assert + failure, and fixes Coverity warnings CID 1444117 and CID 1444118. + Fixes bug 29823; bugfix on 0.2.9.1-alpha. o Minor bugfixes (tor-resolve): - Fix a memory leak in tor-resolve that could happen if Tor gave it @@ -306,8 +305,7 @@ Changes in version 0.4.1.1-alpha - 2019-05-?? - Refactor and encapsulate parts of the codebase that manipulate crypt_path_t objects. Resolves issue 30236. - Refactor several places in our code that coverity incorrectly - believed that we might have memory leaks, so that we can analyze - our software more easily. Closes ticket 30147. + believed might have memory leaks. Closes ticket 30147. - Remove redundant return values in crypto_format, and the associated return value checks elsewhere in the code. Make the implementations in crypto_format consistent, and remove redundant @@ -319,72 +317,45 @@ Changes in version 0.4.1.1-alpha - 2019-05-?? bugfix on 0.3.2.1-alpha. - Simplify v3 onion service link specifier handling code. Fixes bug 23576; bugfix on 0.3.2.1-alpha. - - Split crypto_digest.c into three parts: 1) general code that does - not depend on either NSS or OpenSSL (stays in crypto_digest.c); 2) - code that depends on NSS API (moved to crypto_digest_nss.c); 3) - code that depends on OpenSSL API (moved to - crypto_digest_openssl.c). Resolves ticket 29108. - - Split up the control.c file into several submodules, in - preparation for distributing its current responsibilities - throughout the codebase. Closes ticket 29894. - - Start move responsibility for knowing about periodic events to the - appropriate subsystems, so that the mainloop doesn't need to know - all the periodic events in the rest of the codebase. Implements - tickets 30293 and 30294. + - Split crypto_digest.c into NSS code, OpenSSL code, and shared + code. Resolves ticket 29108. + - Split control.c into several submodules, in preparation for + distributing its current responsibilities throughout the codebase. + Closes ticket 29894. + - Start to move responsibility for knowing about periodic events to + the appropriate subsystems, so that the mainloop doesn't need to + know all the periodic events in the rest of the codebase. + Implements tickets 30293 and 30294. o Documentation: - Document how to find git commits and tags for bug fixes in - CodingStandards.md. And update some changes file documentation. - Closes ticket 30261. + CodingStandards.md. Update some changes file documentation. Closes + ticket 30261. o Removed features: - - Remove linux-tor-prio.sh script from contrib/operator-tools + - Remove the linux-tor-prio.sh script from contrib/operator-tools directory. Resolves issue 29434. - - Remove obsolete OpenSUSE initscript. Resolves issue 30076. + - Remove the obsolete OpenSUSE initscript. Resolves issue 30076. - Remove the obsolete script at contrib/dist/tor.sh.in. Resolves issue 30075. - - Avoid calling monotime_absolute_usec() in circuit padding machines - that do not use token removal or circuit RTT estimation. Fixes bug - 29085; bugfix on 0.4.0.1-alpha. o Code simplification and refactoring (shell scripts): - - Cleanup autogen.sh to silence shellcheck warnings. Closes - ticket 26069. - - Cleanup test_keygen.sh to silence all shellcheck warnings. Closes - ticket 29062. - - Cleanup test_switch_id.sh to silence shellcheck warnings. Closes - ticket 29065. - - Fix issues shellcheck found in test_rebind.sh. Resolves - issue 29063. - - Fix shellcheck warning SC2006 in src/test/fuzz/minimize.sh. - Resolves issue 30079. - - Fix shellcheck warning in test_rust.sh. Fixes issue 29064. - - Fix shellcheck warning in torify script. Resolves issue 29070. - - Fix shellcheck warnings in asciidoc-helper.sh. Resolves - issue 29926. - - Fix shellcheck warnings in fuzz_multi.sh. Resolves issue 30077. - - Fix shellcheck warnings in fuzz_static_testcases.sh. Resolves - ticket 29059. - - Fix shellcheck warnings in nagios-check-tor-authority-cert script. - Resolves issue 29071. - - Fix shellcheck warnings in src/test/fuzz/fixup_filenames.sh. - Resolves issue 30078. - - Fix shellcheck warnings in test-network.sh. Resolves issue 29060. - - Fix shellcheck warnings in test_key_expiration.sh. Resolves - issue 30002. - - Fix shellcheck warnings in zero_length_keys.sh. Resolves - issue 29068. - - Fix test_workqueue_*.sh scripts to silence shellcheck SC2086 - warnings. Fixes issue 29067. + - Clean up many of our shell scripts to fix shellcheck warnings. + These include autogen.sh (ticket 26069), test_keygen.sh (ticket + 29062), test_switch_id.sh (ticket 29065), test_rebind.sh (ticket + 29063), src/test/fuzz/minimize.sh (ticket 30079), test_rust.sh + (ticket 29064), torify (ticket 29070), asciidoc-helper.sh (29926), + fuzz_multi.sh (30077), fuzz_static_testcases.sh (ticket 29059), + nagios-check-tor-authority-cert (ticket 29071), + src/test/fuzz/fixup_filenames.sh (ticket 30078), test-network.sh + (ticket 29060), test_key_expiration.sh (ticket 30002), + zero_length_keys.sh (ticket 29068), and test_workqueue_*.sh + (ticket 29067). o Testing (chutney): - In "make test-network-all", test IPv6-only v3 single onion - services, using the chutney network single-onion-v23-ipv6-md. This - test will not pass until 23588 has been merged. Closes - ticket 27251. - - o Testing (continuous integration): - - In Travis, show stem's tor log after failure. Closes ticket 30234. + services, using the chutney network single-onion-v23-ipv6-md. + Closes ticket 27251. Changes in version 0.4.0.5 - 2019-05-02 From 59b9eecc19877f38b2c9d8b4f7964c6e9875f4c0 Mon Sep 17 00:00:00 2001 From: David Goulet Date: Tue, 7 May 2019 09:16:39 -0400 Subject: [PATCH 1037/2557] sendme: Record cell digest on both client and exit It turns out that only the exit side is validating the authenticated SENDME v1 logic and never the client side. Which means that if a client ever uploaded data towards an exit, the authenticated SENDME logic wouldn't apply. For this to work, we have to record the cell digest client side as well which introduced a new function that supports both type of edges. This also removes a test that is not valid anymore which was that we didn't allow cell recording on an origin circuit (client). Part of #30428 Signed-off-by: David Goulet --- scripts/maint/practracker/exceptions.txt | 2 +- src/core/or/relay.c | 2 +- src/core/or/sendme.c | 49 +++++++++++++++++------- src/core/or/sendme.h | 3 +- src/test/test_sendme.c | 28 ++++---------- 5 files changed, 46 insertions(+), 38 deletions(-) diff --git a/scripts/maint/practracker/exceptions.txt b/scripts/maint/practracker/exceptions.txt index cba3ddc1b3..a83bb62bb7 100644 --- a/scripts/maint/practracker/exceptions.txt +++ b/scripts/maint/practracker/exceptions.txt @@ -122,7 +122,7 @@ problem function-size /src/core/or/policies.c:policy_summarize() 107 problem function-size /src/core/or/protover.c:protover_all_supported() 117 problem file-size /src/core/or/relay.c 3173 problem function-size /src/core/or/relay.c:circuit_receive_relay_cell() 123 -problem function-size /src/core/or/relay.c:relay_send_command_from_edge_() 112 +problem function-size /src/core/or/relay.c:relay_send_command_from_edge_() 116 problem function-size /src/core/or/relay.c:connection_ap_process_end_not_open() 194 problem function-size /src/core/or/relay.c:connection_edge_process_relay_cell_not_open() 139 problem function-size /src/core/or/relay.c:connection_edge_process_relay_cell() 520 diff --git a/src/core/or/relay.c b/src/core/or/relay.c index 8b3a1be18e..91236e5fec 100644 --- a/src/core/or/relay.c +++ b/src/core/or/relay.c @@ -701,7 +701,7 @@ relay_send_command_from_edge_,(streamid_t stream_id, circuit_t *circ, * we need to. This call needs to be after the circuit_package_relay_cell() * because the cell digest is set within that function. */ if (relay_command == RELAY_COMMAND_DATA) { - sendme_record_cell_digest(circ); + sendme_record_cell_digest_on_circ(circ, cpath_layer); } return 0; diff --git a/src/core/or/sendme.c b/src/core/or/sendme.c index e7c65d99e2..586d4d0ae0 100644 --- a/src/core/or/sendme.c +++ b/src/core/or/sendme.c @@ -287,6 +287,21 @@ send_circuit_level_sendme(circuit_t *circ, crypt_path_t *layer_hint, return 0; } +/* Record the cell digest only if the next cell is expected to be a SENDME. */ +static void +record_cell_digest_on_circ(circuit_t *circ, const uint8_t *sendme_digest) +{ + tor_assert(circ); + tor_assert(sendme_digest); + + /* Add the digest to the last seen list in the circuit. */ + if (circ->sendme_last_digests == NULL) { + circ->sendme_last_digests = smartlist_new(); + } + smartlist_add(circ->sendme_last_digests, + tor_memdup(sendme_digest, DIGEST_LEN)); +} + /* * Public API */ @@ -571,31 +586,37 @@ sendme_note_stream_data_packaged(edge_connection_t *conn) return --conn->package_window; } -/* Note the cell digest in the circuit sendme last digests FIFO if applicable. - * It is safe to pass a circuit that isn't meant to track those digests. */ +/* Record the cell digest into the circuit sendme digest list depending on + * which edge we are. The digest is recorded only if we expect the next cell + * that we will receive is a SENDME so we can match the digest. */ void -sendme_record_cell_digest(circuit_t *circ) +sendme_record_cell_digest_on_circ(circuit_t *circ, crypt_path_t *cpath) { - const uint8_t *digest; + int package_window; + uint8_t *sendme_digest; tor_assert(circ); - /* We only keep the cell digest if we are the Exit on that circuit and if - * this cell is the last one before the client should send a SENDME. */ - if (CIRCUIT_IS_ORIGIN(circ)) { - return; + package_window = circ->package_window; + if (cpath) { + package_window = cpath->package_window; } + /* Is this the last cell before a SENDME? The idea is that if the * package_window reaches a multiple of the increment, after this cell, we * should expect a SENDME. */ - if (!sendme_circuit_cell_is_next(circ->package_window)) { + if (!sendme_circuit_cell_is_next(package_window)) { return; } - /* Add the digest to the last seen list in the circuit. */ - digest = relay_crypto_get_sendme_digest(&TO_OR_CIRCUIT(circ)->crypto); - if (circ->sendme_last_digests == NULL) { - circ->sendme_last_digests = smartlist_new(); + /* Getting the digest is expensive so we only do it once we are certain to + * record it on the circuit. */ + if (cpath) { + sendme_digest = cpath_get_sendme_digest(cpath); + } else { + sendme_digest = + relay_crypto_get_sendme_digest(&TO_OR_CIRCUIT(circ)->crypto); } - smartlist_add(circ->sendme_last_digests, tor_memdup(digest, DIGEST_LEN)); + + record_cell_digest_on_circ(circ, sendme_digest); } diff --git a/src/core/or/sendme.h b/src/core/or/sendme.h index ac18bbdd31..2fb73f76d9 100644 --- a/src/core/or/sendme.h +++ b/src/core/or/sendme.h @@ -35,8 +35,9 @@ int sendme_note_circuit_data_packaged(circuit_t *circ, int sendme_note_stream_data_packaged(edge_connection_t *conn); /* Track cell digest. */ -void sendme_record_cell_digest(circuit_t *circ); void sendme_circuit_record_outbound_cell(or_circuit_t *or_circ); +/* Record cell digest on circuit. */ +void sendme_record_cell_digest_on_circ(circuit_t *circ, crypt_path_t *cpath); /* Circuit level information. */ bool sendme_circuit_cell_is_next(int window); diff --git a/src/test/test_sendme.c b/src/test/test_sendme.c index d40fbaf862..463a0ec086 100644 --- a/src/test/test_sendme.c +++ b/src/test/test_sendme.c @@ -46,26 +46,12 @@ static void test_v1_record_digest(void *arg) { or_circuit_t *or_circ = NULL; - origin_circuit_t *orig_circ = NULL; circuit_t *circ = NULL; (void) arg; - /* Create our dummy circuits. */ - orig_circ = origin_circuit_new(); - tt_assert(orig_circ); + /* Create our dummy circuit. */ or_circ = or_circuit_new(1, NULL); - - /* Start by pointing to the origin circuit. */ - circ = TO_CIRCUIT(orig_circ); - circ->purpose = CIRCUIT_PURPOSE_S_REND_JOINED; - - /* We should never note SENDME digest on origin circuit. */ - sendme_record_cell_digest(circ); - tt_assert(!circ->sendme_last_digests); - /* We do not need the origin circuit for now. */ - orig_circ = NULL; - circuit_free_(circ); /* Points it to the OR circuit now. */ circ = TO_CIRCUIT(or_circ); @@ -73,23 +59,23 @@ test_v1_record_digest(void *arg) * in order to catched the CIRCWINDOW_INCREMENT-nth cell. Try something that * shouldn't be noted. */ circ->package_window = CIRCWINDOW_INCREMENT; - sendme_record_cell_digest(circ); + sendme_record_cell_digest_on_circ(circ, NULL); tt_assert(!circ->sendme_last_digests); /* This should work now. Package window at CIRCWINDOW_INCREMENT + 1. */ circ->package_window++; - sendme_record_cell_digest(circ); + sendme_record_cell_digest_on_circ(circ, NULL); tt_assert(circ->sendme_last_digests); tt_int_op(smartlist_len(circ->sendme_last_digests), OP_EQ, 1); /* Next cell in the package window shouldn't do anything. */ circ->package_window++; - sendme_record_cell_digest(circ); + sendme_record_cell_digest_on_circ(circ, NULL); tt_int_op(smartlist_len(circ->sendme_last_digests), OP_EQ, 1); /* The next CIRCWINDOW_INCREMENT should add one more digest. */ circ->package_window = (CIRCWINDOW_INCREMENT * 2) + 1; - sendme_record_cell_digest(circ); + sendme_record_cell_digest_on_circ(circ, NULL); tt_int_op(smartlist_len(circ->sendme_last_digests), OP_EQ, 2); done: @@ -188,7 +174,7 @@ test_v1_build_cell(void *arg) /* Note the wrong digest in the circuit, cell should fail validation. */ circ->package_window = CIRCWINDOW_INCREMENT + 1; - sendme_record_cell_digest(circ); + sendme_record_cell_digest_on_circ(circ, NULL); tt_int_op(smartlist_len(circ->sendme_last_digests), OP_EQ, 1); setup_full_capture_of_logs(LOG_INFO); tt_int_op(sendme_is_valid(circ, payload, sizeof(payload)), OP_EQ, false); @@ -200,7 +186,7 @@ test_v1_build_cell(void *arg) /* Record the cell digest into the circuit, cell should validate. */ memcpy(or_circ->crypto.sendme_digest, digest, sizeof(digest)); circ->package_window = CIRCWINDOW_INCREMENT + 1; - sendme_record_cell_digest(circ); + sendme_record_cell_digest_on_circ(circ, NULL); tt_int_op(smartlist_len(circ->sendme_last_digests), OP_EQ, 1); tt_int_op(sendme_is_valid(circ, payload, sizeof(payload)), OP_EQ, true); /* After a validation, the last digests is always popped out. */ From 69e0d5bfc7d52f223d686bcd87f629f01b03561a Mon Sep 17 00:00:00 2001 From: David Goulet Date: Tue, 7 May 2019 09:19:41 -0400 Subject: [PATCH 1038/2557] sendme: Validate v1 SENDMEs on both client and exit side The validation of the SENDME cell is now done as the very first thing when receiving it for both client and exit. On failure to validate, the circuit is closed as detailed in the specification. Part of #30428 Signed-off-by: David Goulet --- src/core/or/sendme.c | 17 ++++++++--------- src/test/test_relaycell.c | 2 +- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/src/core/or/sendme.c b/src/core/or/sendme.c index 586d4d0ae0..baa57f4f25 100644 --- a/src/core/or/sendme.c +++ b/src/core/or/sendme.c @@ -204,10 +204,10 @@ sendme_is_valid(const circuit_t *circ, const uint8_t *cell_payload, /* Valid cell. */ sendme_cell_free(cell); - return 1; + return true; invalid: sendme_cell_free(cell); - return 0; + return false; } /* Build and encode a version 1 SENDME cell into payload, which must be at @@ -424,6 +424,12 @@ sendme_process_circuit_level(crypt_path_t *layer_hint, tor_assert(circ); tor_assert(cell_payload); + /* Validate the SENDME cell. Depending on the version, different validation + * can be done. An invalid SENDME requires us to close the circuit. */ + if (!sendme_is_valid(circ, cell_payload, cell_payload_len)) { + return -END_CIRC_REASON_TORPROTOCOL; + } + /* If we are the origin of the circuit, we are the Client so we use the * layer hint (the Exit hop) for the package window tracking. */ if (CIRCUIT_IS_ORIGIN(circ)) { @@ -448,13 +454,6 @@ sendme_process_circuit_level(crypt_path_t *layer_hint, * are rate limited. */ circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), cell_payload_len); } else { - /* Validate the SENDME cell. Depending on the version, different - * validation can be done. An invalid SENDME requires us to close the - * circuit. It is only done if we are the Exit of the circuit. */ - if (!sendme_is_valid(circ, cell_payload, cell_payload_len)) { - return -END_CIRC_REASON_TORPROTOCOL; - } - /* We aren't the origin of this circuit so we are the Exit and thus we * track the package window with the circuit object. */ if ((circ->package_window + CIRCWINDOW_INCREMENT) > diff --git a/src/test/test_relaycell.c b/src/test/test_relaycell.c index 0623583511..d6372d3956 100644 --- a/src/test/test_relaycell.c +++ b/src/test/test_relaycell.c @@ -812,7 +812,7 @@ test_circbw_relay(void *arg) ASSERT_UNCOUNTED_BW(); /* Sendme on circuit with non-full window: counted */ - PACK_CELL(0, RELAY_COMMAND_SENDME, "Data1234"); + PACK_CELL(0, RELAY_COMMAND_SENDME, ""); circ->cpath->package_window = 900; connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), edgeconn, circ->cpath); From 44265dd6716887b997bb03d2db1641efd7ae9c19 Mon Sep 17 00:00:00 2001 From: David Goulet Date: Tue, 7 May 2019 09:44:10 -0400 Subject: [PATCH 1039/2557] sendme: Never fallback to v0 if unknown version There was a missing cell version check against our max supported version. In other words, we do not fallback to v0 anymore in case we do know the SENDME version. We can either handle it or not, never fallback to the unauthenticated version in order to avoid gaming the authenticated logic. Add a unit tests making sure we properly test that and also test that we can always handle the default emit and accepted versions. Fixes #30428 Signed-off-by: David Goulet --- src/core/or/sendme.c | 52 +++++++++++++++++++++--------------------- src/core/or/sendme.h | 16 ++++++++++++- src/test/test_sendme.c | 33 +++++++++++++++++++++++++-- 3 files changed, 72 insertions(+), 29 deletions(-) diff --git a/src/core/or/sendme.c b/src/core/or/sendme.c index baa57f4f25..80d6b7d165 100644 --- a/src/core/or/sendme.c +++ b/src/core/or/sendme.c @@ -25,20 +25,6 @@ #include "lib/ctime/di_ops.h" #include "trunnel/sendme.h" -/* The maximum supported version. Above that value, the cell can't be - * recognized as a valid SENDME. */ -#define SENDME_MAX_SUPPORTED_VERSION 1 - -/* The cell version constants for when emitting a cell. */ -#define SENDME_EMIT_MIN_VERSION_DEFAULT 0 -#define SENDME_EMIT_MIN_VERSION_MIN 0 -#define SENDME_EMIT_MIN_VERSION_MAX UINT8_MAX - -/* The cell version constants for when accepting a cell. */ -#define SENDME_ACCEPT_MIN_VERSION_DEFAULT 0 -#define SENDME_ACCEPT_MIN_VERSION_MIN 0 -#define SENDME_ACCEPT_MIN_VERSION_MAX UINT8_MAX - /* Return the minimum version given by the consensus (if any) that should be * used when emitting a SENDME cell. */ STATIC int @@ -123,30 +109,41 @@ cell_v1_is_valid(const sendme_cell_t *cell, const circuit_t *circ) /* Return true iff the given cell version can be handled or if the minimum * accepted version from the consensus is known to us. */ STATIC bool -cell_version_is_valid(uint8_t cell_version) +cell_version_can_be_handled(uint8_t cell_version) { int accept_version = get_accept_min_version(); - /* Can we handle this version? */ + /* We will first check if the consensus minimum accepted version can be + * handled by us and if not, regardless of the cell version we got, we can't + * continue. */ if (accept_version > SENDME_MAX_SUPPORTED_VERSION) { log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, "Unable to accept SENDME version %u (from consensus). " - "We only support <= %d. Probably your tor is too old?", - accept_version, cell_version); + "We only support <= %u. Probably your tor is too old?", + accept_version, SENDME_MAX_SUPPORTED_VERSION); goto invalid; } - /* We only accept a SENDME cell from what the consensus tells us. */ + /* Then, is this version below the accepted version from the consensus? If + * yes, we must not handle it. */ if (cell_version < accept_version) { - log_info(LD_PROTOCOL, "Unacceptable SENDME version %d. Only " + log_info(LD_PROTOCOL, "Unacceptable SENDME version %u. Only " "accepting %u (from consensus). Closing circuit.", cell_version, accept_version); goto invalid; } - return 1; + /* Is this cell version supported by us? */ + if (cell_version > SENDME_MAX_SUPPORTED_VERSION) { + log_info(LD_PROTOCOL, "SENDME cell version %u is not supported by us. " + "We only support <= %u", + cell_version, SENDME_MAX_SUPPORTED_VERSION); + goto invalid; + } + + return true; invalid: - return 0; + return false; } /* Return true iff the encoded SENDME cell in cell_payload of length @@ -183,7 +180,7 @@ sendme_is_valid(const circuit_t *circ, const uint8_t *cell_payload, } /* Validate that we can handle this cell version. */ - if (!cell_version_is_valid(cell_version)) { + if (!cell_version_can_be_handled(cell_version)) { goto invalid; } @@ -195,10 +192,13 @@ sendme_is_valid(const circuit_t *circ, const uint8_t *cell_payload, } break; case 0x00: - /* Fallthrough. Version 0, there is no work to be done on the payload so - * it is necessarily valid if we pass the version validation. */ + /* Version 0, there is no work to be done on the payload so it is + * necessarily valid if we pass the version validation. */ + break; default: - /* Unknown version means we can't handle it so fallback to version 0. */ + log_warn(LD_PROTOCOL, "Unknown SENDME cell version %d received.", + cell_version); + tor_assert_nonfatal_unreached(); break; } diff --git a/src/core/or/sendme.h b/src/core/or/sendme.h index 2fb73f76d9..a71891996b 100644 --- a/src/core/or/sendme.h +++ b/src/core/or/sendme.h @@ -45,6 +45,20 @@ bool sendme_circuit_cell_is_next(int window); /* Private section starts. */ #ifdef SENDME_PRIVATE +/* The maximum supported version. Above that value, the cell can't be + * recognized as a valid SENDME. */ +#define SENDME_MAX_SUPPORTED_VERSION 1 + +/* The cell version constants for when emitting a cell. */ +#define SENDME_EMIT_MIN_VERSION_DEFAULT 0 +#define SENDME_EMIT_MIN_VERSION_MIN 0 +#define SENDME_EMIT_MIN_VERSION_MAX UINT8_MAX + +/* The cell version constants for when accepting a cell. */ +#define SENDME_ACCEPT_MIN_VERSION_DEFAULT 0 +#define SENDME_ACCEPT_MIN_VERSION_MIN 0 +#define SENDME_ACCEPT_MIN_VERSION_MAX UINT8_MAX + /* * Unit tests declaractions. */ @@ -53,7 +67,7 @@ bool sendme_circuit_cell_is_next(int window); STATIC int get_emit_min_version(void); STATIC int get_accept_min_version(void); -STATIC bool cell_version_is_valid(uint8_t cell_version); +STATIC bool cell_version_can_be_handled(uint8_t cell_version); STATIC ssize_t build_cell_payload_v1(const uint8_t *cell_digest, uint8_t *payload); diff --git a/src/test/test_sendme.c b/src/test/test_sendme.c index 463a0ec086..a36c904c28 100644 --- a/src/test/test_sendme.c +++ b/src/test/test_sendme.c @@ -122,9 +122,9 @@ test_v1_consensus_params(void *arg) smartlist_add(current_md_consensus->net_params, (void *) "sendme_accept_min_version=1"); /* Minimum acceptable value is 1. */ - tt_int_op(cell_version_is_valid(1), OP_EQ, true); + tt_int_op(cell_version_can_be_handled(1), OP_EQ, true); /* Minimum acceptable value is 1 so a cell version of 0 is refused. */ - tt_int_op(cell_version_is_valid(0), OP_EQ, false); + tt_int_op(cell_version_can_be_handled(0), OP_EQ, false); done: free_mock_consensus(); @@ -239,6 +239,33 @@ test_cell_payload_pad(void *arg) ; } +static void +test_cell_version_validation(void *arg) +{ + (void) arg; + + /* We currently only support up to SENDME_MAX_SUPPORTED_VERSION so we are + * going to test the boundaries there. */ + + tt_assert(cell_version_can_be_handled(SENDME_MAX_SUPPORTED_VERSION)); + + /* Version below our supported should pass. */ + tt_assert(cell_version_can_be_handled(SENDME_MAX_SUPPORTED_VERSION - 1)); + + /* Extra version from our supported should fail. */ + tt_assert(!cell_version_can_be_handled(SENDME_MAX_SUPPORTED_VERSION + 1)); + + /* Simple check for version 0. */ + tt_assert(cell_version_can_be_handled(0)); + + /* We MUST handle the default cell version that we emit or accept. */ + tt_assert(cell_version_can_be_handled(SENDME_EMIT_MIN_VERSION_DEFAULT)); + tt_assert(cell_version_can_be_handled(SENDME_ACCEPT_MIN_VERSION_DEFAULT)); + + done: + ; +} + struct testcase_t sendme_tests[] = { { "v1_record_digest", test_v1_record_digest, TT_FORK, NULL, NULL }, @@ -248,6 +275,8 @@ struct testcase_t sendme_tests[] = { NULL, NULL }, { "cell_payload_pad", test_cell_payload_pad, TT_FORK, NULL, NULL }, + { "cell_version_validation", test_cell_version_validation, TT_FORK, + NULL, NULL }, END_OF_TESTCASES }; From 3835a3acf57426f692a787e7729de929b40dc62e Mon Sep 17 00:00:00 2001 From: David Goulet Date: Wed, 15 May 2019 10:16:05 -0400 Subject: [PATCH 1040/2557] sendme: Properly record SENDMEs on both edges Turns out that we were only recording the "b_digest" but to have bidirectionnal authenticated SENDMEs, we need to use the "f_digest" in the forward cell situation. Because of the cpath refactoring, this commit plays with the crypt_path_ and relay_crypto_t API a little bit in order to respect the abstractions. Previously, we would record the cell digest as the SENDME digest in the decrypt cell function but to avoid code duplication (both directions needs to record), we now do that right after iff the cell is recognized (at the edge). It is now done in circuit_receive_relay_cell() instead. We now also record the cell digest as the SENDME digest in both relay cell encryption functions since they are split depending on the direction. relay_encrypt_cell_outbound() and relay_encrypt_cell_inbound() need to consider recording the cell digest depending on their direction (f vs b digest). Fixes #30428 Signed-off-by: David Goulet --- scripts/maint/practracker/exceptions.txt | 2 +- src/core/crypto/relay_crypto.c | 31 +++++++----- src/core/crypto/relay_crypto.h | 5 +- src/core/or/crypt_path.c | 18 +++---- src/core/or/crypt_path.h | 3 ++ src/core/or/relay.c | 4 ++ src/core/or/sendme.c | 61 ++++++++++++++++++++---- src/core/or/sendme.h | 5 +- 8 files changed, 93 insertions(+), 36 deletions(-) diff --git a/scripts/maint/practracker/exceptions.txt b/scripts/maint/practracker/exceptions.txt index a83bb62bb7..2671f723ba 100644 --- a/scripts/maint/practracker/exceptions.txt +++ b/scripts/maint/practracker/exceptions.txt @@ -121,7 +121,7 @@ problem file-size /src/core/or/policies.c 3249 problem function-size /src/core/or/policies.c:policy_summarize() 107 problem function-size /src/core/or/protover.c:protover_all_supported() 117 problem file-size /src/core/or/relay.c 3173 -problem function-size /src/core/or/relay.c:circuit_receive_relay_cell() 123 +problem function-size /src/core/or/relay.c:circuit_receive_relay_cell() 127 problem function-size /src/core/or/relay.c:relay_send_command_from_edge_() 116 problem function-size /src/core/or/relay.c:connection_ap_process_end_not_open() 194 problem function-size /src/core/or/relay.c:connection_edge_process_relay_cell_not_open() 139 diff --git a/src/core/crypto/relay_crypto.c b/src/core/crypto/relay_crypto.c index 74cccd2223..8a285131a8 100644 --- a/src/core/crypto/relay_crypto.c +++ b/src/core/crypto/relay_crypto.c @@ -100,12 +100,22 @@ relay_crypto_get_sendme_digest(relay_crypto_t *crypto) return crypto->sendme_digest; } -/** Record the b_digest from crypto and put it in the sendme_digest. */ +/** Record the cell digest, indicated by is_foward_digest or not, as the + * SENDME cell digest. */ void -relay_crypto_record_sendme_digest(relay_crypto_t *crypto) +relay_crypto_record_sendme_digest(relay_crypto_t *crypto, + bool is_foward_digest) { + struct crypto_digest_t *digest; + tor_assert(crypto); - crypto_digest_get_digest(crypto->b_digest, (char *) crypto->sendme_digest, + + digest = crypto->b_digest; + if (is_foward_digest) { + digest = crypto->f_digest; + } + + crypto_digest_get_digest(digest, (char *) crypto->sendme_digest, sizeof(crypto->sendme_digest)); } @@ -161,11 +171,6 @@ relay_decrypt_cell(circuit_t *circ, cell_t *cell, if (relay_digest_matches(cpath_get_incoming_digest(thishop), cell)) { *recognized = 1; *layer_hint = thishop; - /* This cell is for us. Keep a record of this cell because we will - * use it in the next SENDME cell. */ - if (sendme_circuit_cell_is_next(thishop->deliver_window)) { - cpath_sendme_circuit_record_inbound_cell(thishop); - } return 0; } } @@ -213,6 +218,9 @@ relay_encrypt_cell_outbound(cell_t *cell, crypt_path_t *thishop; /* counter for repeated crypts */ cpath_set_cell_forward_digest(layer_hint, cell); + /* Record cell digest as the SENDME digest if need be. */ + sendme_record_sending_cell_digest(TO_CIRCUIT(circ), layer_hint); + thishop = layer_hint; /* moving from farthest to nearest hop */ do { @@ -237,11 +245,8 @@ relay_encrypt_cell_inbound(cell_t *cell, { relay_set_digest(or_circ->crypto.b_digest, cell); - /* We are about to send this cell outbound on the circuit. Keep a record of - * this cell if we are expecting that the next cell is a SENDME. */ - if (sendme_circuit_cell_is_next(TO_CIRCUIT(or_circ)->package_window)) { - sendme_circuit_record_outbound_cell(or_circ); - } + /* Record cell digest as the SENDME digest if need be. */ + sendme_record_sending_cell_digest(TO_CIRCUIT(or_circ), NULL); /* encrypt one layer */ relay_crypt_one_payload(or_circ->crypto.b_crypto, cell->payload); diff --git a/src/core/crypto/relay_crypto.h b/src/core/crypto/relay_crypto.h index 7f09219c7f..9478f8d359 100644 --- a/src/core/crypto/relay_crypto.h +++ b/src/core/crypto/relay_crypto.h @@ -28,7 +28,10 @@ void relay_crypto_clear(relay_crypto_t *crypto); void relay_crypto_assert_ok(const relay_crypto_t *crypto); uint8_t *relay_crypto_get_sendme_digest(relay_crypto_t *crypto); -void relay_crypto_record_sendme_digest(relay_crypto_t *crypto); + +void relay_crypto_record_sendme_digest(relay_crypto_t *crypto, + bool is_foward_digest); + void relay_crypt_one_payload(crypto_cipher_t *cipher, uint8_t *in); diff --git a/src/core/or/crypt_path.c b/src/core/or/crypt_path.c index a4b7190e21..6d5245510f 100644 --- a/src/core/or/crypt_path.c +++ b/src/core/or/crypt_path.c @@ -204,15 +204,6 @@ cpath_set_cell_forward_digest(crypt_path_t *cpath, cell_t *cell) /************ cpath sendme API ***************************/ -/** Keep the current inbound cell digest for the next SENDME digest. This part - * is only done by the client as the circuit came back from the Exit. */ -void -cpath_sendme_circuit_record_inbound_cell(crypt_path_t *cpath) -{ - tor_assert(cpath); - relay_crypto_record_sendme_digest(&cpath->pvt_crypto); -} - /** Return the sendme_digest of this cpath. */ uint8_t * cpath_get_sendme_digest(crypt_path_t *cpath) @@ -220,6 +211,15 @@ cpath_get_sendme_digest(crypt_path_t *cpath) return relay_crypto_get_sendme_digest(&cpath->pvt_crypto); } +/** Record the cell digest, indicated by is_foward_digest or not, as the + * SENDME cell digest. */ +void +cpath_sendme_record_cell_digest(crypt_path_t *cpath, bool is_foward_digest) +{ + tor_assert(cpath); + relay_crypto_record_sendme_digest(&cpath->pvt_crypto, is_foward_digest); +} + /************ other cpath functions ***************************/ /** Return the first non-open hop in cpath, or return NULL if all diff --git a/src/core/or/crypt_path.h b/src/core/or/crypt_path.h index 30c14b3dce..9850610ef7 100644 --- a/src/core/or/crypt_path.h +++ b/src/core/or/crypt_path.h @@ -27,6 +27,9 @@ cpath_crypt_cell(const crypt_path_t *cpath, uint8_t *payload, bool is_decrypt); struct crypto_digest_t * cpath_get_incoming_digest(const crypt_path_t *cpath); +void cpath_sendme_record_cell_digest(crypt_path_t *cpath, + bool is_foward_digest); + void cpath_set_cell_forward_digest(crypt_path_t *cpath, cell_t *cell); diff --git a/src/core/or/relay.c b/src/core/or/relay.c index 91236e5fec..beff225af4 100644 --- a/src/core/or/relay.c +++ b/src/core/or/relay.c @@ -247,6 +247,10 @@ circuit_receive_relay_cell(cell_t *cell, circuit_t *circ, if (recognized) { edge_connection_t *conn = NULL; + /* Recognized cell, the cell digest has been updated, we'll record it for + * the SENDME if need be. */ + sendme_record_received_cell_digest(circ, layer_hint); + if (circ->purpose == CIRCUIT_PURPOSE_PATH_BIAS_TESTING) { if (pathbias_check_probe_response(circ, cell) == -1) { pathbias_count_valid_cells(circ, cell); diff --git a/src/core/or/sendme.c b/src/core/or/sendme.c index 80d6b7d165..eba7c37cdb 100644 --- a/src/core/or/sendme.c +++ b/src/core/or/sendme.c @@ -306,15 +306,6 @@ record_cell_digest_on_circ(circuit_t *circ, const uint8_t *sendme_digest) * Public API */ -/** Keep the current inbound cell digest for the next SENDME digest. This part - * is only done by the client as the circuit came back from the Exit. */ -void -sendme_circuit_record_outbound_cell(or_circuit_t *or_circ) -{ - tor_assert(or_circ); - relay_crypto_record_sendme_digest(&or_circ->crypto); -} - /** Return true iff the next cell for the given cell window is expected to be * a SENDME. * @@ -582,7 +573,8 @@ int sendme_note_stream_data_packaged(edge_connection_t *conn) { tor_assert(conn); - return --conn->package_window; + log_debug(LD_APP, "Stream package_window now %d.", --conn->package_window); + return conn->package_window; } /* Record the cell digest into the circuit sendme digest list depending on @@ -619,3 +611,52 @@ sendme_record_cell_digest_on_circ(circuit_t *circ, crypt_path_t *cpath) record_cell_digest_on_circ(circ, sendme_digest); } + +/* Called once we decrypted a cell and recognized it. Record the cell digest + * as the next sendme digest only if the next cell we'll send on the circuit + * is expected to be a SENDME. */ +void +sendme_record_received_cell_digest(circuit_t *circ, crypt_path_t *cpath) +{ + tor_assert(circ); + + /* Only record if the next cell is expected to be a SENDME. */ + if (!sendme_circuit_cell_is_next(cpath ? cpath->deliver_window : + circ->deliver_window)) { + return; + } + + if (cpath) { + /* Record incoming digest. */ + cpath_sendme_record_cell_digest(cpath, false); + } else { + /* Record foward digest. */ + relay_crypto_record_sendme_digest(&TO_OR_CIRCUIT(circ)->crypto, true); + } +} + +/* Called once we encrypted a cell. Record the cell digest as the next sendme + * digest only if the next cell we expect to receive is a SENDME so we can + * match the digests. */ +void +sendme_record_sending_cell_digest(circuit_t *circ, crypt_path_t *cpath) +{ + tor_assert(circ); + + /* Only record if the next cell is expected to be a SENDME. */ + if (!sendme_circuit_cell_is_next(cpath ? cpath->package_window: + circ->package_window)) { + goto end; + } + + if (cpath) { + /* Record the forward digest. */ + cpath_sendme_record_cell_digest(cpath, true); + } else { + /* Record the incoming digest. */ + relay_crypto_record_sendme_digest(&TO_OR_CIRCUIT(circ)->crypto, false); + } + + end: + return; +} diff --git a/src/core/or/sendme.h b/src/core/or/sendme.h index a71891996b..847fcdd672 100644 --- a/src/core/or/sendme.h +++ b/src/core/or/sendme.h @@ -34,10 +34,11 @@ int sendme_note_circuit_data_packaged(circuit_t *circ, crypt_path_t *layer_hint); int sendme_note_stream_data_packaged(edge_connection_t *conn); -/* Track cell digest. */ -void sendme_circuit_record_outbound_cell(or_circuit_t *or_circ); /* Record cell digest on circuit. */ void sendme_record_cell_digest_on_circ(circuit_t *circ, crypt_path_t *cpath); +/* Record cell digest as the SENDME digest. */ +void sendme_record_received_cell_digest(circuit_t *circ, crypt_path_t *cpath); +void sendme_record_sending_cell_digest(circuit_t *circ, crypt_path_t *cpath); /* Circuit level information. */ bool sendme_circuit_cell_is_next(int window); From 482c4972b996fc7b1a3a8cc13f93c8ecc8748590 Mon Sep 17 00:00:00 2001 From: David Goulet Date: Tue, 21 May 2019 15:19:30 -0400 Subject: [PATCH 1041/2557] sendme: Clarify how sendme_circuit_cell_is_next() works Commit 4ef8470fa5480d3b was actually reverted before because in the end we needed to do this minus 1 check on the window. This commit clarifies that in the code, takes the useful comment changes from 4ef8470fa5480d3b and makes sendme_circuit_cell_is_next() private since it behaves in a very specific way that one external caller might expect. Part of #30428. Signed-off-by: David Goulet --- src/core/or/circuit_st.h | 9 ++++----- src/core/or/sendme.c | 30 ++++++++++++++++++++++-------- src/core/or/sendme.h | 3 --- 3 files changed, 26 insertions(+), 16 deletions(-) diff --git a/src/core/or/circuit_st.h b/src/core/or/circuit_st.h index a68547ecb1..499bf93d6b 100644 --- a/src/core/or/circuit_st.h +++ b/src/core/or/circuit_st.h @@ -115,12 +115,11 @@ struct circuit_t { * list can not contain more than 10 digests of DIGEST_LEN bytes (20). * * At position i in the list, the digest corresponds to the - * ((CIRCWINDOW_INCREMENT * i) - 1)-nth cell received since we expect the - * (CIRCWINDOW_INCREMENT * i)-nth cell to be the SENDME and thus containing - * the previous cell digest. + * (CIRCWINDOW_INCREMENT * i)-nth cell received since we expect a SENDME to + * be received containing that cell digest. * - * For example, position 2 (starting at 0) means that we've received 299 - * cells and the 299th cell digest is kept at index 2. + * For example, position 2 (starting at 0) means that we've received 300 + * cells so the 300th cell digest is kept at index 2. * * At maximum, this list contains 200 bytes plus the smartlist overhead. */ smartlist_t *sendme_last_digests; diff --git a/src/core/or/sendme.c b/src/core/or/sendme.c index eba7c37cdb..d914ba5e2e 100644 --- a/src/core/or/sendme.c +++ b/src/core/or/sendme.c @@ -312,15 +312,29 @@ record_cell_digest_on_circ(circuit_t *circ, const uint8_t *sendme_digest) * We are able to know that because the package or deliver window value minus * one cell (the possible SENDME cell) should be a multiple of the increment * window value. */ -bool -sendme_circuit_cell_is_next(int window) +static bool +circuit_sendme_cell_is_next(int window) { - /* Is this the last cell before a SENDME? The idea is that if the package or - * deliver window reaches a multiple of the increment, after this cell, we - * should expect a SENDME. */ + /* At the start of the window, no SENDME will be expected. */ + if (window == CIRCWINDOW_START) { + return false; + } + + /* Are we at the limit of the increment and if not, we don't expect next + * cell is a SENDME. + * + * We test against the window minus 1 because when we are looking if the + * next cell is a SENDME, the window (either package or deliver) hasn't been + * decremented just yet so when this is called, we are currently processing + * the "window - 1" cell. + * + * This function is used when recording a cell digest and this is done quite + * low in the stack when decrypting or encrypting a cell. The window is only + * updated once the cell is actually put in the outbuf. */ if (((window - 1) % CIRCWINDOW_INCREMENT) != 0) { return false; } + /* Next cell is expected to be a SENDME. */ return true; } @@ -596,7 +610,7 @@ sendme_record_cell_digest_on_circ(circuit_t *circ, crypt_path_t *cpath) /* Is this the last cell before a SENDME? The idea is that if the * package_window reaches a multiple of the increment, after this cell, we * should expect a SENDME. */ - if (!sendme_circuit_cell_is_next(package_window)) { + if (!circuit_sendme_cell_is_next(package_window)) { return; } @@ -621,7 +635,7 @@ sendme_record_received_cell_digest(circuit_t *circ, crypt_path_t *cpath) tor_assert(circ); /* Only record if the next cell is expected to be a SENDME. */ - if (!sendme_circuit_cell_is_next(cpath ? cpath->deliver_window : + if (!circuit_sendme_cell_is_next(cpath ? cpath->deliver_window : circ->deliver_window)) { return; } @@ -644,7 +658,7 @@ sendme_record_sending_cell_digest(circuit_t *circ, crypt_path_t *cpath) tor_assert(circ); /* Only record if the next cell is expected to be a SENDME. */ - if (!sendme_circuit_cell_is_next(cpath ? cpath->package_window: + if (!circuit_sendme_cell_is_next(cpath ? cpath->package_window : circ->package_window)) { goto end; } diff --git a/src/core/or/sendme.h b/src/core/or/sendme.h index 847fcdd672..cdbdf55ac7 100644 --- a/src/core/or/sendme.h +++ b/src/core/or/sendme.h @@ -40,9 +40,6 @@ void sendme_record_cell_digest_on_circ(circuit_t *circ, crypt_path_t *cpath); void sendme_record_received_cell_digest(circuit_t *circ, crypt_path_t *cpath); void sendme_record_sending_cell_digest(circuit_t *circ, crypt_path_t *cpath); -/* Circuit level information. */ -bool sendme_circuit_cell_is_next(int window); - /* Private section starts. */ #ifdef SENDME_PRIVATE From 5479ffabf8ec1a3c63e9f237c6696c4ef0b74a55 Mon Sep 17 00:00:00 2001 From: David Goulet Date: Wed, 22 May 2019 10:37:27 -0400 Subject: [PATCH 1042/2557] sendme: Always pop last SENDME digest from circuit We must not accumulate digests on the circuit if the other end point is using another SENDME version that is not using those digests like v0. This commit makes it that we always pop the digest regardless of the version. Part of #30428 Signed-off-by: David Goulet --- src/core/or/sendme.c | 80 +++++++++++++++++++++++---------------- src/test/test_relaycell.c | 5 +++ src/test/test_sendme.c | 4 ++ 3 files changed, 57 insertions(+), 32 deletions(-) diff --git a/src/core/or/sendme.c b/src/core/or/sendme.c index d914ba5e2e..bfbb65e851 100644 --- a/src/core/or/sendme.c +++ b/src/core/or/sendme.c @@ -47,47 +47,46 @@ get_accept_min_version(void) SENDME_ACCEPT_MIN_VERSION_MAX); } +/* Pop the first cell digset on the given circuit from the SENDME last digests + * list. NULL is returned if the list is uninitialized or empty. + * + * The caller gets ownership of the returned digest thus is responsible for + * freeing the memory. */ +static uint8_t * +pop_first_cell_digest(const circuit_t *circ) +{ + uint8_t *circ_digest; + + tor_assert(circ); + + if (circ->sendme_last_digests == NULL || + smartlist_len(circ->sendme_last_digests) == 0) { + return NULL; + } + + circ_digest = smartlist_get(circ->sendme_last_digests, 0); + smartlist_del_keeporder(circ->sendme_last_digests, 0); + return circ_digest; +} + /* Return true iff the given cell digest matches the first digest in the * circuit sendme list. */ static bool -v1_digest_matches(const circuit_t *circ, const uint8_t *cell_digest) +v1_digest_matches(const uint8_t *circ_digest, const uint8_t *cell_digest) { - bool ret = false; - uint8_t *circ_digest = NULL; - - tor_assert(circ); + tor_assert(circ_digest); tor_assert(cell_digest); - /* We shouldn't have received a SENDME if we have no digests. Log at - * protocol warning because it can be tricked by sending many SENDMEs - * without prior data cell. */ - if (circ->sendme_last_digests == NULL || - smartlist_len(circ->sendme_last_digests) == 0) { - log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, - "We received a SENDME but we have no cell digests to match. " - "Closing circuit."); - goto no_match; - } - - /* Pop the first element that was added (FIFO) and compare it. */ - circ_digest = smartlist_get(circ->sendme_last_digests, 0); - smartlist_del_keeporder(circ->sendme_last_digests, 0); - /* Compare the digest with the one in the SENDME. This cell is invalid * without a perfect match. */ if (tor_memneq(circ_digest, cell_digest, TRUNNEL_SENDME_V1_DIGEST_LEN)) { log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, "SENDME v1 cell digest do not match."); - goto no_match; + return false; } - /* Digests matches! */ - ret = true; - no_match: - /* This digest was popped from the circuit list. Regardless of what happens, - * we have no more use for it. */ - tor_free(circ_digest); - return ret; + /* Digests matches! */ + return true; } /* Return true iff the given decoded SENDME version 1 cell is valid and @@ -97,13 +96,13 @@ v1_digest_matches(const circuit_t *circ, const uint8_t *cell_digest) * cell we saw which tells us that the other side has in fact seen that cell. * See proposal 289 for more details. */ static bool -cell_v1_is_valid(const sendme_cell_t *cell, const circuit_t *circ) +cell_v1_is_valid(const sendme_cell_t *cell, const uint8_t *circ_digest) { tor_assert(cell); - tor_assert(circ); + tor_assert(circ_digest); const uint8_t *cell_digest = sendme_cell_getconstarray_data_v1_digest(cell); - return v1_digest_matches(circ, cell_digest); + return v1_digest_matches(circ_digest, cell_digest); } /* Return true iff the given cell version can be handled or if the minimum @@ -160,6 +159,7 @@ sendme_is_valid(const circuit_t *circ, const uint8_t *cell_payload, size_t cell_payload_len) { uint8_t cell_version; + uint8_t *circ_digest = NULL; sendme_cell_t *cell = NULL; tor_assert(circ); @@ -184,10 +184,24 @@ sendme_is_valid(const circuit_t *circ, const uint8_t *cell_payload, goto invalid; } + /* Pop the first element that was added (FIFO). We do that regardless of the + * version so we don't accumulate on the circuit if v0 is used by the other + * end point. */ + circ_digest = pop_first_cell_digest(circ); + if (circ_digest == NULL) { + /* We shouldn't have received a SENDME if we have no digests. Log at + * protocol warning because it can be tricked by sending many SENDMEs + * without prior data cell. */ + log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, + "We received a SENDME but we have no cell digests to match. " + "Closing circuit."); + goto invalid; + } + /* Validate depending on the version now. */ switch (cell_version) { case 0x01: - if (!cell_v1_is_valid(cell, circ)) { + if (!cell_v1_is_valid(cell, circ_digest)) { goto invalid; } break; @@ -204,9 +218,11 @@ sendme_is_valid(const circuit_t *circ, const uint8_t *cell_payload, /* Valid cell. */ sendme_cell_free(cell); + tor_free(circ_digest); return true; invalid: sendme_cell_free(cell); + tor_free(circ_digest); return false; } diff --git a/src/test/test_relaycell.c b/src/test/test_relaycell.c index d6372d3956..c65279fb25 100644 --- a/src/test/test_relaycell.c +++ b/src/test/test_relaycell.c @@ -17,6 +17,7 @@ #include "core/or/circuitbuild.h" #include "core/or/circuitlist.h" #include "core/or/connection_edge.h" +#include "core/or/sendme.h" #include "core/or/relay.h" #include "test/test.h" #include "test/log_test_helpers.h" @@ -813,6 +814,10 @@ test_circbw_relay(void *arg) /* Sendme on circuit with non-full window: counted */ PACK_CELL(0, RELAY_COMMAND_SENDME, ""); + /* Recording a cell, the window is updated after decryption so off by one in + * order to record and then we process it with the proper window. */ + circ->cpath->package_window = 901; + sendme_record_cell_digest_on_circ(TO_CIRCUIT(circ), circ->cpath); circ->cpath->package_window = 900; connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), edgeconn, circ->cpath); diff --git a/src/test/test_sendme.c b/src/test/test_sendme.c index a36c904c28..fa5ae115ac 100644 --- a/src/test/test_sendme.c +++ b/src/test/test_sendme.c @@ -143,11 +143,13 @@ test_v1_build_cell(void *arg) or_circ = or_circuit_new(1, NULL); circ = TO_CIRCUIT(or_circ); + circ->sendme_last_digests = smartlist_new(); cell_digest = crypto_digest_new(); tt_assert(cell_digest); crypto_digest_add_bytes(cell_digest, "AAAAAAAAAAAAAAAAAAAA", 20); crypto_digest_get_digest(cell_digest, (char *) digest, sizeof(digest)); + smartlist_add(circ->sendme_last_digests, tor_memdup(digest, sizeof(digest))); /* SENDME v1 payload is 3 bytes + 20 bytes digest. See spec. */ ret = build_cell_payload_v1(digest, payload); @@ -157,6 +159,8 @@ test_v1_build_cell(void *arg) /* An empty payload means SENDME version 0 thus valid. */ tt_int_op(sendme_is_valid(circ, payload, 0), OP_EQ, true); + /* Current phoney digest should have been popped. */ + tt_int_op(smartlist_len(circ->sendme_last_digests), OP_EQ, 0); /* An unparseable cell means invalid. */ setup_full_capture_of_logs(LOG_INFO); From 0cad83bea4019b3c448756d718f5b65718e81f6e Mon Sep 17 00:00:00 2001 From: David Goulet Date: Wed, 22 May 2019 11:05:36 -0400 Subject: [PATCH 1043/2557] sendme: Add non fatal asserts for extra safety Two non fatal asserts are added in this commit. First one is to see if the SENDME digest list kept on the circuit for validation ever grows bigger than the maximum number of expected SENDME on a circuit (currently 10). The second one is to know if we ever send more than one SENDME at a time on a circuit. In theory, we shouldn't but if we ever do, the v1 implementation wouldn't work because we only keep one single cell digest (the previous cell to the SENDME) on the circuit/cpath. Thus, sending two SENDME consecutively will lead to a mismatch on the other side because the same cell digest would be use and thus the circuit would collapse. Finally, add an extra debug log in case we emit a v0 which also includes the consensus emit version in that case. Part of #30428 Signed-off-by: David Goulet --- src/core/or/sendme.c | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/core/or/sendme.c b/src/core/or/sendme.c index bfbb65e851..98ed639940 100644 --- a/src/core/or/sendme.c +++ b/src/core/or/sendme.c @@ -64,6 +64,13 @@ pop_first_cell_digest(const circuit_t *circ) return NULL; } + /* More cell digest than the SENDME window is never suppose to happen. The + * cell should have been rejected before reaching this point due to its + * package_window down to 0 leading to a circuit close. Scream loudly but + * still pop the element so we don't memory leak. */ + tor_assert_nonfatal(smartlist_len(circ->sendme_last_digests) <= + CIRCWINDOW_START_MAX / CIRCWINDOW_INCREMENT); + circ_digest = smartlist_get(circ->sendme_last_digests, 0); smartlist_del_keeporder(circ->sendme_last_digests, 0); return circ_digest; @@ -290,6 +297,8 @@ send_circuit_level_sendme(circuit_t *circ, crypt_path_t *layer_hint, default: /* Unknown version, fallback to version 0 meaning no payload. */ payload_len = 0; + log_debug(LD_PROTOCOL, "Emitting SENDME version 0 cell. " + "Consensus emit version is %d", emit_version); break; } @@ -408,6 +417,7 @@ sendme_connection_edge_consider_sending(edge_connection_t *conn) void sendme_circuit_consider_sending(circuit_t *circ, crypt_path_t *layer_hint) { + bool sent_one_sendme = false; const uint8_t *digest; while ((layer_hint ? layer_hint->deliver_window : circ->deliver_window) <= @@ -423,6 +433,12 @@ sendme_circuit_consider_sending(circuit_t *circ, crypt_path_t *layer_hint) if (send_circuit_level_sendme(circ, layer_hint, digest) < 0) { return; /* The circuit's closed, don't continue */ } + /* Current implementation is not suppose to send multiple SENDME at once + * because this means we would use the same relay crypto digest for each + * SENDME leading to a mismatch on the other side and the circuit to + * collapse. Scream loudly if it ever happens so we can address it. */ + tor_assert_nonfatal(!sent_one_sendme); + sent_one_sendme = true; } } From 3a7ed8bc5f2cd1b6665510fcebf9c57829608e81 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 22 May 2019 11:56:02 -0400 Subject: [PATCH 1044/2557] Bump to 0.4.1.1-alpha --- configure.ac | 4 ++-- contrib/win32build/tor-mingw.nsi.in | 2 +- src/win32/orconfig.h | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/configure.ac b/configure.ac index e65e960d7f..9460ae637d 100644 --- a/configure.ac +++ b/configure.ac @@ -4,7 +4,7 @@ dnl Copyright (c) 2007-2019, The Tor Project, Inc. dnl See LICENSE for licensing information AC_PREREQ([2.63]) -AC_INIT([tor],[0.4.1.0-alpha-dev]) +AC_INIT([tor],[0.4.1.1-alpha]) AC_CONFIG_SRCDIR([src/app/main/tor_main.c]) AC_CONFIG_MACRO_DIR([m4]) @@ -14,7 +14,7 @@ AC_CONFIG_MACRO_DIR([m4]) # version number changes. Tor uses it to make sure that it # only shuts down for missing "required protocols" when those protocols # are listed as required by a consensus after this date. -AC_DEFINE(APPROX_RELEASE_DATE, ["2019-02-19"], # for 0.4.1.0-alpha-dev +AC_DEFINE(APPROX_RELEASE_DATE, ["2019-05-22"], # for 0.4.1.1-alpha [Approximate date when this software was released. (Updated when the version changes.)]) # "foreign" means we don't follow GNU package layout standards diff --git a/contrib/win32build/tor-mingw.nsi.in b/contrib/win32build/tor-mingw.nsi.in index 419b5aa58c..7694a4d375 100644 --- a/contrib/win32build/tor-mingw.nsi.in +++ b/contrib/win32build/tor-mingw.nsi.in @@ -8,7 +8,7 @@ !include "LogicLib.nsh" !include "FileFunc.nsh" !insertmacro GetParameters -!define VERSION "0.4.1.0-alpha-dev" +!define VERSION "0.4.1.1-alpha" !define INSTALLER "tor-${VERSION}-win32.exe" !define WEBSITE "https://www.torproject.org/" !define LICENSE "LICENSE" diff --git a/src/win32/orconfig.h b/src/win32/orconfig.h index 892531ceb4..5cf0b1eb0b 100644 --- a/src/win32/orconfig.h +++ b/src/win32/orconfig.h @@ -218,7 +218,7 @@ #define USING_TWOS_COMPLEMENT /* Version number of package */ -#define VERSION "0.4.1.0-alpha-dev" +#define VERSION "0.4.1.1-alpha" From fa57b6cfb0bfd0287530f04f7476d279d9f70006 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 22 May 2019 12:00:20 -0400 Subject: [PATCH 1045/2557] Fold last entry into changelog --- ChangeLog | 14 +++++++++++++- changes/ticket30454 | 10 ---------- 2 files changed, 13 insertions(+), 11 deletions(-) delete mode 100644 changes/ticket30454 diff --git a/ChangeLog b/ChangeLog index b3562bf6a4..f70ce42a7a 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,4 +1,4 @@ -Changes in version 0.4.1.1-alpha - 2019-05-?? +Changes in version 0.4.1.1-alpha - 2019-05-22 This is the first alpha in the 0.4.1.x series. It introduces lightweight circuit padding to make some onion-service circuits harder to distinguish, includes a new "authenticated SENDME" feature to make @@ -53,6 +53,18 @@ Changes in version 0.4.1.1-alpha - 2019-05-?? strong, we are only using it when necessary for performance. Implements tickets 29023 and 29536. + o Major bugfixes (onion service v3): + - Fix an unreachable bug in which an introduction point would could + try to send an INTRODUCE_ACK with a status code that Trunnel would + refuse to encode, leading the relay to assert(). We've + consolidated the ABI values into trunnel now. Fixes bug 30454; + bugfix on 0.3.0.1-alpha. + - Client can now handle unknown status codes from a INTRODUCE_ACK + cells. (The NACK behavior will stay the same.) This will allow us + to extend status codes in the future without breaking the normal + client behavior. Fixes another part of bug 30454; bugfix + on 0.3.0.1-alpha. + o Minor features (circuit padding): - We now use a fast PRNG when scheduling circuit padding. Part of ticket 28636. diff --git a/changes/ticket30454 b/changes/ticket30454 deleted file mode 100644 index 77c45d0feb..0000000000 --- a/changes/ticket30454 +++ /dev/null @@ -1,10 +0,0 @@ - o Major bugfixes (hidden service v3): - - An intro point could try to send an INTRODUCE_ACK with a status code - that it wasn't able to encode leading to a hard assert() of the relay. - Fortunately, that specific code path can not be reached thus this issue - can't be triggered. We've consolidated the ABI values into trunnel now. - Fixes bug 30454; bugfix on 0.3.0.1-alpha. - - HSv3 client will now be able to properly handle unknown status code from - a INTRODUCE_ACK cell (nack) even if they do not know it. The NACK - behavior will stay the same. This will allow us to extend status code if - we want in the future without breaking the normal client behavior. From c93ea09d2983c512b6c7688580fd797d60d7d173 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 22 May 2019 13:57:34 -0400 Subject: [PATCH 1046/2557] More 0.4.1.1-alpha hangelogs edits (credit to seborn here) --- ChangeLog | 68 +++++++++++++++++++++++++++---------------------------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/ChangeLog b/ChangeLog index f70ce42a7a..cda4c11e07 100644 --- a/ChangeLog +++ b/ChangeLog @@ -32,11 +32,11 @@ Changes in version 0.4.1.1-alpha - 2019-05-22 o Major features (flow control): - Implement authenticated SENDMEs as detailed in proposal 289. A SENDME cell now includes the digest of the traffic that it - acknowledges, so that once end point receives the SENDME, it can - confirm the other side's knowledge of the previous cells that were - sent, and prevent certain types of denial-of-service attacks. This - behavior is controlled by two new consensus parameters: see the - proposal for more details. Fixes ticket 26288. + acknowledges, so that once an end point receives the SENDME, it + can confirm the other side's knowledge of the previous cells that + were sent, and prevent certain types of denial-of-service attacks. + This behavior is controlled by two new consensus parameters: see + the proposal for more details. Fixes ticket 26288. o Major features (performance): - Our node selection algorithm now excludes nodes in linear time. @@ -54,12 +54,12 @@ Changes in version 0.4.1.1-alpha - 2019-05-22 Implements tickets 29023 and 29536. o Major bugfixes (onion service v3): - - Fix an unreachable bug in which an introduction point would could - try to send an INTRODUCE_ACK with a status code that Trunnel would - refuse to encode, leading the relay to assert(). We've - consolidated the ABI values into trunnel now. Fixes bug 30454; - bugfix on 0.3.0.1-alpha. - - Client can now handle unknown status codes from a INTRODUCE_ACK + - Fix an unreachable bug in which an introduction point could try to + send an INTRODUCE_ACK with a status code that Trunnel would refuse + to encode, leading the relay to assert(). We've consolidated the + ABI values into Trunnel now. Fixes bug 30454; bugfix + on 0.3.0.1-alpha. + - Clients can now handle unknown status codes from INTRODUCE_ACK cells. (The NACK behavior will stay the same.) This will allow us to extend status codes in the future without breaking the normal client behavior. Fixes another part of bug 30454; bugfix @@ -89,7 +89,7 @@ Changes in version 0.4.1.1-alpha - 2019-05-22 o Minor features (controller): - Add onion service version 3 support to the HSFETCH command. Previously, only version 2 onion services were supported. Closes - ticket 25417. Patch by Neel Chauhan + ticket 25417. Patch by Neel Chauhan. o Minor features (debugging): - Introduce tor_assertf() and tor_assertf_nonfatal() to enable @@ -109,7 +109,7 @@ Changes in version 0.4.1.1-alpha - 2019-05-22 cases. Closes ticket 29542. o Minor features (developer tools): - - Tor's "practracker" test script nows check for files and functions + - Tor's "practracker" test script now checks for files and functions that seem too long and complicated. Existing overlong functions and files are accepted for now, but should eventually be refactored. Closes ticket 29221. @@ -132,7 +132,7 @@ Changes in version 0.4.1.1-alpha - 2019-05-22 o Minor features (IPv6, v3 onion services): - Make v3 onion services put IPv6 addresses in service descriptors. Before this change, service descriptors only contained IPv4 - addressesd. Implements 26992. + addresses. Implements 26992. o Minor features (modularity): - The "--disable-module-dirauth" compile-time option now disables @@ -146,7 +146,7 @@ Changes in version 0.4.1.1-alpha - 2019-05-22 o Minor features (testing): - Tor's unit test code now contains helper functions to replace the PRNG with a deterministic or reproducible version for testing. - Previously, various tests did implemented this in various ways. + Previously, various tests implemented this in various ways. Implements ticket 29732. - We now have a script, cov-test-determinism.sh, to identify places where our unit test coverage has become nondeterministic. Closes @@ -159,12 +159,12 @@ Changes in version 0.4.1.1-alpha - 2019-05-22 about to dump their status to a file. Previously, they set bridges as running in response to a GETINFO command, but those shouldn't modify data structures. Fixes bug 24490; bugfix on 0.2.0.13-alpha. - Patch by Neel Chauhan + Patch by Neel Chauhan. o Minor bugfixes (channel padding statistics): - Channel padding write totals and padding-enabled totals are now counted properly in relay extrainfo descriptors. Fixes bug 29231; - bugfix on 0.3.1.1-alpha + bugfix on 0.3.1.1-alpha. o Minor bugfixes (circuit padding): - Add a "CircuitPadding" torrc option to disable circuit padding. @@ -197,9 +197,9 @@ Changes in version 0.4.1.1-alpha - 2019-05-22 object. Fixes bug 29984; bugfix on 0.2.3.8-alpha. o Minor bugfixes (directory authority, ipv6): - - If we are a directory authority with IPv6 and are marking relays - as running, mark ourselves as reachable on IPv6. Fixes bug 24338; - bugfix on 0.4.0.2-alpha. Patch by Neel Chauhan + - Directory authorities with IPv6 support now always mark themselves + as reachable via IPv6. Fixes bug 24338; bugfix on 0.4.0.2-alpha. + Patch by Neel Chauhan. o Minor bugfixes (documentation): - Improve the documentation for using MapAddress with ".exit". Fixes @@ -212,10 +212,10 @@ Changes in version 0.4.1.1-alpha - 2019-05-22 that contain whitespace. Fixes bug 29635; bugfix on 0.2.3.18-rc. o Minor bugfixes (logging): - - Do not log a warning when running with an OpenSSL version that - that should be compatible with the one we were built with. - Previously, we would warn whenever the version was different. - Fixes bug 30190; bugfix on 0.2.4.2-alpha + - Do not log a warning when running with an OpenSSL version other + than the one Tor was compiled with, if the two versions should be + compatible. Previously, we would warn whenever the version was + different. Fixes bug 30190; bugfix on 0.2.4.2-alpha. - Warn operators when the MyFamily option is set but ContactInfo is missing, as the latter should be set too. Fixes bug 25110; bugfix on 0.3.3.1-alpha. @@ -233,10 +233,10 @@ Changes in version 0.4.1.1-alpha - 2019-05-22 rate-limiting, respond to the controller with a new response, "QUERY_RATE_LIMITED". Previously, we would log QUERY_NO_HSDIR for this case. Fixes bug 28269; bugfix on 0.3.1.1-alpha. Patch by - Neel Chauhan - - When relaunching a circuit to a rendevous service, mark the + Neel Chauhan. + - When relaunching a circuit to a rendezvous service, mark the circuit as needing high-uptime routers as appropriate. Fixes bug - 17357; bugfix on 0.4.0.2-alpha. Patch by Neel Chauhan + 17357; bugfix on 0.4.0.2-alpha. Patch by Neel Chauhan. - Stop ignoring IPv6 link specifiers sent to v3 onion services. (IPv6 support for v3 onion services is still incomplete: see ticket 23493 for details.) Fixes bug 23588; bugfix on @@ -245,9 +245,9 @@ Changes in version 0.4.1.1-alpha - 2019-05-22 o Minor bugfixes (onion services, performance): - When building circuits to onion services, call tor_addr_parse() less often. Previously, we called tor_addr_parse() in - circuit_is_acceptable() even if its output it wasn't used. This + circuit_is_acceptable() even if its output wasn't used. This change should improve performance when building circuits. Fixes - bug 22210; bugfix on 0.2.8.12. Patch by Neel Chauhan + bug 22210; bugfix on 0.2.8.12. Patch by Neel Chauhan. o Minor bugfixes (performance): - When checking whether a node is a bridge, use a fast check to make @@ -270,9 +270,9 @@ Changes in version 0.4.1.1-alpha - 2019-05-22 python3. Fixes bug 29913; bugfix on 0.2.5.3-alpha. o Minor bugfixes (relay): - - If we are are a relay and have IPv6Exit to 1 while ExitRelay is - auto, we act as if ExitRelay is 1. Previously, we ignored IPv6Exit - if ExitRelay was 0 or auto. Fixes bug 29613; bugfix on + - When running as a relay, if IPv6Exit is set to 1 while ExitRelay + is auto, act as if ExitRelay is 1. Previously, we would ignore + IPv6Exit if ExitRelay was 0 or auto. Fixes bug 29613; bugfix on 0.3.5.1-alpha. Patch by Neel Chauhan. o Minor bugfixes (stats): @@ -316,7 +316,7 @@ Changes in version 0.4.1.1-alpha - 2019-05-22 relay subsystem. Closes ticket 30414. - Refactor and encapsulate parts of the codebase that manipulate crypt_path_t objects. Resolves issue 30236. - - Refactor several places in our code that coverity incorrectly + - Refactor several places in our code that Coverity incorrectly believed might have memory leaks. Closes ticket 30147. - Remove redundant return values in crypto_format, and the associated return value checks elsewhere in the code. Make the @@ -341,7 +341,7 @@ Changes in version 0.4.1.1-alpha - 2019-05-22 o Documentation: - Document how to find git commits and tags for bug fixes in - CodingStandards.md. Update some changes file documentation. Closes + CodingStandards.md. Update some file documentation. Closes ticket 30261. o Removed features: From fa410162a3309dba31661f7f22c95bbdc3af66a5 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 22 May 2019 15:19:24 -0400 Subject: [PATCH 1047/2557] circuitpadding tests: Use tt_i64_op() to compare int64_t values Bug not in any released Tor. --- src/test/test_circuitpadding.c | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/test/test_circuitpadding.c b/src/test/test_circuitpadding.c index ca65438430..1ac2bd676e 100644 --- a/src/test/test_circuitpadding.c +++ b/src/test/test_circuitpadding.c @@ -1852,21 +1852,21 @@ test_circuitpadding_state_length(void *arg) circpad_machine_runtime_t *mi = client_side->padding_info[0]; circpad_cell_event_padding_sent(client_side); - tt_int_op(mi->state_length, OP_EQ, 12); + tt_i64_op(mi->state_length, OP_EQ, 12); tt_ptr_op(mi->histogram, OP_EQ, NULL); /* Verify that non-padding does not change our state length */ circpad_cell_event_nonpadding_sent(client_side); - tt_int_op(mi->state_length, OP_EQ, 12); + tt_i64_op(mi->state_length, OP_EQ, 12); /* verify that sending padding changes our state length */ for (uint64_t i = mi->state_length-1; i > 0; i--) { circpad_send_padding_cell_for_callback(mi); - tt_int_op(mi->state_length, OP_EQ, i); + tt_i64_op(mi->state_length, OP_EQ, i); } circpad_send_padding_cell_for_callback(mi); - tt_int_op(mi->state_length, OP_EQ, -1); + tt_i64_op(mi->state_length, OP_EQ, -1); tt_int_op(mi->current_state, OP_EQ, CIRCPAD_STATE_END); /* Restart machine */ @@ -1876,16 +1876,16 @@ test_circuitpadding_state_length(void *arg) client_machine->states[CIRCPAD_STATE_BURST].length_includes_nonpadding = 1; circpad_cell_event_padding_sent(client_side); - tt_int_op(mi->state_length, OP_EQ, 12); + tt_i64_op(mi->state_length, OP_EQ, 12); /* Verify that non-padding does change our state length now */ for (uint64_t i = mi->state_length-1; i > 0; i--) { circpad_cell_event_nonpadding_sent(client_side); - tt_int_op(mi->state_length, OP_EQ, i); + tt_i64_op(mi->state_length, OP_EQ, i); } circpad_cell_event_nonpadding_sent(client_side); - tt_int_op(mi->state_length, OP_EQ, -1); + tt_i64_op(mi->state_length, OP_EQ, -1); tt_int_op(mi->current_state, OP_EQ, CIRCPAD_STATE_END); /* Now, just test token removal when we send padding */ @@ -1895,7 +1895,7 @@ test_circuitpadding_state_length(void *arg) /* Restart machine */ mi->current_state = CIRCPAD_STATE_START; circpad_cell_event_padding_sent(client_side); - tt_int_op(mi->state_length, OP_EQ, 12); + tt_i64_op(mi->state_length, OP_EQ, 12); tt_ptr_op(mi->histogram, OP_NE, NULL); tt_int_op(mi->chosen_bin, OP_EQ, 2); @@ -1907,7 +1907,7 @@ test_circuitpadding_state_length(void *arg) tt_int_op(mi->histogram[2], OP_EQ, i); } - tt_int_op(mi->state_length, OP_EQ, 7); + tt_i64_op(mi->state_length, OP_EQ, 7); tt_int_op(mi->histogram[2], OP_EQ, 1); circpad_send_padding_cell_for_callback(mi); @@ -3009,15 +3009,15 @@ helper_test_hs_machines(bool test_intro_circs) if (test_intro_circs) { /* on the client side, we don't send any padding so * state length is not set */ - tt_int_op(client_machine_runtime->state_length, OP_EQ, -1); + tt_i64_op(client_machine_runtime->state_length, OP_EQ, -1); /* relay side has state limits. check them */ - tt_int_op(relay_machine_runtime->state_length, OP_GE, + tt_i64_op(relay_machine_runtime->state_length, OP_GE, INTRO_MACHINE_MINIMUM_PADDING); - tt_int_op(relay_machine_runtime->state_length, OP_LT, + tt_i64_op(relay_machine_runtime->state_length, OP_LT, INTRO_MACHINE_MAXIMUM_PADDING); } else { - tt_int_op(client_machine_runtime->state_length, OP_EQ, 1); - tt_int_op(relay_machine_runtime->state_length, OP_EQ, 1); + tt_i64_op(client_machine_runtime->state_length, OP_EQ, 1); + tt_i64_op(relay_machine_runtime->state_length, OP_EQ, 1); } if (test_intro_circs) { From a8a0144d1183a3598bffe6c552507c9dcbdcd474 Mon Sep 17 00:00:00 2001 From: Taylor Yu Date: Mon, 1 Apr 2019 14:53:39 -0500 Subject: [PATCH 1048/2557] Multiple subscribers or publishers per subsystem Allow a subsystem to register to publish or subscribe a given message from multiple places. Part of ticket 29976. --- src/lib/pubsub/pubsub_check.c | 26 ++++----------------- src/test/test_pubsub_build.c | 43 ----------------------------------- 2 files changed, 5 insertions(+), 64 deletions(-) diff --git a/src/lib/pubsub/pubsub_check.c b/src/lib/pubsub/pubsub_check.c index a3c22d4f25..bf1196df2c 100644 --- a/src/lib/pubsub/pubsub_check.c +++ b/src/lib/pubsub/pubsub_check.c @@ -172,34 +172,20 @@ pubsub_cfg_dump(const pubsub_cfg_t *cfg, int severity, const char *prefix) /** * Helper: fill a bitarray out with entries corresponding to the - * subsystems listed in items. If any subsystem is listed more than - * once, log a warning. Return 0 on success, -1 on failure. + * subsystems listed in items. **/ -static int +static void get_message_bitarray(const pubsub_adjmap_t *map, - message_id_t msg, const smartlist_t *items, - const char *operation, bitarray_t **out) { - bool ok = true; *out = bitarray_init_zero((unsigned)map->n_subsystems); if (! items) - return 0; + return; SMARTLIST_FOREACH_BEGIN(items, const pubsub_cfg_t *, cfg) { - if (bitarray_is_set(*out, cfg->subsys)) { - log_warn(LD_MESG|LD_BUG, - "Message \"%s\" is configured to be %s by subsystem " - "\"%s\" more than once.", - get_message_id_name(msg), operation, - get_subsys_id_name(cfg->subsys)); - ok = false; - } bitarray_set(*out, cfg->subsys); } SMARTLIST_FOREACH_END(cfg); - - return ok ? 0 : -1; } /** @@ -222,10 +208,8 @@ lint_message_graph(const pubsub_adjmap_t *map, bitarray_t *subscribed_by = NULL; bool ok = true; - if (get_message_bitarray(map, msg, pub, "published", &published_by) < 0) - ok = false; - if (get_message_bitarray(map, msg, sub, "subscribed", &subscribed_by) < 0) - ok = false; + get_message_bitarray(map, pub, &published_by); + get_message_bitarray(map, sub, &subscribed_by); /* Check whether any subsystem is publishing and subscribing the same * message. [??] diff --git a/src/test/test_pubsub_build.c b/src/test/test_pubsub_build.c index ce5bf60080..021323fbf1 100644 --- a/src/test/test_pubsub_build.c +++ b/src/test/test_pubsub_build.c @@ -493,48 +493,6 @@ test_pubsub_build_sub_many(void *arg) tor_free(sysname); } -/* The same subsystem can only declare one publish or subscribe. */ -static void -test_pubsub_build_pubsub_redundant(void *arg) -{ - (void)arg; - pubsub_builder_t *b = NULL; - dispatch_t *dispatcher = NULL; - pubsub_connector_t *c = NULL; - - b = pubsub_builder_new(); - seed_pubsub_builder_basic(b); - pub_binding_t btmp; - - { - c = pubsub_connector_for_subsystem(b, get_subsys_id("sys2")); - DISPATCH_ADD_SUB(c, main, bunch_of_coconuts); - pubsub_add_pub_(c, &btmp, get_channel_id("main"), - get_message_id("yes_we_have_no"), - get_msg_type_id("string"), - 0 /* flags */, - "somewhere.c", 22); - pubsub_connector_free(c); - }; - - setup_full_capture_of_logs(LOG_WARN); - dispatcher = pubsub_builder_finalize(b, NULL); - b = NULL; - tt_assert(dispatcher == NULL); - - expect_log_msg_containing( - "Message \"yes_we_have_no\" is configured to be published by " - "subsystem \"sys2\" more than once."); - expect_log_msg_containing( - "Message \"bunch_of_coconuts\" is configured to be subscribed by " - "subsystem \"sys2\" more than once."); - - done: - pubsub_builder_free(b); - dispatch_free(dispatcher); - teardown_capture_of_logs(); -} - /* It's fine to declare the excl flag. */ static void test_pubsub_build_excl_ok(void *arg) @@ -614,7 +572,6 @@ struct testcase_t pubsub_build_tests[] = { T(pubsub_same, TT_FORK), T(pubsub_multi, TT_FORK), T(sub_many, TT_FORK), - T(pubsub_redundant, TT_FORK), T(excl_ok, TT_FORK), T(excl_bad, TT_FORK), END_OF_TESTCASES From ebe39dcb9225ffe43c6b6d2fe49d4b99d155ff33 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 22 May 2019 18:07:29 -0400 Subject: [PATCH 1049/2557] Now this repository is full of 0.4.1.1-alpha-dev --- configure.ac | 4 ++-- contrib/win32build/tor-mingw.nsi.in | 2 +- src/win32/orconfig.h | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/configure.ac b/configure.ac index 9460ae637d..e7f959f17a 100644 --- a/configure.ac +++ b/configure.ac @@ -4,7 +4,7 @@ dnl Copyright (c) 2007-2019, The Tor Project, Inc. dnl See LICENSE for licensing information AC_PREREQ([2.63]) -AC_INIT([tor],[0.4.1.1-alpha]) +AC_INIT([tor],[0.4.1.1-alpha-dev]) AC_CONFIG_SRCDIR([src/app/main/tor_main.c]) AC_CONFIG_MACRO_DIR([m4]) @@ -14,7 +14,7 @@ AC_CONFIG_MACRO_DIR([m4]) # version number changes. Tor uses it to make sure that it # only shuts down for missing "required protocols" when those protocols # are listed as required by a consensus after this date. -AC_DEFINE(APPROX_RELEASE_DATE, ["2019-05-22"], # for 0.4.1.1-alpha +AC_DEFINE(APPROX_RELEASE_DATE, ["2019-05-22"], # for 0.4.1.1-alpha-dev [Approximate date when this software was released. (Updated when the version changes.)]) # "foreign" means we don't follow GNU package layout standards diff --git a/contrib/win32build/tor-mingw.nsi.in b/contrib/win32build/tor-mingw.nsi.in index 7694a4d375..49958a462d 100644 --- a/contrib/win32build/tor-mingw.nsi.in +++ b/contrib/win32build/tor-mingw.nsi.in @@ -8,7 +8,7 @@ !include "LogicLib.nsh" !include "FileFunc.nsh" !insertmacro GetParameters -!define VERSION "0.4.1.1-alpha" +!define VERSION "0.4.1.1-alpha-dev" !define INSTALLER "tor-${VERSION}-win32.exe" !define WEBSITE "https://www.torproject.org/" !define LICENSE "LICENSE" diff --git a/src/win32/orconfig.h b/src/win32/orconfig.h index 5cf0b1eb0b..442bb81570 100644 --- a/src/win32/orconfig.h +++ b/src/win32/orconfig.h @@ -218,7 +218,7 @@ #define USING_TWOS_COMPLEMENT /* Version number of package */ -#define VERSION "0.4.1.1-alpha" +#define VERSION "0.4.1.1-alpha-dev" From 57ee0e3af98e5dce398e8bc7f6f2b77e53208288 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 23 May 2019 08:24:29 -0400 Subject: [PATCH 1050/2557] Only reject POSTDESCRIPTOR purpose= when the purpose is unrecognized Fixes bug 30580; bugfix on 0.4.1.1-alpha. --- changes/ticket30580 | 4 ++++ src/feature/control/control_cmd.c | 8 +++++--- 2 files changed, 9 insertions(+), 3 deletions(-) create mode 100644 changes/ticket30580 diff --git a/changes/ticket30580 b/changes/ticket30580 new file mode 100644 index 0000000000..9ff1a33e41 --- /dev/null +++ b/changes/ticket30580 @@ -0,0 +1,4 @@ + o Minor bugfixes (controller): + - POSTDESCRIPTOR requests should work again. Previously, they were + broken if a "purpose=" flag was specified. Fixes bug 30580; + bugfix on 0.4.1.1-alpha. diff --git a/src/feature/control/control_cmd.c b/src/feature/control/control_cmd.c index 5555a2c5c4..17d5b0c7f3 100644 --- a/src/feature/control/control_cmd.c +++ b/src/feature/control/control_cmd.c @@ -1049,9 +1049,11 @@ handle_control_postdescriptor(control_connection_t *conn, line = config_line_find_case(args->kwargs, "purpose"); if (line) { purpose = router_purpose_from_string(line->value); - control_printf_endreply(conn, 552, "Unknown purpose \"%s\"", - line->value); - goto done; + if (purpose == ROUTER_PURPOSE_UNKNOWN) { + control_printf_endreply(conn, 552, "Unknown purpose \"%s\"", + line->value); + goto done; + } } line = config_line_find_case(args->kwargs, "cache"); if (line) { From e4d1187584038593a75140d9a8e47024c9eba04c Mon Sep 17 00:00:00 2001 From: Roger Dingledine Date: Wed, 18 Jul 2018 21:00:27 -0400 Subject: [PATCH 1051/2557] refactor logic to decide how much to package from inbuf no actual changes in behavior --- src/core/or/relay.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/core/or/relay.c b/src/core/or/relay.c index 5197ee6138..05fee57a1a 100644 --- a/src/core/or/relay.c +++ b/src/core/or/relay.c @@ -2102,14 +2102,15 @@ connection_edge_package_raw_inbuf(edge_connection_t *conn, int package_partial, if (!bytes_to_process) return 0; - if (!package_partial && bytes_to_process < RELAY_PAYLOAD_SIZE) - return 0; + length = RELAY_PAYLOAD_SIZE; - if (bytes_to_process > RELAY_PAYLOAD_SIZE) { - length = RELAY_PAYLOAD_SIZE; - } else { - length = bytes_to_process; + if (bytes_to_process < length) { /* not a full payload available */ + if (package_partial) + length = bytes_to_process; /* just take whatever's available now */ + else + return 0; /* nothing to do until we have a full payload */ } + stats_n_data_bytes_packaged += length; stats_n_data_cells_packaged += 1; From 530d1179ffe54ad0db2678142154fdd20f71cf53 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 17 May 2019 10:29:35 -0400 Subject: [PATCH 1052/2557] Extract length-deciding function from package_raw_inbuf. --- src/core/or/relay.c | 36 ++++++++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/src/core/or/relay.c b/src/core/or/relay.c index 05fee57a1a..7a121780af 100644 --- a/src/core/or/relay.c +++ b/src/core/or/relay.c @@ -2027,6 +2027,29 @@ uint64_t stats_n_data_cells_received = 0; * ever received were completely full of data. */ uint64_t stats_n_data_bytes_received = 0; +/** + * Helper. Return the number of bytes that should be put into a cell from a + * given edge connection on which n_available bytes are available. + */ +static size_t +connection_edge_get_inbuf_bytes_to_package(size_t n_available, + int package_partial) +{ + if (!n_available) + return 0; + + size_t length = RELAY_PAYLOAD_SIZE; + + if (n_available < length) { /* not a full payload available */ + if (package_partial) + length = n_available; /* just take whatever's available now */ + else + return 0; /* nothing to do until we have a full payload */ + } + + return length; +} + /** If conn has an entire relay payload of bytes on its inbuf (or * package_partial is true), and the appropriate package windows aren't * empty, grab a cell and send it down the circuit. @@ -2099,18 +2122,11 @@ connection_edge_package_raw_inbuf(edge_connection_t *conn, int package_partial, bytes_to_process = connection_get_inbuf_len(TO_CONN(conn)); } - if (!bytes_to_process) + length = connection_edge_get_inbuf_bytes_to_package(bytes_to_process, + package_partial); + if (!length) return 0; - length = RELAY_PAYLOAD_SIZE; - - if (bytes_to_process < length) { /* not a full payload available */ - if (package_partial) - length = bytes_to_process; /* just take whatever's available now */ - else - return 0; /* nothing to do until we have a full payload */ - } - stats_n_data_bytes_packaged += length; stats_n_data_cells_packaged += 1; From 2bb5d8148b2c15e9573a7f041ef631bdf4429b4b Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 15 May 2019 08:27:02 -0400 Subject: [PATCH 1053/2557] In coverage builds, avoid basic-block complexity in log_debug Ordinarily we skip calling log_fn(LOG_DEBUG,...) if debug logging is completely disabled. However, in coverage builds, this means that we get spurious complaints about partially covered basic blocks, in a way that makes our coverage determinism harder to check. --- src/lib/log/log.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/lib/log/log.h b/src/lib/log/log.h index a381220af0..4a3ea0ad55 100644 --- a/src/lib/log/log.h +++ b/src/lib/log/log.h @@ -194,6 +194,11 @@ void tor_log_get_logfile_names(struct smartlist_t *out); extern int log_global_min_severity_; +#ifdef TOR_COVERAGE +/* For coverage builds, we try to avoid our log_debug optimization, since it + * can have weird effects on internal macro coverage. */ +#define debug_logging_enabled() (1) +#else static inline bool debug_logging_enabled(void); /** * Return true iff debug logging is enabled for at least one domain. @@ -202,6 +207,7 @@ static inline bool debug_logging_enabled(void) { return PREDICT_UNLIKELY(log_global_min_severity_ == LOG_DEBUG); } +#endif void log_fn_(int severity, log_domain_mask_t domain, const char *funcname, const char *format, ...) From b88281024579d5f207d15d1a2cc54c113f8a2bde Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 15 May 2019 08:28:25 -0400 Subject: [PATCH 1054/2557] In coverage builds, use branch-free timeradd() and timersub() The ordinary definitions of timeradd() and timersub() contain a branch. However, in coverage builds, this means that we get spurious complaints about partially covered basic blocks, in a way that makes our coverage determinism harder to check. --- src/lib/wallclock/timeval.h | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/lib/wallclock/timeval.h b/src/lib/wallclock/timeval.h index 4967e939bf..33076adc8b 100644 --- a/src/lib/wallclock/timeval.h +++ b/src/lib/wallclock/timeval.h @@ -20,6 +20,27 @@ #include #endif +#ifdef TOR_COVERAGE +/* For coverage builds, we use a slower definition of these macros without + * branches, to make coverage consistent. */ +#undef timeradd +#undef timersub +#define timeradd(tv1,tv2,tvout) \ + do { \ + (tvout)->tv_sec = (tv1)->tv_sec + (tv2)->tv_sec; \ + (tvout)->tv_usec = (tv1)->tv_usec + (tv2)->tv_usec; \ + (tvout)->tv_sec += (tvout)->tv_usec / 1000000; \ + (tvout)->tv_usec %= 1000000; \ + } while (0) +#define timersub(tv1,tv2,tvout) \ + do { \ + (tvout)->tv_sec = (tv1)->tv_sec - (tv2)->tv_sec - 1; \ + (tvout)->tv_usec = (tv1)->tv_usec - (tv2)->tv_usec + 1000000; \ + (tvout)->tv_sec += (tvout)->tv_usec / 1000000; \ + (tvout)->tv_usec %= 1000000; \ + } while (0) +#endif + #ifndef timeradd /** Replacement for timeradd on platforms that do not have it: sets tvout to * the sum of tv1 and tv2. */ From 07ccffa989a8c6ac6ac216b6e729daed14372620 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 15 May 2019 14:58:46 -0400 Subject: [PATCH 1055/2557] Coverage: do not include test-rebind in coverage builds. Because it invokes the Tor mainloop, it does unpredictable things to test coverage of a lot of code that it doesn't actually test at all. (It is more an integration test than anything else.) --- src/test/include.am | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/test/include.am b/src/test/include.am index 824089bc47..85f9c9f880 100644 --- a/src/test/include.am +++ b/src/test/include.am @@ -32,8 +32,15 @@ endif if USEPYTHON TESTSCRIPTS += src/test/test_ntor.sh src/test/test_hs_ntor.sh src/test/test_bt.sh + +if COVERAGE_ENABLED +# ... +else +# Only do this when coverage is not on, since it invokes lots of code +# in a kind of unpredictable way. TESTSCRIPTS += src/test/test_rebind.sh endif +endif TESTS += src/test/test src/test/test-slow src/test/test-memwipe \ src/test/test_workqueue \ From 7893f2cd73310aaf20283f72dc05167a23584ac6 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 23 May 2019 14:36:01 -0400 Subject: [PATCH 1056/2557] cov-test-determinism: use the same RNG seed as in travis.yml We added this facility so that we could get deterministic PRNG behavior for coverage testing on tests that use a replaced PRNG. We need to have our coverage determinism tool test for this as well. --- scripts/test/cov-test-determinism.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/test/cov-test-determinism.sh b/scripts/test/cov-test-determinism.sh index 3b4f372e04..3458f96968 100755 --- a/scripts/test/cov-test-determinism.sh +++ b/scripts/test/cov-test-determinism.sh @@ -25,6 +25,9 @@ else fi if test "$run" = 1; then + # same seed as in travis.yml + TOR_TEST_RNG_SEED="636f766572616765" + export TOR_TEST_RNG_SEED while true; do make reset-gcov CD=coverage-raw/coverage-$(date +%s) From 6d9e47702fe52b0817a593117a7f4a3eecf06ad7 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 24 May 2019 07:42:59 -0400 Subject: [PATCH 1057/2557] changes file for test coverage --- changes/ticket30519 | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 changes/ticket30519 diff --git a/changes/ticket30519 b/changes/ticket30519 new file mode 100644 index 0000000000..efb25b9294 --- /dev/null +++ b/changes/ticket30519 @@ -0,0 +1,4 @@ + o Minor features (testing): + - When running tests in coverage mode, take additional care to make + our coverage deterministic, so that we can accurately track changes in + code coverage. Closes ticket 30519. From 94914e2a4d7f778e2f63493c09984a3721919744 Mon Sep 17 00:00:00 2001 From: Roger Dingledine Date: Sun, 26 May 2019 17:32:42 -0400 Subject: [PATCH 1058/2557] trivial whitespace fixes --- src/core/include.am | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/core/include.am b/src/core/include.am index 66ce2c9650..1a4b9fb8ab 100644 --- a/src/core/include.am +++ b/src/core/include.am @@ -37,7 +37,7 @@ LIBTOR_APP_A_SOURCES = \ src/core/or/circuitmux.c \ src/core/or/circuitmux_ewma.c \ src/core/or/circuitpadding.c \ - src/core/or/circuitpadding_machines.c \ + src/core/or/circuitpadding_machines.c \ src/core/or/circuitstats.c \ src/core/or/circuituse.c \ src/core/or/crypt_path.c \ @@ -185,7 +185,7 @@ MODULE_DIRAUTH_SOURCES = \ src/feature/dirauth/reachability.c \ src/feature/dirauth/recommend_pkg.c \ src/feature/dirauth/shared_random.c \ - src/feature/dirauth/shared_random_state.c \ + src/feature/dirauth/shared_random_state.c \ src/feature/dirauth/voteflags.c if BUILD_MODULE_DIRAUTH @@ -231,7 +231,7 @@ noinst_HEADERS += \ src/core/mainloop/cpuworker.h \ src/core/mainloop/mainloop.h \ src/core/mainloop/mainloop_pubsub.h \ - src/core/mainloop/mainloop_sys.h \ + src/core/mainloop/mainloop_sys.h \ src/core/mainloop/netstatus.h \ src/core/mainloop/periodic.h \ src/core/or/addr_policy_st.h \ @@ -248,7 +248,7 @@ noinst_HEADERS += \ src/core/or/circuitmux_ewma.h \ src/core/or/circuitstats.h \ src/core/or/circuitpadding.h \ - src/core/or/circuitpadding_machines.h \ + src/core/or/circuitpadding_machines.h \ src/core/or/circuituse.h \ src/core/or/command.h \ src/core/or/connection_edge.h \ From 0bc1241494a118d5319207a9f4683b993d389e77 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 17 May 2019 11:03:16 -0400 Subject: [PATCH 1059/2557] Make sure that we send at least some random data in RELAY_DATA cells Proposal 289 prevents SENDME-flooding by requiring the other side to authenticate the data it has received. But this data won't actually be random if they are downloading a known resource. "No problem", we said, "let's fell the empty parts of our cells with some randomness!" and we did that in #26871. Unfortunately, if the relay data payloads are all completely full, there won't be any empty parts for us to randomize. Therefore, we now pick random "randomness windows" between CIRCWINDOW_INCREMENT/2 and CIRCWINDOW_INCREMENT. We remember whether we have sent a cell containing at least 16 bytes of randomness in that window. If we haven't, then when the window is exhausted, we send one. (This window approach is designed to lower the number of rng checks we have to do. The number 16 is pulled out of a hat to change the attacker's guessing difficulty to "impossible".) Implements 28646. --- changes/ticket26846 | 6 ++ scripts/maint/practracker/exceptions.txt | 2 +- src/core/or/circuit_st.h | 11 ++++ src/core/or/circuitlist.c | 1 + src/core/or/circuitlist.h | 2 +- src/core/or/relay.c | 76 +++++++++++++++++++++--- src/core/or/relay.h | 1 + 7 files changed, 90 insertions(+), 9 deletions(-) create mode 100644 changes/ticket26846 diff --git a/changes/ticket26846 b/changes/ticket26846 new file mode 100644 index 0000000000..a18e231798 --- /dev/null +++ b/changes/ticket26846 @@ -0,0 +1,6 @@ + o Minor features (authenticated SENDME): + - Ensure that there is enough randomness on every circuit + to prevent an attacker from successfully predicting what SENDME cells + they will need to send: at a random interval, if we have not send + randomness already, leave some extra space at the end of a cell that + we can fill with random bytes. Closes ticket 26846. diff --git a/scripts/maint/practracker/exceptions.txt b/scripts/maint/practracker/exceptions.txt index 712e59dc5b..3a32e97beb 100644 --- a/scripts/maint/practracker/exceptions.txt +++ b/scripts/maint/practracker/exceptions.txt @@ -120,7 +120,7 @@ problem function-size /src/core/or/connection_or.c:connection_or_compute_authent problem file-size /src/core/or/policies.c 3249 problem function-size /src/core/or/policies.c:policy_summarize() 107 problem function-size /src/core/or/protover.c:protover_all_supported() 117 -problem file-size /src/core/or/relay.c 3173 +problem file-size /src/core/or/relay.c 3250 problem function-size /src/core/or/relay.c:circuit_receive_relay_cell() 127 problem function-size /src/core/or/relay.c:relay_send_command_from_edge_() 116 problem function-size /src/core/or/relay.c:connection_ap_process_end_not_open() 194 diff --git a/src/core/or/circuit_st.h b/src/core/or/circuit_st.h index 499bf93d6b..3c7b931614 100644 --- a/src/core/or/circuit_st.h +++ b/src/core/or/circuit_st.h @@ -92,6 +92,10 @@ struct circuit_t { /** True iff this circuit has received a DESTROY cell in either direction */ unsigned int received_destroy : 1; + /** True iff we have sent a sufficiently random data cell since last + * we reset send_randomness_after_n_cells. */ + unsigned int have_sent_sufficiently_random_cell : 1; + uint8_t state; /**< Current status of this circuit. */ uint8_t purpose; /**< Why are we creating this circuit? */ @@ -104,6 +108,13 @@ struct circuit_t { * circuit-level sendme cells to indicate that we're willing to accept * more. */ int deliver_window; + /** + * How many cells do we have until we need to send one that contains + * sufficient randomness? Used to ensure that authenticated SENDME cells + * will reflect some unpredictable information. + **/ + uint16_t send_randomness_after_n_cells; + /** FIFO containing the digest of the cells that are just before a SENDME is * sent by the client. It is done at the last cell before our package_window * goes down to 0 which is when we expect a SENDME. diff --git a/src/core/or/circuitlist.c b/src/core/or/circuitlist.c index 72952a8a52..ebbe7f0824 100644 --- a/src/core/or/circuitlist.c +++ b/src/core/or/circuitlist.c @@ -993,6 +993,7 @@ init_circuit_base(circuit_t *circ) circ->package_window = circuit_initial_package_window(); circ->deliver_window = CIRCWINDOW_START; + circuit_reset_sendme_randomness(circ); cell_queue_init(&circ->n_chan_cells); smartlist_add(circuit_get_global_list(), circ); diff --git a/src/core/or/circuitlist.h b/src/core/or/circuitlist.h index 6f5fce4875..80c1f7ac4e 100644 --- a/src/core/or/circuitlist.h +++ b/src/core/or/circuitlist.h @@ -218,7 +218,7 @@ void circuit_mark_all_dirty_circs_as_unusable(void); void circuit_synchronize_written_or_bandwidth(const circuit_t *c, circuit_channel_direction_t dir); MOCK_DECL(void, circuit_mark_for_close_, (circuit_t *circ, int reason, - int line, const char *file)); + int line, const char *cfile)); int circuit_get_cpath_len(origin_circuit_t *circ); int circuit_get_cpath_opened_len(const origin_circuit_t *); void circuit_clear_cpath(origin_circuit_t *circ); diff --git a/src/core/or/relay.c b/src/core/or/relay.c index 7a121780af..c48147dff8 100644 --- a/src/core/or/relay.c +++ b/src/core/or/relay.c @@ -533,6 +533,10 @@ relay_command_to_string(uint8_t command) } } +/** When padding a cell with randomness, leave this many zeros after the + * payload. */ +#define CELL_PADDING_GAP 4 + /** Return the offset where the padding should start. The data_len is * the relay payload length expected to be put in the cell. It can not be * bigger than RELAY_PAYLOAD_SIZE else this function assert(). @@ -556,7 +560,7 @@ get_pad_cell_offset(size_t data_len) /* If the offset is larger than the cell payload size, we return an offset * of zero indicating that no padding needs to be added. */ - size_t offset = RELAY_HEADER_SIZE + data_len + 4; + size_t offset = RELAY_HEADER_SIZE + data_len + CELL_PADDING_GAP; if (offset >= CELL_PAYLOAD_SIZE) { return 0; } @@ -2027,27 +2031,82 @@ uint64_t stats_n_data_cells_received = 0; * ever received were completely full of data. */ uint64_t stats_n_data_bytes_received = 0; +/** + * Called when initializing a circuit, or when we have reached the end of the + * window in which we need to send some randomness so that incoming sendme + * cells will be unpredictable. Resets the flags and picks a new window. + */ +void +circuit_reset_sendme_randomness(circuit_t *circ) +{ + circ->have_sent_sufficiently_random_cell = 0; + circ->send_randomness_after_n_cells = CIRCWINDOW_INCREMENT / 2 + + crypto_fast_rng_get_uint(get_thread_fast_rng(), CIRCWINDOW_INCREMENT / 2); +} + +/** + * Any relay data payload containing fewer than this many real bytes is + * considered to have enough randomness to. + **/ +#define RELAY_PAYLOAD_LENGTH_FOR_RANDOM_SENDMES \ + (RELAY_PAYLOAD_SIZE - CELL_PADDING_GAP - 16) + /** * Helper. Return the number of bytes that should be put into a cell from a * given edge connection on which n_available bytes are available. */ static size_t connection_edge_get_inbuf_bytes_to_package(size_t n_available, - int package_partial) + int package_partial, + circuit_t *on_circuit) { if (!n_available) return 0; - size_t length = RELAY_PAYLOAD_SIZE; + /* Do we need to force this payload to have space for randomness? */ + const bool force_random_bytes = + (on_circuit->send_randomness_after_n_cells == 0) && + (! on_circuit->have_sent_sufficiently_random_cell); - if (n_available < length) { /* not a full payload available */ + /* At most how much would we like to send in this cell? */ + size_t target_length; + if (force_random_bytes) { + target_length = RELAY_PAYLOAD_LENGTH_FOR_RANDOM_SENDMES; + } else { + target_length = RELAY_PAYLOAD_SIZE; + } + + /* Decide how many bytes we will actually put into this cell. */ + size_t package_length; + if (n_available >= target_length) { /* A full payload is available. */ + package_length = target_length; + } else { /* not a full payload available */ if (package_partial) - length = n_available; /* just take whatever's available now */ + package_length = n_available; /* just take whatever's available now */ else return 0; /* nothing to do until we have a full payload */ } - return length; + /* If we reach this point, we will be definitely sending the cell. */ + tor_assert_nonfatal(package_length > 0); + + if (package_length <= RELAY_PAYLOAD_LENGTH_FOR_RANDOM_SENDMES) { + /* This cell will have enough randomness in the padding to make a future + * sendme cell unpredictable. */ + on_circuit->have_sent_sufficiently_random_cell = 1; + } + + if (on_circuit->send_randomness_after_n_cells == 0) { + /* Either this cell, or some previous cell, had enough padding to + * ensure sendme unpredictability. */ + tor_assert_nonfatal(on_circuit->have_sent_sufficiently_random_cell); + /* Pick a new interval in which we need to send randomness. */ + circuit_reset_sendme_randomness(on_circuit); + } + + --on_circuit->send_randomness_after_n_cells; + + return package_length; } /** If conn has an entire relay payload of bytes on its inbuf (or @@ -2123,10 +2182,13 @@ connection_edge_package_raw_inbuf(edge_connection_t *conn, int package_partial, } length = connection_edge_get_inbuf_bytes_to_package(bytes_to_process, - package_partial); + package_partial, circ); if (!length) return 0; + /* If we reach this point, we will definitely be packaging bytes into + * a cell. */ + stats_n_data_bytes_packaged += length; stats_n_data_cells_packaged += 1; diff --git a/src/core/or/relay.h b/src/core/or/relay.h index 97d5d6d0f2..0fc308f7df 100644 --- a/src/core/or/relay.h +++ b/src/core/or/relay.h @@ -42,6 +42,7 @@ int connection_edge_package_raw_inbuf(edge_connection_t *conn, int package_partial, int *max_cells); void connection_edge_consider_sending_sendme(edge_connection_t *conn); +void circuit_reset_sendme_randomness(circuit_t *circ); extern uint64_t stats_n_data_cells_packaged; extern uint64_t stats_n_data_bytes_packaged; From fcd51fd49f8c30ab8d5d1d099d43e510187150c6 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 23 May 2019 09:29:24 -0400 Subject: [PATCH 1060/2557] Tests for deciding how full our relay cells should be --- src/core/or/relay.c | 2 +- src/core/or/relay.h | 3 ++ src/test/test_sendme.c | 79 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 83 insertions(+), 1 deletion(-) diff --git a/src/core/or/relay.c b/src/core/or/relay.c index c48147dff8..9f90a09699 100644 --- a/src/core/or/relay.c +++ b/src/core/or/relay.c @@ -2055,7 +2055,7 @@ circuit_reset_sendme_randomness(circuit_t *circ) * Helper. Return the number of bytes that should be put into a cell from a * given edge connection on which n_available bytes are available. */ -static size_t +STATIC size_t connection_edge_get_inbuf_bytes_to_package(size_t n_available, int package_partial, circuit_t *on_circuit) diff --git a/src/core/or/relay.h b/src/core/or/relay.h index 0fc308f7df..79036f97bd 100644 --- a/src/core/or/relay.h +++ b/src/core/or/relay.h @@ -123,6 +123,9 @@ STATIC int connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, edge_connection_t *conn, crypt_path_t *layer_hint); STATIC size_t get_pad_cell_offset(size_t payload_len); +STATIC size_t connection_edge_get_inbuf_bytes_to_package(size_t n_available, + int package_partial, + circuit_t *on_circuit); #endif /* defined(RELAY_PRIVATE) */ diff --git a/src/test/test_sendme.c b/src/test/test_sendme.c index fa5ae115ac..eb402232bc 100644 --- a/src/test/test_sendme.c +++ b/src/test/test_sendme.c @@ -270,6 +270,84 @@ test_cell_version_validation(void *arg) ; } +/* check our decisions about how much stuff to put into relay cells. */ +static void +test_package_payload_len(void *arg) +{ + (void)arg; + /* this is not a real circuit: it only has the fields needed for this + * test. */ + circuit_t *c = tor_malloc_zero(sizeof(circuit_t)); + + /* check initial conditions. */ + circuit_reset_sendme_randomness(c); + tt_assert(! c->have_sent_sufficiently_random_cell); + tt_int_op(c->send_randomness_after_n_cells, OP_GE, CIRCWINDOW_INCREMENT / 2); + tt_int_op(c->send_randomness_after_n_cells, OP_LT, CIRCWINDOW_INCREMENT); + + /* We have a bunch of cells before we need to send randomness, so the first + * few can be packaged full. */ + int initial = c->send_randomness_after_n_cells; + size_t n = connection_edge_get_inbuf_bytes_to_package(10000, 0, c); + tt_uint_op(RELAY_PAYLOAD_SIZE, OP_EQ, n); + n = connection_edge_get_inbuf_bytes_to_package(95000, 1, c); + tt_uint_op(RELAY_PAYLOAD_SIZE, OP_EQ, n); + tt_int_op(c->send_randomness_after_n_cells, OP_EQ, initial - 2); + + /* If package_partial isn't set, we won't package a partially full cell at + * all. */ + n = connection_edge_get_inbuf_bytes_to_package(RELAY_PAYLOAD_SIZE-1, 0, c); + tt_int_op(n, OP_EQ, 0); + /* no change in our state, since nothing was sent. */ + tt_assert(! c->have_sent_sufficiently_random_cell); + tt_int_op(c->send_randomness_after_n_cells, OP_EQ, initial - 2); + + /* If package_partial is set and the partial cell is not going to have + * _enough_ randomness, we package it, but we don't consider ourselves to + * have sent a sufficiently random cell. */ + n = connection_edge_get_inbuf_bytes_to_package(RELAY_PAYLOAD_SIZE-1, 1, c); + tt_int_op(n, OP_EQ, RELAY_PAYLOAD_SIZE-1); + tt_assert(! c->have_sent_sufficiently_random_cell); + tt_int_op(c->send_randomness_after_n_cells, OP_EQ, initial - 3); + + /* Make sure we set have_set_sufficiently_random_cell as appropriate. */ + n = connection_edge_get_inbuf_bytes_to_package(RELAY_PAYLOAD_SIZE-64, 1, c); + tt_int_op(n, OP_EQ, RELAY_PAYLOAD_SIZE-64); + tt_assert(c->have_sent_sufficiently_random_cell); + tt_int_op(c->send_randomness_after_n_cells, OP_EQ, initial - 4); + + /* Now let's look at what happens when we get down to zero. Since we have + * sent a sufficiently random cell, we will not force this one to have a gap. + */ + c->send_randomness_after_n_cells = 0; + n = connection_edge_get_inbuf_bytes_to_package(10000, 1, c); + tt_int_op(n, OP_EQ, RELAY_PAYLOAD_SIZE); + /* Now these will be reset. */ + tt_assert(! c->have_sent_sufficiently_random_cell); + tt_int_op(c->send_randomness_after_n_cells, OP_GE, + CIRCWINDOW_INCREMENT / 2 - 1); + + /* What would happen if we hadn't sent a sufficiently random cell? */ + c->send_randomness_after_n_cells = 0; + n = connection_edge_get_inbuf_bytes_to_package(10000, 1, c); + const size_t reduced_payload_size = RELAY_PAYLOAD_SIZE - 4 - 16; + tt_int_op(n, OP_EQ, reduced_payload_size); + /* Now these will be reset. */ + tt_assert(! c->have_sent_sufficiently_random_cell); + tt_int_op(c->send_randomness_after_n_cells, OP_GE, + CIRCWINDOW_INCREMENT / 2 - 1); + + /* Here is a fun case: if it's time to package a small cell, then + * package_partial==0 should mean we accept that many bytes. + */ + c->send_randomness_after_n_cells = 0; + n = connection_edge_get_inbuf_bytes_to_package(reduced_payload_size, 0, c); + tt_int_op(n, OP_EQ, reduced_payload_size); + + done: + tor_free(c); +} + struct testcase_t sendme_tests[] = { { "v1_record_digest", test_v1_record_digest, TT_FORK, NULL, NULL }, @@ -281,6 +359,7 @@ struct testcase_t sendme_tests[] = { NULL, NULL }, { "cell_version_validation", test_cell_version_validation, TT_FORK, NULL, NULL }, + { "package_payload_len", test_package_payload_len, 0, NULL, NULL }, END_OF_TESTCASES }; From 24a2352d56d807320c45fcdd8c74435bda4302c2 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 28 May 2019 19:45:50 -0400 Subject: [PATCH 1061/2557] Trivial fix for a trivial warning with gcc 9.1.1 Fix on 4e3d144fb0940d8ee5a89427d471ea3656e8e122; bug not in any released Tor. --- src/lib/net/address.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/net/address.c b/src/lib/net/address.c index 7dec4c8e25..546af800a9 100644 --- a/src/lib/net/address.c +++ b/src/lib/net/address.c @@ -339,7 +339,7 @@ tor_addr_to_str(char *dest, const tor_addr_t *addr, size_t len, int decorate) break; case AF_INET6: /* Shortest addr [ :: ] + \0 */ - if (len < (3 + (decorate ? 2 : 0))) + if (len < (3u + (decorate ? 2 : 0))) return NULL; if (decorate) From 7971b3a5a6fc8b964339b144ae33faf9db1c3869 Mon Sep 17 00:00:00 2001 From: Taylor R Campbell Date: Sat, 25 May 2019 03:55:24 +0000 Subject: [PATCH 1062/2557] Use MAP_INHERIT_ZERO or MAP_INHERIT_NONE if available. Fixes assertion failure in tests on NetBSD: slow/prob_distr/stochastic_log_logistic: [forking] May 25 03:56:58.091 [err] tor_assertion_failed_(): Bug: src/lib/crypt_ops/crypto_rand_fast.c:184: crypto_fast_rng_new_from_seed: Assertion inherit != INHERIT_RES_KEEP failed; aborting. (on Tor 0.4.1.1-alpha-dev 29955f13e5bc8e61) May 25 03:56:58.091 [err] Bug: Assertion inherit != INHERIT_RES_KEEP failed in crypto_fast_rng_new_from_seed at src/lib/crypt_ops/crypto_rand_fast.c:184: . (Stack trace not available) (on Tor 0.4.1.1-alpha-dev 29955f13e5bc8e61) [Lost connection!] --- src/lib/malloc/map_anon.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/lib/malloc/map_anon.c b/src/lib/malloc/map_anon.c index 2fc6e89ea2..2430f7ad11 100644 --- a/src/lib/malloc/map_anon.c +++ b/src/lib/malloc/map_anon.c @@ -50,11 +50,15 @@ #ifdef INHERIT_ZERO #define FLAG_ZERO INHERIT_ZERO +#elif defined(MAP_INHERIT_ZERO) +#define FLAG_ZERO MAP_INHERIT_ZERO #endif #ifdef INHERIT_NONE #define FLAG_NOINHERIT INHERIT_NONE #elif defined(VM_INHERIT_NONE) #define FLAG_NOINHERIT VM_INHERIT_NONE +#elif defined(MAP_INHERIT_NONE) +#define FLAG_NOINHERIT MAP_INHERIT_NONE #endif #elif defined(HAVE_MADVISE) From bdf685e47614ef5dca935b9fe9a608ffdd63a816 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 29 May 2019 09:05:26 -0400 Subject: [PATCH 1063/2557] Changes file for bug 30614 --- changes/bug30614 | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 changes/bug30614 diff --git a/changes/bug30614 b/changes/bug30614 new file mode 100644 index 0000000000..9f904bd115 --- /dev/null +++ b/changes/bug30614 @@ -0,0 +1,4 @@ + o Minor bugfixes (NetBSD): + - Fix usage of minherit() on NetBSD and other platforms that define + MAP_INHERIT_{ZERO,NONE} instead of INHERIT_{ZERO,NONE}. Fixes bug + 30614; bugfix on 0.4.0.2-alpha. Patch from Taylor Campbell. From 3789f22bcbfbc6de415a838e4c4bfb2555c7d6c3 Mon Sep 17 00:00:00 2001 From: David Goulet Date: Tue, 28 May 2019 09:44:06 -0400 Subject: [PATCH 1064/2557] hs: Implement a helper to repurpose a circuit When we repurpose a hidden service circuit, we need to clean up from the HS circuit map and any HS related data structured contained in the circuit. This commit adds an helper function that does it when repurposing a hidden service circuit. Fixes #29034 Signed-off-by: David Goulet --- changes/bug29034 | 5 +++++ src/core/or/circuituse.c | 6 ++++++ src/feature/hs/hs_circuit.c | 31 +++++++++++++++++++++++++++++++ src/feature/hs/hs_circuit.h | 1 + src/feature/rend/rendcommon.c | 11 +++++++++++ src/feature/rend/rendcommon.h | 2 ++ 6 files changed, 56 insertions(+) create mode 100644 changes/bug29034 diff --git a/changes/bug29034 b/changes/bug29034 new file mode 100644 index 0000000000..816b615009 --- /dev/null +++ b/changes/bug29034 @@ -0,0 +1,5 @@ + o Major bugfixes (Onion service reachability): + - Properly clean up the introduction point map and associated state when + circuits change purpose from onion service circuits to pathbias, + measurement, or other circuit types. This should fix some instances of + introduction point failure. Fixes bug 29034; bugfix on 0.3.2.1-alpha. diff --git a/src/core/or/circuituse.c b/src/core/or/circuituse.c index 02bfa15fb3..465d64215b 100644 --- a/src/core/or/circuituse.c +++ b/src/core/or/circuituse.c @@ -3052,6 +3052,12 @@ circuit_change_purpose(circuit_t *circ, uint8_t new_purpose) if (circ->purpose == new_purpose) return; + /* Take specific actions if we are repurposing a hidden service circuit. */ + if (circuit_purpose_is_hidden_service(circ->purpose) && + !circuit_purpose_is_hidden_service(new_purpose)) { + hs_circ_repurpose(circ); + } + if (CIRCUIT_IS_ORIGIN(circ)) { char old_purpose_desc[80] = ""; diff --git a/src/feature/hs/hs_circuit.c b/src/feature/hs/hs_circuit.c index e3873d2f18..2e59a357b3 100644 --- a/src/feature/hs/hs_circuit.c +++ b/src/feature/hs/hs_circuit.c @@ -24,6 +24,7 @@ #include "feature/nodelist/describe.h" #include "feature/nodelist/nodelist.h" #include "feature/rend/rendservice.h" +#include "feature/rend/rendcommon.h" #include "feature/stats/rephist.h" #include "lib/crypt_ops/crypto_dh.h" #include "lib/crypt_ops/crypto_rand.h" @@ -1269,3 +1270,33 @@ hs_circ_cleanup(circuit_t *circ) hs_circuitmap_remove_circuit(circ); } } + +/* The given circuit will be repurposed so take the appropriate actions. A + * cleanup from the HS maps and of all HS related structures is done. + * + * Once this function returns, the circuit can be safely repurposed. */ +void +hs_circ_repurpose(circuit_t *circ) +{ + origin_circuit_t *origin_circ; + + tor_assert(circ); + + /* Only repurposing an origin circuit is possible for HS. */ + if (!CIRCUIT_IS_ORIGIN(circ)) { + return; + } + origin_circ = TO_ORIGIN_CIRCUIT(circ); + + /* First, cleanup the circuit from the HS maps. */ + hs_circ_cleanup(circ); + + /* Depending on the version, different cleanup is done. */ + if (origin_circ->rend_data) { + /* v2. */ + rend_circ_cleanup(origin_circ); + } else if (origin_circ->hs_ident) { + /* v3. */ + hs_ident_circuit_free(origin_circ->hs_ident); + } +} diff --git a/src/feature/hs/hs_circuit.h b/src/feature/hs/hs_circuit.h index b8d8b25add..0786f3ee45 100644 --- a/src/feature/hs/hs_circuit.h +++ b/src/feature/hs/hs_circuit.h @@ -16,6 +16,7 @@ /* Cleanup function when the circuit is closed or/and freed. */ void hs_circ_cleanup(circuit_t *circ); +void hs_circ_repurpose(circuit_t *circ); /* Circuit API. */ int hs_circ_service_intro_has_opened(hs_service_t *service, diff --git a/src/feature/rend/rendcommon.c b/src/feature/rend/rendcommon.c index de48af795f..b10a5863b4 100644 --- a/src/feature/rend/rendcommon.c +++ b/src/feature/rend/rendcommon.c @@ -1045,3 +1045,14 @@ rend_circuit_pk_digest_eq(const origin_circuit_t *ocirc, match: return 1; } + +/* Cleanup the given circuit of all HS v2 data structure. */ +void +rend_circ_cleanup(origin_circuit_t *circ) +{ + tor_assert(circ); + + /* Both fields are set to NULL with these. */ + crypto_pk_free(circ->intro_key); + rend_data_free(circ->rend_data); +} diff --git a/src/feature/rend/rendcommon.h b/src/feature/rend/rendcommon.h index f136863c7a..c9a04846d7 100644 --- a/src/feature/rend/rendcommon.h +++ b/src/feature/rend/rendcommon.h @@ -71,6 +71,8 @@ int rend_non_anonymous_mode_enabled(const or_options_t *options); void assert_circ_anonymity_ok(const origin_circuit_t *circ, const or_options_t *options); +void rend_circ_cleanup(origin_circuit_t *circ); + #ifdef RENDCOMMON_PRIVATE STATIC int From 2d66250d8acfc61210442df94fe04df09c2dbab6 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 29 May 2019 10:10:57 -0400 Subject: [PATCH 1065/2557] Remove want_cmddata from HSFETCH, which does not in fact want data This looks a copy-and-paste error to me. Fixes bug 30646; bugfix on 0.4.1.1-alpha. --- changes/bug30646 | 4 ++++ src/feature/control/control_cmd.c | 1 - 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 changes/bug30646 diff --git a/changes/bug30646 b/changes/bug30646 new file mode 100644 index 0000000000..e95a54e3ef --- /dev/null +++ b/changes/bug30646 @@ -0,0 +1,4 @@ + o Minor bugfixes (controller): + - Repair the HSFETCH command so that it works again. Previously, it + expected a body when it shouldn't have. Fixes bug 30646; bugfix on + 0.4.1.1-alpha. diff --git a/src/feature/control/control_cmd.c b/src/feature/control/control_cmd.c index 17d5b0c7f3..abb579bd43 100644 --- a/src/feature/control/control_cmd.c +++ b/src/feature/control/control_cmd.c @@ -1385,7 +1385,6 @@ static const control_cmd_syntax_t hsfetch_syntax = { .min_args = 1, .max_args = 1, .accept_keywords = true, .allowed_keywords = hsfetch_keywords, - .want_cmddata = true, }; /** Implementation for the HSFETCH command. */ From ba9b0319b00692038e5d4aa2eff7cf5d6947e659 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 29 May 2019 11:25:47 -0400 Subject: [PATCH 1066/2557] Shutdown libevent _after_ the subsystems. This is necessary since shutting down libevent frees some pointer that the subsystems want to free themselves. A longer term solution will be to turn the evloop module into a subsystem itself, but for now it is best to do the minimal fix. Fixes bug 30629; bugfix on 0.4.1.1-alpha. --- src/app/main/shutdown.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/app/main/shutdown.c b/src/app/main/shutdown.c index e4dcaa1324..cc0091a9ab 100644 --- a/src/app/main/shutdown.c +++ b/src/app/main/shutdown.c @@ -157,10 +157,11 @@ tor_free_all(int postfork) if (!postfork) { release_lockfile(); } - tor_libevent_free_all(); subsystems_shutdown(); + tor_libevent_free_all(); + /* Stuff in util.c and address.c*/ if (!postfork) { esc_router_info(NULL); From 66eae4afffb35afe891ec14a3389a484ecb7b373 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 29 May 2019 16:08:33 -0400 Subject: [PATCH 1067/2557] Check whether gcc-hardening is runnable, and log an error if not Closes ticket 27530. --- changes/ticket27530 | 4 ++++ configure.ac | 11 +++++++++++ 2 files changed, 15 insertions(+) create mode 100644 changes/ticket27530 diff --git a/changes/ticket27530 b/changes/ticket27530 new file mode 100644 index 0000000000..8ae4f52668 --- /dev/null +++ b/changes/ticket27530 @@ -0,0 +1,4 @@ + o Minor features (compilation): + - Log a more useful error message when we are compiling and one of the + compile-time hardening options we have selected can be linked but + not executed. Closes ticket 27530. diff --git a/configure.ac b/configure.ac index e7f959f17a..1ecf82c662 100644 --- a/configure.ac +++ b/configure.ac @@ -1188,6 +1188,17 @@ m4_ifdef([AS_VAR_IF],[ TOR_CHECK_LDFLAGS(-pie, "$all_ldflags_for_check", "$all_libs_for_check") fi TOR_TRY_COMPILE_WITH_CFLAGS(-fwrapv, also_link, CFLAGS_FWRAPV="-fwrapv", true) + + AC_MSG_CHECKING([whether we can run hardened binaries]) + AC_RUN_IFELSE([AC_LANG_PROGRAM([], [return 0;])], + [AC_MSG_RESULT([yes])], + [AC_MSG_RESULT([no]) + AC_MSG_ERROR([dnl + We can link with compiler hardening options, but we can't run with them. + That's a bad sign! If you must, you can pass --disable-gcc-hardening to + configure, but it would be better to figure out what the underlying problem + is.])], + [AC_MSG_RESULT([cross])]) fi if test "$fragile_hardening" = "yes"; then From c390f1dd7ef6f458fff9064c82f3804581de07a9 Mon Sep 17 00:00:00 2001 From: teor Date: Thu, 15 Nov 2018 12:05:28 +1000 Subject: [PATCH 1068/2557] doc: Add a new Travis CI cron job when there's a new maint branch Part of 28453. --- doc/HACKING/ReleasingTor.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/doc/HACKING/ReleasingTor.md b/doc/HACKING/ReleasingTor.md index 4c87a366cc..c7cd0bc111 100644 --- a/doc/HACKING/ReleasingTor.md +++ b/doc/HACKING/ReleasingTor.md @@ -240,7 +240,11 @@ new Tor release: `maint-x.y.z` branch to "newversion-dev", and do a `merge -s ours` merge to avoid taking that change into master. -2. Forward-port the ChangeLog (and ReleaseNotes if appropriate) to the +2. If there is a new `maint-x.y.z` branch, create a Travis CI cron job that + builds the release every week. (It's ok to skip the weekly build if the + branch was updated in the last 24 hours.) + +3. Forward-port the ChangeLog (and ReleaseNotes if appropriate) to the master branch. -3. Keep an eye on the blog post, to moderate comments and answer questions. +4. Keep an eye on the blog post, to moderate comments and answer questions. From 6263d9e13f51d7f4f077034cb7701ba530c4fd6c Mon Sep 17 00:00:00 2001 From: teor Date: Thu, 15 Nov 2018 12:06:29 +1000 Subject: [PATCH 1069/2557] doc: Add End of Life Tor instructions Closes 28453. --- doc/HACKING/EndOfLifeTor.md | 52 +++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 doc/HACKING/EndOfLifeTor.md diff --git a/doc/HACKING/EndOfLifeTor.md b/doc/HACKING/EndOfLifeTor.md new file mode 100644 index 0000000000..36150bca6d --- /dev/null +++ b/doc/HACKING/EndOfLifeTor.md @@ -0,0 +1,52 @@ + +End of Life on an old release series +------------------------------------ + +Here are the steps that the maintainer should take when an old Tor release +series reaches End of Life: + +=== 0. Preliminaries + +0. A few months before End of Life: + Write a deprecation announcement. + Send the announcement out with every new release announcement. + +1. A month before End of Life: + Send the announcement to tor-announce, tor-talk, tor-relays, and the + packagers. + +2. A few days before End of Life: + Get at least two of weasel/arma/Sebastian to + remove the old version number from their approved versions list. + Give them a few days to do this if you can. + +=== 1. On the day + +1. Open tickets to remove the release from: + - the jenkins builds + - tor's Travis CI cron jobs + - chutney's Travis CI tests (#) + - stem's Travis CI tests (#) + +2. Close the milestone in Trac. To do this, go to Trac, log in, + select "Admin" near the top of the screen, then select "Milestones" from + the menu on the left. Click on the milestone for this version, and + select the "Completed" checkbox. By convention, we select the date as + the End of Life date. + +3. Replace NNN-backport with NNN-unreached-backport in all open trac tickets. + +4. If there are any remaining tickets in the milestone: + - merge_ready tickets are for backports: + - if there are no supported releases for the backport, close the ticket + - if there is an earlier (LTS) release for the backport, move the ticket + to that release + - other tickets should be closed (if we won't fix them) or moved to a + supported release (if we will fix them) + +5. Double-check: did the version get un-recommended in the consensus yet? + If not, don't announce until they have the up-to-date versions, or people + will get confused. + +6. Mail the end of life announcement to tor-announce, the packagers list, + and tor-relays. The current list of packagers is in ReleasingTor.md. From 3ba2e04ee7ee74d20a76536b195833caf227cc32 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 30 May 2019 09:05:35 -0400 Subject: [PATCH 1070/2557] Minor tweaks on EOL process First, clarify that this is for planned deprecations, not security-related issues. Second, we actually _don't_ want to remove the versions from the approved list before the EOL date, or people will start getting warnings too early. --- doc/HACKING/EndOfLifeTor.md | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/doc/HACKING/EndOfLifeTor.md b/doc/HACKING/EndOfLifeTor.md index 36150bca6d..502b862fc9 100644 --- a/doc/HACKING/EndOfLifeTor.md +++ b/doc/HACKING/EndOfLifeTor.md @@ -3,7 +3,9 @@ End of Life on an old release series ------------------------------------ Here are the steps that the maintainer should take when an old Tor release -series reaches End of Life: +series reaches End of Life. Note that they are _only_ for entire series that +have reached their planned EOL: they do not apply to security-related +deprecations of individual versions. === 0. Preliminaries @@ -15,11 +17,6 @@ series reaches End of Life: Send the announcement to tor-announce, tor-talk, tor-relays, and the packagers. -2. A few days before End of Life: - Get at least two of weasel/arma/Sebastian to - remove the old version number from their approved versions list. - Give them a few days to do this if you can. - === 1. On the day 1. Open tickets to remove the release from: @@ -44,9 +41,8 @@ series reaches End of Life: - other tickets should be closed (if we won't fix them) or moved to a supported release (if we will fix them) -5. Double-check: did the version get un-recommended in the consensus yet? - If not, don't announce until they have the up-to-date versions, or people - will get confused. - -6. Mail the end of life announcement to tor-announce, the packagers list, +5. Mail the end of life announcement to tor-announce, the packagers list, and tor-relays. The current list of packagers is in ReleasingTor.md. + +6. Get at least two of weasel/arma/Sebastian to remove the old version number + from their approved versions list. From d41427b054e981152659754ef2b1ee44bb76d722 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 30 May 2019 09:07:45 -0400 Subject: [PATCH 1071/2557] A couple of suggestions from dgoulet on EndOfLife.md --- doc/HACKING/EndOfLifeTor.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/doc/HACKING/EndOfLifeTor.md b/doc/HACKING/EndOfLifeTor.md index 502b862fc9..c7492f0ca0 100644 --- a/doc/HACKING/EndOfLifeTor.md +++ b/doc/HACKING/EndOfLifeTor.md @@ -44,5 +44,7 @@ deprecations of individual versions. 5. Mail the end of life announcement to tor-announce, the packagers list, and tor-relays. The current list of packagers is in ReleasingTor.md. -6. Get at least two of weasel/arma/Sebastian to remove the old version number - from their approved versions list. +6. Ask at least two of weasel/arma/Sebastian to remove the old version + number from their approved versions list. + +7. Update the CoreTorReleases wiki page. From 8f3430fc28be9ac16909fd02e0d48aad4ccc128e Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 30 May 2019 12:58:12 -0400 Subject: [PATCH 1072/2557] changes file for 30629 --- changes/bug30629 | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 changes/bug30629 diff --git a/changes/bug30629 b/changes/bug30629 new file mode 100644 index 0000000000..59fa96ee68 --- /dev/null +++ b/changes/bug30629 @@ -0,0 +1,6 @@ + o Minor bugfixes (shutdown, libevent, memory safety): + - Avoid use-after-free bugs when shutting down, by making sure that we + shut down libevent only after shutting down all of its users. We + believe these are harmless in practice, since they only occur on the + shutdown path, and do not involve any attacker-controlled data. Fixes + bug 30629; bugfix on 0.4.1.1-alpha. From de3eb8c5e40fb45821c54cc237e95c98b39d0e2a Mon Sep 17 00:00:00 2001 From: Mike Perry Date: Thu, 30 May 2019 15:15:01 -0700 Subject: [PATCH 1073/2557] Bug 30649: Check that machine is absent before warn --- src/core/or/circuitpadding.c | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/src/core/or/circuitpadding.c b/src/core/or/circuitpadding.c index b2315d822f..ae32713bfa 100644 --- a/src/core/or/circuitpadding.c +++ b/src/core/or/circuitpadding.c @@ -254,17 +254,25 @@ circpad_marked_circuit_for_padding(circuit_t *circ, int reason) return 0; // No machine wanted to keep the circuit open; mark for close } -/** Free all the machineinfos in circ that match machine_num. */ -static void +/** + * Free all the machineinfos in circ that match machine_num. + * + * Returns true if any machineinfos with that number were freed. + * False otherwise. */ +static int free_circ_machineinfos_with_machine_num(circuit_t *circ, int machine_num) { + int found = 0; FOR_EACH_CIRCUIT_MACHINE_BEGIN(i) { if (circ->padding_machine[i] && circ->padding_machine[i]->machine_num == machine_num) { circpad_circuit_machineinfo_free_idx(circ, i); circ->padding_machine[i] = NULL; + found = 1; } } FOR_EACH_CIRCUIT_MACHINE_END; + + return found; } /** @@ -2811,7 +2819,10 @@ circpad_handle_padding_negotiate(circuit_t *circ, cell_t *cell) if (negotiate->command == CIRCPAD_COMMAND_STOP) { /* Free the machine corresponding to this machine type */ - free_circ_machineinfos_with_machine_num(circ, negotiate->machine_type); + if (free_circ_machineinfos_with_machine_num(circ, + negotiate->machine_type)) { + goto done; + } log_fn(LOG_WARN, LD_CIRC, "Received circuit padding stop command for unknown machine."); goto err; From 9bef75a29a009f057c69c730b6bf3eccfc0c4b45 Mon Sep 17 00:00:00 2001 From: Mike Perry Date: Thu, 30 May 2019 15:18:16 -0700 Subject: [PATCH 1074/2557] Bug 30649: Changes file. --- changes/bug30649 | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 changes/bug30649 diff --git a/changes/bug30649 b/changes/bug30649 new file mode 100644 index 0000000000..3f5768bcb4 --- /dev/null +++ b/changes/bug30649 @@ -0,0 +1,4 @@ + o Minor bugfixes (circuit padding): + - On relays, properly check that a padding machine is absent before + logging a warn about it being absent. Fixes bug 30649; + bugfix on 0.4.1.1-alpha. From 6263755728f62b2535535f68960e601dccaead9d Mon Sep 17 00:00:00 2001 From: Mike Perry Date: Thu, 30 May 2019 16:20:56 -0700 Subject: [PATCH 1075/2557] Make some warns into protocol warns I'm not sure I agree with this option. --- src/core/or/circuitpadding.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/core/or/circuitpadding.c b/src/core/or/circuitpadding.c index ae32713bfa..84650a5a98 100644 --- a/src/core/or/circuitpadding.c +++ b/src/core/or/circuitpadding.c @@ -2805,14 +2805,14 @@ circpad_handle_padding_negotiate(circuit_t *circ, cell_t *cell) circpad_negotiate_t *negotiate; if (CIRCUIT_IS_ORIGIN(circ)) { - log_fn(LOG_WARN, LD_PROTOCOL, + log_fn(LOG_WARN, LD_CIRC, "Padding negotiate cell unsupported at origin."); return -1; } if (circpad_negotiate_parse(&negotiate, cell->payload+RELAY_HEADER_SIZE, CELL_PAYLOAD_SIZE-RELAY_HEADER_SIZE) < 0) { - log_fn(LOG_WARN, LD_CIRC, + log_fn(LOG_PROTOCOL_WARN, LD_CIRC, "Received malformed PADDING_NEGOTIATE cell; dropping."); return -1; } @@ -2823,7 +2823,7 @@ circpad_handle_padding_negotiate(circuit_t *circ, cell_t *cell) negotiate->machine_type)) { goto done; } - log_fn(LOG_WARN, LD_CIRC, + log_fn(LOG_PROTOCOL_WARN, LD_CIRC, "Received circuit padding stop command for unknown machine."); goto err; } else if (negotiate->command == CIRCPAD_COMMAND_START) { @@ -2864,21 +2864,21 @@ circpad_handle_padding_negotiated(circuit_t *circ, cell_t *cell, circpad_negotiated_t *negotiated; if (!CIRCUIT_IS_ORIGIN(circ)) { - log_fn(LOG_WARN, LD_PROTOCOL, + log_fn(LOG_PROTOCOL_WARN, LD_CIRC, "Padding negotiated cell unsupported at non-origin."); return -1; } /* Verify this came from the expected hop */ if (!circpad_padding_is_from_expected_hop(circ, layer_hint)) { - log_fn(LOG_WARN, LD_PROTOCOL, + log_fn(LOG_WARN, LD_CIRC, "Padding negotiated cell from wrong hop!"); return -1; } if (circpad_negotiated_parse(&negotiated, cell->payload+RELAY_HEADER_SIZE, CELL_PAYLOAD_SIZE-RELAY_HEADER_SIZE) < 0) { - log_fn(LOG_WARN, LD_CIRC, + log_fn(LOG_PROTOCOL_WARN, LD_CIRC, "Received malformed PADDING_NEGOTIATED cell; " "dropping."); return -1; From 5d4b4f948a338a41966bb2e15af5875d4df8a08f Mon Sep 17 00:00:00 2001 From: rl1987 Date: Fri, 31 May 2019 09:35:19 +0300 Subject: [PATCH 1076/2557] Mention Travis/Appveyor/Jenkins URLs in ReleasingTor.md --- changes/doc30630 | 3 +++ doc/HACKING/ReleasingTor.md | 9 +++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) create mode 100644 changes/doc30630 diff --git a/changes/doc30630 b/changes/doc30630 new file mode 100644 index 0000000000..0fbd8d4dd4 --- /dev/null +++ b/changes/doc30630 @@ -0,0 +1,3 @@ + o Documentation: + - Mention URLs for Travis/Appveyor/Jenkins in ReleasingTor.md. Closes + ticket 30630. diff --git a/doc/HACKING/ReleasingTor.md b/doc/HACKING/ReleasingTor.md index 4c87a366cc..d9864dba27 100644 --- a/doc/HACKING/ReleasingTor.md +++ b/doc/HACKING/ReleasingTor.md @@ -20,8 +20,11 @@ new Tor release: === I. Make sure it works -1. Make sure that CI passes: have a look at Travis, Appveyor, and - Jenkins. Make sure you're looking at the right branches. +1. Make sure that CI passes: have a look at Travis + (https://travis-ci.org/torproject/tor/branches), Appveyor + (https://ci.appveyor.com/project/torproject/tor/history), and + Jenkins (https://jenkins.torproject.org/view/tor/). + Make sure you're looking at the right branches. If there are any unexplained failures, try to fix them or figure them out. @@ -244,3 +247,5 @@ new Tor release: master branch. 3. Keep an eye on the blog post, to moderate comments and answer questions. + + From 41b94722e5c93ec06911f9c63296a65ce295c1ea Mon Sep 17 00:00:00 2001 From: David Goulet Date: Fri, 31 May 2019 10:43:01 -0400 Subject: [PATCH 1077/2557] test: Add test_hs_circ.c for HS circuit testing For now, only tests HS circuit repurpose function. Part of #29034 Signed-off-by: David Goulet --- src/test/include.am | 1 + src/test/test.c | 1 + src/test/test.h | 1 + src/test/test_hs_circ.c | 70 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 73 insertions(+) create mode 100644 src/test/test_hs_circ.c diff --git a/src/test/include.am b/src/test/include.am index ecb7689579..955016bee8 100644 --- a/src/test/include.am +++ b/src/test/include.am @@ -132,6 +132,7 @@ src_test_test_SOURCES += \ src/test/test_hs_common.c \ src/test/test_hs_config.c \ src/test/test_hs_cell.c \ + src/test/test_hs_circ.c \ src/test/test_hs_ntor.c \ src/test/test_hs_service.c \ src/test/test_hs_client.c \ diff --git a/src/test/test.c b/src/test/test.c index 58b468775c..70c5558150 100644 --- a/src/test/test.c +++ b/src/test/test.c @@ -877,6 +877,7 @@ struct testgroup_t testgroups[] = { { "legacy_hs/", hs_tests }, { "hs_cache/", hs_cache }, { "hs_cell/", hs_cell_tests }, + { "hs_circ/", hs_circ_tests }, { "hs_common/", hs_common_tests }, { "hs_config/", hs_config_tests }, { "hs_control/", hs_control_tests }, diff --git a/src/test/test.h b/src/test/test.h index aacc9dba87..8a5c675bc3 100644 --- a/src/test/test.h +++ b/src/test/test.h @@ -216,6 +216,7 @@ extern struct testcase_t geoip_tests[]; extern struct testcase_t hs_tests[]; extern struct testcase_t hs_cache[]; extern struct testcase_t hs_cell_tests[]; +extern struct testcase_t hs_circ_tests[]; extern struct testcase_t hs_common_tests[]; extern struct testcase_t hs_config_tests[]; extern struct testcase_t hs_control_tests[]; diff --git a/src/test/test_hs_circ.c b/src/test/test_hs_circ.c new file mode 100644 index 0000000000..af28af2573 --- /dev/null +++ b/src/test/test_hs_circ.c @@ -0,0 +1,70 @@ +/* Copyright (c) 2017-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file test_hs_circ.c + * \brief Test hidden service circuit functionality. + */ + +#define CIRCUITLIST_PRIVATE + +#include "test/test.h" +#include "test/test_helpers.h" +#include "test/log_test_helpers.h" + +#include "core/or/circuitbuild.h" +#include "core/or/circuitlist.h" +#include "core/or/circuituse.h" +#include "core/or/origin_circuit_st.h" + +#include "feature/hs/hs_circuit.h" +#include "feature/hs/hs_circuitmap.h" + +static void +test_circuit_repurpose(void *arg) +{ + origin_circuit_t *intro_circ = NULL; + const origin_circuit_t *search; + ed25519_keypair_t kp; + + (void) arg; + + hs_init(); + + intro_circ = origin_circuit_init(CIRCUIT_PURPOSE_S_ESTABLISH_INTRO, 0); + tt_assert(intro_circ); + ed25519_keypair_generate(&kp, 0); + + /* Register circuit in global map and make sure it is actually there. */ + hs_circuitmap_register_intro_circ_v3_service_side(intro_circ, + &kp.pubkey); + tt_assert(TO_CIRCUIT(intro_circ)->hs_token); + search = hs_circuitmap_get_intro_circ_v3_service_side(&kp.pubkey); + tt_mem_op(search, OP_EQ, intro_circ, sizeof(origin_circuit_t)); + + /* Setup circuit HS ident. We don't care about the service pubkey. */ + intro_circ->hs_ident = hs_ident_circuit_new(&kp.pubkey, + HS_IDENT_CIRCUIT_INTRO); + tt_assert(intro_circ->hs_ident); + + /* Trigger a repurpose. State should be cleaned up. */ + hs_circ_repurpose(TO_CIRCUIT(intro_circ)); + + /* Removed from map. */ + search = hs_circuitmap_get_intro_circ_v3_service_side(&kp.pubkey); + tt_assert(!search); + /* HS identifier has been removed. */ + tt_assert(!intro_circ->hs_ident); + + done: + circuit_free_(TO_CIRCUIT(intro_circ)); + hs_free_all(); +} + +struct testcase_t hs_circ_tests[] = { + { "repurpose", test_circuit_repurpose, TT_FORK, + NULL, NULL }, + + END_OF_TESTCASES +}; + From 5e594831c761626022c323e48a8df9cb9deb2291 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 31 May 2019 12:41:44 -0400 Subject: [PATCH 1078/2557] Practracker excpetions. --- scripts/maint/practracker/exceptions.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/maint/practracker/exceptions.txt b/scripts/maint/practracker/exceptions.txt index 3a32e97beb..f0c176560b 100644 --- a/scripts/maint/practracker/exceptions.txt +++ b/scripts/maint/practracker/exceptions.txt @@ -32,7 +32,7 @@ problem file-size /src/app/config/config.c 8520 problem include-count /src/app/config/config.c 88 problem function-size /src/app/config/config.c:options_act_reversible() 296 -problem function-size /src/app/config/config.c:options_act() 588 +problem function-size /src/app/config/config.c:options_act() 590 problem function-size /src/app/config/config.c:resolve_my_address() 192 problem function-size /src/app/config/config.c:options_validate() 1220 problem function-size /src/app/config/config.c:options_init_from_torrc() 210 @@ -136,7 +136,7 @@ problem function-size /src/feature/client/addressmap.c:addressmap_rewrite() 112 problem function-size /src/feature/client/bridges.c:rewrite_node_address_for_bridge() 126 problem function-size /src/feature/client/circpathbias.c:pathbias_measure_close_rate() 108 problem function-size /src/feature/client/dnsserv.c:evdns_server_callback() 153 -problem file-size /src/feature/client/entrynodes.c 3817 +problem file-size /src/feature/client/entrynodes.c 3820 problem function-size /src/feature/client/entrynodes.c:entry_guards_upgrade_waiting_circuits() 153 problem function-size /src/feature/client/entrynodes.c:entry_guard_parse_from_state() 246 problem function-size /src/feature/client/transports.c:handle_proxy_line() 108 From 180048e013c06ee67c053186aab46ff94cea0489 Mon Sep 17 00:00:00 2001 From: Roger Dingledine Date: Sat, 1 Jun 2019 15:37:51 -0400 Subject: [PATCH 1079/2557] fix some simple typos --- ChangeLog | 6 +++--- ReleaseNotes | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/ChangeLog b/ChangeLog index cda4c11e07..86f39bfd71 100644 --- a/ChangeLog +++ b/ChangeLog @@ -641,7 +641,7 @@ Changes in version 0.4.0.3-alpha - 2019-03-22 Changes in version 0.3.5.8 - 2019-02-21 - Tor 0.3.5.8 backports serveral fixes from later releases, including fixes + Tor 0.3.5.8 backports several fixes from later releases, including fixes for an annoying SOCKS-parsing bug that affected users in earlier 0.3.5.x releases. @@ -660,7 +660,7 @@ Changes in version 0.3.5.8 - 2019-02-21 o Major bugfixes (networking, backport from 0.4.0.2-alpha): - Gracefully handle empty username/password fields in SOCKS5 - username/password auth messsage and allow SOCKS5 handshake to + username/password auth message and allow SOCKS5 handshake to continue. Previously, we had rejected these handshakes, breaking certain applications. Fixes bug 29175; bugfix on 0.3.5.1-alpha. @@ -867,7 +867,7 @@ Changes in version 0.4.0.2-alpha - 2019-02-21 - Check that bugfix versions in changes files look like Tor versions from the versions spec. Warn when bugfixes claim to be on a future release. Closes ticket 27761. - - Provide a git pre-commit hook that disallows commiting if we have + - Provide a git pre-commit hook that disallows committing if we have any failures in our code and changelog formatting checks. It is now available in scripts/maint/pre-commit.git-hook. Implements feature 28976. diff --git a/ReleaseNotes b/ReleaseNotes index badc5e6d0c..788804236b 100644 --- a/ReleaseNotes +++ b/ReleaseNotes @@ -71,7 +71,7 @@ Changes in version 0.4.0.5 - 2019-05-02 o Major bugfixes (networking): - Gracefully handle empty username/password fields in SOCKS5 - username/password auth messsage and allow SOCKS5 handshake to + username/password auth message and allow SOCKS5 handshake to continue. Previously, we had rejected these handshakes, breaking certain applications. Fixes bug 29175; bugfix on 0.3.5.1-alpha. @@ -140,7 +140,7 @@ Changes in version 0.4.0.5 - 2019-05-02 - Check that bugfix versions in changes files look like Tor versions from the versions spec. Warn when bugfixes claim to be on a future release. Closes ticket 27761. - - Provide a git pre-commit hook that disallows commiting if we have + - Provide a git pre-commit hook that disallows committing if we have any failures in our code and changelog formatting checks. It is now available in scripts/maint/pre-commit.git-hook. Implements feature 28976. @@ -602,7 +602,7 @@ Changes in version 0.4.0.5 - 2019-05-02 Changes in version 0.3.5.8 - 2019-02-21 - Tor 0.3.5.8 backports serveral fixes from later releases, including fixes + Tor 0.3.5.8 backports several fixes from later releases, including fixes for an annoying SOCKS-parsing bug that affected users in earlier 0.3.5.x releases. From 33382184b67d43b859de2f50d24cc7955b9f0db7 Mon Sep 17 00:00:00 2001 From: David Goulet Date: Mon, 3 Jun 2019 14:31:51 -0400 Subject: [PATCH 1080/2557] sendme: Do not decrement window in a log_debug() If "Log debug ..." is not set, the decrement never happens. This lead to the package/deliver window to be out of sync at the stream level and thus breaking the connection after 50+ cells. Fixes #30628 Signed-off-by: David Goulet --- changes/ticket30628 | 5 +++++ src/core/or/sendme.c | 4 +++- 2 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 changes/ticket30628 diff --git a/changes/ticket30628 b/changes/ticket30628 new file mode 100644 index 0000000000..7128c5acf7 --- /dev/null +++ b/changes/ticket30628 @@ -0,0 +1,5 @@ + o Major bugfixes (Flow Control, SENDME): + - The decrement of the stream-level package window was done in a log_debug() + statement meaning that if the debug logs were not enabled, the decrement + would never happen and thus the window would be out of sync with the other + end point. Fixes bug 30628; bugfix on 0.4.1.1-alpha. diff --git a/src/core/or/sendme.c b/src/core/or/sendme.c index 7d409a16ad..47ac95f3cf 100644 --- a/src/core/or/sendme.c +++ b/src/core/or/sendme.c @@ -619,7 +619,9 @@ int sendme_note_stream_data_packaged(edge_connection_t *conn) { tor_assert(conn); - log_debug(LD_APP, "Stream package_window now %d.", --conn->package_window); + + --conn->package_window; + log_debug(LD_APP, "Stream package_window now %d.", conn->package_window); return conn->package_window; } From dc5cdde60c5efb1a37d919134e28b6445e235765 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 4 Jun 2019 08:29:43 -0400 Subject: [PATCH 1081/2557] update practracker --- scripts/maint/practracker/exceptions.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/maint/practracker/exceptions.txt b/scripts/maint/practracker/exceptions.txt index 3a32e97beb..e0db60efbf 100644 --- a/scripts/maint/practracker/exceptions.txt +++ b/scripts/maint/practracker/exceptions.txt @@ -54,7 +54,7 @@ problem function-size /src/app/main/main.c:sandbox_init_filter() 291 problem function-size /src/app/main/main.c:run_tor_main_loop() 105 problem function-size /src/app/main/ntmain.c:nt_service_install() 125 problem include-count /src/app/main/shutdown.c 52 -problem file-size /src/core/mainloop/connection.c 5560 +problem file-size /src/core/mainloop/connection.c 5570 problem include-count /src/core/mainloop/connection.c 62 problem function-size /src/core/mainloop/connection.c:connection_free_minimal() 185 problem function-size /src/core/mainloop/connection.c:connection_listener_new() 328 From 8cb6b2b9ab74d187e4839d298fbce06dafeb33f0 Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Wed, 5 Jun 2019 14:56:28 +0300 Subject: [PATCH 1082/2557] Fix typo in #29670 changes file. --- changes/bug29670 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changes/bug29670 b/changes/bug29670 index 00b0c33327..288bbbf62f 100644 --- a/changes/bug29670 +++ b/changes/bug29670 @@ -1,4 +1,4 @@ o Minor bugfixes (configuration, proxies): - Fix a bug that prevented us from supporting SOCKS5 proxies that want - authentication along with configued (but unused!) + authentication along with configured (but unused!) ClientTransportPlugins. Fixes bug 29670; bugfix on 0.2.6.1-alpha. From e8aab46a3a95e6d424553625c2a671d79c583d22 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Fri, 17 May 2019 19:24:26 +0300 Subject: [PATCH 1083/2557] Document TOR_UPSTREAM_REMOTE_NAME env variable --- scripts/git/pre-push.git-hook | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/git/pre-push.git-hook b/scripts/git/pre-push.git-hook index 00e048029e..c9e72a4d43 100755 --- a/scripts/git/pre-push.git-hook +++ b/scripts/git/pre-push.git-hook @@ -8,6 +8,9 @@ # # To install this script, copy it into .git/hooks/pre-push path in your # local copy of git repository. Make sure it has permission to execute. +# Furthermore, make sure that TOR_UPSTREAM_REMOTE_NAME environment +# variable is set to local name of git remote that corresponds to upstream +# repository on e.g. git.torproject.org. # # The following sample script was used as starting point: # https://github.com/git/git/blob/master/templates/hooks--pre-push.sample From ae490189f8fc5291152ce9a49b5b8a5b0e1ddf1a Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 5 Jun 2019 09:06:43 -0400 Subject: [PATCH 1084/2557] practracker update --- scripts/maint/practracker/exceptions.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/maint/practracker/exceptions.txt b/scripts/maint/practracker/exceptions.txt index 1445475f5f..c596482504 100644 --- a/scripts/maint/practracker/exceptions.txt +++ b/scripts/maint/practracker/exceptions.txt @@ -92,7 +92,7 @@ problem function-size /src/core/or/circuitlist.c:circuits_handle_oom() 117 problem function-size /src/core/or/circuitmux.c:circuitmux_set_policy() 110 problem function-size /src/core/or/circuitmux.c:circuitmux_attach_circuit() 114 problem function-size /src/core/or/circuitstats.c:circuit_build_times_parse_state() 124 -problem file-size /src/core/or/circuituse.c 3156 +problem file-size /src/core/or/circuituse.c 3162 problem function-size /src/core/or/circuituse.c:circuit_is_acceptable() 133 problem function-size /src/core/or/circuituse.c:circuit_expire_building() 394 problem function-size /src/core/or/circuituse.c:circuit_log_ancient_one_hop_circuits() 126 From 0982d0136996179e1c4bcf26d2890e12301f6aad Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 5 Jun 2019 09:24:18 -0400 Subject: [PATCH 1085/2557] Start a changelog for 0.4.1.2-alpha --- ChangeLog | 115 ++++++++++++++++++++++++++++++++++++++++++++ changes/bug29034 | 5 -- changes/bug29670 | 4 -- changes/bug29875 | 11 ----- changes/bug30286 | 4 -- changes/bug30316 | 4 -- changes/bug30561 | 6 --- changes/bug30614 | 4 -- changes/bug30629 | 6 --- changes/bug30646 | 4 -- changes/ticket26846 | 6 --- changes/ticket28878 | 11 ----- changes/ticket29617 | 4 -- changes/ticket29702 | 4 -- changes/ticket30150 | 4 -- changes/ticket30519 | 4 -- changes/ticket30539 | 4 -- changes/ticket30580 | 4 -- changes/ticket30628 | 5 -- 19 files changed, 115 insertions(+), 94 deletions(-) delete mode 100644 changes/bug29034 delete mode 100644 changes/bug29670 delete mode 100644 changes/bug29875 delete mode 100644 changes/bug30286 delete mode 100644 changes/bug30316 delete mode 100644 changes/bug30561 delete mode 100644 changes/bug30614 delete mode 100644 changes/bug30629 delete mode 100644 changes/bug30646 delete mode 100644 changes/ticket26846 delete mode 100644 changes/ticket28878 delete mode 100644 changes/ticket29617 delete mode 100644 changes/ticket29702 delete mode 100644 changes/ticket30150 delete mode 100644 changes/ticket30519 delete mode 100644 changes/ticket30539 delete mode 100644 changes/ticket30580 delete mode 100644 changes/ticket30628 diff --git a/ChangeLog b/ChangeLog index 86f39bfd71..f3eac65d52 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,118 @@ +Changes in version 0.4.1.2-alpha - 2019-06-05 + Tor 0.4.1.2-alpha resolves numerous bugs--some of them from the previous + alpha, and some much older. It also contains minor testing improvements, + and an improvement to the security of our authenticated sendme + implementation. + + o Major bugfixes (bridges): + - Consider our directory information to have changed when our list of + bridges changes. Previously, Tor would not re-compute the status of its + directory information when bridges changed, and therefore would not + realize that it was no longer able to build circuits. Fixes part of bug + 29875. + - Do not count previously configured working bridges towards our total of + working bridges. Previously, when Tor's list of bridges changed, it + would think that the old bridges were still usable, and delay fetching + router descriptors for the new ones. Fixes part of bug 29875; bugfix + on 0.3.0.1-alpha. + + o Major bugfixes (Flow Control, SENDME): + - The decrement of the stream-level package window was done in a log_debug() + statement meaning that if the debug logs were not enabled, the decrement + would never happen and thus the window would be out of sync with the other + end point. Fixes bug 30628; bugfix on 0.4.1.1-alpha. + + o Major bugfixes (Onion service reachability): + - Properly clean up the introduction point map and associated state when + circuits change purpose from onion service circuits to pathbias, + measurement, or other circuit types. This should fix some instances of + introduction point failure. Fixes bug 29034; bugfix on 0.3.2.1-alpha. + + o Minor features (authenticated SENDME): + - Ensure that there is enough randomness on every circuit + to prevent an attacker from successfully predicting what SENDME cells + they will need to send: at a random interval, if we have not send + randomness already, leave some extra space at the end of a cell that + we can fill with random bytes. Closes ticket 26846. + + o Minor features (continuous integration): + - When running coverage builds on Travis, we now set TOR_TEST_RNG_SEED, + to avoid RNG-based coverage differences. + Part of ticket 28878. + + o Minor features (maintenance): + - Add a new "make autostyle" target that developers can use to + apply all automatic Tor style and consistency conversions to the + codebase. Closes ticket 30539. + + o Minor features (testing): + - The circuitpadding tests now use a reproducible RNG implementation, + so that if a test fails, we can learn why. Part of ticket 28878. + - Tor's tests now support an environment variable, TOR_TEST_RNG_SEED, + to set the RNG seed for tests that use a reproducible RNG. + Part of ticket 28878. + - When running tests in coverage mode, take additional care to make + our coverage deterministic, so that we can accurately track changes in + code coverage. Closes ticket 30519. + + o Minor bugfixes (configuration, proxies): + - Fix a bug that prevented us from supporting SOCKS5 proxies that want + authentication along with configured (but unused!) + ClientTransportPlugins. Fixes bug 29670; bugfix on 0.2.6.1-alpha. + + o Minor bugfixes (controller): + - POSTDESCRIPTOR requests should work again. Previously, they were + broken if a "purpose=" flag was specified. Fixes bug 30580; + bugfix on 0.4.1.1-alpha. + - Repair the HSFETCH command so that it works again. Previously, it + expected a body when it shouldn't have. Fixes bug 30646; bugfix on + 0.4.1.1-alpha. + + o Minor bugfixes (developer tooling): + - Fix pre-push hook to refrain from rejecting fixup and squash commits + when pushing to non-upstream git remote. Fixes bug 30286; bugfix on + 0.4.0.1-alpha. + + o Minor bugfixes (directory authority): + - Move the "bandwidth-file-headers" line in directory authority votes + so that it conforms to dir-spec.txt. Fixes bug 30316; bugfix on + 0.3.5.1-alpha. + + o Minor bugfixes (NetBSD): + - Fix usage of minherit() on NetBSD and other platforms that define + MAP_INHERIT_{ZERO,NONE} instead of INHERIT_{ZERO,NONE}. Fixes bug + 30614; bugfix on 0.4.0.2-alpha. Patch from Taylor Campbell. + + o Minor bugfixes (out-of-memory handler): + - When purging the DNS cache because of an out-of-memory condition, + try purging just the older entries at first. Previously, we would + purge the whole thing. Fixes bug 29617; bugfix on 0.3.5.1-alpha. + + o Minor bugfixes (portability): + - Avoid crashing in our tor_vasprintf() implementation on systems that + define neither vasprintf() nor _vscprintf(). (This bug has been here + long enough that we question whether people are running Tor on such + systems, but we're applying the fix out of caution.) Fixes bug 30561; + bugfix on 0.2.8.2-alpha. Found and fixed by Tobias Stoeckmann. + + o Minor bugfixes (shutdown, libevent, memory safety): + - Avoid use-after-free bugs when shutting down, by making sure that we + shut down libevent only after shutting down all of its users. We + believe these are harmless in practice, since they only occur on the + shutdown path, and do not involve any attacker-controlled data. Fixes + bug 30629; bugfix on 0.4.1.1-alpha. + + o Minor bugfixes (static analysis): + - Fix several spurious Coverity warnings about the unit tests, to lower our + chances of missing any real warnings in the future. Fixes bug 30150; + bugfix on 0.3.5.1-alpha and various other Tor versions. + + o Testing: + - Specify torrc paths (with empty files) when launching tor in + integration tests; refrain from reading user and system torrcs. + Resolves issue 29702. + + Changes in version 0.4.1.1-alpha - 2019-05-22 This is the first alpha in the 0.4.1.x series. It introduces lightweight circuit padding to make some onion-service circuits harder diff --git a/changes/bug29034 b/changes/bug29034 deleted file mode 100644 index 816b615009..0000000000 --- a/changes/bug29034 +++ /dev/null @@ -1,5 +0,0 @@ - o Major bugfixes (Onion service reachability): - - Properly clean up the introduction point map and associated state when - circuits change purpose from onion service circuits to pathbias, - measurement, or other circuit types. This should fix some instances of - introduction point failure. Fixes bug 29034; bugfix on 0.3.2.1-alpha. diff --git a/changes/bug29670 b/changes/bug29670 deleted file mode 100644 index 288bbbf62f..0000000000 --- a/changes/bug29670 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (configuration, proxies): - - Fix a bug that prevented us from supporting SOCKS5 proxies that want - authentication along with configured (but unused!) - ClientTransportPlugins. Fixes bug 29670; bugfix on 0.2.6.1-alpha. diff --git a/changes/bug29875 b/changes/bug29875 deleted file mode 100644 index 58a1c871cd..0000000000 --- a/changes/bug29875 +++ /dev/null @@ -1,11 +0,0 @@ - o Major bugfixes (bridges): - - Do not count previously configured working bridges towards our total of - working bridges. Previously, when Tor's list of bridges changed, it - would think that the old bridges were still usable, and delay fetching - router descriptors for the new ones. Fixes part of bug 29875; bugfix - on 0.3.0.1-alpha. - - Consider our directory information to have changed when our list of - bridges changes. Previously, Tor would not re-compute the status of its - directory information when bridges changed, and therefore would not - realize that it was no longer able to build circuits. Fixes part of bug - 29875. diff --git a/changes/bug30286 b/changes/bug30286 deleted file mode 100644 index f2fc67a484..0000000000 --- a/changes/bug30286 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (developer tooling): - - Fix pre-push hook to refrain from rejecting fixup and squash commits - when pushing to non-upstream git remote. Fixes bug 30286; bugfix on - 0.4.0.1-alpha. diff --git a/changes/bug30316 b/changes/bug30316 deleted file mode 100644 index 3e396318ad..0000000000 --- a/changes/bug30316 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (directory authority): - - Move the "bandwidth-file-headers" line in directory authority votes - so that it conforms to dir-spec.txt. Fixes bug 30316; bugfix on - 0.3.5.1-alpha. diff --git a/changes/bug30561 b/changes/bug30561 deleted file mode 100644 index afb3f02c62..0000000000 --- a/changes/bug30561 +++ /dev/null @@ -1,6 +0,0 @@ - o Minor bugfixes (portability): - - Avoid crashing in our tor_vasprintf() implementation on systems that - define neither vasprintf() nor _vscprintf(). (This bug has been here - long enough that we question whether people are running Tor on such - systems, but we're applying the fix out of caution.) Fixes bug 30561; - bugfix on 0.2.8.2-alpha. Found and fixed by Tobias Stoeckmann. diff --git a/changes/bug30614 b/changes/bug30614 deleted file mode 100644 index 9f904bd115..0000000000 --- a/changes/bug30614 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (NetBSD): - - Fix usage of minherit() on NetBSD and other platforms that define - MAP_INHERIT_{ZERO,NONE} instead of INHERIT_{ZERO,NONE}. Fixes bug - 30614; bugfix on 0.4.0.2-alpha. Patch from Taylor Campbell. diff --git a/changes/bug30629 b/changes/bug30629 deleted file mode 100644 index 59fa96ee68..0000000000 --- a/changes/bug30629 +++ /dev/null @@ -1,6 +0,0 @@ - o Minor bugfixes (shutdown, libevent, memory safety): - - Avoid use-after-free bugs when shutting down, by making sure that we - shut down libevent only after shutting down all of its users. We - believe these are harmless in practice, since they only occur on the - shutdown path, and do not involve any attacker-controlled data. Fixes - bug 30629; bugfix on 0.4.1.1-alpha. diff --git a/changes/bug30646 b/changes/bug30646 deleted file mode 100644 index e95a54e3ef..0000000000 --- a/changes/bug30646 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (controller): - - Repair the HSFETCH command so that it works again. Previously, it - expected a body when it shouldn't have. Fixes bug 30646; bugfix on - 0.4.1.1-alpha. diff --git a/changes/ticket26846 b/changes/ticket26846 deleted file mode 100644 index a18e231798..0000000000 --- a/changes/ticket26846 +++ /dev/null @@ -1,6 +0,0 @@ - o Minor features (authenticated SENDME): - - Ensure that there is enough randomness on every circuit - to prevent an attacker from successfully predicting what SENDME cells - they will need to send: at a random interval, if we have not send - randomness already, leave some extra space at the end of a cell that - we can fill with random bytes. Closes ticket 26846. diff --git a/changes/ticket28878 b/changes/ticket28878 deleted file mode 100644 index 73ca47c72f..0000000000 --- a/changes/ticket28878 +++ /dev/null @@ -1,11 +0,0 @@ - o Minor features (testing): - - The circuitpadding tests now use a reproducible RNG implementation, - so that if a test fails, we can learn why. Part of ticket 28878. - - Tor's tests now support an environment variable, TOR_TEST_RNG_SEED, - to set the RNG seed for tests that use a reproducible RNG. - Part of ticket 28878. - - o Minor features (continuous integration): - - When running coverage builds on Travis, we now set TOR_TEST_RNG_SEED, - to avoid RNG-based coverage differences. - Part of ticket 28878. diff --git a/changes/ticket29617 b/changes/ticket29617 deleted file mode 100644 index 4d50ea9627..0000000000 --- a/changes/ticket29617 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (out-of-memory handler): - - When purging the DNS cache because of an out-of-memory condition, - try purging just the older entries at first. Previously, we would - purge the whole thing. Fixes bug 29617; bugfix on 0.3.5.1-alpha. diff --git a/changes/ticket29702 b/changes/ticket29702 deleted file mode 100644 index e1cc1f867b..0000000000 --- a/changes/ticket29702 +++ /dev/null @@ -1,4 +0,0 @@ - o Testing: - - Specify torrc paths (with empty files) when launching tor in - integration tests; refrain from reading user and system torrcs. - Resolves issue 29702. diff --git a/changes/ticket30150 b/changes/ticket30150 deleted file mode 100644 index 70808fa5db..0000000000 --- a/changes/ticket30150 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (static analysis): - - Fix several spurious Coverity warnings about the unit tests, to lower our - chances of missing any real warnings in the future. Fixes bug 30150; - bugfix on 0.3.5.1-alpha and various other Tor versions. diff --git a/changes/ticket30519 b/changes/ticket30519 deleted file mode 100644 index efb25b9294..0000000000 --- a/changes/ticket30519 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor features (testing): - - When running tests in coverage mode, take additional care to make - our coverage deterministic, so that we can accurately track changes in - code coverage. Closes ticket 30519. diff --git a/changes/ticket30539 b/changes/ticket30539 deleted file mode 100644 index 8d105eacb9..0000000000 --- a/changes/ticket30539 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor features (maintenance): - - Add a new "make autostyle" target that developers can use to - apply all automatic Tor style and consistency conversions to the - codebase. Closes ticket 30539. diff --git a/changes/ticket30580 b/changes/ticket30580 deleted file mode 100644 index 9ff1a33e41..0000000000 --- a/changes/ticket30580 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (controller): - - POSTDESCRIPTOR requests should work again. Previously, they were - broken if a "purpose=" flag was specified. Fixes bug 30580; - bugfix on 0.4.1.1-alpha. diff --git a/changes/ticket30628 b/changes/ticket30628 deleted file mode 100644 index 7128c5acf7..0000000000 --- a/changes/ticket30628 +++ /dev/null @@ -1,5 +0,0 @@ - o Major bugfixes (Flow Control, SENDME): - - The decrement of the stream-level package window was done in a log_debug() - statement meaning that if the debug logs were not enabled, the decrement - would never happen and thus the window would be out of sync with the other - end point. Fixes bug 30628; bugfix on 0.4.1.1-alpha. From d2de899bf4f954e0886b88f4ee2562a13f49a716 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 5 Jun 2019 09:24:42 -0400 Subject: [PATCH 1086/2557] Rewrap the changelog --- ChangeLog | 128 ++++++++++++++++++++++++++++-------------------------- 1 file changed, 66 insertions(+), 62 deletions(-) diff --git a/ChangeLog b/ChangeLog index f3eac65d52..6387d6dbb1 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,82 +1,84 @@ Changes in version 0.4.1.2-alpha - 2019-06-05 - Tor 0.4.1.2-alpha resolves numerous bugs--some of them from the previous - alpha, and some much older. It also contains minor testing improvements, - and an improvement to the security of our authenticated sendme - implementation. + Tor 0.4.1.2-alpha resolves numerous bugs--some of them from the + previous alpha, and some much older. It also contains minor testing + improvements, and an improvement to the security of our authenticated + sendme implementation. o Major bugfixes (bridges): - - Consider our directory information to have changed when our list of - bridges changes. Previously, Tor would not re-compute the status of its - directory information when bridges changed, and therefore would not - realize that it was no longer able to build circuits. Fixes part of bug - 29875. - - Do not count previously configured working bridges towards our total of - working bridges. Previously, when Tor's list of bridges changed, it - would think that the old bridges were still usable, and delay fetching - router descriptors for the new ones. Fixes part of bug 29875; bugfix - on 0.3.0.1-alpha. + - Consider our directory information to have changed when our list + of bridges changes. Previously, Tor would not re-compute the + status of its directory information when bridges changed, and + therefore would not realize that it was no longer able to build + circuits. Fixes part of bug 29875. + - Do not count previously configured working bridges towards our + total of working bridges. Previously, when Tor's list of bridges + changed, it would think that the old bridges were still usable, + and delay fetching router descriptors for the new ones. Fixes part + of bug 29875; bugfix on 0.3.0.1-alpha. o Major bugfixes (Flow Control, SENDME): - - The decrement of the stream-level package window was done in a log_debug() - statement meaning that if the debug logs were not enabled, the decrement - would never happen and thus the window would be out of sync with the other - end point. Fixes bug 30628; bugfix on 0.4.1.1-alpha. + - The decrement of the stream-level package window was done in a + log_debug() statement meaning that if the debug logs were not + enabled, the decrement would never happen and thus the window + would be out of sync with the other end point. Fixes bug 30628; + bugfix on 0.4.1.1-alpha. o Major bugfixes (Onion service reachability): - - Properly clean up the introduction point map and associated state when - circuits change purpose from onion service circuits to pathbias, - measurement, or other circuit types. This should fix some instances of - introduction point failure. Fixes bug 29034; bugfix on 0.3.2.1-alpha. + - Properly clean up the introduction point map and associated state + when circuits change purpose from onion service circuits to + pathbias, measurement, or other circuit types. This should fix + some instances of introduction point failure. Fixes bug 29034; + bugfix on 0.3.2.1-alpha. o Minor features (authenticated SENDME): - - Ensure that there is enough randomness on every circuit - to prevent an attacker from successfully predicting what SENDME cells - they will need to send: at a random interval, if we have not send - randomness already, leave some extra space at the end of a cell that - we can fill with random bytes. Closes ticket 26846. + - Ensure that there is enough randomness on every circuit to prevent + an attacker from successfully predicting what SENDME cells they + will need to send: at a random interval, if we have not send + randomness already, leave some extra space at the end of a cell + that we can fill with random bytes. Closes ticket 26846. o Minor features (continuous integration): - - When running coverage builds on Travis, we now set TOR_TEST_RNG_SEED, - to avoid RNG-based coverage differences. - Part of ticket 28878. + - When running coverage builds on Travis, we now set + TOR_TEST_RNG_SEED, to avoid RNG-based coverage differences. Part + of ticket 28878. o Minor features (maintenance): - - Add a new "make autostyle" target that developers can use to - apply all automatic Tor style and consistency conversions to the + - Add a new "make autostyle" target that developers can use to apply + all automatic Tor style and consistency conversions to the codebase. Closes ticket 30539. o Minor features (testing): - The circuitpadding tests now use a reproducible RNG implementation, so that if a test fails, we can learn why. Part of ticket 28878. - Tor's tests now support an environment variable, TOR_TEST_RNG_SEED, - to set the RNG seed for tests that use a reproducible RNG. - Part of ticket 28878. + to set the RNG seed for tests that use a reproducible RNG. Part of + ticket 28878. - When running tests in coverage mode, take additional care to make - our coverage deterministic, so that we can accurately track changes in - code coverage. Closes ticket 30519. + our coverage deterministic, so that we can accurately track + changes in code coverage. Closes ticket 30519. o Minor bugfixes (configuration, proxies): - - Fix a bug that prevented us from supporting SOCKS5 proxies that want - authentication along with configured (but unused!) + - Fix a bug that prevented us from supporting SOCKS5 proxies that + want authentication along with configured (but unused!) ClientTransportPlugins. Fixes bug 29670; bugfix on 0.2.6.1-alpha. o Minor bugfixes (controller): - POSTDESCRIPTOR requests should work again. Previously, they were - broken if a "purpose=" flag was specified. Fixes bug 30580; - bugfix on 0.4.1.1-alpha. + broken if a "purpose=" flag was specified. Fixes bug 30580; bugfix + on 0.4.1.1-alpha. - Repair the HSFETCH command so that it works again. Previously, it - expected a body when it shouldn't have. Fixes bug 30646; bugfix on - 0.4.1.1-alpha. + expected a body when it shouldn't have. Fixes bug 30646; bugfix + on 0.4.1.1-alpha. o Minor bugfixes (developer tooling): - - Fix pre-push hook to refrain from rejecting fixup and squash commits - when pushing to non-upstream git remote. Fixes bug 30286; bugfix on - 0.4.0.1-alpha. + - Fix pre-push hook to refrain from rejecting fixup and squash + commits when pushing to non-upstream git remote. Fixes bug 30286; + bugfix on 0.4.0.1-alpha. o Minor bugfixes (directory authority): - - Move the "bandwidth-file-headers" line in directory authority votes - so that it conforms to dir-spec.txt. Fixes bug 30316; bugfix on - 0.3.5.1-alpha. + - Move the "bandwidth-file-headers" line in directory authority + votes so that it conforms to dir-spec.txt. Fixes bug 30316; bugfix + on 0.3.5.1-alpha. o Minor bugfixes (NetBSD): - Fix usage of minherit() on NetBSD and other platforms that define @@ -85,27 +87,29 @@ Changes in version 0.4.1.2-alpha - 2019-06-05 o Minor bugfixes (out-of-memory handler): - When purging the DNS cache because of an out-of-memory condition, - try purging just the older entries at first. Previously, we would + try purging just the older entries at first. Previously, we would purge the whole thing. Fixes bug 29617; bugfix on 0.3.5.1-alpha. o Minor bugfixes (portability): - - Avoid crashing in our tor_vasprintf() implementation on systems that - define neither vasprintf() nor _vscprintf(). (This bug has been here - long enough that we question whether people are running Tor on such - systems, but we're applying the fix out of caution.) Fixes bug 30561; - bugfix on 0.2.8.2-alpha. Found and fixed by Tobias Stoeckmann. + - Avoid crashing in our tor_vasprintf() implementation on systems + that define neither vasprintf() nor _vscprintf(). (This bug has + been here long enough that we question whether people are running + Tor on such systems, but we're applying the fix out of caution.) + Fixes bug 30561; bugfix on 0.2.8.2-alpha. Found and fixed by + Tobias Stoeckmann. o Minor bugfixes (shutdown, libevent, memory safety): - - Avoid use-after-free bugs when shutting down, by making sure that we - shut down libevent only after shutting down all of its users. We - believe these are harmless in practice, since they only occur on the - shutdown path, and do not involve any attacker-controlled data. Fixes - bug 30629; bugfix on 0.4.1.1-alpha. + - Avoid use-after-free bugs when shutting down, by making sure that + we shut down libevent only after shutting down all of its users. + We believe these are harmless in practice, since they only occur + on the shutdown path, and do not involve any attacker-controlled + data. Fixes bug 30629; bugfix on 0.4.1.1-alpha. o Minor bugfixes (static analysis): - - Fix several spurious Coverity warnings about the unit tests, to lower our - chances of missing any real warnings in the future. Fixes bug 30150; - bugfix on 0.3.5.1-alpha and various other Tor versions. + - Fix several spurious Coverity warnings about the unit tests, to + lower our chances of missing any real warnings in the future. + Fixes bug 30150; bugfix on 0.3.5.1-alpha and various other + Tor versions. o Testing: - Specify torrc paths (with empty files) when launching tor in From d1b02456c107256ee562b36b0ef2f5544eb27cee Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 5 Jun 2019 09:25:21 -0400 Subject: [PATCH 1087/2557] Bump to 0.4.1.2-alpha --- configure.ac | 4 ++-- contrib/win32build/tor-mingw.nsi.in | 2 +- src/win32/orconfig.h | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/configure.ac b/configure.ac index e7f959f17a..f068d16462 100644 --- a/configure.ac +++ b/configure.ac @@ -4,7 +4,7 @@ dnl Copyright (c) 2007-2019, The Tor Project, Inc. dnl See LICENSE for licensing information AC_PREREQ([2.63]) -AC_INIT([tor],[0.4.1.1-alpha-dev]) +AC_INIT([tor],[0.4.1.2-alpha]) AC_CONFIG_SRCDIR([src/app/main/tor_main.c]) AC_CONFIG_MACRO_DIR([m4]) @@ -14,7 +14,7 @@ AC_CONFIG_MACRO_DIR([m4]) # version number changes. Tor uses it to make sure that it # only shuts down for missing "required protocols" when those protocols # are listed as required by a consensus after this date. -AC_DEFINE(APPROX_RELEASE_DATE, ["2019-05-22"], # for 0.4.1.1-alpha-dev +AC_DEFINE(APPROX_RELEASE_DATE, ["2019-06-05"], # for 0.4.1.2-alpha [Approximate date when this software was released. (Updated when the version changes.)]) # "foreign" means we don't follow GNU package layout standards diff --git a/contrib/win32build/tor-mingw.nsi.in b/contrib/win32build/tor-mingw.nsi.in index 49958a462d..99783c40f2 100644 --- a/contrib/win32build/tor-mingw.nsi.in +++ b/contrib/win32build/tor-mingw.nsi.in @@ -8,7 +8,7 @@ !include "LogicLib.nsh" !include "FileFunc.nsh" !insertmacro GetParameters -!define VERSION "0.4.1.1-alpha-dev" +!define VERSION "0.4.1.2-alpha" !define INSTALLER "tor-${VERSION}-win32.exe" !define WEBSITE "https://www.torproject.org/" !define LICENSE "LICENSE" diff --git a/src/win32/orconfig.h b/src/win32/orconfig.h index 442bb81570..a3daa98de0 100644 --- a/src/win32/orconfig.h +++ b/src/win32/orconfig.h @@ -218,7 +218,7 @@ #define USING_TWOS_COMPLEMENT /* Version number of package */ -#define VERSION "0.4.1.1-alpha-dev" +#define VERSION "0.4.1.2-alpha" From 60213a3621c5fa354fd7b3f3feb1a2a336d5c9ce Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 5 Jun 2019 09:33:35 -0400 Subject: [PATCH 1088/2557] Run "make autostyle." --- src/app/config/config.c | 8 +++---- src/app/config/or_options_st.h | 2 +- src/app/config/or_state_st.h | 2 +- src/app/config/statefile.h | 2 +- src/app/main/subsysmgr.h | 2 +- src/core/crypto/onion_crypto.h | 2 +- src/core/mainloop/connection.c | 10 ++++---- src/core/mainloop/mainloop.c | 4 ++-- src/core/mainloop/mainloop.h | 6 ++--- src/core/mainloop/mainloop_pubsub.c | 6 ++--- src/core/mainloop/mainloop_pubsub.h | 2 +- src/core/mainloop/mainloop_sys.h | 2 +- src/core/mainloop/netstatus.h | 2 +- src/core/or/addr_policy_st.h | 2 +- src/core/or/address_set.h | 2 +- src/core/or/cell_queue_st.h | 2 +- src/core/or/cell_st.h | 2 +- src/core/or/circuit_st.h | 2 +- src/core/or/circuitpadding.c | 4 ++-- src/core/or/circuitpadding.h | 6 ++--- src/core/or/circuitpadding_machines.h | 4 ++-- src/core/or/connection_st.h | 2 +- src/core/or/cpath_build_state_st.h | 2 +- src/core/or/crypt_path.h | 2 +- src/core/or/crypt_path_reference_st.h | 2 +- src/core/or/crypt_path_st.h | 4 ++-- src/core/or/destroy_cell_queue_st.h | 2 +- src/core/or/dos.h | 4 ++-- src/core/or/edge_connection_st.h | 2 +- src/core/or/entry_connection_st.h | 2 +- src/core/or/entry_port_cfg_st.h | 2 +- src/core/or/extend_info_st.h | 2 +- src/core/or/half_edge_st.h | 2 +- src/core/or/listener_connection_st.h | 2 +- src/core/or/ocirc_event.h | 2 +- src/core/or/ocirc_event_sys.h | 2 +- src/core/or/or_circuit_st.h | 2 +- src/core/or/or_connection_st.h | 2 +- src/core/or/or_handshake_certs_st.h | 2 +- src/core/or/or_handshake_state_st.h | 2 +- src/core/or/or_periodic.c | 2 +- src/core/or/orconn_event.h | 2 +- src/core/or/orconn_event_sys.h | 2 +- src/core/or/origin_circuit_st.h | 2 +- src/core/or/port_cfg_st.h | 2 +- src/core/or/relay_crypto_st.h | 2 +- src/core/or/sendme.h | 4 ++-- src/core/or/server_port_cfg_st.h | 2 +- src/core/or/socks_request_st.h | 2 +- src/core/or/tor_version_st.h | 2 +- src/core/or/var_cell_st.h | 2 +- src/feature/api/tor_api.c | 4 ++-- src/feature/api/tor_api.h | 2 +- src/feature/control/btrack_circuit.h | 2 +- src/feature/control/btrack_orconn.h | 4 ++-- src/feature/control/btrack_orconn_cevent.h | 2 +- src/feature/control/btrack_orconn_maps.h | 2 +- src/feature/control/btrack_sys.h | 2 +- src/feature/control/control_connection_st.h | 2 +- src/feature/control/control_getinfo.h | 2 +- src/feature/control/fmt_serverstatus.h | 2 +- src/feature/control/getinfo_geoip.h | 2 +- src/feature/dirauth/authmode.h | 6 ++--- src/feature/dirauth/bridgeauth.h | 2 +- src/feature/dirauth/bwauth.h | 2 +- src/feature/dirauth/dirauth_periodic.h | 6 ++--- src/feature/dirauth/dirauth_sys.h | 2 +- src/feature/dirauth/dirvote.h | 4 ++-- src/feature/dirauth/dsigs_parse.h | 2 +- src/feature/dirauth/guardfraction.h | 2 +- .../dirauth/ns_detached_signatures_st.h | 2 +- src/feature/dirauth/process_descs.h | 2 +- src/feature/dirauth/reachability.h | 2 +- src/feature/dirauth/recommend_pkg.h | 4 ++-- src/feature/dirauth/shared_random.h | 4 ++-- src/feature/dirauth/vote_microdesc_hash_st.h | 2 +- src/feature/dirauth/voteflags.h | 6 ++--- src/feature/dircache/cached_dir_st.h | 2 +- src/feature/dircache/consdiffmgr.c | 2 +- src/feature/dircache/dircache.h | 2 +- src/feature/dirclient/dir_server_st.h | 2 +- src/feature/dirclient/dirclient.h | 2 +- src/feature/dirclient/dlstatus.h | 2 +- src/feature/dirclient/download_status_st.h | 2 +- src/feature/dircommon/dir_connection_st.h | 2 +- src/feature/dircommon/vote_timing_st.h | 2 +- src/feature/dircommon/voting_schedule.h | 2 +- src/feature/dirparse/microdesc_parse.h | 2 +- src/feature/dirparse/ns_parse.h | 4 ++-- src/feature/dirparse/sigcommon.h | 2 +- src/feature/dirparse/signing.h | 2 +- src/feature/dirparse/unparseable.h | 2 +- src/feature/hibernate/hibernate.c | 4 ++-- src/feature/hs/hs_service.h | 2 +- src/feature/hs/hs_stats.h | 2 +- src/feature/hs/hsdir_index_st.h | 2 +- src/feature/hs_common/shared_random_client.h | 2 +- src/feature/keymgt/loadkey.h | 2 +- src/feature/nodelist/authcert.h | 2 +- src/feature/nodelist/authority_cert_st.h | 2 +- src/feature/nodelist/desc_store_st.h | 2 +- src/feature/nodelist/describe.h | 2 +- src/feature/nodelist/dirlist.h | 2 +- src/feature/nodelist/document_signature_st.h | 2 +- src/feature/nodelist/extrainfo_st.h | 2 +- src/feature/nodelist/microdesc_st.h | 2 +- src/feature/nodelist/networkstatus.c | 2 +- .../nodelist/networkstatus_sr_info_st.h | 2 +- src/feature/nodelist/networkstatus_st.h | 2 +- .../nodelist/networkstatus_voter_info_st.h | 2 +- src/feature/nodelist/nickname.h | 2 +- src/feature/nodelist/node_select.h | 4 ++-- src/feature/nodelist/node_st.h | 2 +- src/feature/nodelist/nodefamily.h | 2 +- src/feature/nodelist/nodefamily_st.h | 2 +- src/feature/nodelist/routerinfo.h | 2 +- src/feature/nodelist/routerinfo_st.h | 2 +- src/feature/nodelist/routerlist_st.h | 2 +- src/feature/nodelist/routerstatus_st.h | 2 +- src/feature/nodelist/signed_descriptor_st.h | 2 +- src/feature/nodelist/vote_routerstatus_st.h | 2 +- src/feature/relay/dns.c | 2 +- src/feature/relay/onion_queue.h | 2 +- src/feature/relay/relay_periodic.c | 2 +- src/feature/relay/router.c | 2 +- src/feature/relay/selftest.h | 2 +- src/feature/rend/rend_authorized_client_st.h | 2 +- .../rend_encoded_v2_service_descriptor_st.h | 2 +- src/feature/rend/rend_intro_point_st.h | 2 +- src/feature/rend/rend_service_descriptor_st.h | 2 +- src/feature/rend/rendparse.h | 2 +- src/feature/stats/predict_ports.h | 2 +- src/feature/stats/rephist.h | 2 +- src/lib/arch/bytes.h | 6 ++--- src/lib/cc/compat_compiler.h | 2 +- src/lib/cc/ctassert.h | 6 ++--- src/lib/cc/torint.h | 8 +++---- src/lib/compress/compress_zstd.c | 16 ++++++------- src/lib/container/bitarray.h | 2 +- src/lib/container/map.h | 2 +- src/lib/container/namemap.h | 2 +- src/lib/container/namemap_st.h | 2 +- src/lib/container/order.h | 2 +- src/lib/container/smartlist.h | 2 +- src/lib/crypt_ops/crypto_cipher.h | 2 +- src/lib/crypt_ops/crypto_dh_openssl.c | 8 +++---- src/lib/crypt_ops/crypto_digest.c | 16 ++++++------- src/lib/crypt_ops/crypto_digest_openssl.c | 24 +++++++++---------- src/lib/crypt_ops/crypto_hkdf.c | 10 ++++---- src/lib/crypt_ops/crypto_init.c | 2 +- src/lib/crypt_ops/crypto_init.h | 2 +- src/lib/crypt_ops/crypto_nss_mgt.h | 4 ++-- src/lib/crypt_ops/crypto_ope.c | 4 ++-- src/lib/crypt_ops/crypto_ope.h | 4 ++-- src/lib/crypt_ops/crypto_openssl_mgt.c | 4 ++-- src/lib/crypt_ops/crypto_openssl_mgt.h | 2 +- src/lib/crypt_ops/crypto_rand.c | 12 +++++----- src/lib/crypt_ops/crypto_rand.h | 4 ++-- src/lib/crypt_ops/crypto_rand_fast.c | 12 +++++----- src/lib/crypt_ops/crypto_rsa.c | 2 +- src/lib/crypt_ops/crypto_rsa.h | 8 +++---- src/lib/crypt_ops/crypto_rsa_nss.c | 2 +- src/lib/crypt_ops/crypto_s2k.c | 4 ++-- src/lib/crypt_ops/crypto_util.c | 2 +- src/lib/crypt_ops/digestset.h | 2 +- src/lib/defs/dh_sizes.h | 2 +- src/lib/defs/digest_sizes.h | 2 +- src/lib/defs/time.h | 2 +- src/lib/defs/x25519_sizes.h | 2 +- src/lib/dispatch/dispatch.h | 2 +- src/lib/dispatch/dispatch_cfg.h | 2 +- src/lib/dispatch/dispatch_cfg_st.h | 2 +- src/lib/dispatch/dispatch_naming.h | 2 +- src/lib/dispatch/dispatch_st.h | 4 ++-- src/lib/dispatch/msgtypes.h | 2 +- src/lib/encoding/binascii.h | 2 +- src/lib/encoding/keyval.h | 2 +- src/lib/encoding/pem.h | 2 +- src/lib/encoding/qstring.h | 2 +- src/lib/encoding/time_fmt.h | 2 +- src/lib/err/torerr.h | 2 +- src/lib/evloop/token_bucket.h | 4 ++-- src/lib/fs/conffile.h | 2 +- src/lib/fs/dir.h | 2 +- src/lib/fs/files.h | 14 +++++------ src/lib/fs/lockfile.h | 2 +- src/lib/fs/mmap.c | 2 +- src/lib/fs/mmap.h | 2 +- src/lib/fs/path.h | 2 +- src/lib/fs/userdb.h | 4 ++-- src/lib/fs/winlib.h | 4 ++-- src/lib/geoip/country.h | 2 +- src/lib/intmath/addsub.h | 2 +- src/lib/intmath/logic.h | 2 +- src/lib/intmath/weakrng.h | 2 +- src/lib/lock/compat_mutex.h | 2 +- src/lib/log/escape.h | 2 +- src/lib/log/log.c | 4 ++-- src/lib/log/log.h | 2 +- src/lib/log/ratelim.h | 2 +- src/lib/log/util_bug.h | 6 ++--- src/lib/log/win32err.h | 2 +- src/lib/malloc/malloc.h | 4 ++-- src/lib/malloc/map_anon.c | 18 +++++++------- src/lib/malloc/map_anon.h | 2 +- src/lib/math/fp.h | 2 +- src/lib/math/laplace.h | 2 +- src/lib/math/prob_distr.h | 6 ++--- src/lib/meminfo/meminfo.h | 2 +- src/lib/net/alertsock.h | 2 +- src/lib/net/buffers_net.h | 2 +- src/lib/net/gethostname.h | 2 +- src/lib/net/inaddr.h | 2 +- src/lib/net/inaddr_st.h | 2 +- src/lib/net/nettypes.h | 2 +- src/lib/net/resolve.c | 4 ++-- src/lib/net/resolve.h | 2 +- src/lib/net/socket.c | 8 +++---- src/lib/net/socket.h | 2 +- src/lib/net/socketpair.c | 4 ++-- src/lib/net/socketpair.h | 2 +- src/lib/net/socks5_status.h | 2 +- src/lib/osinfo/uname.h | 2 +- src/lib/process/daemon.h | 2 +- src/lib/process/env.h | 2 +- src/lib/process/pidfile.h | 2 +- src/lib/process/process.c | 10 ++++---- src/lib/process/process.h | 6 ++--- src/lib/process/process_unix.c | 2 +- src/lib/process/process_unix.h | 6 ++--- src/lib/process/process_win32.c | 4 ++-- src/lib/process/process_win32.h | 6 ++--- src/lib/process/setuid.h | 2 +- src/lib/process/winprocess_sys.c | 2 +- src/lib/pubsub/pub_binding_st.h | 2 +- src/lib/pubsub/pubsub.h | 2 +- src/lib/pubsub/pubsub_build.h | 2 +- src/lib/pubsub/pubsub_builder_st.h | 4 ++-- src/lib/pubsub/pubsub_connect.h | 2 +- src/lib/pubsub/pubsub_flags.h | 2 +- src/lib/pubsub/pubsub_macros.h | 2 +- src/lib/pubsub/pubsub_publish.h | 2 +- src/lib/smartlist_core/smartlist_core.h | 2 +- src/lib/smartlist_core/smartlist_split.h | 2 +- src/lib/string/compat_string.h | 8 +++---- src/lib/string/parse_int.h | 2 +- src/lib/string/printf.h | 2 +- src/lib/string/scanf.h | 2 +- src/lib/subsys/subsys.h | 2 +- src/lib/term/getpass.h | 2 +- src/lib/testsupport/testsupport.h | 2 +- src/lib/thread/numcpus.h | 2 +- src/lib/time/compat_time.c | 10 ++++---- src/lib/time/compat_time.h | 2 +- src/lib/time/tvdiff.h | 2 +- src/lib/tls/nss_countbytes.h | 2 +- src/lib/tls/tortls.h | 10 ++++---- src/lib/tls/tortls_internal.h | 6 ++--- src/lib/tls/tortls_openssl.c | 14 +++++------ src/lib/tls/tortls_st.h | 4 ++-- src/lib/tls/x509.h | 6 ++--- src/lib/tls/x509_internal.h | 2 +- src/lib/tls/x509_nss.c | 8 +++---- src/lib/tls/x509_openssl.c | 4 ++-- src/lib/trace/debug.h | 2 +- src/lib/trace/events.h | 6 ++--- src/lib/trace/trace.h | 2 +- src/lib/wallclock/approx_time.h | 2 +- src/lib/wallclock/time_to_tm.h | 2 +- src/lib/wallclock/timeval.h | 4 ++-- src/lib/wallclock/tor_gettimeofday.h | 2 +- src/test/bench.c | 4 ++-- src/test/fuzz/fuzzing.h | 2 +- src/test/fuzz/fuzzing_common.c | 4 ++-- src/test/ptr_helpers.c | 2 +- src/test/ptr_helpers.h | 2 +- src/test/test-memwipe.c | 2 +- src/test/test_circuitpadding.c | 4 ++-- src/test/test_config.c | 6 ++--- src/test/test_connection.h | 2 +- src/test/test_crypto.c | 10 ++++---- src/test/test_dir_common.h | 2 +- src/test/test_dns.c | 4 ++-- src/test/test_dos.c | 2 +- src/test/test_link_handshake.c | 4 ++-- src/test/test_periodic_event.c | 2 +- src/test/test_process.c | 4 ++-- src/test/test_process_slow.c | 2 +- src/test/test_protover.c | 10 ++++---- src/test/test_rng.c | 2 +- src/test/test_shared_random.c | 2 +- src/test/test_tortls.c | 6 ++--- src/test/test_tortls.h | 2 +- src/test/test_tortls_openssl.c | 4 ++-- src/test/test_util.c | 10 ++++---- src/tools/tor-gencert.c | 2 +- 296 files changed, 480 insertions(+), 480 deletions(-) diff --git a/src/app/config/config.c b/src/app/config/config.c index d1b9d06bb3..4cbe81026c 100644 --- a/src/app/config/config.c +++ b/src/app/config/config.c @@ -3551,7 +3551,7 @@ options_validate(or_options_t *old_options, or_options_t *options, tor_free(t); t = format_recommended_version_list(options->RecommendedServerVersions, 1); tor_free(t); -#endif +#endif /* defined(HAVE_MODULE_DIRAUTH) */ if (options->UseEntryGuards) { log_info(LD_CONFIG, "Authoritative directory servers can't set " @@ -3577,7 +3577,7 @@ options_validate(or_options_t *old_options, or_options_t *options, if (options->GuardfractionFile && !old_options) { dirserv_read_guardfraction_file(options->GuardfractionFile, NULL); } -#endif +#endif /* defined(HAVE_MODULE_DIRAUTH) */ } if (options->AuthoritativeDir && !options->DirPort_set) @@ -4605,7 +4605,7 @@ compute_real_max_mem_in_queues(const uint64_t val, int log_guess) #else /* On a 32-bit platform, we can't have 8GB of ram. */ #define RAM_IS_VERY_LARGE(x) (0) -#endif +#endif /* SIZEOF_SIZE_T > 4 */ if (RAM_IS_VERY_LARGE(ram)) { /* If we have 8 GB, or more, RAM available, we set the MaxMemInQueues @@ -5777,7 +5777,7 @@ options_init_logs(const or_options_t *old_options, or_options_t *options, #else log_warn(LD_CONFIG, "Android logging is not supported" " on this system. Sorry."); -#endif // HAVE_ANDROID_LOG_H. +#endif /* defined(HAVE_ANDROID_LOG_H) */ goto cleanup; } } diff --git a/src/app/config/or_options_st.h b/src/app/config/or_options_st.h index 4e03bec7fa..2ee2d15674 100644 --- a/src/app/config/or_options_st.h +++ b/src/app/config/or_options_st.h @@ -1110,4 +1110,4 @@ struct or_options_t { int DormantCanceledByStartup; }; -#endif +#endif /* !defined(TOR_OR_OPTIONS_ST_H) */ diff --git a/src/app/config/or_state_st.h b/src/app/config/or_state_st.h index cdb9b38287..f45c6196cc 100644 --- a/src/app/config/or_state_st.h +++ b/src/app/config/or_state_st.h @@ -96,4 +96,4 @@ struct or_state_t { int Dormant; }; -#endif +#endif /* !defined(TOR_OR_STATE_ST_H) */ diff --git a/src/app/config/statefile.h b/src/app/config/statefile.h index 1950078450..515c90a52f 100644 --- a/src/app/config/statefile.h +++ b/src/app/config/statefile.h @@ -31,6 +31,6 @@ STATIC struct config_line_t *get_transport_in_state_by_name( STATIC void or_state_free_(or_state_t *state); #define or_state_free(st) FREE_AND_NULL(or_state_t, or_state_free_, (st)) STATIC or_state_t *or_state_new(void); -#endif +#endif /* defined(STATEFILE_PRIVATE) */ #endif /* !defined(TOR_STATEFILE_H) */ diff --git a/src/app/main/subsysmgr.h b/src/app/main/subsysmgr.h index 4ac44afca7..d4426614e3 100644 --- a/src/app/main/subsysmgr.h +++ b/src/app/main/subsysmgr.h @@ -26,4 +26,4 @@ void subsystems_prefork(void); void subsystems_postfork(void); void subsystems_thread_cleanup(void); -#endif +#endif /* !defined(TOR_SUBSYSMGR_T) */ diff --git a/src/core/crypto/onion_crypto.h b/src/core/crypto/onion_crypto.h index 1cddde3610..7abdd6538e 100644 --- a/src/core/crypto/onion_crypto.h +++ b/src/core/crypto/onion_crypto.h @@ -44,4 +44,4 @@ void server_onion_keys_free_(server_onion_keys_t *keys); #define server_onion_keys_free(keys) \ FREE_AND_NULL(server_onion_keys_t, server_onion_keys_free_, (keys)) -#endif +#endif /* !defined(TOR_ONION_CRYPTO_H) */ diff --git a/src/core/mainloop/connection.c b/src/core/mainloop/connection.c index 487f6d4a55..127f08683f 100644 --- a/src/core/mainloop/connection.c +++ b/src/core/mainloop/connection.c @@ -1476,7 +1476,7 @@ connection_listener_new(const struct sockaddr *listensockaddr, goto err; } } -#endif /* __APPLE__ */ +#endif /* !defined(__APPLE__) */ #endif /* defined(HAVE_SYS_UN_H) */ } else { log_err(LD_BUG, "Got unexpected address family %d.", @@ -2856,7 +2856,7 @@ retry_listener_ports(smartlist_t *old_conns, SMARTLIST_DEL_CURRENT(old_conns, conn); break; } -#endif +#endif /* defined(ENABLE_LISTENER_REBIND) */ } } SMARTLIST_FOREACH_END(wanted); @@ -2958,7 +2958,7 @@ retry_all_listeners(smartlist_t *new_conns, int close_all_noncontrol) conn_type_to_string(old_conn->type), old_conn->address, old_conn->port, new_conn->address, new_conn->port); } SMARTLIST_FOREACH_END(r); -#endif +#endif /* defined(ENABLE_LISTENER_REBIND) */ /* Any members that were still in 'listeners' don't correspond to * any configured port. Kill 'em. */ @@ -3957,9 +3957,9 @@ update_send_buffer_size(tor_socket_t sock) &isb, sizeof(isb), &bytesReturned, NULL, NULL)) { setsockopt(sock, SOL_SOCKET, SO_SNDBUF, (const char*)&isb, sizeof(isb)); } -#else +#else /* !(defined(_WIN32)) */ (void) sock; -#endif +#endif /* defined(_WIN32) */ } /** Try to flush more bytes onto conn-\>s. diff --git a/src/core/mainloop/mainloop.c b/src/core/mainloop/mainloop.c index 82042e8498..c051b11566 100644 --- a/src/core/mainloop/mainloop.c +++ b/src/core/mainloop/mainloop.c @@ -754,7 +754,7 @@ tor_shutdown_event_loop_for_restart_cb( tor_event_free(tor_shutdown_event_loop_for_restart_event); tor_shutdown_event_loop_and_exit(0); } -#endif +#endif /* defined(ENABLE_RESTART_DEBUGGING) */ /** * After finishing the current callback (if any), shut down the main loop, @@ -2375,7 +2375,7 @@ do_main_loop(void) event_add(tor_shutdown_event_loop_for_restart_event, &restart_after); } } -#endif +#endif /* defined(ENABLE_RESTART_DEBUGGING) */ return run_main_loop_until_done(); } diff --git a/src/core/mainloop/mainloop.h b/src/core/mainloop/mainloop.h index cdc2bf8608..caef736c15 100644 --- a/src/core/mainloop/mainloop.h +++ b/src/core/mainloop/mainloop.h @@ -112,7 +112,7 @@ extern smartlist_t *connection_array; /* We need the periodic_event_item_t definition. */ #include "core/mainloop/periodic.h" extern periodic_event_item_t mainloop_periodic_events[]; -#endif -#endif /* defined(MAIN_PRIVATE) */ +#endif /* defined(TOR_UNIT_TESTS) */ +#endif /* defined(MAINLOOP_PRIVATE) */ -#endif +#endif /* !defined(TOR_MAINLOOP_H) */ diff --git a/src/core/mainloop/mainloop_pubsub.c b/src/core/mainloop/mainloop_pubsub.c index 724a3115c8..53275d8119 100644 --- a/src/core/mainloop/mainloop_pubsub.c +++ b/src/core/mainloop/mainloop_pubsub.c @@ -6,9 +6,9 @@ #include "orconfig.h" -#include "src/core/or/or.h" -#include "src/core/mainloop/mainloop.h" -#include "src/core/mainloop/mainloop_pubsub.h" +#include "core/or/or.h" +#include "core/mainloop/mainloop.h" +#include "core/mainloop/mainloop_pubsub.h" #include "lib/container/smartlist.h" #include "lib/dispatch/dispatch.h" diff --git a/src/core/mainloop/mainloop_pubsub.h b/src/core/mainloop/mainloop_pubsub.h index a31b2b4ba7..365a3dd565 100644 --- a/src/core/mainloop/mainloop_pubsub.h +++ b/src/core/mainloop/mainloop_pubsub.h @@ -21,4 +21,4 @@ int tor_mainloop_set_delivery_strategy(const char *msg_channel_name, deliv_strategy_t strategy); void tor_mainloop_disconnect_pubsub(void); -#endif +#endif /* !defined(TOR_MAINLOOP_PUBSUB_H) */ diff --git a/src/core/mainloop/mainloop_sys.h b/src/core/mainloop/mainloop_sys.h index 14c567278c..fa74fe5d4b 100644 --- a/src/core/mainloop/mainloop_sys.h +++ b/src/core/mainloop/mainloop_sys.h @@ -9,4 +9,4 @@ extern const struct subsys_fns_t sys_mainloop; -#endif +#endif /* !defined(MAINLOOP_SYS_H) */ diff --git a/src/core/mainloop/netstatus.h b/src/core/mainloop/netstatus.h index aba631e2fb..e8469ff558 100644 --- a/src/core/mainloop/netstatus.h +++ b/src/core/mainloop/netstatus.h @@ -21,4 +21,4 @@ void netstatus_flush_to_state(or_state_t *state, time_t now); void netstatus_load_from_state(const or_state_t *state, time_t now); void netstatus_note_clock_jumped(time_t seconds_diff); -#endif +#endif /* !defined(TOR_NETSTATUS_H) */ diff --git a/src/core/or/addr_policy_st.h b/src/core/or/addr_policy_st.h index a75f1a731d..11442d29b4 100644 --- a/src/core/or/addr_policy_st.h +++ b/src/core/or/addr_policy_st.h @@ -43,4 +43,4 @@ struct addr_policy_t { uint16_t prt_max; /**< Highest port number to accept/reject. */ }; -#endif +#endif /* !defined(TOR_ADDR_POLICY_ST_H) */ diff --git a/src/core/or/address_set.h b/src/core/or/address_set.h index 7a9e71628e..95608a9a53 100644 --- a/src/core/or/address_set.h +++ b/src/core/or/address_set.h @@ -28,4 +28,4 @@ void address_set_add_ipv4h(address_set_t *set, uint32_t addr); int address_set_probably_contains(const address_set_t *set, const struct tor_addr_t *addr); -#endif +#endif /* !defined(TOR_ADDRESS_SET_H) */ diff --git a/src/core/or/cell_queue_st.h b/src/core/or/cell_queue_st.h index 130b95a011..7ba339b965 100644 --- a/src/core/or/cell_queue_st.h +++ b/src/core/or/cell_queue_st.h @@ -26,4 +26,4 @@ struct cell_queue_t { int n; /**< The number of cells in the queue. */ }; -#endif +#endif /* !defined(PACKED_CELL_ST_H) */ diff --git a/src/core/or/cell_st.h b/src/core/or/cell_st.h index 7ab7eceb50..c4eec4f4b5 100644 --- a/src/core/or/cell_st.h +++ b/src/core/or/cell_st.h @@ -16,5 +16,5 @@ struct cell_t { uint8_t payload[CELL_PAYLOAD_SIZE]; /**< Cell body. */ }; -#endif +#endif /* !defined(CELL_ST_H) */ diff --git a/src/core/or/circuit_st.h b/src/core/or/circuit_st.h index 3c7b931614..eae3c908d5 100644 --- a/src/core/or/circuit_st.h +++ b/src/core/or/circuit_st.h @@ -229,4 +229,4 @@ struct circuit_t { struct circpad_machine_runtime_t *padding_info[CIRCPAD_MAX_MACHINES]; }; -#endif +#endif /* !defined(CIRCUIT_ST_H) */ diff --git a/src/core/or/circuitpadding.c b/src/core/or/circuitpadding.c index b2315d822f..9210fa4e37 100644 --- a/src/core/or/circuitpadding.c +++ b/src/core/or/circuitpadding.c @@ -2595,7 +2595,7 @@ circpad_circ_responder_machine_init(void) circpad_register_padding_machine(circ_responder_machine, relay_padding_machines); } -#endif +#endif /* defined(TOR_UNIT_TESTS) */ /** * Initialize all of our padding machines. @@ -2977,4 +2977,4 @@ circpad_string_to_machine(const char *str) return NULL; } -#endif +#endif /* 0 */ diff --git a/src/core/or/circuitpadding.h b/src/core/or/circuitpadding.h index 0dc66246d9..3cf40e11db 100644 --- a/src/core/or/circuitpadding.h +++ b/src/core/or/circuitpadding.h @@ -10,7 +10,7 @@ #ifndef TOR_CIRCUITPADDING_H #define TOR_CIRCUITPADDING_H -#include "src/trunnel/circpad_negotiation.h" +#include "trunnel/circpad_negotiation.h" #include "lib/evloop/timers.h" struct circuit_t; @@ -803,6 +803,6 @@ extern smartlist_t *relay_padding_machines; #endif -#endif +#endif /* defined(CIRCUITPADDING_PRIVATE) */ -#endif +#endif /* !defined(TOR_CIRCUITPADDING_H) */ diff --git a/src/core/or/circuitpadding_machines.h b/src/core/or/circuitpadding_machines.h index c44a70f2cc..3c9798d42d 100644 --- a/src/core/or/circuitpadding_machines.h +++ b/src/core/or/circuitpadding_machines.h @@ -30,6 +30,6 @@ void circpad_machine_client_hide_rend_circuits(smartlist_t *machines_sl); * The actual value will be sampled between the min and max.*/ #define INTRO_MACHINE_MAXIMUM_PADDING 10 -#endif +#endif /* defined(CIRCUITPADDING_MACHINES_PRIVATE) */ -#endif +#endif /* !defined(TOR_CIRCUITPADDING_MACHINES_H) */ diff --git a/src/core/or/connection_st.h b/src/core/or/connection_st.h index d1430eda14..1c42a56d6b 100644 --- a/src/core/or/connection_st.h +++ b/src/core/or/connection_st.h @@ -146,4 +146,4 @@ struct connection_t { * directory connection. */ #define DIR_CONN_IS_SERVER(conn) ((conn)->purpose == DIR_PURPOSE_SERVER) -#endif +#endif /* !defined(CONNECTION_ST_H) */ diff --git a/src/core/or/cpath_build_state_st.h b/src/core/or/cpath_build_state_st.h index dbe596d851..4572a10430 100644 --- a/src/core/or/cpath_build_state_st.h +++ b/src/core/or/cpath_build_state_st.h @@ -34,5 +34,5 @@ struct cpath_build_state_t { time_t expiry_time; }; -#endif +#endif /* !defined(CIRCUIT_BUILD_STATE_ST_ST_H) */ diff --git a/src/core/or/crypt_path.h b/src/core/or/crypt_path.h index 9850610ef7..7a95fec2b4 100644 --- a/src/core/or/crypt_path.h +++ b/src/core/or/crypt_path.h @@ -43,4 +43,4 @@ uint8_t *cpath_get_sendme_digest(crypt_path_t *cpath); unsigned int cpath_get_n_hops(crypt_path_t **head_ptr); #endif /* defined(TOR_UNIT_TESTS) */ -#endif +#endif /* !defined(CRYPT_PATH_H) */ diff --git a/src/core/or/crypt_path_reference_st.h b/src/core/or/crypt_path_reference_st.h index 3d79f26c1c..1827022b4e 100644 --- a/src/core/or/crypt_path_reference_st.h +++ b/src/core/or/crypt_path_reference_st.h @@ -19,5 +19,5 @@ struct crypt_path_reference_t { crypt_path_t *cpath; }; -#endif +#endif /* !defined(CRYPT_PATH_REFERENCE_ST_H) */ diff --git a/src/core/or/crypt_path_st.h b/src/core/or/crypt_path_st.h index d18d23e939..249ac6aaa3 100644 --- a/src/core/or/crypt_path_st.h +++ b/src/core/or/crypt_path_st.h @@ -35,7 +35,7 @@ struct onion_handshake_state_t { /* Helper macro to access private members of a struct. */ #define pvt_crypto CRYPT_PATH_PRIV_FIELD(crypto) -#endif +#endif /* defined(CRYPT_PATH_PRIVATE) */ /** Holds accounting information for a single step in the layered encryption * performed by a circuit. Used only at the client edge of a circuit. */ @@ -82,4 +82,4 @@ struct crypt_path_t { relay_crypto_t CRYPT_PATH_PRIV_FIELD(crypto); }; -#endif +#endif /* !defined(CRYPT_PATH_ST_H) */ diff --git a/src/core/or/destroy_cell_queue_st.h b/src/core/or/destroy_cell_queue_st.h index 56630670ba..e917afc700 100644 --- a/src/core/or/destroy_cell_queue_st.h +++ b/src/core/or/destroy_cell_queue_st.h @@ -23,5 +23,5 @@ struct destroy_cell_queue_t { int n; /**< The number of cells in the queue. */ }; -#endif +#endif /* !defined(DESTROY_CELL_QUEUE_ST_H) */ diff --git a/src/core/or/dos.h b/src/core/or/dos.h index 95448d0530..b5154a7cd2 100644 --- a/src/core/or/dos.h +++ b/src/core/or/dos.h @@ -134,7 +134,7 @@ MOCK_DECL(STATIC unsigned int, get_param_cc_enabled, MOCK_DECL(STATIC unsigned int, get_param_conn_enabled, (const networkstatus_t *ns)); -#endif /* TOR_DOS_PRIVATE */ +#endif /* defined(DOS_PRIVATE) */ -#endif /* TOR_DOS_H */ +#endif /* !defined(TOR_DOS_H) */ diff --git a/src/core/or/edge_connection_st.h b/src/core/or/edge_connection_st.h index 1665b8589f..8922a3a9cf 100644 --- a/src/core/or/edge_connection_st.h +++ b/src/core/or/edge_connection_st.h @@ -73,5 +73,5 @@ struct edge_connection_t { uint64_t dirreq_id; }; -#endif +#endif /* !defined(EDGE_CONNECTION_ST_H) */ diff --git a/src/core/or/entry_connection_st.h b/src/core/or/entry_connection_st.h index 45621fadbf..e65c545d17 100644 --- a/src/core/or/entry_connection_st.h +++ b/src/core/or/entry_connection_st.h @@ -96,5 +96,5 @@ struct entry_connection_t { /** Cast a entry_connection_t subtype pointer to a edge_connection_t **/ #define ENTRY_TO_EDGE_CONN(c) (&(((c))->edge_)) -#endif +#endif /* !defined(ENTRY_CONNECTION_ST_H) */ diff --git a/src/core/or/entry_port_cfg_st.h b/src/core/or/entry_port_cfg_st.h index 87dfb331e5..b84838d44f 100644 --- a/src/core/or/entry_port_cfg_st.h +++ b/src/core/or/entry_port_cfg_st.h @@ -50,5 +50,5 @@ struct entry_port_cfg_t { }; -#endif +#endif /* !defined(ENTRY_PORT_CFG_ST_H) */ diff --git a/src/core/or/extend_info_st.h b/src/core/or/extend_info_st.h index bc7a77b1b2..7704ff16b5 100644 --- a/src/core/or/extend_info_st.h +++ b/src/core/or/extend_info_st.h @@ -27,4 +27,4 @@ struct extend_info_t { curve25519_public_key_t curve25519_onion_key; }; -#endif +#endif /* !defined(EXTEND_INFO_ST_H) */ diff --git a/src/core/or/half_edge_st.h b/src/core/or/half_edge_st.h index d4617be108..1fe47ad3f1 100644 --- a/src/core/or/half_edge_st.h +++ b/src/core/or/half_edge_st.h @@ -30,5 +30,5 @@ typedef struct half_edge_t { int connected_pending : 1; } half_edge_t; -#endif +#endif /* !defined(HALF_EDGE_ST_H) */ diff --git a/src/core/or/listener_connection_st.h b/src/core/or/listener_connection_st.h index 8989a39dc8..1250d9c9b4 100644 --- a/src/core/or/listener_connection_st.h +++ b/src/core/or/listener_connection_st.h @@ -21,5 +21,5 @@ struct listener_connection_t { }; -#endif +#endif /* !defined(LISTENER_CONNECTION_ST_H) */ diff --git a/src/core/or/ocirc_event.h b/src/core/or/ocirc_event.h index 0b125c2898..59ec9e27cb 100644 --- a/src/core/or/ocirc_event.h +++ b/src/core/or/ocirc_event.h @@ -86,4 +86,4 @@ void ocirc_event_subscribe(ocirc_event_rcvr_t fn); void ocirc_event_publish(const ocirc_event_msg_t *msg); #endif -#endif /* defined(TOR_OCIRC_EVENT_STATE_H) */ +#endif /* !defined(TOR_OCIRC_EVENT_H) */ diff --git a/src/core/or/ocirc_event_sys.h b/src/core/or/ocirc_event_sys.h index 9d4bfe5333..61180496da 100644 --- a/src/core/or/ocirc_event_sys.h +++ b/src/core/or/ocirc_event_sys.h @@ -10,4 +10,4 @@ extern const struct subsys_fns_t sys_ocirc_event; -#endif /* defined(TOR_OCIRC_EVENT_H) */ +#endif /* !defined(TOR_OCIRC_EVENT_SYS_H) */ diff --git a/src/core/or/or_circuit_st.h b/src/core/or/or_circuit_st.h index 062e4ac854..6789668224 100644 --- a/src/core/or/or_circuit_st.h +++ b/src/core/or/or_circuit_st.h @@ -71,5 +71,5 @@ struct or_circuit_t { uint64_t total_cell_waiting_time; }; -#endif +#endif /* !defined(OR_CIRCUIT_ST_H) */ diff --git a/src/core/or/or_connection_st.h b/src/core/or/or_connection_st.h index a5ce844bff..051fcd00d3 100644 --- a/src/core/or/or_connection_st.h +++ b/src/core/or/or_connection_st.h @@ -91,4 +91,4 @@ struct or_connection_t { uint64_t bytes_xmitted, bytes_xmitted_by_tls; }; -#endif +#endif /* !defined(OR_CONNECTION_ST_H) */ diff --git a/src/core/or/or_handshake_certs_st.h b/src/core/or/or_handshake_certs_st.h index a93b7104aa..9deb6d6d59 100644 --- a/src/core/or/or_handshake_certs_st.h +++ b/src/core/or/or_handshake_certs_st.h @@ -37,4 +37,4 @@ struct or_handshake_certs_t { size_t ed_rsa_crosscert_len; }; -#endif +#endif /* !defined(OR_HANDSHAKE_CERTS_ST) */ diff --git a/src/core/or/or_handshake_state_st.h b/src/core/or/or_handshake_state_st.h index 09a8a34179..472ce8a302 100644 --- a/src/core/or/or_handshake_state_st.h +++ b/src/core/or/or_handshake_state_st.h @@ -74,5 +74,5 @@ struct or_handshake_state_t { or_handshake_certs_t *certs; }; -#endif +#endif /* !defined(OR_HANDSHAKE_STATE_ST) */ diff --git a/src/core/or/or_periodic.c b/src/core/or/or_periodic.c index 93dfa8cf8b..fe28c99192 100644 --- a/src/core/or/or_periodic.c +++ b/src/core/or/or_periodic.c @@ -18,7 +18,7 @@ #include "core/or/circuituse.h" #include "core/or/or_periodic.h" -#include "src/feature/relay/routermode.h" +#include "feature/relay/routermode.h" #define DECLARE_EVENT(name, roles, flags) \ static periodic_event_item_t name ## _event = \ diff --git a/src/core/or/orconn_event.h b/src/core/or/orconn_event.h index 80289d53e6..d6635793db 100644 --- a/src/core/or/orconn_event.h +++ b/src/core/or/orconn_event.h @@ -117,4 +117,4 @@ void orconn_event_subscribe(orconn_event_rcvr_t); void orconn_event_publish(const orconn_event_msg_t *); #endif -#endif /* defined(TOR_ORCONN_EVENT_H) */ +#endif /* !defined(TOR_ORCONN_EVENT_H) */ diff --git a/src/core/or/orconn_event_sys.h b/src/core/or/orconn_event_sys.h index bfb0a3ac4a..9703b2e3d1 100644 --- a/src/core/or/orconn_event_sys.h +++ b/src/core/or/orconn_event_sys.h @@ -9,4 +9,4 @@ extern const struct subsys_fns_t sys_orconn_event; -#endif /* defined(TOR_ORCONN_SYS_H) */ +#endif /* !defined(TOR_ORCONN_EVENT_SYS_H) */ diff --git a/src/core/or/origin_circuit_st.h b/src/core/or/origin_circuit_st.h index daa5f41dad..01bbc84ae2 100644 --- a/src/core/or/origin_circuit_st.h +++ b/src/core/or/origin_circuit_st.h @@ -295,4 +295,4 @@ struct origin_circuit_t { }; -#endif +#endif /* !defined(ORIGIN_CIRCUIT_ST_H) */ diff --git a/src/core/or/port_cfg_st.h b/src/core/or/port_cfg_st.h index b67091ce32..e9e82bb1de 100644 --- a/src/core/or/port_cfg_st.h +++ b/src/core/or/port_cfg_st.h @@ -31,5 +31,5 @@ struct port_cfg_t { char unix_addr[FLEXIBLE_ARRAY_MEMBER]; }; -#endif +#endif /* !defined(PORT_CFG_ST_H) */ diff --git a/src/core/or/relay_crypto_st.h b/src/core/or/relay_crypto_st.h index 1f243ccdc8..83bbd329a6 100644 --- a/src/core/or/relay_crypto_st.h +++ b/src/core/or/relay_crypto_st.h @@ -30,4 +30,4 @@ struct relay_crypto_t { }; #undef crypto_cipher_t -#endif +#endif /* !defined(RELAY_CRYPTO_ST_H) */ diff --git a/src/core/or/sendme.h b/src/core/or/sendme.h index cdbdf55ac7..20477103fd 100644 --- a/src/core/or/sendme.h +++ b/src/core/or/sendme.h @@ -73,8 +73,8 @@ STATIC bool sendme_is_valid(const circuit_t *circ, const uint8_t *cell_payload, size_t cell_payload_len); -#endif /* TOR_UNIT_TESTS */ +#endif /* defined(TOR_UNIT_TESTS) */ -#endif /* SENDME_PRIVATE */ +#endif /* defined(SENDME_PRIVATE) */ #endif /* !defined(TOR_SENDME_H) */ diff --git a/src/core/or/server_port_cfg_st.h b/src/core/or/server_port_cfg_st.h index bd026af7ee..0738735c61 100644 --- a/src/core/or/server_port_cfg_st.h +++ b/src/core/or/server_port_cfg_st.h @@ -16,5 +16,5 @@ struct server_port_cfg_t { unsigned int bind_ipv6_only : 1; }; -#endif +#endif /* !defined(SERVER_PORT_CFG_ST_H) */ diff --git a/src/core/or/socks_request_st.h b/src/core/or/socks_request_st.h index 5922870c61..9fb941ff7e 100644 --- a/src/core/or/socks_request_st.h +++ b/src/core/or/socks_request_st.h @@ -74,4 +74,4 @@ struct socks_request_t { uint8_t socks5_atyp; /* SOCKS5 address type */ }; -#endif +#endif /* !defined(SOCKS_REQUEST_ST_H) */ diff --git a/src/core/or/tor_version_st.h b/src/core/or/tor_version_st.h index 716429bd32..c5bdcaf07b 100644 --- a/src/core/or/tor_version_st.h +++ b/src/core/or/tor_version_st.h @@ -28,5 +28,5 @@ struct tor_version_t { char git_tag[DIGEST_LEN]; }; -#endif +#endif /* !defined(TOR_VERSION_ST_H) */ diff --git a/src/core/or/var_cell_st.h b/src/core/or/var_cell_st.h index 4287c83f6d..607c0d6c83 100644 --- a/src/core/or/var_cell_st.h +++ b/src/core/or/var_cell_st.h @@ -19,5 +19,5 @@ struct var_cell_t { uint8_t payload[FLEXIBLE_ARRAY_MEMBER]; }; -#endif +#endif /* !defined(VAR_CELL_ST_H) */ diff --git a/src/feature/api/tor_api.c b/src/feature/api/tor_api.c index 697397d46b..fd9d241353 100644 --- a/src/feature/api/tor_api.c +++ b/src/feature/api/tor_api.c @@ -40,10 +40,10 @@ #define raw_socketpair tor_ersatz_socketpair #define raw_closesocket closesocket #define snprintf _snprintf -#else +#else /* !(defined(_WIN32)) */ #define raw_socketpair socketpair #define raw_closesocket close -#endif +#endif /* defined(_WIN32) */ #ifdef HAVE_UNISTD_H #include diff --git a/src/feature/api/tor_api.h b/src/feature/api/tor_api.h index 2bf130c376..cb84853a52 100644 --- a/src/feature/api/tor_api.h +++ b/src/feature/api/tor_api.h @@ -55,7 +55,7 @@ typedef SOCKET tor_control_socket_t; #else typedef int tor_control_socket_t; #define INVALID_TOR_CONTROL_SOCKET (-1) -#endif +#endif /* defined(_WIN32) */ /** DOCDOC */ tor_control_socket_t tor_main_configuration_setup_control_socket( diff --git a/src/feature/control/btrack_circuit.h b/src/feature/control/btrack_circuit.h index c40822f1f1..9e06fefb07 100644 --- a/src/feature/control/btrack_circuit.h +++ b/src/feature/control/btrack_circuit.h @@ -12,4 +12,4 @@ int btrack_circ_init(void); void btrack_circ_fini(void); -#endif /* defined(TOR_BTRACK_CIRCUIT_H) */ +#endif /* !defined(TOR_BTRACK_CIRCUIT_H) */ diff --git a/src/feature/control/btrack_orconn.h b/src/feature/control/btrack_orconn.h index 6ab4892a78..f8f5c1096c 100644 --- a/src/feature/control/btrack_orconn.h +++ b/src/feature/control/btrack_orconn.h @@ -30,9 +30,9 @@ typedef struct bt_orconn_t { bool is_onehop; /**< Is this for a one-hop circuit? */ } bt_orconn_t; -#endif /* defined(BTRACK_ORCONN_PRIVATE) */ +#endif /* defined(BTRACK_ORCONN_PRIVATE) */ int btrack_orconn_init(void); void btrack_orconn_fini(void); -#endif /* defined(TOR_BTRACK_ORCONN_H) */ +#endif /* !defined(TOR_BTRACK_ORCONN_H) */ diff --git a/src/feature/control/btrack_orconn_cevent.h b/src/feature/control/btrack_orconn_cevent.h index 954b452451..afec55581e 100644 --- a/src/feature/control/btrack_orconn_cevent.h +++ b/src/feature/control/btrack_orconn_cevent.h @@ -15,4 +15,4 @@ void bto_cevent_anyconn(const bt_orconn_t *); void bto_cevent_apconn(const bt_orconn_t *); void bto_cevent_reset(void); -#endif /* defined(TOR_BTRACK_ORCONN_CEVENT_H) */ +#endif /* !defined(TOR_BTRACK_ORCONN_CEVENT_H) */ diff --git a/src/feature/control/btrack_orconn_maps.h b/src/feature/control/btrack_orconn_maps.h index 2065eb61b2..c2043fa153 100644 --- a/src/feature/control/btrack_orconn_maps.h +++ b/src/feature/control/btrack_orconn_maps.h @@ -15,4 +15,4 @@ bt_orconn_t *bto_find_or_new(uint64_t, uint64_t); void bto_init_maps(void); void bto_clear_maps(void); -#endif /* defined(TOR_BTRACK_ORCONN_MAPS_H) */ +#endif /* !defined(TOR_BTRACK_ORCONN_MAPS_H) */ diff --git a/src/feature/control/btrack_sys.h b/src/feature/control/btrack_sys.h index fad35b41db..3f831d0640 100644 --- a/src/feature/control/btrack_sys.h +++ b/src/feature/control/btrack_sys.h @@ -11,4 +11,4 @@ extern const struct subsys_fns_t sys_btrack; -#endif /* defined(TOR_BTRACK_SYS_H) */ +#endif /* !defined(TOR_BTRACK_SYS_H) */ diff --git a/src/feature/control/control_connection_st.h b/src/feature/control/control_connection_st.h index cace6bb36f..c9164f03b3 100644 --- a/src/feature/control/control_connection_st.h +++ b/src/feature/control/control_connection_st.h @@ -44,4 +44,4 @@ struct control_connection_t { char *current_cmd; }; -#endif +#endif /* !defined(CONTROL_CONNECTION_ST_H) */ diff --git a/src/feature/control/control_getinfo.h b/src/feature/control/control_getinfo.h index 2d56586f6d..52978686d8 100644 --- a/src/feature/control/control_getinfo.h +++ b/src/feature/control/control_getinfo.h @@ -58,4 +58,4 @@ STATIC int getinfo_helper_current_time( const char **errmsg); #endif /* defined(CONTROL_GETINFO_PRIVATE) */ -#endif /* !defined(TOR_CONTROL_GETINFO) */ +#endif /* !defined(TOR_CONTROL_GETINFO_H) */ diff --git a/src/feature/control/fmt_serverstatus.h b/src/feature/control/fmt_serverstatus.h index 4b95e5b59f..d9190cb7e1 100644 --- a/src/feature/control/fmt_serverstatus.h +++ b/src/feature/control/fmt_serverstatus.h @@ -15,4 +15,4 @@ int list_server_status_v1(smartlist_t *routers, char **router_status_out, int for_controller); -#endif +#endif /* !defined(TOR_FMT_SERVERSTATUS_H) */ diff --git a/src/feature/control/getinfo_geoip.h b/src/feature/control/getinfo_geoip.h index fe22137859..94759d0d18 100644 --- a/src/feature/control/getinfo_geoip.h +++ b/src/feature/control/getinfo_geoip.h @@ -11,4 +11,4 @@ int getinfo_helper_geoip(control_connection_t *control_conn, const char *question, char **answer, const char **errmsg); -#endif +#endif /* !defined(TOR_GETINFO_GEOIP_H) */ diff --git a/src/feature/dirauth/authmode.h b/src/feature/dirauth/authmode.h index 876a1f947b..48afc3cdb4 100644 --- a/src/feature/dirauth/authmode.h +++ b/src/feature/dirauth/authmode.h @@ -29,7 +29,7 @@ authdir_mode_v3(const or_options_t *options) #define have_module_dirauth() (1) -#else /* HAVE_MODULE_DIRAUTH */ +#else /* !(defined(HAVE_MODULE_DIRAUTH)) */ #define authdir_mode(options) (((void)(options)),0) #define authdir_mode_handles_descs(options,purpose) \ @@ -41,6 +41,6 @@ authdir_mode_v3(const or_options_t *options) #define have_module_dirauth() (0) -#endif /* HAVE_MODULE_DIRAUTH */ +#endif /* defined(HAVE_MODULE_DIRAUTH) */ -#endif /* TOR_MODE_H */ +#endif /* !defined(TOR_DIRAUTH_MODE_H) */ diff --git a/src/feature/dirauth/bridgeauth.h b/src/feature/dirauth/bridgeauth.h index cc80fd6375..4905e9c3ee 100644 --- a/src/feature/dirauth/bridgeauth.h +++ b/src/feature/dirauth/bridgeauth.h @@ -9,4 +9,4 @@ void bridgeauth_dump_bridge_status_to_file(time_t now); -#endif +#endif /* !defined(TOR_DIRAUTH_BRIDGEAUTH_H) */ diff --git a/src/feature/dirauth/bwauth.h b/src/feature/dirauth/bwauth.h index 8b7acc4a1c..81c8affbd7 100644 --- a/src/feature/dirauth/bwauth.h +++ b/src/feature/dirauth/bwauth.h @@ -55,4 +55,4 @@ STATIC void dirserv_cache_measured_bw(const measured_bw_line_t *parsed_line, STATIC void dirserv_expire_measured_bw_cache(time_t now); #endif /* defined(BWAUTH_PRIVATE) */ -#endif +#endif /* !defined(TOR_BWAUTH_H) */ diff --git a/src/feature/dirauth/dirauth_periodic.h b/src/feature/dirauth/dirauth_periodic.h index de14cbb3c8..1124fae952 100644 --- a/src/feature/dirauth/dirauth_periodic.h +++ b/src/feature/dirauth/dirauth_periodic.h @@ -12,7 +12,7 @@ void dirauth_register_periodic_events(void); void reschedule_dirvote(const or_options_t *options); -#else +#else /* !(defined(HAVE_MODULE_DIRAUTH)) */ static inline void reschedule_dirvote(const or_options_t *options) @@ -20,6 +20,6 @@ reschedule_dirvote(const or_options_t *options) (void)options; } -#endif +#endif /* defined(HAVE_MODULE_DIRAUTH) */ -#endif +#endif /* !defined(DIRVOTE_PERIODIC_H) */ diff --git a/src/feature/dirauth/dirauth_sys.h b/src/feature/dirauth/dirauth_sys.h index e10f4c9589..4e9b6a2ab4 100644 --- a/src/feature/dirauth/dirauth_sys.h +++ b/src/feature/dirauth/dirauth_sys.h @@ -9,4 +9,4 @@ extern const struct subsys_fns_t sys_dirauth; -#endif +#endif /* !defined(DIRAUTH_SYS_H) */ diff --git a/src/feature/dirauth/dirvote.h b/src/feature/dirauth/dirvote.h index f9de5ebc41..a0cfe0a34c 100644 --- a/src/feature/dirauth/dirvote.h +++ b/src/feature/dirauth/dirvote.h @@ -128,7 +128,7 @@ struct config_line_t; char *format_recommended_version_list(const struct config_line_t *line, int warn); -#else /* HAVE_MODULE_DIRAUTH */ +#else /* !(defined(HAVE_MODULE_DIRAUTH)) */ static inline time_t dirvote_act(const or_options_t *options, time_t now) @@ -193,7 +193,7 @@ dirvote_add_signatures(const char *detached_signatures_body, return 0; } -#endif /* HAVE_MODULE_DIRAUTH */ +#endif /* defined(HAVE_MODULE_DIRAUTH) */ /* Item access */ MOCK_DECL(const char*, dirvote_get_pending_consensus, diff --git a/src/feature/dirauth/dsigs_parse.h b/src/feature/dirauth/dsigs_parse.h index fec51ba488..0cc53072f8 100644 --- a/src/feature/dirauth/dsigs_parse.h +++ b/src/feature/dirauth/dsigs_parse.h @@ -19,4 +19,4 @@ void ns_detached_signatures_free_(ns_detached_signatures_t *s); #define ns_detached_signatures_free(s) \ FREE_AND_NULL(ns_detached_signatures_t, ns_detached_signatures_free_, (s)) -#endif +#endif /* !defined(TOR_DSIGS_PARSE_H) */ diff --git a/src/feature/dirauth/guardfraction.h b/src/feature/dirauth/guardfraction.h index 72404907a4..9f01ded838 100644 --- a/src/feature/dirauth/guardfraction.h +++ b/src/feature/dirauth/guardfraction.h @@ -21,4 +21,4 @@ dirserv_read_guardfraction_file_from_str(const char *guardfraction_file_str, int dirserv_read_guardfraction_file(const char *fname, smartlist_t *vote_routerstatuses); -#endif +#endif /* !defined(TOR_GUARDFRACTION_H) */ diff --git a/src/feature/dirauth/ns_detached_signatures_st.h b/src/feature/dirauth/ns_detached_signatures_st.h index 0f92be2f0d..61d20b7525 100644 --- a/src/feature/dirauth/ns_detached_signatures_st.h +++ b/src/feature/dirauth/ns_detached_signatures_st.h @@ -18,5 +18,5 @@ struct ns_detached_signatures_t { * document_signature_t */ }; -#endif +#endif /* !defined(NS_DETACHED_SIGNATURES_ST_H) */ diff --git a/src/feature/dirauth/process_descs.h b/src/feature/dirauth/process_descs.h index 510e54f813..001c866eba 100644 --- a/src/feature/dirauth/process_descs.h +++ b/src/feature/dirauth/process_descs.h @@ -36,4 +36,4 @@ void dirserv_set_node_flags_from_authoritative_status(node_t *node, int dirserv_would_reject_router(const routerstatus_t *rs); -#endif +#endif /* !defined(TOR_RECV_UPLOADS_H) */ diff --git a/src/feature/dirauth/reachability.h b/src/feature/dirauth/reachability.h index 5a938673ff..873a3f9a23 100644 --- a/src/feature/dirauth/reachability.h +++ b/src/feature/dirauth/reachability.h @@ -33,4 +33,4 @@ int dirserv_should_launch_reachability_test(const routerinfo_t *ri, void dirserv_single_reachability_test(time_t now, routerinfo_t *router); void dirserv_test_reachability(time_t now); -#endif +#endif /* !defined(TOR_REACHABILITY_H) */ diff --git a/src/feature/dirauth/recommend_pkg.h b/src/feature/dirauth/recommend_pkg.h index 1f97d50177..af17e945e8 100644 --- a/src/feature/dirauth/recommend_pkg.h +++ b/src/feature/dirauth/recommend_pkg.h @@ -24,6 +24,6 @@ validate_recommended_package_line(const char *line) return 0; } -#endif +#endif /* defined(HAVE_MODULE_DIRAUTH) */ -#endif +#endif /* !defined(TOR_RECOMMEND_PKG_H) */ diff --git a/src/feature/dirauth/shared_random.h b/src/feature/dirauth/shared_random.h index 0b45ad1ed7..1d8fa89b0f 100644 --- a/src/feature/dirauth/shared_random.h +++ b/src/feature/dirauth/shared_random.h @@ -110,7 +110,7 @@ int sr_init(int save_to_disk); void sr_save_and_cleanup(void); void sr_act_post_consensus(const networkstatus_t *consensus); -#else /* HAVE_MODULE_DIRAUTH */ +#else /* !(defined(HAVE_MODULE_DIRAUTH)) */ static inline int sr_init(int save_to_disk) @@ -131,7 +131,7 @@ sr_act_post_consensus(const networkstatus_t *consensus) (void) consensus; } -#endif /* HAVE_MODULE_DIRAUTH */ +#endif /* defined(HAVE_MODULE_DIRAUTH) */ /* Public methods used only by dirauth code. */ diff --git a/src/feature/dirauth/vote_microdesc_hash_st.h b/src/feature/dirauth/vote_microdesc_hash_st.h index 92acdf1157..7869f92b4f 100644 --- a/src/feature/dirauth/vote_microdesc_hash_st.h +++ b/src/feature/dirauth/vote_microdesc_hash_st.h @@ -18,5 +18,5 @@ struct vote_microdesc_hash_t { char *microdesc_hash_line; }; -#endif +#endif /* !defined(VOTE_MICRODESC_HASH_ST_H) */ diff --git a/src/feature/dirauth/voteflags.h b/src/feature/dirauth/voteflags.h index ee809a290d..c4f36e7817 100644 --- a/src/feature/dirauth/voteflags.h +++ b/src/feature/dirauth/voteflags.h @@ -25,7 +25,7 @@ void dirauth_set_routerstatus_from_routerinfo(routerstatus_t *rs, int listbadexits); void dirserv_compute_performance_thresholds(digestmap_t *omit_as_sybil); -#endif +#endif /* defined(HAVE_MODULE_DIRAUTH) */ void dirserv_set_bridges_running(time_t now); @@ -34,6 +34,6 @@ void dirserv_set_bridges_running(time_t now); * StaleDesc flag. */ #define DESC_IS_STALE_INTERVAL (18*60*60) STATIC void dirserv_set_routerstatus_testing(routerstatus_t *rs); -#endif +#endif /* defined(VOTEFLAGS_PRIVATE) */ -#endif +#endif /* !defined(TOR_VOTEFLAGS_H) */ diff --git a/src/feature/dircache/cached_dir_st.h b/src/feature/dircache/cached_dir_st.h index 71dca8c3a2..a28802f905 100644 --- a/src/feature/dircache/cached_dir_st.h +++ b/src/feature/dircache/cached_dir_st.h @@ -21,5 +21,5 @@ struct cached_dir_t { int refcnt; /**< Reference count for this cached_dir_t. */ }; -#endif +#endif /* !defined(CACHED_DIR_ST_H) */ diff --git a/src/feature/dircache/consdiffmgr.c b/src/feature/dircache/consdiffmgr.c index 6b16307e3c..397efa0341 100644 --- a/src/feature/dircache/consdiffmgr.c +++ b/src/feature/dircache/consdiffmgr.c @@ -525,7 +525,7 @@ consdiffmgr_add_consensus_nulterm(const char *consensus, tor_free(ctmp); return r; } -#endif +#endif /* defined(TOR_UNIT_TESTS) */ /** * Given a buffer containing a networkstatus consensus, and the results of diff --git a/src/feature/dircache/dircache.h b/src/feature/dircache/dircache.h index 236ea649ef..de0d205f6a 100644 --- a/src/feature/dircache/dircache.h +++ b/src/feature/dircache/dircache.h @@ -38,6 +38,6 @@ STATIC int parse_hs_version_from_post(const char *url, const char *prefix, const char **end_pos); STATIC unsigned parse_accept_encoding_header(const char *h); -#endif +#endif /* defined(DIRCACHE_PRIVATE) */ #endif /* !defined(TOR_DIRCACHE_H) */ diff --git a/src/feature/dirclient/dir_server_st.h b/src/feature/dirclient/dir_server_st.h index 2f5706cdd9..8e35532435 100644 --- a/src/feature/dirclient/dir_server_st.h +++ b/src/feature/dirclient/dir_server_st.h @@ -51,4 +51,4 @@ struct dir_server_t { **/ }; -#endif +#endif /* !defined(DIR_SERVER_ST_H) */ diff --git a/src/feature/dirclient/dirclient.h b/src/feature/dirclient/dirclient.h index 1a93265dc3..be4374c7cf 100644 --- a/src/feature/dirclient/dirclient.h +++ b/src/feature/dirclient/dirclient.h @@ -167,6 +167,6 @@ STATIC int handle_response_fetch_consensus(dir_connection_t *conn, STATIC dirinfo_type_t dir_fetch_type(int dir_purpose, int router_purpose, const char *resource); -#endif +#endif /* defined(DIRCLIENT_PRIVATE) */ #endif /* !defined(TOR_DIRCLIENT_H) */ diff --git a/src/feature/dirclient/dlstatus.h b/src/feature/dirclient/dlstatus.h index 99e0d0225b..681712b059 100644 --- a/src/feature/dirclient/dlstatus.h +++ b/src/feature/dirclient/dlstatus.h @@ -53,6 +53,6 @@ STATIC void next_random_exponential_delay_range(int *low_bound_out, /* no more than triple the previous delay */ #define DIR_TEST_NET_RANDOM_MULTIPLIER (2) -#endif +#endif /* defined(DLSTATUS_PRIVATE) */ #endif /* !defined(TOR_DLSTATUS_H) */ diff --git a/src/feature/dirclient/download_status_st.h b/src/feature/dirclient/download_status_st.h index 11555a1dcc..39a5ad2860 100644 --- a/src/feature/dirclient/download_status_st.h +++ b/src/feature/dirclient/download_status_st.h @@ -61,5 +61,5 @@ struct download_status_t { * only updated if backoff == 1 */ }; -#endif +#endif /* !defined(DOWNLOAD_STATUS_ST_H) */ diff --git a/src/feature/dircommon/dir_connection_st.h b/src/feature/dircommon/dir_connection_st.h index 8c59cc7a46..a858560c29 100644 --- a/src/feature/dircommon/dir_connection_st.h +++ b/src/feature/dircommon/dir_connection_st.h @@ -64,4 +64,4 @@ struct dir_connection_t { #endif /* defined(MEASUREMENTS_21206) */ }; -#endif +#endif /* !defined(DIR_CONNECTION_ST_H) */ diff --git a/src/feature/dircommon/vote_timing_st.h b/src/feature/dircommon/vote_timing_st.h index 47b90ab009..814a325314 100644 --- a/src/feature/dircommon/vote_timing_st.h +++ b/src/feature/dircommon/vote_timing_st.h @@ -20,5 +20,5 @@ struct vote_timing_t { int dist_delay; }; -#endif +#endif /* !defined(VOTE_TIMING_ST_H) */ diff --git a/src/feature/dircommon/voting_schedule.h b/src/feature/dircommon/voting_schedule.h index bafd81184e..d78c7ee2da 100644 --- a/src/feature/dircommon/voting_schedule.h +++ b/src/feature/dircommon/voting_schedule.h @@ -61,5 +61,5 @@ time_t voting_schedule_get_start_of_next_interval(time_t now, int offset); time_t voting_schedule_get_next_valid_after_time(void); -#endif /* TOR_VOTING_SCHEDULE_H */ +#endif /* !defined(TOR_VOTING_SCHEDULE_H) */ diff --git a/src/feature/dirparse/microdesc_parse.h b/src/feature/dirparse/microdesc_parse.h index 23a90084b1..95af85544a 100644 --- a/src/feature/dirparse/microdesc_parse.h +++ b/src/feature/dirparse/microdesc_parse.h @@ -17,4 +17,4 @@ smartlist_t *microdescs_parse_from_string(const char *s, const char *eos, saved_location_t where, smartlist_t *invalid_digests_out); -#endif +#endif /* !defined(TOR_MICRODESC_PARSE_H) */ diff --git a/src/feature/dirparse/ns_parse.h b/src/feature/dirparse/ns_parse.h index dedfa6fc88..0cf2cc88d0 100644 --- a/src/feature/dirparse/ns_parse.h +++ b/src/feature/dirparse/ns_parse.h @@ -42,6 +42,6 @@ STATIC routerstatus_t *routerstatus_parse_entry_from_string( vote_routerstatus_t *vote_rs, int consensus_method, consensus_flavor_t flav); -#endif +#endif /* defined(NS_PARSE_PRIVATE) */ -#endif +#endif /* !defined(TOR_NS_PARSE_H) */ diff --git a/src/feature/dirparse/sigcommon.h b/src/feature/dirparse/sigcommon.h index fdd8e839a9..b6b34e8f62 100644 --- a/src/feature/dirparse/sigcommon.h +++ b/src/feature/dirparse/sigcommon.h @@ -43,6 +43,6 @@ MOCK_DECL(STATIC int, signed_digest_equals, MOCK_DECL(STATIC int, router_compute_hash_final,(char *digest, const char *start, size_t len, digest_algorithm_t alg)); -#endif +#endif /* defined(SIGCOMMON_PRIVATE) */ #endif /* !defined(TOR_SIGCOMMON_H) */ diff --git a/src/feature/dirparse/signing.h b/src/feature/dirparse/signing.h index 2e3699baf8..8b119b4eb2 100644 --- a/src/feature/dirparse/signing.h +++ b/src/feature/dirparse/signing.h @@ -20,4 +20,4 @@ int router_append_dirobj_signature(char *buf, size_t buf_len, const char *digest, size_t digest_len, crypto_pk_t *private_key); -#endif +#endif /* !defined(TOR_SIGNING_H) */ diff --git a/src/feature/dirparse/unparseable.h b/src/feature/dirparse/unparseable.h index 853fe8cb0f..49e047961f 100644 --- a/src/feature/dirparse/unparseable.h +++ b/src/feature/dirparse/unparseable.h @@ -51,6 +51,6 @@ EXTERN(struct smartlist_t *, descs_dumped) MOCK_DECL(STATIC dumped_desc_t *, dump_desc_populate_one_file, (const char *dirname, const char *f)); STATIC void dump_desc_populate_fifo_from_directory(const char *dirname); -#endif +#endif /* defined(UNPARSEABLE_PRIVATE) */ #endif /* !defined(TOR_UNPARSEABLE_H) */ diff --git a/src/feature/hibernate/hibernate.c b/src/feature/hibernate/hibernate.c index 7351e5e002..674fe3c813 100644 --- a/src/feature/hibernate/hibernate.c +++ b/src/feature/hibernate/hibernate.c @@ -57,7 +57,7 @@ hibernating, phase 2: * Coverity. Here's a kludge to unconfuse it. */ # define __INCLUDE_LEVEL__ 2 -# endif /* defined(__COVERITY__) && !defined(__INCLUDE_LEVEL__) */ +#endif /* defined(__COVERITY__) && !defined(__INCLUDE_LEVEL__) */ #include #endif /* defined(HAVE_SYSTEMD) */ @@ -893,7 +893,7 @@ hibernate_begin(hibernate_state_t new_state, time_t now) */ sd_notifyf(0, "EXTEND_TIMEOUT_USEC=%" PRIu64, ((uint64_t)(options->ShutdownWaitLength) + 30) * TOR_USEC_PER_SEC); -#endif +#endif /* defined(HAVE_SYSTEMD) */ } else { /* soft limit reached */ hibernate_end_time = interval_end_time; } diff --git a/src/feature/hs/hs_service.h b/src/feature/hs/hs_service.h index 8d7f773219..22aa00b2d7 100644 --- a/src/feature/hs/hs_service.h +++ b/src/feature/hs/hs_service.h @@ -361,7 +361,7 @@ STATIC hs_service_t *get_first_service(void); STATIC hs_service_intro_point_t *service_intro_point_find_by_ident( const hs_service_t *service, const hs_ident_circuit_t *ident); -#endif +#endif /* defined(TOR_UNIT_TESTS) */ /* Service accessors. */ STATIC hs_service_t *find_service(hs_service_ht *map, diff --git a/src/feature/hs/hs_stats.h b/src/feature/hs/hs_stats.h index ca048e2123..6700eca15b 100644 --- a/src/feature/hs/hs_stats.h +++ b/src/feature/hs/hs_stats.h @@ -15,4 +15,4 @@ uint32_t hs_stats_get_n_introduce2_v2_cells(void); void hs_stats_note_service_rendezvous_launch(void); uint32_t hs_stats_get_n_rendezvous_launches(void); -#endif +#endif /* !defined(TOR_HS_STATS_H) */ diff --git a/src/feature/hs/hsdir_index_st.h b/src/feature/hs/hsdir_index_st.h index 7d4116d8bb..6c86c02f47 100644 --- a/src/feature/hs/hsdir_index_st.h +++ b/src/feature/hs/hsdir_index_st.h @@ -20,5 +20,5 @@ struct hsdir_index_t { uint8_t store_second[DIGEST256_LEN]; }; -#endif +#endif /* !defined(HSDIR_INDEX_ST_H) */ diff --git a/src/feature/hs_common/shared_random_client.h b/src/feature/hs_common/shared_random_client.h index 95fe2c65ab..c90c52cfea 100644 --- a/src/feature/hs_common/shared_random_client.h +++ b/src/feature/hs_common/shared_random_client.h @@ -44,5 +44,5 @@ time_t get_start_time_of_current_round(void); #endif /* TOR_UNIT_TESTS */ -#endif /* TOR_SHARED_RANDOM_CLIENT_H */ +#endif /* !defined(TOR_SHARED_RANDOM_CLIENT_H) */ diff --git a/src/feature/keymgt/loadkey.h b/src/feature/keymgt/loadkey.h index 8beee57a20..0a5af0b804 100644 --- a/src/feature/keymgt/loadkey.h +++ b/src/feature/keymgt/loadkey.h @@ -52,4 +52,4 @@ int read_encrypted_secret_key(ed25519_secret_key_t *out, int write_encrypted_secret_key(const ed25519_secret_key_t *out, const char *fname); -#endif +#endif /* !defined(TOR_LOADKEY_H) */ diff --git a/src/feature/nodelist/authcert.h b/src/feature/nodelist/authcert.h index 2effdb06e6..071293f9ee 100644 --- a/src/feature/nodelist/authcert.h +++ b/src/feature/nodelist/authcert.h @@ -57,4 +57,4 @@ MOCK_DECL(download_status_t *, download_status_for_authority_id_and_sk, void authcert_free_all(void); -#endif +#endif /* !defined(TOR_AUTHCERT_H) */ diff --git a/src/feature/nodelist/authority_cert_st.h b/src/feature/nodelist/authority_cert_st.h index 68a84bc452..bf9b690c24 100644 --- a/src/feature/nodelist/authority_cert_st.h +++ b/src/feature/nodelist/authority_cert_st.h @@ -28,5 +28,5 @@ struct authority_cert_t { uint16_t dir_port; }; -#endif +#endif /* !defined(AUTHORITY_CERT_ST_H) */ diff --git a/src/feature/nodelist/desc_store_st.h b/src/feature/nodelist/desc_store_st.h index b04a1abc7d..4d1378cdfa 100644 --- a/src/feature/nodelist/desc_store_st.h +++ b/src/feature/nodelist/desc_store_st.h @@ -36,4 +36,4 @@ struct desc_store_t { size_t bytes_dropped; }; -#endif +#endif /* !defined(DESC_STORE_ST_H) */ diff --git a/src/feature/nodelist/describe.h b/src/feature/nodelist/describe.h index 018af6470e..d29192200e 100644 --- a/src/feature/nodelist/describe.h +++ b/src/feature/nodelist/describe.h @@ -22,4 +22,4 @@ const char *node_describe(const struct node_t *node); const char *router_describe(const struct routerinfo_t *ri); const char *routerstatus_describe(const struct routerstatus_t *ri); -#endif +#endif /* !defined(TOR_DESCRIBE_H) */ diff --git a/src/feature/nodelist/dirlist.h b/src/feature/nodelist/dirlist.h index 9fabd0a44a..b6dda32d85 100644 --- a/src/feature/nodelist/dirlist.h +++ b/src/feature/nodelist/dirlist.h @@ -44,4 +44,4 @@ void dir_server_add(dir_server_t *ent); void clear_dir_servers(void); void dirlist_free_all(void); -#endif +#endif /* !defined(TOR_DIRLIST_H) */ diff --git a/src/feature/nodelist/document_signature_st.h b/src/feature/nodelist/document_signature_st.h index 66e32c422f..ac2a803252 100644 --- a/src/feature/nodelist/document_signature_st.h +++ b/src/feature/nodelist/document_signature_st.h @@ -25,5 +25,5 @@ struct document_signature_t { * as good. */ }; -#endif +#endif /* !defined(DOCUMENT_SIGNATURE_ST_H) */ diff --git a/src/feature/nodelist/extrainfo_st.h b/src/feature/nodelist/extrainfo_st.h index c54277b05e..22c708f018 100644 --- a/src/feature/nodelist/extrainfo_st.h +++ b/src/feature/nodelist/extrainfo_st.h @@ -26,5 +26,5 @@ struct extrainfo_t { size_t pending_sig_len; }; -#endif +#endif /* !defined(EXTRAINFO_ST_H) */ diff --git a/src/feature/nodelist/microdesc_st.h b/src/feature/nodelist/microdesc_st.h index 367e6a3ef6..c8265cb778 100644 --- a/src/feature/nodelist/microdesc_st.h +++ b/src/feature/nodelist/microdesc_st.h @@ -78,4 +78,4 @@ struct microdesc_t { struct short_policy_t *ipv6_exit_policy; }; -#endif +#endif /* !defined(MICRODESC_ST_H) */ diff --git a/src/feature/nodelist/networkstatus.c b/src/feature/nodelist/networkstatus.c index c7e337309e..2db293a8af 100644 --- a/src/feature/nodelist/networkstatus.c +++ b/src/feature/nodelist/networkstatus.c @@ -1772,7 +1772,7 @@ reload_consensus_from_file(const char *fname, flavor, flags, source_dir); tor_free(content); } -#endif +#endif /* defined(_WIN32) */ if (rv < -1) { log_warn(LD_GENERAL, "Couldn't set consensus from cache file %s", escaped(fname)); diff --git a/src/feature/nodelist/networkstatus_sr_info_st.h b/src/feature/nodelist/networkstatus_sr_info_st.h index 677d8ed811..420c3d61e4 100644 --- a/src/feature/nodelist/networkstatus_sr_info_st.h +++ b/src/feature/nodelist/networkstatus_sr_info_st.h @@ -19,5 +19,5 @@ struct networkstatus_sr_info_t { smartlist_t *commits; }; -#endif +#endif /* !defined(NETWORKSTATUS_SR_INFO_ST_H) */ diff --git a/src/feature/nodelist/networkstatus_st.h b/src/feature/nodelist/networkstatus_st.h index 5c1eea3259..6e84c170d6 100644 --- a/src/feature/nodelist/networkstatus_st.h +++ b/src/feature/nodelist/networkstatus_st.h @@ -104,4 +104,4 @@ struct networkstatus_t { uint8_t bw_file_digest256[DIGEST256_LEN]; }; -#endif +#endif /* !defined(NETWORKSTATUS_ST_H) */ diff --git a/src/feature/nodelist/networkstatus_voter_info_st.h b/src/feature/nodelist/networkstatus_voter_info_st.h index 4037fcdeca..66af82a8e3 100644 --- a/src/feature/nodelist/networkstatus_voter_info_st.h +++ b/src/feature/nodelist/networkstatus_voter_info_st.h @@ -27,4 +27,4 @@ struct networkstatus_voter_info_t { smartlist_t *sigs; }; -#endif +#endif /* !defined(NETWORKSTATUS_VOTER_INFO_ST_H) */ diff --git a/src/feature/nodelist/nickname.h b/src/feature/nodelist/nickname.h index 9bdc6b50e8..78db2a5f91 100644 --- a/src/feature/nodelist/nickname.h +++ b/src/feature/nodelist/nickname.h @@ -16,4 +16,4 @@ int is_legal_nickname(const char *s); int is_legal_nickname_or_hexdigest(const char *s); int is_legal_hexdigest(const char *s); -#endif +#endif /* !defined(TOR_NICKNAME_H) */ diff --git a/src/feature/nodelist/node_select.h b/src/feature/nodelist/node_select.h index ed7450b92c..d8b4aca5c1 100644 --- a/src/feature/nodelist/node_select.h +++ b/src/feature/nodelist/node_select.h @@ -97,6 +97,6 @@ STATIC const routerstatus_t *router_pick_directory_server_impl( int *n_busy_out); STATIC int router_is_already_dir_fetching(const tor_addr_port_t *ap, int serverdesc, int microdesc); -#endif +#endif /* defined(NODE_SELECT_PRIVATE) */ -#endif +#endif /* !defined(TOR_NODE_SELECT_H) */ diff --git a/src/feature/nodelist/node_st.h b/src/feature/nodelist/node_st.h index 53ffde29e4..c63a535a19 100644 --- a/src/feature/nodelist/node_st.h +++ b/src/feature/nodelist/node_st.h @@ -99,4 +99,4 @@ struct node_t { struct hsdir_index_t hsdir_index; }; -#endif +#endif /* !defined(NODE_ST_H) */ diff --git a/src/feature/nodelist/nodefamily.h b/src/feature/nodelist/nodefamily.h index bc5dafce03..31b71e77a0 100644 --- a/src/feature/nodelist/nodefamily.h +++ b/src/feature/nodelist/nodefamily.h @@ -47,4 +47,4 @@ char *nodefamily_canonicalize(const char *s, const uint8_t *rsa_id_self, void nodefamily_free_all(void); -#endif +#endif /* !defined(TOR_NODEFAMILY_H) */ diff --git a/src/feature/nodelist/nodefamily_st.h b/src/feature/nodelist/nodefamily_st.h index be533da824..20390c9308 100644 --- a/src/feature/nodelist/nodefamily_st.h +++ b/src/feature/nodelist/nodefamily_st.h @@ -45,4 +45,4 @@ struct nodefamily_t { #define NODEFAMILY_MEMBER_PTR(nf, i) \ (&((nf)->family_members[(i) * NODEFAMILY_MEMBER_LEN])) -#endif +#endif /* !defined(TOR_NODEFAMILY_ST_H) */ diff --git a/src/feature/nodelist/routerinfo.h b/src/feature/nodelist/routerinfo.h index bfa28c7754..ca66e660b3 100644 --- a/src/feature/nodelist/routerinfo.h +++ b/src/feature/nodelist/routerinfo.h @@ -24,4 +24,4 @@ smartlist_t *router_get_all_orports(const routerinfo_t *ri); const char *router_purpose_to_string(uint8_t p); uint8_t router_purpose_from_string(const char *s); -#endif +#endif /* !defined(TOR_ROUTERINFO_H) */ diff --git a/src/feature/nodelist/routerinfo_st.h b/src/feature/nodelist/routerinfo_st.h index 59656818c1..59fd56d0a0 100644 --- a/src/feature/nodelist/routerinfo_st.h +++ b/src/feature/nodelist/routerinfo_st.h @@ -112,4 +112,4 @@ struct routerinfo_t { uint8_t purpose; }; -#endif +#endif /* !defined(ROUTERINFO_ST_H) */ diff --git a/src/feature/nodelist/routerlist_st.h b/src/feature/nodelist/routerlist_st.h index 7446ead3cb..10b919a1bf 100644 --- a/src/feature/nodelist/routerlist_st.h +++ b/src/feature/nodelist/routerlist_st.h @@ -36,5 +36,5 @@ struct routerlist_t { desc_store_t extrainfo_store; }; -#endif +#endif /* !defined(ROUTERLIST_ST_H) */ diff --git a/src/feature/nodelist/routerstatus_st.h b/src/feature/nodelist/routerstatus_st.h index 8d91b45e11..46337c9e52 100644 --- a/src/feature/nodelist/routerstatus_st.h +++ b/src/feature/nodelist/routerstatus_st.h @@ -78,5 +78,5 @@ struct routerstatus_t { }; -#endif +#endif /* !defined(ROUTERSTATUS_ST_H) */ diff --git a/src/feature/nodelist/signed_descriptor_st.h b/src/feature/nodelist/signed_descriptor_st.h index bdcebf184a..64c28f7440 100644 --- a/src/feature/nodelist/signed_descriptor_st.h +++ b/src/feature/nodelist/signed_descriptor_st.h @@ -57,5 +57,5 @@ struct signed_descriptor_t { unsigned int send_unencrypted : 1; }; -#endif +#endif /* !defined(SIGNED_DESCRIPTOR_ST_H) */ diff --git a/src/feature/nodelist/vote_routerstatus_st.h b/src/feature/nodelist/vote_routerstatus_st.h index 366754c166..0d909da260 100644 --- a/src/feature/nodelist/vote_routerstatus_st.h +++ b/src/feature/nodelist/vote_routerstatus_st.h @@ -38,4 +38,4 @@ struct vote_routerstatus_t { uint8_t ed25519_id[ED25519_PUBKEY_LEN]; }; -#endif +#endif /* !defined(VOTE_ROUTERSTATUS_ST_H) */ diff --git a/src/feature/relay/dns.c b/src/feature/relay/dns.c index f68efc9096..05b97e0ae2 100644 --- a/src/feature/relay/dns.c +++ b/src/feature/relay/dns.c @@ -1394,7 +1394,7 @@ configured_nameserver_address(const size_t idx) return NULL; } -#endif +#endif /* defined(HAVE_EVDNS_BASE_GET_NAMESERVER_ADDR) */ /** Configure eventdns nameservers if force is true, or if the configuration * has changed since the last time we called this function, or if we failed on diff --git a/src/feature/relay/onion_queue.h b/src/feature/relay/onion_queue.h index 0df921e057..cf478bc1a0 100644 --- a/src/feature/relay/onion_queue.h +++ b/src/feature/relay/onion_queue.h @@ -20,4 +20,4 @@ int onion_num_pending(uint16_t handshake_type); void onion_pending_remove(or_circuit_t *circ); void clear_pending_onions(void); -#endif +#endif /* !defined(TOR_ONION_QUEUE_H) */ diff --git a/src/feature/relay/relay_periodic.c b/src/feature/relay/relay_periodic.c index 8908b57415..b48b495895 100644 --- a/src/feature/relay/relay_periodic.c +++ b/src/feature/relay/relay_periodic.c @@ -24,7 +24,7 @@ #include "feature/relay/routerkeys.h" #include "feature/relay/routermode.h" #include "feature/relay/selftest.h" -#include "src/feature/stats/predict_ports.h" +#include "feature/stats/predict_ports.h" #include "lib/crypt_ops/crypto_rand.h" diff --git a/src/feature/relay/router.c b/src/feature/relay/router.c index 82dad3191d..e0daf34db2 100644 --- a/src/feature/relay/router.c +++ b/src/feature/relay/router.c @@ -353,7 +353,7 @@ set_server_identity_key_digest_testing(const uint8_t *digest) { memcpy(server_identitykey_digest, digest, DIGEST_LEN); } -#endif +#endif /* defined(TOR_UNIT_TESTS) */ /** Make sure that we have set up our identity keys to match or not match as * appropriate, and die with an assertion if we have not. */ diff --git a/src/feature/relay/selftest.h b/src/feature/relay/selftest.h index a80ec8936e..aea77ec791 100644 --- a/src/feature/relay/selftest.h +++ b/src/feature/relay/selftest.h @@ -21,4 +21,4 @@ void router_orport_found_reachable(void); void router_dirport_found_reachable(void); void router_perform_bandwidth_test(int num_circs, time_t now); -#endif +#endif /* !defined(TOR_SELFTEST_H) */ diff --git a/src/feature/rend/rend_authorized_client_st.h b/src/feature/rend/rend_authorized_client_st.h index 7bd4f2fe8c..51a1798fcb 100644 --- a/src/feature/rend/rend_authorized_client_st.h +++ b/src/feature/rend/rend_authorized_client_st.h @@ -14,5 +14,5 @@ struct rend_authorized_client_t { crypto_pk_t *client_key; }; -#endif +#endif /* !defined(REND_AUTHORIZED_CLIENT_ST_H) */ diff --git a/src/feature/rend/rend_encoded_v2_service_descriptor_st.h b/src/feature/rend/rend_encoded_v2_service_descriptor_st.h index 05ff145d53..bd8a60f0d9 100644 --- a/src/feature/rend/rend_encoded_v2_service_descriptor_st.h +++ b/src/feature/rend/rend_encoded_v2_service_descriptor_st.h @@ -13,5 +13,5 @@ struct rend_encoded_v2_service_descriptor_t { char *desc_str; /**< Descriptor string. */ }; -#endif +#endif /* !defined(REND_ENCODED_V2_SERVICE_DESCRIPTOR_ST_H) */ diff --git a/src/feature/rend/rend_intro_point_st.h b/src/feature/rend/rend_intro_point_st.h index de6987e569..4882b62752 100644 --- a/src/feature/rend/rend_intro_point_st.h +++ b/src/feature/rend/rend_intro_point_st.h @@ -73,4 +73,4 @@ struct rend_intro_point_t { unsigned int circuit_established:1; }; -#endif +#endif /* !defined(REND_INTRO_POINT_ST_H) */ diff --git a/src/feature/rend/rend_service_descriptor_st.h b/src/feature/rend/rend_service_descriptor_st.h index aeb3178064..ff7627ce96 100644 --- a/src/feature/rend/rend_service_descriptor_st.h +++ b/src/feature/rend/rend_service_descriptor_st.h @@ -30,5 +30,5 @@ struct rend_service_descriptor_t { smartlist_t *successful_uploads; }; -#endif +#endif /* !defined(REND_SERVICE_DESCRIPTOR_ST_H) */ diff --git a/src/feature/rend/rendparse.h b/src/feature/rend/rendparse.h index 0cef931e90..b1ccce9b6c 100644 --- a/src/feature/rend/rendparse.h +++ b/src/feature/rend/rendparse.h @@ -29,4 +29,4 @@ int rend_parse_introduction_points(rend_service_descriptor_t *parsed, size_t intro_points_encoded_size); int rend_parse_client_keys(strmap_t *parsed_clients, const char *str); -#endif +#endif /* !defined(TOR_REND_PARSE_H) */ diff --git a/src/feature/stats/predict_ports.h b/src/feature/stats/predict_ports.h index 272344da2f..45b206c23a 100644 --- a/src/feature/stats/predict_ports.h +++ b/src/feature/stats/predict_ports.h @@ -27,4 +27,4 @@ int rep_hist_circbuilding_dormant(time_t now); int predicted_ports_prediction_time_remaining(time_t now); void predicted_ports_free_all(void); -#endif +#endif /* !defined(TOR_PREDICT_PORTS_H) */ diff --git a/src/feature/stats/rephist.h b/src/feature/stats/rephist.h index 3accc8c610..0d72946382 100644 --- a/src/feature/stats/rephist.h +++ b/src/feature/stats/rephist.h @@ -103,7 +103,7 @@ typedef struct bw_array_t bw_array_t; STATIC uint64_t find_largest_max(bw_array_t *b); STATIC void commit_max(bw_array_t *b); STATIC void advance_obs(bw_array_t *b); -#endif +#endif /* defined(REPHIST_PRIVATE) */ /** * Represents the type of a cell for padding accounting diff --git a/src/lib/arch/bytes.h b/src/lib/arch/bytes.h index fa82241b28..b8b6288139 100644 --- a/src/lib/arch/bytes.h +++ b/src/lib/arch/bytes.h @@ -129,7 +129,7 @@ tor_ntohll(uint64_t a) { return a; } -#else +#else /* !(defined(WORDS_BIGENDIAN)) */ static inline uint16_t tor_htons(uint16_t a) { @@ -177,6 +177,6 @@ tor_ntohll(uint64_t a) { return tor_htonll(a); } -#endif +#endif /* defined(WORDS_BIGENDIAN) */ -#endif +#endif /* !defined(TOR_BYTES_H) */ diff --git a/src/lib/cc/compat_compiler.h b/src/lib/cc/compat_compiler.h index 18b76cc1a1..a8d1593214 100644 --- a/src/lib/cc/compat_compiler.h +++ b/src/lib/cc/compat_compiler.h @@ -229,4 +229,4 @@ #define EAT_SEMICOLON \ struct dummy_semicolon_eater__ -#endif /* !defined(TOR_COMPAT_H) */ +#endif /* !defined(TOR_COMPAT_COMPILER_H) */ diff --git a/src/lib/cc/ctassert.h b/src/lib/cc/ctassert.h index e42976360f..bedf0b83a6 100644 --- a/src/lib/cc/ctassert.h +++ b/src/lib/cc/ctassert.h @@ -22,7 +22,7 @@ /* If C11 is available, just use _Static_assert. */ #define CTASSERT(x) _Static_assert((x), #x) -#else +#else /* !(__STDC_VERSION__ >= 201112L) */ /* * If C11 is not available, expand __COUNTER__, or __INCLUDE_LEVEL__ @@ -42,12 +42,12 @@ #else /* hope it's unique enough */ #define CTASSERT(x) CTASSERT_EXPN((x), l, __LINE__) -#endif +#endif /* defined(__COUNTER__) || ... */ #define CTASSERT_EXPN(x, a, b) CTASSERT_DECL(x, a, b) #define CTASSERT_DECL(x, a, b) \ typedef char tor_ctassert_##a##_##b[(x) ? 1 : -1] ATTR_UNUSED -#endif +#endif /* __STDC_VERSION__ >= 201112L */ #endif /* !defined(TOR_CTASSERT_H) */ diff --git a/src/lib/cc/torint.h b/src/lib/cc/torint.h index 9a66aada18..523f378ed7 100644 --- a/src/lib/cc/torint.h +++ b/src/lib/cc/torint.h @@ -96,9 +96,9 @@ typedef int32_t ssize_t; # else # define TOR_PRIuSZ PRIu32 # endif -#else +#else /* !(defined(_WIN32)) */ # define TOR_PRIuSZ "zu" -#endif +#endif /* defined(_WIN32) */ #ifdef _WIN32 # ifdef _WIN64 @@ -106,9 +106,9 @@ typedef int32_t ssize_t; # else # define TOR_PRIdSZ PRId32 # endif -#else +#else /* !(defined(_WIN32)) */ # define TOR_PRIdSZ "zd" -#endif +#endif /* defined(_WIN32) */ #ifndef SSIZE_MAX #if (SIZEOF_SIZE_T == 4) diff --git a/src/lib/compress/compress_zstd.c b/src/lib/compress/compress_zstd.c index 45d0d4d602..a99ea67e0b 100644 --- a/src/lib/compress/compress_zstd.c +++ b/src/lib/compress/compress_zstd.c @@ -25,7 +25,7 @@ * all invocations of zstd's static-only functions in a check to make sure * that the compile-time version matches the run-time version. */ #define ZSTD_STATIC_LINKING_ONLY -#endif +#endif /* defined(ENABLE_ZSTD_ADVANCED_APIS) */ #ifdef HAVE_ZSTD #ifdef HAVE_CFLAG_WUNUSED_CONST_VARIABLE @@ -35,7 +35,7 @@ DISABLE_GCC_WARNING(unused-const-variable) #ifdef HAVE_CFLAG_WUNUSED_CONST_VARIABLE ENABLE_GCC_WARNING(unused-const-variable) #endif -#endif +#endif /* defined(HAVE_ZSTD) */ /** Total number of bytes allocated for Zstandard state. */ static atomic_counter_t total_zstd_allocation; @@ -77,7 +77,7 @@ tor_zstd_format_version(char *buf, size_t buflen, unsigned version_number) version_number / 100 % 100, version_number % 100); } -#endif +#endif /* defined(HAVE_ZSTD) */ #define VERSION_STR_MAX_LEN 16 /* more than enough space for 99.99.99 */ @@ -125,9 +125,9 @@ tor_zstd_can_use_static_apis(void) } #endif return (ZSTD_VERSION_NUMBER == ZSTD_versionNumber()); -#else +#else /* !(defined(ZSTD_STATIC_LINKING_ONLY) && defined(HAVE_ZSTD)) */ return 0; -#endif +#endif /* defined(ZSTD_STATIC_LINKING_ONLY) && defined(HAVE_ZSTD) */ } /** Internal Zstandard state for incremental compression/decompression. @@ -237,7 +237,7 @@ tor_zstd_state_size_precalc(int compress, int preset) #endif } } -#endif +#endif /* defined(ZSTD_STATIC_LINKING_ONLY) */ return tor_zstd_state_size_precalc_fake(compress, preset); } #endif /* defined(HAVE_ZSTD) */ @@ -527,7 +527,7 @@ tor_zstd_warn_if_version_mismatched(void) "For safety, we'll avoid using advanced zstd functionality.", header_version, runtime_version); } -#endif +#endif /* defined(HAVE_ZSTD) && defined(ENABLE_ZSTD_ADVANCED_APIS) */ } #ifdef TOR_UNIT_TESTS @@ -538,4 +538,4 @@ tor_zstd_set_static_apis_disabled_for_testing(int disabled) { static_apis_disable_for_testing = disabled; } -#endif +#endif /* defined(TOR_UNIT_TESTS) */ diff --git a/src/lib/container/bitarray.h b/src/lib/container/bitarray.h index 910d5fea65..45992796af 100644 --- a/src/lib/container/bitarray.h +++ b/src/lib/container/bitarray.h @@ -83,4 +83,4 @@ bitarray_is_set(bitarray_t *b, int bit) return b[bit >> BITARRAY_SHIFT] & (1u << (bit & BITARRAY_MASK)); } -#endif /* !defined(TOR_CONTAINER_H) */ +#endif /* !defined(TOR_BITARRAY_H) */ diff --git a/src/lib/container/map.h b/src/lib/container/map.h index d61b1ec18f..9da1d3072c 100644 --- a/src/lib/container/map.h +++ b/src/lib/container/map.h @@ -258,4 +258,4 @@ void* strmap_remove_lc(strmap_t *map, const char *key); return digestmap_iter_done((digestmap_iter_t*)iter); \ } -#endif /* !defined(TOR_CONTAINER_H) */ +#endif /* !defined(TOR_MAP_H) */ diff --git a/src/lib/container/namemap.h b/src/lib/container/namemap.h index 97792e13ba..b96bc13f3a 100644 --- a/src/lib/container/namemap.h +++ b/src/lib/container/namemap.h @@ -32,4 +32,4 @@ unsigned namemap_get_or_create_id(namemap_t *map, size_t namemap_get_size(const namemap_t *map); void namemap_clear(namemap_t *map); -#endif +#endif /* !defined(TOR_NAMEMAP_H) */ diff --git a/src/lib/container/namemap_st.h b/src/lib/container/namemap_st.h index 5717352fa2..5008fd5855 100644 --- a/src/lib/container/namemap_st.h +++ b/src/lib/container/namemap_st.h @@ -31,4 +31,4 @@ struct namemap_t { /** Macro to initialize a namemap. */ #define NAMEMAP_INIT() { HT_INITIALIZER(), NULL } -#endif +#endif /* !defined(NAMEMAP_ST_H) */ diff --git a/src/lib/container/order.h b/src/lib/container/order.h index a176d6d8a6..3f2fd054a0 100644 --- a/src/lib/container/order.h +++ b/src/lib/container/order.h @@ -57,4 +57,4 @@ third_quartile_uint32(uint32_t *array, int n_elements) return find_nth_uint32(array, n_elements, (n_elements*3)/4); } -#endif /* !defined(TOR_CONTAINER_H) */ +#endif /* !defined(TOR_ORDER_H) */ diff --git a/src/lib/container/smartlist.h b/src/lib/container/smartlist.h index 77682db03e..81b0151433 100644 --- a/src/lib/container/smartlist.h +++ b/src/lib/container/smartlist.h @@ -165,4 +165,4 @@ char *smartlist_join_strings2(smartlist_t *sl, const char *join, } \ STMT_END -#endif /* !defined(TOR_CONTAINER_H) */ +#endif /* !defined(TOR_SMARTLIST_H) */ diff --git a/src/lib/crypt_ops/crypto_cipher.h b/src/lib/crypt_ops/crypto_cipher.h index cc4fbf7a41..88d63c1df2 100644 --- a/src/lib/crypt_ops/crypto_cipher.h +++ b/src/lib/crypt_ops/crypto_cipher.h @@ -54,4 +54,4 @@ int crypto_cipher_decrypt_with_iv(const char *key, char *to, size_t tolen, const char *from, size_t fromlen); -#endif /* !defined(TOR_CRYPTO_H) */ +#endif /* !defined(TOR_CRYPTO_CIPHER_H) */ diff --git a/src/lib/crypt_ops/crypto_dh_openssl.c b/src/lib/crypt_ops/crypto_dh_openssl.c index 8c6388fd5d..75cee1b596 100644 --- a/src/lib/crypt_ops/crypto_dh_openssl.c +++ b/src/lib/crypt_ops/crypto_dh_openssl.c @@ -34,7 +34,7 @@ static int tor_check_dh_key(int severity, const BIGNUM *bn); struct crypto_dh_t { DH *dh; /**< The openssl DH object */ }; -#endif +#endif /* !defined(ENABLE_NSS) */ static DH *new_openssl_dh_from_params(BIGNUM *p, BIGNUM *g); @@ -100,7 +100,7 @@ crypto_validate_dh_params(const BIGNUM *p, const BIGNUM *g) DH_free(dh); return ret; } -#endif +#endif /* 0 */ /** * Helper: convert hex to a bignum, and return it. Assert that the @@ -202,7 +202,7 @@ crypto_dh_new(int dh_type) tor_free(res); // sets res to NULL. return res; } -#endif +#endif /* !defined(ENABLE_NSS) */ /** Create and return a new openssl DH from a given prime and generator. */ static DH * @@ -461,7 +461,7 @@ crypto_dh_free_(crypto_dh_t *dh) DH_free(dh->dh); tor_free(dh); } -#endif +#endif /* !defined(ENABLE_NSS) */ void crypto_dh_free_all_openssl(void) diff --git a/src/lib/crypt_ops/crypto_digest.c b/src/lib/crypt_ops/crypto_digest.c index 9da135e9c4..64a7d2d52c 100644 --- a/src/lib/crypt_ops/crypto_digest.c +++ b/src/lib/crypt_ops/crypto_digest.c @@ -149,9 +149,9 @@ struct crypto_xof_t { * outside the tests yet. */ EVP_MD_CTX *ctx; -#else +#else /* !(defined(OPENSSL_HAS_SHAKE3_EVP)) */ keccak_state s; -#endif +#endif /* defined(OPENSSL_HAS_SHAKE3_EVP) */ }; /** Allocate a new XOF object backed by SHAKE-256. The security level @@ -169,9 +169,9 @@ crypto_xof_new(void) tor_assert(xof->ctx); int r = EVP_DigestInit(xof->ctx, EVP_shake256()); tor_assert(r == 1); -#else +#else /* !(defined(OPENSSL_HAS_SHAKE256)) */ keccak_xof_init(&xof->s, 256); -#endif +#endif /* defined(OPENSSL_HAS_SHAKE256) */ return xof; } @@ -188,7 +188,7 @@ crypto_xof_add_bytes(crypto_xof_t *xof, const uint8_t *data, size_t len) #else int i = keccak_xof_absorb(&xof->s, data, len); tor_assert(i == 0); -#endif +#endif /* defined(OPENSSL_HAS_SHAKE256) */ } /** Squeeze bytes out of a XOF object. Calling this routine will render @@ -203,7 +203,7 @@ crypto_xof_squeeze_bytes(crypto_xof_t *xof, uint8_t *out, size_t len) #else int i = keccak_xof_squeeze(&xof->s, out, len); tor_assert(i == 0); -#endif +#endif /* defined(OPENSSL_HAS_SHAKE256) */ } /** Cleanse and deallocate a XOF object. */ @@ -236,10 +236,10 @@ crypto_xof(uint8_t *output, size_t output_len, r = EVP_DigestFinalXOF(ctx, output, output_len); tor_assert(r == 1); EVP_MD_CTX_free(ctx); -#else +#else /* !(defined(OPENSSL_HAS_SHA3)) */ crypto_xof_t *xof = crypto_xof_new(); crypto_xof_add_bytes(xof, input, input_len); crypto_xof_squeeze_bytes(xof, output, output_len); crypto_xof_free(xof); -#endif +#endif /* defined(OPENSSL_HAS_SHA3) */ } diff --git a/src/lib/crypt_ops/crypto_digest_openssl.c b/src/lib/crypt_ops/crypto_digest_openssl.c index a1c92351fc..c631b0eac0 100644 --- a/src/lib/crypt_ops/crypto_digest_openssl.c +++ b/src/lib/crypt_ops/crypto_digest_openssl.c @@ -70,7 +70,7 @@ crypto_digest256(char *digest, const char *m, size_t len, #else ret = (sha3_256((uint8_t *)digest, DIGEST256_LEN,(const uint8_t *)m, len) > -1); -#endif +#endif /* defined(OPENSSL_HAS_SHA3) */ } if (!ret) @@ -100,7 +100,7 @@ crypto_digest512(char *digest, const char *m, size_t len, #else ret = (sha3_512((uint8_t*)digest, DIGEST512_LEN, (const uint8_t*)m, len) > -1); -#endif +#endif /* defined(OPENSSL_HAS_SHA3) */ } if (!ret) @@ -167,7 +167,7 @@ crypto_digest_alloc_bytes(digest_algorithm_t alg) case DIGEST_SHA3_256: /* Fall through */ case DIGEST_SHA3_512: return END_OF_FIELD(d.sha3); -#endif +#endif /* defined(OPENSSL_HAS_SHA3) */ default: tor_assert(0); // LCOV_EXCL_LINE return 0; // LCOV_EXCL_LINE @@ -212,14 +212,14 @@ crypto_digest_new_internal(digest_algorithm_t algorithm) return NULL; } break; -#else +#else /* !(defined(OPENSSL_HAS_SHA3)) */ case DIGEST_SHA3_256: keccak_digest_init(&r->d.sha3, 256); break; case DIGEST_SHA3_512: keccak_digest_init(&r->d.sha3, 512); break; -#endif +#endif /* defined(OPENSSL_HAS_SHA3) */ default: tor_assert_unreached(); } @@ -271,7 +271,7 @@ crypto_digest_free_(crypto_digest_t *digest) EVP_MD_CTX_free(digest->d.md); } } -#endif +#endif /* defined(OPENSSL_HAS_SHA3) */ size_t bytes = crypto_digest_alloc_bytes(digest->algorithm); memwipe(digest, 0, bytes); tor_free(digest); @@ -310,12 +310,12 @@ crypto_digest_add_bytes(crypto_digest_t *digest, const char *data, tor_assert(r); } break; -#else +#else /* !(defined(OPENSSL_HAS_SHA3)) */ case DIGEST_SHA3_256: /* FALLSTHROUGH */ case DIGEST_SHA3_512: keccak_digest_update(&digest->d.sha3, (const uint8_t *)data, len); break; -#endif +#endif /* defined(OPENSSL_HAS_SHA3) */ default: /* LCOV_EXCL_START */ tor_fragile_assert(); @@ -354,12 +354,12 @@ crypto_digest_get_digest(crypto_digest_t *digest, EVP_MD_CTX_free(tmp); tor_assert(res == 1); goto done; -#else +#else /* !(defined(OPENSSL_HAS_SHA3)) */ /* Tiny-Keccak handles copying into a temporary ctx, and also can handle * short output buffers by truncating appropriately. */ keccak_digest_sum(&digest->d.sha3, (uint8_t *)out, out_len); return; -#endif +#endif /* defined(OPENSSL_HAS_SHA3) */ } const size_t alloc_bytes = crypto_digest_alloc_bytes(digest->algorithm); @@ -412,7 +412,7 @@ crypto_digest_dup(const crypto_digest_t *digest) result->d.md = EVP_MD_CTX_new(); EVP_MD_CTX_copy(result->d.md, digest->d.md); } -#endif +#endif /* defined(OPENSSL_HAS_SHA3) */ return result; } @@ -458,7 +458,7 @@ crypto_digest_assign(crypto_digest_t *into, EVP_MD_CTX_copy(into->d.md, from->d.md); return; } -#endif +#endif /* defined(OPENSSL_HAS_SHA3) */ memcpy(into,from,alloc_bytes); } diff --git a/src/lib/crypt_ops/crypto_hkdf.c b/src/lib/crypt_ops/crypto_hkdf.c index fd2e701651..e0f3d65ad1 100644 --- a/src/lib/crypt_ops/crypto_hkdf.c +++ b/src/lib/crypt_ops/crypto_hkdf.c @@ -25,7 +25,7 @@ #include #define HAVE_OPENSSL_HKDF 1 #endif -#endif +#endif /* defined(ENABLE_OPENSSL) */ #include @@ -109,7 +109,7 @@ crypto_expand_key_material_rfc5869_sha256_openssl( return 0; } -#else +#else /* !(defined(HAVE_OPENSSL_HKDF)) */ /** * Perform RFC5869 HKDF computation using our own legacy implementation. @@ -166,7 +166,7 @@ crypto_expand_key_material_rfc5869_sha256_legacy( memwipe(mac, 0, sizeof(mac)); return 0; } -#endif +#endif /* defined(HAVE_OPENSSL_HKDF) */ /** Expand some secret key material according to RFC5869, using SHA256 as the * underlying hash. The key_in_len bytes at key_in are the @@ -191,11 +191,11 @@ crypto_expand_key_material_rfc5869_sha256( salt_in_len, info_in, info_in_len, key_out, key_out_len); -#else +#else /* !(defined(HAVE_OPENSSL_HKDF)) */ return crypto_expand_key_material_rfc5869_sha256_legacy(key_in, key_in_len, salt_in, salt_in_len, info_in, info_in_len, key_out, key_out_len); -#endif +#endif /* defined(HAVE_OPENSSL_HKDF) */ } diff --git a/src/lib/crypt_ops/crypto_init.c b/src/lib/crypt_ops/crypto_init.c index 5c2780b2ca..a16bf4e11a 100644 --- a/src/lib/crypt_ops/crypto_init.c +++ b/src/lib/crypt_ops/crypto_init.c @@ -99,7 +99,7 @@ crypto_global_init(int useAccel, const char *accelName, const char *accelDir) (void)useAccel; (void)accelName; (void)accelDir; -#endif +#endif /* defined(ENABLE_OPENSSL) */ #ifdef ENABLE_NSS if (crypto_nss_late_init() < 0) return -1; diff --git a/src/lib/crypt_ops/crypto_init.h b/src/lib/crypt_ops/crypto_init.h index 540d08eb56..8de3eb03ed 100644 --- a/src/lib/crypt_ops/crypto_init.h +++ b/src/lib/crypt_ops/crypto_init.h @@ -33,4 +33,4 @@ const char *crypto_get_header_version_string(void); int tor_is_using_nss(void); -#endif /* !defined(TOR_CRYPTO_H) */ +#endif /* !defined(TOR_CRYPTO_INIT_H) */ diff --git a/src/lib/crypt_ops/crypto_nss_mgt.h b/src/lib/crypt_ops/crypto_nss_mgt.h index 72fd2a1229..4cfa9b42a4 100644 --- a/src/lib/crypt_ops/crypto_nss_mgt.h +++ b/src/lib/crypt_ops/crypto_nss_mgt.h @@ -29,6 +29,6 @@ void crypto_nss_global_cleanup(void); void crypto_nss_prefork(void); void crypto_nss_postfork(void); -#endif +#endif /* defined(ENABLE_NSS) */ -#endif /* !defined(TOR_CRYPTO_NSS_H) */ +#endif /* !defined(TOR_CRYPTO_NSS_MGT_H) */ diff --git a/src/lib/crypt_ops/crypto_ope.c b/src/lib/crypt_ops/crypto_ope.c index 2186d2a939..4bd4b35706 100644 --- a/src/lib/crypt_ops/crypto_ope.c +++ b/src/lib/crypt_ops/crypto_ope.c @@ -57,9 +57,9 @@ ope_val_from_le(ope_val_t x) ((x) >> 8) | (((x)&0xff) << 8); } -#else +#else /* !(defined(WORDS_BIGENDIAN)) */ #define ope_val_from_le(x) (x) -#endif +#endif /* defined(WORDS_BIGENDIAN) */ /** * Return a new AES256-CTR stream cipher object for ope, ready to yield diff --git a/src/lib/crypt_ops/crypto_ope.h b/src/lib/crypt_ops/crypto_ope.h index 610d956335..9778dfe0f0 100644 --- a/src/lib/crypt_ops/crypto_ope.h +++ b/src/lib/crypt_ops/crypto_ope.h @@ -41,6 +41,6 @@ struct aes_cnt_cipher; STATIC struct aes_cnt_cipher *ope_get_cipher(const crypto_ope_t *ope, uint32_t initial_idx); STATIC uint64_t sum_values_from_cipher(struct aes_cnt_cipher *c, size_t n); -#endif +#endif /* defined(CRYPTO_OPE_PRIVATE) */ -#endif +#endif /* !defined(CRYPTO_OPE_H) */ diff --git a/src/lib/crypt_ops/crypto_openssl_mgt.c b/src/lib/crypt_ops/crypto_openssl_mgt.c index c97815f9a4..9ec59e7c81 100644 --- a/src/lib/crypt_ops/crypto_openssl_mgt.c +++ b/src/lib/crypt_ops/crypto_openssl_mgt.c @@ -200,10 +200,10 @@ crypto_openssl_early_init(void) OPENSSL_INIT_LOAD_CRYPTO_STRINGS | OPENSSL_INIT_ADD_ALL_CIPHERS | OPENSSL_INIT_ADD_ALL_DIGESTS, NULL); -#else +#else /* !(defined(OPENSSL_1_1_API)) */ ERR_load_crypto_strings(); OpenSSL_add_all_algorithms(); -#endif +#endif /* defined(OPENSSL_1_1_API) */ setup_openssl_threading(); diff --git a/src/lib/crypt_ops/crypto_openssl_mgt.h b/src/lib/crypt_ops/crypto_openssl_mgt.h index a3dd03aa04..111a2d12ed 100644 --- a/src/lib/crypt_ops/crypto_openssl_mgt.h +++ b/src/lib/crypt_ops/crypto_openssl_mgt.h @@ -84,6 +84,6 @@ int crypto_openssl_late_init(int useAccel, const char *accelName, void crypto_openssl_thread_cleanup(void); void crypto_openssl_global_cleanup(void); -#endif /* ENABLE_OPENSSL */ +#endif /* defined(ENABLE_OPENSSL) */ #endif /* !defined(TOR_CRYPTO_OPENSSL_H) */ diff --git a/src/lib/crypt_ops/crypto_rand.c b/src/lib/crypt_ops/crypto_rand.c index 79c8ed1eed..a80a98f267 100644 --- a/src/lib/crypt_ops/crypto_rand.c +++ b/src/lib/crypt_ops/crypto_rand.c @@ -47,7 +47,7 @@ DISABLE_GCC_WARNING(redundant-decls) #include #include ENABLE_GCC_WARNING(redundant-decls) -#endif +#endif /* defined(ENABLE_OPENSSL) */ #ifdef ENABLE_NSS #include @@ -419,7 +419,7 @@ crypto_seed_openssl_rng(void) else return -1; } -#endif +#endif /* defined(ENABLE_OPENSSL) */ #ifdef ENABLE_NSS /** @@ -442,7 +442,7 @@ crypto_seed_nss_rng(void) return load_entropy_ok ? 0 : -1; } -#endif +#endif /* defined(ENABLE_NSS) */ /** * Seed the RNG for any and all crypto libraries that we're using with bytes @@ -520,13 +520,13 @@ crypto_rand_unmocked(char *to, size_t n) #undef BUFLEN } -#else +#else /* !(defined(ENABLE_NSS)) */ int r = RAND_bytes((unsigned char*)to, (int)n); /* We consider a PRNG failure non-survivable. Let's assert so that we get a * stack trace about where it happened. */ tor_assert(r >= 0); -#endif +#endif /* defined(ENABLE_NSS) */ } /** @@ -627,6 +627,6 @@ crypto_force_rand_ssleay(void) RAND_set_rand_method(default_method); return 1; } -#endif +#endif /* defined(ENABLE_OPENSSL) */ return 0; } diff --git a/src/lib/crypt_ops/crypto_rand.h b/src/lib/crypt_ops/crypto_rand.h index 528f238fa5..a019287aa9 100644 --- a/src/lib/crypt_ops/crypto_rand.h +++ b/src/lib/crypt_ops/crypto_rand.h @@ -87,7 +87,7 @@ crypto_fast_rng_t *get_thread_fast_rng(void); void destroy_thread_fast_rng(void); void crypto_rand_fast_init(void); void crypto_rand_fast_shutdown(void); -#endif +#endif /* defined(CRYPTO_PRIVATE) */ #if defined(TOR_UNIT_TESTS) /* Used for white-box testing */ @@ -96,7 +96,7 @@ size_t crypto_fast_rng_get_bytes_used_per_stream(void); void crypto_fast_rng_disable_reseed(crypto_fast_rng_t *rng); /* To override the prng for testing. */ crypto_fast_rng_t *crypto_replace_thread_fast_rng(crypto_fast_rng_t *rng); -#endif +#endif /* defined(TOR_UNIT_TESTS) */ #ifdef CRYPTO_RAND_PRIVATE diff --git a/src/lib/crypt_ops/crypto_rand_fast.c b/src/lib/crypt_ops/crypto_rand_fast.c index b71ade81bd..c7f71a17c9 100644 --- a/src/lib/crypt_ops/crypto_rand_fast.c +++ b/src/lib/crypt_ops/crypto_rand_fast.c @@ -182,7 +182,7 @@ crypto_fast_rng_new_from_seed(const uint8_t *seed) /* We decided above that noinherit would always do _something_. Assert here * that we were correct. */ tor_assert(inherit != INHERIT_RES_KEEP); -#endif +#endif /* defined(CHECK_PID) || ... */ return result; } @@ -196,7 +196,7 @@ crypto_fast_rng_disable_reseed(crypto_fast_rng_t *rng) { rng->n_till_reseed = -1; } -#endif +#endif /* defined(TOR_UNIT_TESTS) */ /** * Helper: create a crypto_cipher_t object from SEED_LEN bytes of @@ -251,7 +251,7 @@ crypto_fast_rng_refill(crypto_fast_rng_t *rng) #else /* If testing is disabled, this shouldn't be able to become negative. */ tor_assert_unreached(); -#endif +#endif /* defined(TOR_UNIT_TESTS) */ } /* Now fill rng->buf with output from our stream cipher, initialized from * that seed value. */ @@ -302,7 +302,7 @@ crypto_fast_rng_getbytes_impl(crypto_fast_rng_t *rng, uint8_t *out, */ tor_assert(rng->owner == getpid()); } -#endif +#endif /* defined(CHECK_PID) */ size_t bytes_to_yield = n; @@ -356,7 +356,7 @@ crypto_fast_rng_get_bytes_used_per_stream(void) { return BUFLEN; } -#endif +#endif /* defined(TOR_UNIT_TESTS) */ /** * Thread-local instance for our fast RNG. @@ -409,7 +409,7 @@ crypto_replace_thread_fast_rng(crypto_fast_rng_t *rng) tor_threadlocal_set(&thread_rng, rng); return old_rng; } -#endif +#endif /* defined(TOR_UNIT_TESTS) */ /** * Initialize the global thread-local key that will be used to keep track diff --git a/src/lib/crypt_ops/crypto_rsa.c b/src/lib/crypt_ops/crypto_rsa.c index c9189b0dfc..c39d2e18d1 100644 --- a/src/lib/crypt_ops/crypto_rsa.c +++ b/src/lib/crypt_ops/crypto_rsa.c @@ -59,7 +59,7 @@ crypto_get_rsa_padding(int padding) default: tor_assert(0); return -1; // LCOV_EXCL_LINE } } -#endif +#endif /* defined(ENABLE_OPENSSL) */ /** Compare the public-key components of a and b. Return non-zero iff * a==b. A NULL key is considered to be distinct from all non-NULL diff --git a/src/lib/crypt_ops/crypto_rsa.h b/src/lib/crypt_ops/crypto_rsa.h index c1ea767f85..e9bfec2f85 100644 --- a/src/lib/crypt_ops/crypto_rsa.h +++ b/src/lib/crypt_ops/crypto_rsa.h @@ -119,7 +119,7 @@ struct rsa_st *crypto_pk_get_openssl_rsa_(crypto_pk_t *env); crypto_pk_t *crypto_new_pk_from_openssl_rsa_(struct rsa_st *rsa); MOCK_DECL(struct evp_pkey_st *, crypto_pk_get_openssl_evp_pkey_,( crypto_pk_t *env,int private)); -#endif +#endif /* defined(ENABLE_OPENSSL) */ #ifdef ENABLE_NSS struct SECKEYPublicKeyStr; @@ -129,7 +129,7 @@ const struct SECKEYPublicKeyStr *crypto_pk_get_nss_pubkey( const crypto_pk_t *key); const struct SECKEYPrivateKeyStr *crypto_pk_get_nss_privkey( const crypto_pk_t *key); -#endif +#endif /* defined(ENABLE_NSS) */ void crypto_pk_assign_public(crypto_pk_t *dest, const crypto_pk_t *src); void crypto_pk_assign_private(crypto_pk_t *dest, const crypto_pk_t *src); @@ -140,6 +140,6 @@ struct SECItemStr; STATIC int secitem_uint_cmp(const struct SECItemStr *a, const struct SECItemStr *b); #endif -#endif +#endif /* defined(TOR_UNIT_TESTS) */ -#endif +#endif /* !defined(TOR_CRYPTO_RSA_H) */ diff --git a/src/lib/crypt_ops/crypto_rsa_nss.c b/src/lib/crypt_ops/crypto_rsa_nss.c index ad2ad38b66..612b7a0e64 100644 --- a/src/lib/crypt_ops/crypto_rsa_nss.c +++ b/src/lib/crypt_ops/crypto_rsa_nss.c @@ -156,7 +156,7 @@ crypto_pk_get_openssl_evp_pkey_,(crypto_pk_t *pk, int private)) tor_free(buf); return result; } -#endif +#endif /* defined(ENABLE_OPENSSL) */ /** Allocate and return storage for a public key. The key itself will not yet * be set. diff --git a/src/lib/crypt_ops/crypto_s2k.c b/src/lib/crypt_ops/crypto_s2k.c index 42276597d4..5cf98e3e64 100644 --- a/src/lib/crypt_ops/crypto_s2k.c +++ b/src/lib/crypt_ops/crypto_s2k.c @@ -285,7 +285,7 @@ secret_to_key_compute_key(uint8_t *key_out, size_t key_out_len, if (rv < 0) return S2K_FAILED; return (int)key_out_len; -#else +#else /* !(defined(ENABLE_OPENSSL)) */ SECItem passItem = { .type = siBuffer, .data = (unsigned char *) secret, .len = (int)secret_len }; @@ -325,7 +325,7 @@ secret_to_key_compute_key(uint8_t *key_out, size_t key_out_len, if (alg) SECOID_DestroyAlgorithmID(alg, PR_TRUE); return rv; -#endif +#endif /* defined(ENABLE_OPENSSL) */ } case S2K_TYPE_SCRYPT: { diff --git a/src/lib/crypt_ops/crypto_util.c b/src/lib/crypt_ops/crypto_util.c index 67a1a9eb92..5e3f4a87a1 100644 --- a/src/lib/crypt_ops/crypto_util.c +++ b/src/lib/crypt_ops/crypto_util.c @@ -30,7 +30,7 @@ DISABLE_GCC_WARNING(redundant-decls) #include #include ENABLE_GCC_WARNING(redundant-decls) -#endif +#endif /* defined(ENABLE_OPENSSL) */ #include "lib/log/log.h" #include "lib/log/util_bug.h" diff --git a/src/lib/crypt_ops/digestset.h b/src/lib/crypt_ops/digestset.h index 91d53a0542..7d6d687342 100644 --- a/src/lib/crypt_ops/digestset.h +++ b/src/lib/crypt_ops/digestset.h @@ -26,4 +26,4 @@ void digestset_add(digestset_t *set, const char *addr); int digestset_probably_contains(const digestset_t *set, const char *addr); -#endif +#endif /* !defined(TOR_DIGESTSET_H) */ diff --git a/src/lib/defs/dh_sizes.h b/src/lib/defs/dh_sizes.h index a2ffbc51c2..b0d1eba0c5 100644 --- a/src/lib/defs/dh_sizes.h +++ b/src/lib/defs/dh_sizes.h @@ -19,4 +19,4 @@ /** Length of our legacy DH keys. */ #define DH1024_KEY_LEN (1024/8) -#endif +#endif /* !defined(TOR_DH_SIZES_H) */ diff --git a/src/lib/defs/digest_sizes.h b/src/lib/defs/digest_sizes.h index 525e5209d6..a0dd97a74d 100644 --- a/src/lib/defs/digest_sizes.h +++ b/src/lib/defs/digest_sizes.h @@ -24,4 +24,4 @@ /** Length of the output of our 64-bit optimized message digests (SHA512). */ #define DIGEST512_LEN 64 -#endif +#endif /* !defined(TOR_DIGEST_SIZES_H) */ diff --git a/src/lib/defs/time.h b/src/lib/defs/time.h index c25f5022c5..459afbf42d 100644 --- a/src/lib/defs/time.h +++ b/src/lib/defs/time.h @@ -20,4 +20,4 @@ /* How many nanoseconds per millisecond */ #define TOR_NSEC_PER_MSEC (1000*1000) -#endif +#endif /* !defined(TOR_TIME_DEFS_H) */ diff --git a/src/lib/defs/x25519_sizes.h b/src/lib/defs/x25519_sizes.h index 8933a8866b..6431f0a2dd 100644 --- a/src/lib/defs/x25519_sizes.h +++ b/src/lib/defs/x25519_sizes.h @@ -33,4 +33,4 @@ #define ED25519_BASE64_LEN 43 #define ED25519_SIG_BASE64_LEN 86 -#endif +#endif /* !defined(TOR_X25519_SIZES_H) */ diff --git a/src/lib/dispatch/dispatch.h b/src/lib/dispatch/dispatch.h index 8e62e8f168..a9e655409a 100644 --- a/src/lib/dispatch/dispatch.h +++ b/src/lib/dispatch/dispatch.h @@ -111,4 +111,4 @@ void dispatch_free_msg_(const dispatch_t *d, msg_t *msg); char *dispatch_fmt_msg_data(const dispatch_t *d, const msg_t *msg); -#endif +#endif /* !defined(TOR_DISPATCH_H) */ diff --git a/src/lib/dispatch/dispatch_cfg.h b/src/lib/dispatch/dispatch_cfg.h index 2c755e39bc..61fade7240 100644 --- a/src/lib/dispatch/dispatch_cfg.h +++ b/src/lib/dispatch/dispatch_cfg.h @@ -36,4 +36,4 @@ int dcfg_add_recv(dispatch_cfg_t *cfg, message_id_t msg, void dcfg_free_(dispatch_cfg_t *cfg); -#endif +#endif /* !defined(TOR_DISPATCH_CFG_H) */ diff --git a/src/lib/dispatch/dispatch_cfg_st.h b/src/lib/dispatch/dispatch_cfg_st.h index d004fe5934..57b6f0347f 100644 --- a/src/lib/dispatch/dispatch_cfg_st.h +++ b/src/lib/dispatch/dispatch_cfg_st.h @@ -22,4 +22,4 @@ struct dispatch_cfg_t { struct smartlist_t *recv_by_msg; }; -#endif +#endif /* !defined(TOR_DISPATCH_CFG_ST_H) */ diff --git a/src/lib/dispatch/dispatch_naming.h b/src/lib/dispatch/dispatch_naming.h index c116d2184d..fd6c83cc12 100644 --- a/src/lib/dispatch/dispatch_naming.h +++ b/src/lib/dispatch/dispatch_naming.h @@ -43,4 +43,4 @@ size_t get_num_msg_type_ids(void); void dispatch_naming_init(void); -#endif +#endif /* !defined(TOR_DISPATCH_NAMING_H) */ diff --git a/src/lib/dispatch/dispatch_st.h b/src/lib/dispatch/dispatch_st.h index 568107b700..ee42518b5a 100644 --- a/src/lib/dispatch/dispatch_st.h +++ b/src/lib/dispatch/dispatch_st.h @@ -103,6 +103,6 @@ struct dispatch_t { dispatch_typefns_t *typefns; }; -#endif +#endif /* defined(DISPATCH_PRIVATE) */ -#endif +#endif /* !defined(TOR_DISPATCH_ST_H) */ diff --git a/src/lib/dispatch/msgtypes.h b/src/lib/dispatch/msgtypes.h index 4e79e592a6..b4c4a10248 100644 --- a/src/lib/dispatch/msgtypes.h +++ b/src/lib/dispatch/msgtypes.h @@ -77,4 +77,4 @@ typedef struct dispatch_typefns_t { char *(*fmt_fn)(msg_aux_data_t); } dispatch_typefns_t; -#endif +#endif /* !defined(TOR_DISPATCH_MSGTYPES_H) */ diff --git a/src/lib/encoding/binascii.h b/src/lib/encoding/binascii.h index 44998bb85b..40c5593b11 100644 --- a/src/lib/encoding/binascii.h +++ b/src/lib/encoding/binascii.h @@ -58,4 +58,4 @@ size_t base32_encoded_size(size_t srclen); void base16_encode(char *dest, size_t destlen, const char *src, size_t srclen); int base16_decode(char *dest, size_t destlen, const char *src, size_t srclen); -#endif /* !defined(TOR_UTIL_FORMAT_H) */ +#endif /* !defined(TOR_BINASCII_H) */ diff --git a/src/lib/encoding/keyval.h b/src/lib/encoding/keyval.h index cd327b7a82..dcddfa3396 100644 --- a/src/lib/encoding/keyval.h +++ b/src/lib/encoding/keyval.h @@ -14,4 +14,4 @@ int string_is_key_value(int severity, const char *string); -#endif +#endif /* !defined(TOR_KEYVAL_H) */ diff --git a/src/lib/encoding/pem.h b/src/lib/encoding/pem.h index 0bbb06a794..6b20350aa8 100644 --- a/src/lib/encoding/pem.h +++ b/src/lib/encoding/pem.h @@ -23,4 +23,4 @@ int pem_encode(char *dest, size_t destlen, const uint8_t *src, size_t srclen, int pem_decode(uint8_t *dest, size_t destlen, const char *src, size_t srclen, const char *objtype); -#endif +#endif /* !defined(TOR_PEM_H) */ diff --git a/src/lib/encoding/qstring.h b/src/lib/encoding/qstring.h index fe15b655f1..840e1044ce 100644 --- a/src/lib/encoding/qstring.h +++ b/src/lib/encoding/qstring.h @@ -15,4 +15,4 @@ const char *decode_qstring(const char *start, size_t in_len_max, char **out, size_t *out_len); -#endif +#endif /* !defined(TOR_ENCODING_QSTRING_H) */ diff --git a/src/lib/encoding/time_fmt.h b/src/lib/encoding/time_fmt.h index 0ddeca57fc..d14bc1f902 100644 --- a/src/lib/encoding/time_fmt.h +++ b/src/lib/encoding/time_fmt.h @@ -41,4 +41,4 @@ int parse_iso_time_nospace(const char *cp, time_t *t); int parse_http_time(const char *buf, struct tm *tm); int format_time_interval(char *out, size_t out_len, long interval); -#endif +#endif /* !defined(TOR_TIME_FMT_H) */ diff --git a/src/lib/err/torerr.h b/src/lib/err/torerr.h index 0badaf7c6d..c2da6697a9 100644 --- a/src/lib/err/torerr.h +++ b/src/lib/err/torerr.h @@ -45,4 +45,4 @@ void tor_log_sigsafe_err_set_granularity(int ms); int format_hex_number_sigsafe(unsigned long x, char *buf, int max_len); int format_dec_number_sigsafe(unsigned long x, char *buf, int max_len); -#endif /* !defined(TOR_TORLOG_H) */ +#endif /* !defined(TOR_TORERR_H) */ diff --git a/src/lib/evloop/token_bucket.h b/src/lib/evloop/token_bucket.h index 9398d2baa3..1ce6f1bf94 100644 --- a/src/lib/evloop/token_bucket.h +++ b/src/lib/evloop/token_bucket.h @@ -112,6 +112,6 @@ token_bucket_rw_get_write(const token_bucket_rw_t *bucket) STATIC uint32_t rate_per_sec_to_rate_per_step(uint32_t rate); -#endif +#endif /* defined(TOKEN_BUCKET_PRIVATE) */ -#endif /* TOR_TOKEN_BUCKET_H */ +#endif /* !defined(TOR_TOKEN_BUCKET_H) */ diff --git a/src/lib/fs/conffile.h b/src/lib/fs/conffile.h index 7af9119dbb..29115e1085 100644 --- a/src/lib/fs/conffile.h +++ b/src/lib/fs/conffile.h @@ -20,4 +20,4 @@ int config_get_lines_include(const char *string, struct config_line_t **result, int extended, int *has_include, struct smartlist_t *opened_lst); -#endif /* !defined(TOR_CONFLINE_H) */ +#endif /* !defined(TOR_CONFFILE_H) */ diff --git a/src/lib/fs/dir.h b/src/lib/fs/dir.h index 826bc2dfc5..9ff81faa42 100644 --- a/src/lib/fs/dir.h +++ b/src/lib/fs/dir.h @@ -30,4 +30,4 @@ MOCK_DECL(int, check_private_dir, (const char *dirname, cpd_check_t check, MOCK_DECL(struct smartlist_t *, tor_listdir, (const char *dirname)); -#endif +#endif /* !defined(TOR_DIR_H) */ diff --git a/src/lib/fs/files.h b/src/lib/fs/files.h index 52c94c914f..81dba8c140 100644 --- a/src/lib/fs/files.h +++ b/src/lib/fs/files.h @@ -27,7 +27,7 @@ #ifdef HAVE_SYS_STAT_H #include #endif -#endif +#endif /* defined(_WIN32) */ #ifndef O_BINARY #define O_BINARY 0 @@ -108,7 +108,7 @@ char *read_file_to_str_until_eof(int fd, size_t max_bytes_to_read, * Tor is built for unit tests, or when Tor is built on an operating system * without its own getdelim(). */ ssize_t compat_getdelim_(char **lineptr, size_t *n, int delim, FILE *stream); -#endif +#endif /* !defined(HAVE_GETDELIM) || defined(TOR_UNIT_TESTS) */ #ifdef HAVE_GETDELIM /** @@ -123,10 +123,10 @@ ssize_t compat_getdelim_(char **lineptr, size_t *n, int delim, FILE *stream); */ #define tor_getdelim(lineptr, n, delim, stream) \ getdelim((lineptr), (n), (delim), (stream)) -#else +#else /* !(defined(HAVE_GETDELIM)) */ #define tor_getdelim(lineptr, n, delim, stream) \ compat_getdelim_((lineptr), (n), (delim), (stream)) -#endif +#endif /* defined(HAVE_GETDELIM) */ #ifdef HAVE_GETLINE /** @@ -137,9 +137,9 @@ ssize_t compat_getdelim_(char **lineptr, size_t *n, int delim, FILE *stream); */ #define tor_getline(lineptr, n, stream) \ getline((lineptr), (n), (stream)) -#else +#else /* !(defined(HAVE_GETLINE)) */ #define tor_getline(lineptr, n, stream) \ tor_getdelim((lineptr), (n), '\n', (stream)) -#endif +#endif /* defined(HAVE_GETLINE) */ -#endif +#endif /* !defined(TOR_FS_H) */ diff --git a/src/lib/fs/lockfile.h b/src/lib/fs/lockfile.h index 8aeee4cc7f..fc0281e253 100644 --- a/src/lib/fs/lockfile.h +++ b/src/lib/fs/lockfile.h @@ -17,4 +17,4 @@ tor_lockfile_t *tor_lockfile_lock(const char *filename, int blocking, int *locked_out); void tor_lockfile_unlock(tor_lockfile_t *lockfile); -#endif +#endif /* !defined(TOR_LOCKFILE_H) */ diff --git a/src/lib/fs/mmap.c b/src/lib/fs/mmap.c index daaee1f9b1..f71c0cff7a 100644 --- a/src/lib/fs/mmap.c +++ b/src/lib/fs/mmap.c @@ -237,4 +237,4 @@ tor_munmap_file(tor_mmap_t *handle) } #else #error "cannot implement tor_mmap_file" -#endif /* defined(HAVE_MMAP) || ... || ... */ +#endif /* defined(HAVE_MMAP) || defined(RUNNING_DOXYGEN) || ... */ diff --git a/src/lib/fs/mmap.h b/src/lib/fs/mmap.h index 18fb18a13c..61aad544b2 100644 --- a/src/lib/fs/mmap.h +++ b/src/lib/fs/mmap.h @@ -38,4 +38,4 @@ typedef struct tor_mmap_t { tor_mmap_t *tor_mmap_file(const char *filename); int tor_munmap_file(tor_mmap_t *handle); -#endif +#endif /* !defined(TOR_MMAP_H) */ diff --git a/src/lib/fs/path.h b/src/lib/fs/path.h index 4675ac84e8..28a1838b88 100644 --- a/src/lib/fs/path.h +++ b/src/lib/fs/path.h @@ -27,4 +27,4 @@ void clean_fname_for_stat(char *name); int get_parent_directory(char *fname); char *make_path_absolute(char *fname); -#endif +#endif /* !defined(TOR_PATH_H) */ diff --git a/src/lib/fs/userdb.h b/src/lib/fs/userdb.h index 5c39794873..5e5ddb89a3 100644 --- a/src/lib/fs/userdb.h +++ b/src/lib/fs/userdb.h @@ -21,6 +21,6 @@ struct passwd; const struct passwd *tor_getpwnam(const char *username); const struct passwd *tor_getpwuid(uid_t uid); char *get_user_homedir(const char *username); -#endif +#endif /* !defined(_WIN32) */ -#endif +#endif /* !defined(TOR_USERDB_H) */ diff --git a/src/lib/fs/winlib.h b/src/lib/fs/winlib.h index 64a22439e5..7237226c76 100644 --- a/src/lib/fs/winlib.h +++ b/src/lib/fs/winlib.h @@ -17,6 +17,6 @@ #include HANDLE load_windows_system_library(const TCHAR *library_name); -#endif +#endif /* defined(_WIN32) */ -#endif +#endif /* !defined(TOR_WINLIB_H) */ diff --git a/src/lib/geoip/country.h b/src/lib/geoip/country.h index 9a8911d494..a24a1c4c0d 100644 --- a/src/lib/geoip/country.h +++ b/src/lib/geoip/country.h @@ -13,4 +13,4 @@ typedef int16_t country_t; #define COUNTRY_MAX INT16_MAX -#endif +#endif /* !defined(TOR_COUNTRY_H) */ diff --git a/src/lib/intmath/addsub.h b/src/lib/intmath/addsub.h index 83efa82919..3f745d457d 100644 --- a/src/lib/intmath/addsub.h +++ b/src/lib/intmath/addsub.h @@ -16,4 +16,4 @@ uint32_t tor_add_u32_nowrap(uint32_t a, uint32_t b); -#endif /* !defined(TOR_INTMATH_MULDIV_H) */ +#endif /* !defined(TOR_INTMATH_ADDSUB_H) */ diff --git a/src/lib/intmath/logic.h b/src/lib/intmath/logic.h index a4cecd69cc..b2f77462e1 100644 --- a/src/lib/intmath/logic.h +++ b/src/lib/intmath/logic.h @@ -17,4 +17,4 @@ /** Macro: true if two values have different boolean values. */ #define bool_neq(a,b) (!(a)!=!(b)) -#endif +#endif /* !defined(HAVE_TOR_LOGIC_H) */ diff --git a/src/lib/intmath/weakrng.h b/src/lib/intmath/weakrng.h index e26bf58cbb..40941e59b2 100644 --- a/src/lib/intmath/weakrng.h +++ b/src/lib/intmath/weakrng.h @@ -28,4 +28,4 @@ int32_t tor_weak_random_range(tor_weak_rng_t *rng, int32_t top); * n */ #define tor_weak_random_one_in_n(rng, n) (0==tor_weak_random_range((rng),(n))) -#endif +#endif /* !defined(TOR_WEAKRNG_H) */ diff --git a/src/lib/lock/compat_mutex.h b/src/lib/lock/compat_mutex.h index b63ce24024..e0c3d7cb78 100644 --- a/src/lib/lock/compat_mutex.h +++ b/src/lib/lock/compat_mutex.h @@ -48,7 +48,7 @@ typedef struct tor_mutex_t { #else /** No-threads only: Dummy variable so that tor_mutex_t takes up space. */ int _unused; -#endif /* defined(USE_WIN32_MUTEX) || ... */ +#endif /* defined(USE_WIN32_THREADS) || ... */ } tor_mutex_t; tor_mutex_t *tor_mutex_new(void); diff --git a/src/lib/log/escape.h b/src/lib/log/escape.h index 2f726186c5..0b9fc3406b 100644 --- a/src/lib/log/escape.h +++ b/src/lib/log/escape.h @@ -20,4 +20,4 @@ char *esc_for_log(const char *string) ATTR_MALLOC; char *esc_for_log_len(const char *chars, size_t n) ATTR_MALLOC; const char *escaped(const char *string); -#endif /* !defined(TOR_TORLOG_H) */ +#endif /* !defined(TOR_ESCAPE_H) */ diff --git a/src/lib/log/log.c b/src/lib/log/log.c index 84e3eafc27..c68f335da6 100644 --- a/src/lib/log/log.c +++ b/src/lib/log/log.c @@ -155,7 +155,7 @@ severity_to_android_log_priority(int severity) // LCOV_EXCL_STOP } } -#endif // HAVE_ANDROID_LOG_H. +#endif /* defined(HAVE_ANDROID_LOG_H) */ /** A mutex to guard changes to logfiles and logging. */ static tor_mutex_t log_mutex; @@ -1233,7 +1233,7 @@ add_android_log(const log_severity_list_t *severity, UNLOCK_LOGS(); return 0; } -#endif // HAVE_ANDROID_LOG_H. +#endif /* defined(HAVE_ANDROID_LOG_H) */ /** If level is a valid log severity, return the corresponding * numeric value. Otherwise, return -1. */ diff --git a/src/lib/log/log.h b/src/lib/log/log.h index 4a3ea0ad55..3db2169584 100644 --- a/src/lib/log/log.h +++ b/src/lib/log/log.h @@ -207,7 +207,7 @@ static inline bool debug_logging_enabled(void) { return PREDICT_UNLIKELY(log_global_min_severity_ == LOG_DEBUG); } -#endif +#endif /* defined(TOR_COVERAGE) */ void log_fn_(int severity, log_domain_mask_t domain, const char *funcname, const char *format, ...) diff --git a/src/lib/log/ratelim.h b/src/lib/log/ratelim.h index 48edd7c849..1db54ba726 100644 --- a/src/lib/log/ratelim.h +++ b/src/lib/log/ratelim.h @@ -50,4 +50,4 @@ typedef struct ratelim_t { char *rate_limit_log(ratelim_t *lim, time_t now); -#endif +#endif /* !defined(TOR_RATELIM_H) */ diff --git a/src/lib/log/util_bug.h b/src/lib/log/util_bug.h index fb223b35f4..546ae1e3ef 100644 --- a/src/lib/log/util_bug.h +++ b/src/lib/log/util_bug.h @@ -80,10 +80,10 @@ tor__assert_tmp_value__; \ } ) #define ASSERT_PREDICT_LIKELY_(e) ASSERT_PREDICT_UNLIKELY_(e) -#else +#else /* !(defined(TOR_UNIT_TESTS) && defined(__GNUC__)) */ #define ASSERT_PREDICT_UNLIKELY_(e) PREDICT_UNLIKELY(e) #define ASSERT_PREDICT_LIKELY_(e) PREDICT_LIKELY(e) -#endif +#endif /* defined(TOR_UNIT_TESTS) && defined(__GNUC__) */ /* Sometimes we don't want to use assertions during branch coverage tests; it * leads to tons of unreached branches which in reality are only assertions we @@ -96,7 +96,7 @@ (void)(a); \ (void)(fmt); \ STMT_END -#else +#else /* !(defined(TOR_UNIT_TESTS) && ... */ /** Like assert(3), but send assertion failures to the log as well as to * stderr. */ #define tor_assert(expr) tor_assertf(expr, NULL) diff --git a/src/lib/log/win32err.h b/src/lib/log/win32err.h index 33413dfd15..ecfa88792d 100644 --- a/src/lib/log/win32err.h +++ b/src/lib/log/win32err.h @@ -19,4 +19,4 @@ char *format_win32_error(DWORD err); #endif -#endif +#endif /* !defined(TOR_WIN32ERR_H) */ diff --git a/src/lib/malloc/malloc.h b/src/lib/malloc/malloc.h index ef6b509ca4..8c81d30dd5 100644 --- a/src/lib/malloc/malloc.h +++ b/src/lib/malloc/malloc.h @@ -48,12 +48,12 @@ void tor_free_(void *mem); raw_free(*tor_free__tmpvar); \ *tor_free__tmpvar=NULL; \ STMT_END -#else +#else /* !(defined(__GNUC__)) */ #define tor_free(p) STMT_BEGIN \ raw_free(p); \ (p)=NULL; \ STMT_END -#endif +#endif /* defined(__GNUC__) */ #define tor_malloc(size) tor_malloc_(size) #define tor_malloc_zero(size) tor_malloc_zero_(size) diff --git a/src/lib/malloc/map_anon.c b/src/lib/malloc/map_anon.c index e2c41ab9c3..219bd706cc 100644 --- a/src/lib/malloc/map_anon.c +++ b/src/lib/malloc/map_anon.c @@ -59,7 +59,7 @@ #define FLAG_NOINHERIT VM_INHERIT_NONE #elif defined(MAP_INHERIT_NONE) #define FLAG_NOINHERIT MAP_INHERIT_NONE -#endif +#endif /* defined(INHERIT_NONE) || ... */ #elif defined(HAVE_MADVISE) @@ -72,7 +72,7 @@ #define FLAG_NOINHERIT MADV_DONTFORK #endif -#endif +#endif /* defined(HAVE_MINHERIT) || ... */ /** * Helper: try to prevent the sz bytes at mem from being swapped @@ -91,7 +91,7 @@ lock_mem(void *mem, size_t sz) (void) sz; return 0; -#endif +#endif /* defined(_WIN32) || ... */ } /** @@ -108,7 +108,7 @@ nodump_mem(void *mem, size_t sz) (void) mem; (void) sz; return 0; -#endif +#endif /* defined(MADV_DONTDUMP) */ } /** @@ -130,19 +130,19 @@ noinherit_mem(void *mem, size_t sz, inherit_res_t *inherit_result_out) *inherit_result_out = INHERIT_RES_ZERO; return 0; } -#endif +#endif /* defined(FLAG_ZERO) */ #ifdef FLAG_NOINHERIT int r2 = MINHERIT(mem, sz, FLAG_NOINHERIT); if (r2 == 0) { *inherit_result_out = INHERIT_RES_DROP; } return r2; -#else +#else /* !(defined(FLAG_NOINHERIT)) */ (void)inherit_result_out; (void)mem; (void)sz; return 0; -#endif +#endif /* defined(FLAG_NOINHERIT) */ } /** @@ -199,7 +199,7 @@ tor_mmap_anonymous(size_t sz, unsigned flags, raw_assert(ptr != NULL); #else ptr = tor_malloc_zero(sz); -#endif +#endif /* defined(_WIN32) || ... */ if (flags & ANONMAP_PRIVATE) { int lock_result = lock_mem(ptr, sz); @@ -234,5 +234,5 @@ tor_munmap_anonymous(void *mapping, size_t sz) #else (void)sz; tor_free(mapping); -#endif +#endif /* defined(_WIN32) || ... */ } diff --git a/src/lib/malloc/map_anon.h b/src/lib/malloc/map_anon.h index 6c02cd6c16..4c4690e12f 100644 --- a/src/lib/malloc/map_anon.h +++ b/src/lib/malloc/map_anon.h @@ -62,7 +62,7 @@ typedef enum { #define NOINHERIT_CAN_FAIL #else #define NOINHERIT_CAN_FAIL -#endif +#endif /* defined(_WIN32) || ... */ void *tor_mmap_anonymous(size_t sz, unsigned flags, inherit_res_t *inherit_result_out); diff --git a/src/lib/math/fp.h b/src/lib/math/fp.h index cb24649e6c..a73789c945 100644 --- a/src/lib/math/fp.h +++ b/src/lib/math/fp.h @@ -21,4 +21,4 @@ int64_t tor_llround(double d) ATTR_CONST; int64_t clamp_double_to_int64(double number); int tor_isinf(double x); -#endif +#endif /* !defined(TOR_FP_H) */ diff --git a/src/lib/math/laplace.h b/src/lib/math/laplace.h index e8651e5197..02b0e890f0 100644 --- a/src/lib/math/laplace.h +++ b/src/lib/math/laplace.h @@ -19,4 +19,4 @@ int64_t sample_laplace_distribution(double mu, double b, double p); int64_t add_laplace_noise(int64_t signal, double random, double delta_f, double epsilon); -#endif +#endif /* !defined(TOR_LAPLACE_H) */ diff --git a/src/lib/math/prob_distr.h b/src/lib/math/prob_distr.h index 8fccf8d015..7254dc8623 100644 --- a/src/lib/math/prob_distr.h +++ b/src/lib/math/prob_distr.h @@ -58,7 +58,7 @@ struct dist { #else #define TYPE_CHECK_OBJ(OPS, OBJ, TYPE) \ (0*sizeof(&(OBJ) - (const TYPE *)&(OBJ))) -#endif +#endif /* defined(__COVERITY__) */ /** * Typed initializer element for struct dist using the specified struct @@ -248,6 +248,6 @@ STATIC double icdf_genpareto(double p, double mu, double sigma, double xi); STATIC double isf_genpareto(double p, double mu, double sigma, double xi); STATIC double sample_genpareto(uint32_t s, double p0, double xi); -#endif +#endif /* defined(PROB_DISTR_PRIVATE) */ -#endif +#endif /* !defined(TOR_PROB_DISTR_H) */ diff --git a/src/lib/meminfo/meminfo.h b/src/lib/meminfo/meminfo.h index 2d64e1ab06..9580640f4d 100644 --- a/src/lib/meminfo/meminfo.h +++ b/src/lib/meminfo/meminfo.h @@ -18,4 +18,4 @@ void tor_log_mallinfo(int severity); MOCK_DECL(int, get_total_system_memory, (size_t *mem_out)); -#endif +#endif /* !defined(TOR_MEMINFO_H) */ diff --git a/src/lib/net/alertsock.h b/src/lib/net/alertsock.h index c45f42be81..4d0d0dd57c 100644 --- a/src/lib/net/alertsock.h +++ b/src/lib/net/alertsock.h @@ -42,4 +42,4 @@ typedef struct alert_sockets_t { int alert_sockets_create(alert_sockets_t *socks_out, uint32_t flags); void alert_sockets_close(alert_sockets_t *socks); -#endif +#endif /* !defined(TOR_ALERTSOCK_H) */ diff --git a/src/lib/net/buffers_net.h b/src/lib/net/buffers_net.h index a3a90172a1..5058dd0a26 100644 --- a/src/lib/net/buffers_net.h +++ b/src/lib/net/buffers_net.h @@ -31,4 +31,4 @@ int buf_read_from_pipe(struct buf_t *buf, int fd, size_t at_most, int buf_flush_to_pipe(struct buf_t *buf, int fd, size_t sz, size_t *buf_flushlen); -#endif /* !defined(TOR_BUFFERS_H) */ +#endif /* !defined(TOR_BUFFERS_NET_H) */ diff --git a/src/lib/net/gethostname.h b/src/lib/net/gethostname.h index 69b0528bc0..b3b77b0589 100644 --- a/src/lib/net/gethostname.h +++ b/src/lib/net/gethostname.h @@ -16,4 +16,4 @@ MOCK_DECL(int,tor_gethostname,(char *name, size_t namelen)); -#endif +#endif /* !defined(TOR_GETHOSTNAME_H) */ diff --git a/src/lib/net/inaddr.h b/src/lib/net/inaddr.h index 36352b65ea..602573944c 100644 --- a/src/lib/net/inaddr.h +++ b/src/lib/net/inaddr.h @@ -24,4 +24,4 @@ int tor_inet_ntoa(const struct in_addr *in, char *buf, size_t buf_len); const char *tor_inet_ntop(int af, const void *src, char *dst, size_t len); int tor_inet_pton(int af, const char *src, void *dst); -#endif +#endif /* !defined(TOR_INADDR_H) */ diff --git a/src/lib/net/inaddr_st.h b/src/lib/net/inaddr_st.h index 806f2c096a..230f29a63a 100644 --- a/src/lib/net/inaddr_st.h +++ b/src/lib/net/inaddr_st.h @@ -104,4 +104,4 @@ struct sockaddr_in6 { }; #endif /* !defined(HAVE_STRUCT_SOCKADDR_IN6) */ -#endif /* TOR_INADDR_ST_H */ +#endif /* !defined(TOR_INADDR_ST_H) */ diff --git a/src/lib/net/nettypes.h b/src/lib/net/nettypes.h index 6209bbe18a..0eb352c657 100644 --- a/src/lib/net/nettypes.h +++ b/src/lib/net/nettypes.h @@ -41,4 +41,4 @@ typedef int socklen_t; #define TOR_INVALID_SOCKET (-1) #endif /* defined(_WIN32) */ -#endif +#endif /* !defined(TOR_NET_TYPES_H) */ diff --git a/src/lib/net/resolve.c b/src/lib/net/resolve.c index 49c263faa2..2dda491d14 100644 --- a/src/lib/net/resolve.c +++ b/src/lib/net/resolve.c @@ -421,7 +421,7 @@ tor_make_getaddrinfo_cache_active(void) { sandbox_getaddrinfo_is_active = 1; } -#else +#else /* !(defined(USE_SANDBOX_GETADDRINFO)) */ void sandbox_disable_getaddrinfo_cache(void) { @@ -430,4 +430,4 @@ void tor_make_getaddrinfo_cache_active(void) { } -#endif +#endif /* defined(USE_SANDBOX_GETADDRINFO) */ diff --git a/src/lib/net/resolve.h b/src/lib/net/resolve.h index 0fb77f1661..d11c902a91 100644 --- a/src/lib/net/resolve.h +++ b/src/lib/net/resolve.h @@ -55,4 +55,4 @@ void tor_free_getaddrinfo_cache(void); void sandbox_disable_getaddrinfo_cache(void); void tor_make_getaddrinfo_cache_active(void); -#endif +#endif /* !defined(TOR_RESOLVE_H) */ diff --git a/src/lib/net/socket.c b/src/lib/net/socket.c index f978deeab8..e824a05045 100644 --- a/src/lib/net/socket.c +++ b/src/lib/net/socket.c @@ -84,9 +84,9 @@ check_network_configuration(bool server_mode) "so your relay makes it harder to figure out how busy it is."); } } -#else +#else /* !(defined(__FreeBSD__)) */ (void) server_mode; -#endif +#endif /* defined(__FreeBSD__) */ } /* When set_max_file_sockets() is called, update this with the max file @@ -487,11 +487,11 @@ tor_socketpair(int family, int type, int protocol, tor_socket_t fd[2]) r = socketpair(family, type, protocol, fd); if (r < 0) return -errno; -#else +#else /* !(defined(HAVE_SOCKETPAIR) && !defined(_WIN32)) */ r = tor_ersatz_socketpair(family, type, protocol, fd); if (r < 0) return -r; -#endif +#endif /* defined(HAVE_SOCKETPAIR) && !defined(_WIN32) */ #if defined(FD_CLOEXEC) if (SOCKET_OK(fd[0])) { diff --git a/src/lib/net/socket.h b/src/lib/net/socket.h index 86ae336dfb..193ad91e4c 100644 --- a/src/lib/net/socket.h +++ b/src/lib/net/socket.h @@ -116,4 +116,4 @@ const char *tor_socket_strerror(int e); #define SIO_IDEAL_SEND_BACKLOG_QUERY 0x4004747b #endif -#endif +#endif /* !defined(TOR_SOCKET_H) */ diff --git a/src/lib/net/socketpair.c b/src/lib/net/socketpair.c index 15c706bec7..3be7b26f7f 100644 --- a/src/lib/net/socketpair.c +++ b/src/lib/net/socketpair.c @@ -22,11 +22,11 @@ #include #define socket_errno() (WSAGetLastError()) #define SOCKET_EPROTONOSUPPORT WSAEPROTONOSUPPORT -#else +#else /* !(defined(_WIN32)) */ #define closesocket(x) close(x) #define socket_errno() (errno) #define SOCKET_EPROTONOSUPPORT EPROTONOSUPPORT -#endif +#endif /* defined(_WIN32) */ #ifdef NEED_ERSATZ_SOCKETPAIR diff --git a/src/lib/net/socketpair.h b/src/lib/net/socketpair.h index 6be0803881..5820606973 100644 --- a/src/lib/net/socketpair.h +++ b/src/lib/net/socketpair.h @@ -16,4 +16,4 @@ int tor_ersatz_socketpair(int family, int type, int protocol, tor_socket_t fd[2]); #endif -#endif +#endif /* !defined(TOR_SOCKETPAIR_H) */ diff --git a/src/lib/net/socks5_status.h b/src/lib/net/socks5_status.h index e55242ce66..e55119e0b0 100644 --- a/src/lib/net/socks5_status.h +++ b/src/lib/net/socks5_status.h @@ -29,4 +29,4 @@ typedef enum { SOCKS5_ADDRESS_TYPE_NOT_SUPPORTED = 0x08, } socks5_reply_status_t; -#endif +#endif /* !defined(TOR_SOCKS5_STATUS_H) */ diff --git a/src/lib/osinfo/uname.h b/src/lib/osinfo/uname.h index fcce629074..443a30d358 100644 --- a/src/lib/osinfo/uname.h +++ b/src/lib/osinfo/uname.h @@ -15,4 +15,4 @@ MOCK_DECL(const char *, get_uname,(void)); -#endif +#endif /* !defined(HAVE_TOR_UNAME_H) */ diff --git a/src/lib/process/daemon.h b/src/lib/process/daemon.h index 20920e0aae..423c498837 100644 --- a/src/lib/process/daemon.h +++ b/src/lib/process/daemon.h @@ -18,4 +18,4 @@ int finish_daemon(const char *desired_cwd); bool start_daemon_has_been_called(void); -#endif +#endif /* !defined(TOR_DAEMON_H) */ diff --git a/src/lib/process/env.h b/src/lib/process/env.h index 15d59351e0..19c2235970 100644 --- a/src/lib/process/env.h +++ b/src/lib/process/env.h @@ -38,4 +38,4 @@ void set_environment_variable_in_smartlist(struct smartlist_t *env_vars, const char *new_var, void (*free_old)(void*), int free_p); -#endif +#endif /* !defined(TOR_ENV_H) */ diff --git a/src/lib/process/pidfile.h b/src/lib/process/pidfile.h index dfeb39e046..af59041f80 100644 --- a/src/lib/process/pidfile.h +++ b/src/lib/process/pidfile.h @@ -13,4 +13,4 @@ int write_pidfile(const char *filename); -#endif +#endif /* !defined(TOR_PIDFILE_H) */ diff --git a/src/lib/process/process.c b/src/lib/process/process.c index 422942dc83..631c7169f1 100644 --- a/src/lib/process/process.c +++ b/src/lib/process/process.c @@ -83,7 +83,7 @@ struct process_t { #else /** Our Win32 process handle. */ process_win32_t *win32_process; -#endif +#endif /* !defined(_WIN32) */ }; /** Convert a given process status in status to its string @@ -205,7 +205,7 @@ process_new(const char *command) #else /* Prepare our Win32 process handle. */ process->win32_process = process_win32_new(); -#endif +#endif /* !defined(_WIN32) */ smartlist_add(processes, process); @@ -240,7 +240,7 @@ process_free_(process_t *process) #else /* Cleanup our Win32 process handle. */ process_win32_free(process->win32_process); -#endif +#endif /* !defined(_WIN32) */ smartlist_remove(processes, process); @@ -513,7 +513,7 @@ process_get_unix_process(const process_t *process) tor_assert(process->unix_process); return process->unix_process; } -#else +#else /* !(!defined(_WIN32)) */ /** Get the internal handle for Windows backend. */ process_win32_t * process_get_win32_process(const process_t *process) @@ -522,7 +522,7 @@ process_get_win32_process(const process_t *process) tor_assert(process->win32_process); return process->win32_process; } -#endif +#endif /* !defined(_WIN32) */ /** Write size bytes of data to the given process's standard * input. */ diff --git a/src/lib/process/process.h b/src/lib/process/process.h index 14069923a0..05c091a5bf 100644 --- a/src/lib/process/process.h +++ b/src/lib/process/process.h @@ -111,7 +111,7 @@ struct process_unix_t *process_get_unix_process(const process_t *process); #else struct process_win32_t; struct process_win32_t *process_get_win32_process(const process_t *process); -#endif +#endif /* !defined(_WIN32) */ void process_write(process_t *process, const uint8_t *data, size_t size); @@ -140,6 +140,6 @@ STATIC void process_read_buffer(process_t *process, STATIC void process_read_lines(process_t *process, buf_t *buffer, process_read_callback_t callback); -#endif /* defined(PROCESS_PRIVATE). */ +#endif /* defined(PROCESS_PRIVATE) */ -#endif /* defined(TOR_PROCESS_H). */ +#endif /* !defined(TOR_PROCESS_H) */ diff --git a/src/lib/process/process_unix.c b/src/lib/process/process_unix.c index 790ab897e9..17ade87463 100644 --- a/src/lib/process/process_unix.c +++ b/src/lib/process/process_unix.c @@ -702,4 +702,4 @@ process_unix_close_file_descriptors(process_unix_t *unix_process) return success; } -#endif /* defined(_WIN32). */ +#endif /* !defined(_WIN32) */ diff --git a/src/lib/process/process_unix.h b/src/lib/process/process_unix.h index a1d8f72993..da40b3e567 100644 --- a/src/lib/process/process_unix.h +++ b/src/lib/process/process_unix.h @@ -61,8 +61,8 @@ STATIC int process_unix_read_handle(process_t *, process_unix_handle_t *, buf_t *); STATIC bool process_unix_close_file_descriptors(process_unix_t *); -#endif /* defined(PROCESS_UNIX_PRIVATE). */ +#endif /* defined(PROCESS_UNIX_PRIVATE) */ -#endif /* defined(_WIN32). */ +#endif /* !defined(_WIN32) */ -#endif /* defined(TOR_PROCESS_UNIX_H). */ +#endif /* !defined(TOR_PROCESS_UNIX_H) */ diff --git a/src/lib/process/process_win32.c b/src/lib/process/process_win32.c index ddbe76bfd9..624333d4a3 100644 --- a/src/lib/process/process_win32.c +++ b/src/lib/process/process_win32.c @@ -741,7 +741,7 @@ process_win32_cleanup_handle(process_win32_handle_t *handle) format_win32_error(error_code)); } } -#endif +#endif /* 0 */ /* Close our handle. */ if (handle->pipe != INVALID_HANDLE_VALUE) { @@ -1084,4 +1084,4 @@ tor_join_win_cmdline(const char *argv[]) return joined_argv; } -#endif /* ! defined(_WIN32). */ +#endif /* defined(_WIN32) */ diff --git a/src/lib/process/process_win32.h b/src/lib/process/process_win32.h index d79dde157e..a50d86df5b 100644 --- a/src/lib/process/process_win32.h +++ b/src/lib/process/process_win32.h @@ -90,8 +90,8 @@ STATIC bool process_win32_handle_read_completion(process_win32_handle_t *, STATIC char *format_win_cmdline_argument(const char *arg); STATIC char *tor_join_win_cmdline(const char *argv[]); -#endif /* defined(PROCESS_WIN32_PRIVATE). */ +#endif /* defined(PROCESS_WIN32_PRIVATE) */ -#endif /* ! defined(_WIN32). */ +#endif /* defined(_WIN32) */ -#endif /* defined(TOR_PROCESS_WIN32_H). */ +#endif /* !defined(TOR_PROCESS_WIN32_H) */ diff --git a/src/lib/process/setuid.h b/src/lib/process/setuid.h index 7d03e1f025..a2125d2d06 100644 --- a/src/lib/process/setuid.h +++ b/src/lib/process/setuid.h @@ -19,4 +19,4 @@ int have_capability_support(void); #define SWITCH_ID_WARN_IF_NO_CAPS (1<<1) int switch_id(const char *user, unsigned flags); -#endif +#endif /* !defined(TOR_SETUID_H) */ diff --git a/src/lib/process/winprocess_sys.c b/src/lib/process/winprocess_sys.c index 1266babca8..48c0888658 100644 --- a/src/lib/process/winprocess_sys.c +++ b/src/lib/process/winprocess_sys.c @@ -51,7 +51,7 @@ subsys_winprocess_initialize(void) return 0; } -#else /* !defined(_WIN32) */ +#else /* !(defined(_WIN32)) */ #define WINPROCESS_SYS_ENABLED false #define subsys_winprocess_initialize NULL #endif /* defined(_WIN32) */ diff --git a/src/lib/pubsub/pub_binding_st.h b/src/lib/pubsub/pub_binding_st.h index 4f5df8ff38..d841bf3f54 100644 --- a/src/lib/pubsub/pub_binding_st.h +++ b/src/lib/pubsub/pub_binding_st.h @@ -35,4 +35,4 @@ typedef struct pub_binding_t { msg_t msg_template; } pub_binding_t; -#endif +#endif /* !defined(TOR_PUB_BINDING_ST_H) */ diff --git a/src/lib/pubsub/pubsub.h b/src/lib/pubsub/pubsub.h index 08e3f42f7c..5346b07517 100644 --- a/src/lib/pubsub/pubsub.h +++ b/src/lib/pubsub/pubsub.h @@ -86,4 +86,4 @@ #include "lib/pubsub/pubsub_macros.h" #include "lib/pubsub/pubsub_publish.h" -#endif +#endif /* !defined(TOR_PUBSUB_PUBSUB_H) */ diff --git a/src/lib/pubsub/pubsub_build.h b/src/lib/pubsub/pubsub_build.h index 93aad50b28..5a0c5f5bd3 100644 --- a/src/lib/pubsub/pubsub_build.h +++ b/src/lib/pubsub/pubsub_build.h @@ -89,4 +89,4 @@ void pubsub_items_clear_bindings(pubsub_items_t *items); FREE_AND_NULL(pubsub_items_t, pubsub_items_free_, (cfg)) void pubsub_items_free_(pubsub_items_t *cfg); -#endif +#endif /* !defined(TOR_PUBSUB_BUILD_H) */ diff --git a/src/lib/pubsub/pubsub_builder_st.h b/src/lib/pubsub/pubsub_builder_st.h index cedeb02b16..545aa3f3ef 100644 --- a/src/lib/pubsub/pubsub_builder_st.h +++ b/src/lib/pubsub/pubsub_builder_st.h @@ -156,6 +156,6 @@ typedef struct pubsub_adjmap_t { struct smartlist_t **sub_by_msg; } pubsub_adjmap_t; -#endif +#endif /* defined(PUBSUB_PRIVATE) */ -#endif +#endif /* !defined(TOR_PUBSUB_BUILDER_ST_H) */ diff --git a/src/lib/pubsub/pubsub_connect.h b/src/lib/pubsub/pubsub_connect.h index bdcb33d2f5..0ad106044e 100644 --- a/src/lib/pubsub/pubsub_connect.h +++ b/src/lib/pubsub/pubsub_connect.h @@ -51,4 +51,4 @@ int pubsub_connector_register_type_(struct pubsub_connector_t *, const char *file, unsigned line); -#endif +#endif /* !defined(TOR_PUBSUB_CONNECT_H) */ diff --git a/src/lib/pubsub/pubsub_flags.h b/src/lib/pubsub/pubsub_flags.h index d07a06be7b..53c6e49565 100644 --- a/src/lib/pubsub/pubsub_flags.h +++ b/src/lib/pubsub/pubsub_flags.h @@ -29,4 +29,4 @@ */ #define DISP_FLAG_STUB (1u<<1) -#endif +#endif /* !defined(TOR_PUBSUB_FLAGS_H) */ diff --git a/src/lib/pubsub/pubsub_macros.h b/src/lib/pubsub/pubsub_macros.h index d091e40dfa..357e59fd54 100644 --- a/src/lib/pubsub/pubsub_macros.h +++ b/src/lib/pubsub/pubsub_macros.h @@ -370,4 +370,4 @@ __FILE__, \ __LINE__) -#endif +#endif /* !defined(TOR_DISPATCH_MSG_H) */ diff --git a/src/lib/pubsub/pubsub_publish.h b/src/lib/pubsub/pubsub_publish.h index 0250fd0760..0686a465de 100644 --- a/src/lib/pubsub/pubsub_publish.h +++ b/src/lib/pubsub/pubsub_publish.h @@ -12,4 +12,4 @@ struct pub_binding_t; int pubsub_pub_(const struct pub_binding_t *pub, msg_aux_data_t auxdata); -#endif +#endif /* !defined(TOR_PUBSUB_PUBLISH_H) */ diff --git a/src/lib/smartlist_core/smartlist_core.h b/src/lib/smartlist_core/smartlist_core.h index a1a195f312..795741c447 100644 --- a/src/lib/smartlist_core/smartlist_core.h +++ b/src/lib/smartlist_core/smartlist_core.h @@ -98,4 +98,4 @@ void smartlist_del(smartlist_t *sl, int idx); void smartlist_del_keeporder(smartlist_t *sl, int idx); void smartlist_insert(smartlist_t *sl, int idx, void *val); -#endif /* !defined(TOR_CONTAINER_H) */ +#endif /* !defined(TOR_SMARTLIST_CORE_H) */ diff --git a/src/lib/smartlist_core/smartlist_split.h b/src/lib/smartlist_core/smartlist_split.h index 4f72376125..e2cc7245b7 100644 --- a/src/lib/smartlist_core/smartlist_split.h +++ b/src/lib/smartlist_core/smartlist_split.h @@ -17,4 +17,4 @@ int smartlist_split_string(smartlist_t *sl, const char *str, const char *sep, int flags, int max); -#endif +#endif /* !defined(TOR_SMARTLIST_SPLIT_H) */ diff --git a/src/lib/string/compat_string.h b/src/lib/string/compat_string.h index a0e37bb6dc..4f30bf5392 100644 --- a/src/lib/string/compat_string.h +++ b/src/lib/string/compat_string.h @@ -25,14 +25,14 @@ static inline int strncasecmp(const char *a, const char *b, size_t n); static inline int strncasecmp(const char *a, const char *b, size_t n) { return _strnicmp(a,b,n); } -#endif +#endif /* !defined(HAVE_STRNCASECMP) */ #ifndef HAVE_STRCASECMP static inline int strcasecmp(const char *a, const char *b); static inline int strcasecmp(const char *a, const char *b) { return _stricmp(a,b); } -#endif -#endif +#endif /* !defined(HAVE_STRCASECMP) */ +#endif /* defined(_WIN32) */ #if defined __APPLE__ /* On OSX 10.9 and later, the overlap-checking code for strlcat would @@ -59,4 +59,4 @@ char *tor_strtok_r_impl(char *str, const char *sep, char **lasts); #define tor_strtok_r(str, sep, lasts) tor_strtok_r_impl(str, sep, lasts) #endif -#endif +#endif /* !defined(TOR_COMPAT_STRING_H) */ diff --git a/src/lib/string/parse_int.h b/src/lib/string/parse_int.h index 925547942e..50d48b44c5 100644 --- a/src/lib/string/parse_int.h +++ b/src/lib/string/parse_int.h @@ -22,4 +22,4 @@ double tor_parse_double(const char *s, double min, double max, int *ok, uint64_t tor_parse_uint64(const char *s, int base, uint64_t min, uint64_t max, int *ok, char **next); -#endif +#endif /* !defined(TOR_PARSE_INT_H) */ diff --git a/src/lib/string/printf.h b/src/lib/string/printf.h index 2cc13d6bee..6e90770f81 100644 --- a/src/lib/string/printf.h +++ b/src/lib/string/printf.h @@ -27,4 +27,4 @@ int tor_asprintf(char **strp, const char *fmt, ...) int tor_vasprintf(char **strp, const char *fmt, va_list args) CHECK_PRINTF(2,0); -#endif /* !defined(TOR_UTIL_STRING_H) */ +#endif /* !defined(TOR_UTIL_PRINTF_H) */ diff --git a/src/lib/string/scanf.h b/src/lib/string/scanf.h index 6673173de5..b642e242db 100644 --- a/src/lib/string/scanf.h +++ b/src/lib/string/scanf.h @@ -21,4 +21,4 @@ int tor_vsscanf(const char *buf, const char *pattern, va_list ap) \ int tor_sscanf(const char *buf, const char *pattern, ...) CHECK_SCANF(2, 3); -#endif /* !defined(TOR_UTIL_STRING_H) */ +#endif /* !defined(TOR_UTIL_SCANF_H) */ diff --git a/src/lib/subsys/subsys.h b/src/lib/subsys/subsys.h index d78bb4a602..21f984f32d 100644 --- a/src/lib/subsys/subsys.h +++ b/src/lib/subsys/subsys.h @@ -92,4 +92,4 @@ typedef struct subsys_fns_t { * less than this value. */ #define SUBSYS_LEVEL_LIBS -10 -#endif +#endif /* !defined(TOR_SUBSYS_T) */ diff --git a/src/lib/term/getpass.h b/src/lib/term/getpass.h index a9c146ea8f..aa597ec423 100644 --- a/src/lib/term/getpass.h +++ b/src/lib/term/getpass.h @@ -15,4 +15,4 @@ ssize_t tor_getpass(const char *prompt, char *output, size_t buflen); -#endif +#endif /* !defined(TOR_GETPASS_H) */ diff --git a/src/lib/testsupport/testsupport.h b/src/lib/testsupport/testsupport.h index 9363a9ba66..631ec0228c 100644 --- a/src/lib/testsupport/testsupport.h +++ b/src/lib/testsupport/testsupport.h @@ -21,7 +21,7 @@ * tests. */ #define STATIC #define EXTERN(type, name) extern type name; -#else +#else /* !(defined(TOR_UNIT_TESTS)) */ #define STATIC static #define EXTERN(type, name) #endif /* defined(TOR_UNIT_TESTS) */ diff --git a/src/lib/thread/numcpus.h b/src/lib/thread/numcpus.h index 3f0a29ce7c..2f1ea16eb9 100644 --- a/src/lib/thread/numcpus.h +++ b/src/lib/thread/numcpus.h @@ -13,4 +13,4 @@ int compute_num_cpus(void); -#endif +#endif /* !defined(TOR_NUMCPUS_H) */ diff --git a/src/lib/time/compat_time.c b/src/lib/time/compat_time.c index 70802770cc..7136eaba67 100644 --- a/src/lib/time/compat_time.c +++ b/src/lib/time/compat_time.c @@ -304,7 +304,7 @@ monotime_coarse_get(monotime_coarse_t *out) #endif /* defined(TOR_UNIT_TESTS) */ out->abstime_ = mach_approximate_time(); } -#endif +#endif /* defined(HAVE_MACH_APPROXIMATE_TIME) */ /** * Return the number of nanoseconds between start and end. @@ -767,7 +767,7 @@ monotime_coarse_zero(monotime_coarse_t *out) { memset(out, 0, sizeof(*out)); } -#endif +#endif /* defined(MONOTIME_COARSE_TYPE_IS_DIFFERENT) */ int64_t monotime_diff_usec(const monotime_t *start, @@ -833,7 +833,7 @@ monotime_coarse_absolute_msec(void) { return monotime_coarse_absolute_nsec() / ONE_MILLION; } -#else +#else /* !(defined(MONOTIME_COARSE_FN_IS_DIFFERENT)) */ #define initialized_at_coarse initialized_at #endif /* defined(MONOTIME_COARSE_FN_IS_DIFFERENT) */ @@ -865,7 +865,7 @@ monotime_msec_to_approx_coarse_stamp_units(uint64_t msec) mach_time_info.numer; return abstime_val >> monotime_shift; } -#else +#else /* !(defined(__APPLE__)) */ uint64_t monotime_coarse_stamp_units_to_approx_msec(uint64_t units) { @@ -876,4 +876,4 @@ monotime_msec_to_approx_coarse_stamp_units(uint64_t msec) { return (msec * STAMP_TICKS_PER_SECOND) / 1000; } -#endif +#endif /* defined(__APPLE__) */ diff --git a/src/lib/time/compat_time.h b/src/lib/time/compat_time.h index 360d92e5c9..8c7661d7cb 100644 --- a/src/lib/time/compat_time.h +++ b/src/lib/time/compat_time.h @@ -344,7 +344,7 @@ monotime_coarse_diff_msec32(const monotime_coarse_t *start, #else #define USING_32BIT_MSEC_HACK return monotime_coarse_diff_msec32_(start, end); -#endif +#endif /* SIZEOF_VOID_P == 8 */ } #ifdef TOR_UNIT_TESTS diff --git a/src/lib/time/tvdiff.h b/src/lib/time/tvdiff.h index 724af1528a..657cb99553 100644 --- a/src/lib/time/tvdiff.h +++ b/src/lib/time/tvdiff.h @@ -20,4 +20,4 @@ int64_t tv_to_msec(const struct timeval *tv); time_t time_diff(const time_t from, const time_t to); -#endif +#endif /* !defined(TOR_TVDIFF_H) */ diff --git a/src/lib/tls/nss_countbytes.h b/src/lib/tls/nss_countbytes.h index 8b31603923..47f220c4c1 100644 --- a/src/lib/tls/nss_countbytes.h +++ b/src/lib/tls/nss_countbytes.h @@ -22,4 +22,4 @@ int tor_get_prfiledesc_byte_counts(struct PRFileDesc *fd, uint64_t *n_read_out, uint64_t *n_written_out); -#endif +#endif /* !defined(TOR_NSS_COUNTBYTES_H) */ diff --git a/src/lib/tls/tortls.h b/src/lib/tls/tortls.h index 8efc7a1c98..9e195c6af2 100644 --- a/src/lib/tls/tortls.h +++ b/src/lib/tls/tortls.h @@ -25,12 +25,12 @@ struct ssl_ctx_st; struct ssl_session_st; typedef struct ssl_ctx_st tor_tls_context_impl_t; typedef struct ssl_st tor_tls_impl_t; -#else +#else /* !(defined(ENABLE_OPENSSL)) */ struct PRFileDesc; typedef struct PRFileDesc tor_tls_context_impl_t; typedef struct PRFileDesc tor_tls_impl_t; -#endif -#endif +#endif /* defined(ENABLE_OPENSSL) */ +#endif /* defined(TORTLS_PRIVATE) */ struct tor_x509_cert_t; @@ -144,9 +144,9 @@ void check_no_tls_errors_(const char *fname, int line); void tor_tls_log_one_error(tor_tls_t *tls, unsigned long err, int severity, int domain, const char *doing); -#else +#else /* !(defined(ENABLE_OPENSSL)) */ #define check_no_tls_errors() STMT_NIL -#endif +#endif /* defined(ENABLE_OPENSSL) */ int tor_tls_get_my_certs(int server, const struct tor_x509_cert_t **link_cert_out, diff --git a/src/lib/tls/tortls_internal.h b/src/lib/tls/tortls_internal.h index 071c506561..866483a94c 100644 --- a/src/lib/tls/tortls_internal.h +++ b/src/lib/tls/tortls_internal.h @@ -61,8 +61,8 @@ STATIC int tor_tls_session_secret_cb(struct ssl_st *ssl, void *secret, void *arg); STATIC int find_cipher_by_id(const SSL *ssl, const SSL_METHOD *m, uint16_t cipher); -#endif -#endif +#endif /* defined(TORTLS_OPENSSL_PRIVATE) */ +#endif /* defined(ENABLE_OPENSSL) */ #ifdef TOR_UNIT_TESTS extern int tor_tls_object_ex_data_index; @@ -73,4 +73,4 @@ extern uint64_t total_bytes_written_over_tls; extern uint64_t total_bytes_written_by_tls; #endif /* defined(TOR_UNIT_TESTS) */ -#endif /* defined(TORTLS_INTERNAL_H) */ +#endif /* !defined(TORTLS_INTERNAL_H) */ diff --git a/src/lib/tls/tortls_openssl.c b/src/lib/tls/tortls_openssl.c index b40f948a3b..04027104e1 100644 --- a/src/lib/tls/tortls_openssl.c +++ b/src/lib/tls/tortls_openssl.c @@ -25,7 +25,7 @@ * and mess things up, in at least some openssl versions. */ #include #include -#endif +#endif /* defined(_WIN32) */ #include "lib/crypt_ops/crypto_cipher.h" #include "lib/crypt_ops/crypto_rand.h" @@ -318,7 +318,7 @@ tor_tls_init(void) #else SSL_library_init(); SSL_load_error_strings(); -#endif +#endif /* defined(OPENSSL_1_1_API) */ #if (SIZEOF_VOID_P >= 8 && \ OPENSSL_VERSION_NUMBER >= OPENSSL_V_SERIES(1,0,1)) @@ -383,7 +383,7 @@ static const char SERVER_CIPHER_LIST[] = * conclude that it has no valid ciphers if it's running with TLS1.3. */ TLS1_3_TXT_AES_128_GCM_SHA256 ":" -#endif +#endif /* defined(TLS1_3_TXT_AES_128_GCM_SHA256) */ TLS1_TXT_DHE_RSA_WITH_AES_256_SHA ":" TLS1_TXT_DHE_RSA_WITH_AES_128_SHA; @@ -657,7 +657,7 @@ tor_tls_context_new(crypto_pk_t *identity, unsigned int key_lifetime, if (r < 0) goto error; } -#else +#else /* !(defined(SSL_CTX_set1_groups_list) || ...) */ if (! is_client) { int nid; EC_KEY *ec_key; @@ -673,7 +673,7 @@ tor_tls_context_new(crypto_pk_t *identity, unsigned int key_lifetime, SSL_CTX_set_tmp_ecdh(result->ctx, ec_key); EC_KEY_free(ec_key); } -#endif +#endif /* defined(SSL_CTX_set1_groups_list) || ...) SSL_CTX_set_verify(result->ctx, SSL_VERIFY_PEER, always_accept_verify_cb); /* let us realloc bufs that we're writing from */ @@ -1062,7 +1062,7 @@ tor_tls_new(tor_socket_t sock, int isServer) /* We can't actually use TLS 1.3 until this bug is fixed. */ SSL_set_max_proto_version(result->ssl, TLS1_2_VERSION); } -#endif +#endif /* defined(SSL_CTRL_SET_MAX_PROTO_VERSION) */ if (!SSL_set_cipher_list(result->ssl, isServer ? SERVER_CIPHER_LIST : CLIENT_CIPHER_LIST)) { @@ -1728,7 +1728,7 @@ tor_tls_export_key_material,(tor_tls_t *tls, uint8_t *secrets_out, else return -1; } -#endif +#endif /* defined(TLS1_3_VERSION) */ return (r == 1) ? 0 : -1; } diff --git a/src/lib/tls/tortls_st.h b/src/lib/tls/tortls_st.h index 3f7ea8ac6a..73f6e6ecca 100644 --- a/src/lib/tls/tortls_st.h +++ b/src/lib/tls/tortls_st.h @@ -64,7 +64,7 @@ struct tor_tls_t { void (*negotiated_callback)(tor_tls_t *tls, void *arg); /** Argument to pass to negotiated_callback. */ void *callback_arg; -#endif +#endif /* defined(ENABLE_OPENSSL) */ #ifdef ENABLE_NSS /** Last values retried from tor_get_prfiledesc_byte_counts(). */ uint64_t last_write_count; @@ -72,4 +72,4 @@ struct tor_tls_t { #endif }; -#endif +#endif /* !defined(TOR_TORTLS_ST_H) */ diff --git a/src/lib/tls/x509.h b/src/lib/tls/x509.h index 5e6660de5c..0390a5464d 100644 --- a/src/lib/tls/x509.h +++ b/src/lib/tls/x509.h @@ -35,7 +35,7 @@ struct tor_x509_cert_t { common_digests_t cert_digests; common_digests_t pkey_digests; }; -#endif +#endif /* defined(TOR_X509_PRIVATE) */ void tor_tls_pick_certificate_lifetime(time_t now, unsigned cert_lifetime, @@ -47,7 +47,7 @@ tor_x509_cert_t *tor_x509_cert_replace_expiration( const tor_x509_cert_t *inp, time_t new_expiration_time, crypto_pk_t *signing_key); -#endif +#endif /* defined(TOR_UNIT_TESTS) */ tor_x509_cert_t *tor_x509_cert_dup(const tor_x509_cert_t *cert); @@ -72,4 +72,4 @@ int tor_tls_cert_is_valid(int severity, time_t now, int check_rsa_1024); -#endif +#endif /* !defined(TOR_X509_H) */ diff --git a/src/lib/tls/x509_internal.h b/src/lib/tls/x509_internal.h index bf2bec9689..f858baae98 100644 --- a/src/lib/tls/x509_internal.h +++ b/src/lib/tls/x509_internal.h @@ -50,4 +50,4 @@ int tor_x509_cert_set_cached_der_encoding(tor_x509_cert_t *cert); #define tor_x509_cert_set_cached_der_encoding(cert) (0) #endif -#endif +#endif /* !defined(TOR_X509_INTERNAL_H) */ diff --git a/src/lib/tls/x509_nss.c b/src/lib/tls/x509_nss.c index fb4af54c52..e04afaf07b 100644 --- a/src/lib/tls/x509_nss.c +++ b/src/lib/tls/x509_nss.c @@ -120,13 +120,13 @@ tor_tls_create_certificate_internal(crypto_pk_t *rsa, der.data, der.len, (SECKEYPrivateKey *)signing_key,//const &cert->signature); -#else +#else /* !(0) */ s = SEC_DerSignData(cert->arena, &signed_der, der.data, der.len, (SECKEYPrivateKey *)signing_key,//const SEC_OID_PKCS1_SHA256_WITH_RSA_ENCRYPTION); -#endif +#endif /* 0 */ if (s != SECSuccess) goto err; @@ -145,7 +145,7 @@ tor_tls_create_certificate_internal(crypto_pk_t *rsa, &result_cert->signatureWrap, issuer_pk, NULL); tor_assert(cert_ok == SECSuccess); } -#endif +#endif /* 1 */ err: if (subject_spki) @@ -455,4 +455,4 @@ tor_x509_cert_replace_expiration(const tor_x509_cert_t *inp, return newcert ? tor_x509_cert_new(newcert) : NULL; } -#endif +#endif /* defined(TOR_UNIT_TESTS) */ diff --git a/src/lib/tls/x509_openssl.c b/src/lib/tls/x509_openssl.c index a344279c22..03f65049cf 100644 --- a/src/lib/tls/x509_openssl.c +++ b/src/lib/tls/x509_openssl.c @@ -59,12 +59,12 @@ ENABLE_GCC_WARNING(redundant-decls) #define X509_get_notAfter(cert) \ X509_getm_notAfter(cert) #endif -#else /* ! OPENSSL_VERSION_NUMBER >= OPENSSL_V_SERIES(1,1,0) */ +#else /* !(defined(OPENSSL_1_1_API)) */ #define X509_get_notBefore_const(cert) \ ((const ASN1_TIME*) X509_get_notBefore((X509 *)cert)) #define X509_get_notAfter_const(cert) \ ((const ASN1_TIME*) X509_get_notAfter((X509 *)cert)) -#endif +#endif /* defined(OPENSSL_1_1_API) */ /** Return a newly allocated X509 name with commonName cname. */ static X509_NAME * diff --git a/src/lib/trace/debug.h b/src/lib/trace/debug.h index e35616cf50..92bb95c883 100644 --- a/src/lib/trace/debug.h +++ b/src/lib/trace/debug.h @@ -27,4 +27,4 @@ "\"" XSTR(subsystem) "\" hit. " \ "(line "XSTR(__LINE__) ")") -#endif /* TOR_TRACE_LOG_DEBUG_H */ +#endif /* !defined(TOR_TRACE_LOG_DEBUG_H) */ diff --git a/src/lib/trace/events.h b/src/lib/trace/events.h index 1e1e7b9d16..0674f7d501 100644 --- a/src/lib/trace/events.h +++ b/src/lib/trace/events.h @@ -34,12 +34,12 @@ #include "lib/trace/debug.h" #endif -#else /* TOR_EVENT_TRACING_ENABLED */ +#else /* !(defined(TOR_EVENT_TRACING_ENABLED)) */ /* Reaching this point, we NOP every event declaration because event tracing * is not been enabled at compile time. */ #define tor_trace(subsystem, name, args...) -#endif /* TOR_EVENT_TRACING_ENABLED */ +#endif /* defined(TOR_EVENT_TRACING_ENABLED) */ -#endif /* TOR_TRACE_EVENTS_H */ +#endif /* !defined(TOR_TRACE_EVENTS_H) */ diff --git a/src/lib/trace/trace.h b/src/lib/trace/trace.h index 606d435568..5001b28a1d 100644 --- a/src/lib/trace/trace.h +++ b/src/lib/trace/trace.h @@ -11,4 +11,4 @@ void tor_trace_init(void); -#endif // TOR_TRACE_TRACE_H +#endif /* !defined(TOR_TRACE_TRACE_H) */ diff --git a/src/lib/wallclock/approx_time.h b/src/lib/wallclock/approx_time.h index e6b53f2c27..e7da160122 100644 --- a/src/lib/wallclock/approx_time.h +++ b/src/lib/wallclock/approx_time.h @@ -22,4 +22,4 @@ time_t approx_time(void); void update_approx_time(time_t now); #endif /* defined(TIME_IS_FAST) */ -#endif +#endif /* !defined(TOR_APPROX_TIME_H) */ diff --git a/src/lib/wallclock/time_to_tm.h b/src/lib/wallclock/time_to_tm.h index abe78c0efe..da27fcaba1 100644 --- a/src/lib/wallclock/time_to_tm.h +++ b/src/lib/wallclock/time_to_tm.h @@ -19,4 +19,4 @@ struct tm *tor_localtime_r_msg(const time_t *timep, struct tm *result, struct tm *tor_gmtime_r_msg(const time_t *timep, struct tm *result, char **err_out); -#endif +#endif /* !defined(TOR_WALLCLOCK_TIME_TO_TM_H) */ diff --git a/src/lib/wallclock/timeval.h b/src/lib/wallclock/timeval.h index 33076adc8b..e632d04a04 100644 --- a/src/lib/wallclock/timeval.h +++ b/src/lib/wallclock/timeval.h @@ -39,7 +39,7 @@ (tvout)->tv_sec += (tvout)->tv_usec / 1000000; \ (tvout)->tv_usec %= 1000000; \ } while (0) -#endif +#endif /* defined(TOR_COVERAGE) */ #ifndef timeradd /** Replacement for timeradd on platforms that do not have it: sets tvout to @@ -83,4 +83,4 @@ ((tv1)->tv_sec op (tv2)->tv_sec)) #endif /* !defined(timercmp) */ -#endif +#endif /* !defined(TOR_TIMEVAL_H) */ diff --git a/src/lib/wallclock/tor_gettimeofday.h b/src/lib/wallclock/tor_gettimeofday.h index c7fff9747a..6fec2fc893 100644 --- a/src/lib/wallclock/tor_gettimeofday.h +++ b/src/lib/wallclock/tor_gettimeofday.h @@ -17,4 +17,4 @@ struct timeval; MOCK_DECL(void, tor_gettimeofday, (struct timeval *timeval)); -#endif +#endif /* !defined(TOR_GETTIMEOFDAY_H) */ diff --git a/src/test/bench.c b/src/test/bench.c index 65fa617cbd..cf732df593 100644 --- a/src/test/bench.c +++ b/src/test/bench.c @@ -22,7 +22,7 @@ #include #include #include -#endif +#endif /* defined(ENABLE_OPENSSL) */ #include "core/or/circuitlist.h" #include "app/config/config.h" @@ -701,7 +701,7 @@ bench_ecdh_p224(void) { bench_ecdh_impl(NID_secp224r1, "P-224"); } -#endif +#endif /* defined(ENABLE_OPENSSL) */ static void bench_md_parse(void) diff --git a/src/test/fuzz/fuzzing.h b/src/test/fuzz/fuzzing.h index 150ac4aa7d..2d278825ec 100644 --- a/src/test/fuzz/fuzzing.h +++ b/src/test/fuzz/fuzzing.h @@ -9,5 +9,5 @@ int fuzz_main(const uint8_t *data, size_t sz); void disable_signature_checking(void); -#endif /* FUZZING_H */ +#endif /* !defined(FUZZING_H) */ diff --git a/src/test/fuzz/fuzzing_common.c b/src/test/fuzz/fuzzing_common.c index 387c865a9b..6d0f9d7d60 100644 --- a/src/test/fuzz/fuzzing_common.c +++ b/src/test/fuzz/fuzzing_common.c @@ -137,7 +137,7 @@ LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) return fuzz_main(Data, Size); } -#else /* Not LLVM_FUZZ, so AFL. */ +#else /* !(defined(LLVM_FUZZ)) */ int main(int argc, char **argv) @@ -194,4 +194,4 @@ main(int argc, char **argv) return 0; } -#endif +#endif /* defined(LLVM_FUZZ) */ diff --git a/src/test/ptr_helpers.c b/src/test/ptr_helpers.c index 296238feeb..a55ab437fa 100644 --- a/src/test/ptr_helpers.c +++ b/src/test/ptr_helpers.c @@ -3,7 +3,7 @@ * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ -#include "ptr_helpers.h" +#include "test/ptr_helpers.h" /** * Cast (inptr_t value) to a void pointer. diff --git a/src/test/ptr_helpers.h b/src/test/ptr_helpers.h index 67776b1006..7349bddd51 100644 --- a/src/test/ptr_helpers.h +++ b/src/test/ptr_helpers.h @@ -20,4 +20,4 @@ cast_uintptr_to_voidstar(uintptr_t x); uintptr_t cast_voidstar_to_uintptr(void *x); -#endif +#endif /* !defined(TOR_PTR_HELPERS_H) */ diff --git a/src/test/test-memwipe.c b/src/test/test-memwipe.c index 43754ed1c2..3f952e484f 100644 --- a/src/test/test-memwipe.c +++ b/src/test/test-memwipe.c @@ -49,7 +49,7 @@ const char *s = NULL; * us do bad things, such as access freed buffers, without crashing. */ extern const char *malloc_options; const char *malloc_options = "sufjj"; -#endif +#endif /* defined(OpenBSD) */ static unsigned fill_a_buffer_memset(void) diff --git a/src/test/test_circuitpadding.c b/src/test/test_circuitpadding.c index a2d192203e..236f4a192d 100644 --- a/src/test/test_circuitpadding.c +++ b/src/test/test_circuitpadding.c @@ -6,7 +6,7 @@ #define CRYPT_PATH_PRIVATE #include "core/or/or.h" -#include "test.h" +#include "test/test.h" #include "lib/testsupport/testsupport.h" #include "core/or/connection_or.h" #include "core/or/channel.h" @@ -2329,7 +2329,7 @@ test_circuitpadding_circuitsetup_machine(void *arg) return; } -#endif +#endif /* 0 */ /** Helper function: Initializes a padding machine where every state uses the * uniform probability distribution. */ diff --git a/src/test/test_config.c b/src/test/test_config.c index 6cfb7b764b..12b78f806a 100644 --- a/src/test/test_config.c +++ b/src/test/test_config.c @@ -5066,7 +5066,7 @@ test_config_include_no_permission(void *data) chmod(dir, 0700); tor_free(dir); } -#endif +#endif /* !defined(_WIN32) */ static void test_config_include_recursion_before_after(void *data) @@ -5698,7 +5698,7 @@ test_config_compute_max_mem_in_queues(void *data) #else /* We are on a 32-bit system. */ tt_u64_op(compute_real_max_mem_in_queues(0, 0), OP_EQ, GIGABYTE(1)); -#endif +#endif /* SIZEOF_VOID_P >= 8 */ /* We are able to detect the amount of RAM on the system. */ total_system_memory_return = 0; @@ -5739,7 +5739,7 @@ test_config_compute_max_mem_in_queues(void *data) /* We will at maximum get MAX_DEFAULT_MEMORY_QUEUE_SIZE here. */ tt_u64_op(compute_real_max_mem_in_queues(0, 0), OP_EQ, MAX_DEFAULT_MEMORY_QUEUE_SIZE); -#endif +#endif /* SIZEOF_SIZE_T > 4 */ done: UNMOCK(get_total_system_memory); diff --git a/src/test/test_connection.h b/src/test/test_connection.h index 027e405d89..40121e6d38 100644 --- a/src/test/test_connection.h +++ b/src/test/test_connection.h @@ -14,4 +14,4 @@ void test_conn_lookup_addr_helper(const char *address, int family, tor_addr_t *addr); -#endif +#endif /* !defined(TOR_TEST_CONNECTION_H) */ diff --git a/src/test/test_crypto.c b/src/test/test_crypto.c index 872da3d2c5..178a9a5097 100644 --- a/src/test/test_crypto.c +++ b/src/test/test_crypto.c @@ -32,7 +32,7 @@ DISABLE_GCC_WARNING(redundant-decls) #include ENABLE_GCC_WARNING(redundant-decls) -#endif +#endif /* defined(ENABLE_OPENSSL) */ /** Run unit tests for Diffie-Hellman functionality. */ static void @@ -190,7 +190,7 @@ test_crypto_dh(void *arg) DH_get0_key(dh4, &pk, &sk); #else pk = dh4->pub_key; -#endif +#endif /* defined(OPENSSL_1_1_API) */ tt_assert(pk); tt_int_op(BN_num_bytes(pk), OP_LE, DH1024_KEY_LEN); tt_int_op(BN_num_bytes(pk), OP_GT, 0); @@ -207,7 +207,7 @@ test_crypto_dh(void *arg) tt_int_op(s1len, OP_GT, 0); tt_mem_op(s1, OP_EQ, s2, s1len); } -#endif +#endif /* defined(ENABLE_OPENSSL) */ done: crypto_dh_free(dh1); @@ -219,7 +219,7 @@ test_crypto_dh(void *arg) DH_free(dh4); if (pubkey_tmp) BN_free(pubkey_tmp); -#endif +#endif /* defined(ENABLE_OPENSSL) */ } static void @@ -248,7 +248,7 @@ test_crypto_openssl_version(void *arg) tt_int_op(a, OP_GE, 0); tt_int_op(b, OP_GE, 0); tt_int_op(c, OP_GE, 0); -#endif +#endif /* defined(ENABLE_NSS) */ done: ; diff --git a/src/test/test_dir_common.h b/src/test/test_dir_common.h index ab99ed36f4..619dc83eb9 100644 --- a/src/test/test_dir_common.h +++ b/src/test/test_dir_common.h @@ -52,4 +52,4 @@ int dir_common_construct_vote_3(networkstatus_t **vote, networkstatus_t **vote_out, int *n_vrs, time_t now, int clear_rl); -#endif +#endif /* !defined(TOR_TEST_DIR_COMMON_H) */ diff --git a/src/test/test_dns.c b/src/test/test_dns.c index 231e6965f7..51ff8729d0 100644 --- a/src/test/test_dns.c +++ b/src/test/test_dns.c @@ -67,7 +67,7 @@ NS(test_main)(void *arg) tt_assert(tor_addr_family(nameserver_addr) == AF_INET); tt_assert(tor_addr_eq_ipv4h(nameserver_addr, 0x7f000001)); -#endif +#endif /* !defined(_WIN32) */ UNMOCK(get_options); @@ -77,7 +77,7 @@ NS(test_main)(void *arg) } #undef NS_SUBMODULE -#endif +#endif /* defined(HAVE_EVDNS_BASE_GET_NAMESERVER_ADDR) */ #define NS_SUBMODULE clip_ttl diff --git a/src/test/test_dos.c b/src/test/test_dos.c index 4756c5014e..bda9908e6c 100644 --- a/src/test/test_dos.c +++ b/src/test/test_dos.c @@ -411,7 +411,7 @@ test_dos_bucket_refill(void *arg) } tt_uint_op(current_circ_count, OP_EQ, 0); tt_uint_op(dos_stats->cc_stats.circuit_bucket, OP_EQ, current_circ_count); -#endif +#endif /* SIZEOF_TIME_T == 8 */ done: tor_free(chan); diff --git a/src/test/test_link_handshake.c b/src/test/test_link_handshake.c index 4e4c86aa0a..5e78e1ce4d 100644 --- a/src/test/test_link_handshake.c +++ b/src/test/test_link_handshake.c @@ -948,7 +948,7 @@ test_link_handshake_send_authchallenge(void *arg) #else tt_int_op(36, OP_EQ, cell1->payload_len); tt_int_op(36, OP_EQ, cell2->payload_len); -#endif +#endif /* defined(HAVE_WORKING_TOR_TLS_GET_TLSSECRETS) */ tt_int_op(0, OP_EQ, cell1->circ_id); tt_int_op(0, OP_EQ, cell2->circ_id); tt_int_op(CELL_AUTH_CHALLENGE, OP_EQ, cell1->command); @@ -960,7 +960,7 @@ test_link_handshake_send_authchallenge(void *arg) #else tt_mem_op("\x00\x01\x00\x03", OP_EQ, cell1->payload + 32, 4); tt_mem_op("\x00\x01\x00\x03", OP_EQ, cell2->payload + 32, 4); -#endif +#endif /* defined(HAVE_WORKING_TOR_TLS_GET_TLSSECRETS) */ tt_mem_op(cell1->payload, OP_NE, cell2->payload, 32); done: diff --git a/src/test/test_periodic_event.c b/src/test/test_periodic_event.c index 961a8be698..267156a908 100644 --- a/src/test/test_periodic_event.c +++ b/src/test/test_periodic_event.c @@ -108,7 +108,7 @@ test_pe_launch(void *arg) periodic_event_item_t *item = &periodic_events[i]; tt_int_op(periodic_event_is_enabled(item), OP_EQ, 0); } -#endif +#endif /* 0 */ initialize_periodic_events(); periodic_events_connect_all(); diff --git a/src/test/test_process.c b/src/test/test_process.c index 7cc01d2442..7836312761 100644 --- a/src/test/test_process.c +++ b/src/test/test_process.c @@ -594,7 +594,7 @@ test_unix(void *arg) done: process_free(process); -#endif +#endif /* !defined(_WIN32) */ } static void @@ -649,7 +649,7 @@ test_win32(void *arg) done: tor_free(joined_argv); process_free(process); -#endif +#endif /* defined(_WIN32) */ } struct testcase_t process_tests[] = { diff --git a/src/test/test_process_slow.c b/src/test/test_process_slow.c index 1322d7b833..91252c725d 100644 --- a/src/test/test_process_slow.c +++ b/src/test/test_process_slow.c @@ -135,7 +135,7 @@ get_win32_test_binary_path(void) done: return NULL; } -#endif +#endif /* defined(_WIN32) */ static void main_loop_timeout_cb(periodic_timer_t *timer, void *data) diff --git a/src/test/test_protover.c b/src/test/test_protover.c index 63c508bd13..1759aef97d 100644 --- a/src/test/test_protover.c +++ b/src/test/test_protover.c @@ -22,7 +22,7 @@ test_protover_parse(void *arg) tt_skip(); done: ; -#else +#else /* !(defined(HAVE_RUST)) */ char *re_encoded = NULL; const char *orig = "Foo=1,3 Bar=3 Baz= Quux=9-12,14,15-16,900"; @@ -89,7 +89,7 @@ test_protover_parse(void *arg) SMARTLIST_FOREACH(elts, proto_entry_t *, ent, proto_entry_free(ent)); smartlist_free(elts); tor_free(re_encoded); -#endif +#endif /* defined(HAVE_RUST) */ } static void @@ -133,7 +133,7 @@ test_protover_parse_fail(void *arg) "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); tt_ptr_op(elts, OP_EQ, NULL); -#endif +#endif /* defined(HAVE_RUST) */ done: ; } @@ -335,7 +335,7 @@ test_protover_all_supported(void *arg) "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" "aaaaaaaaaaaa=1-65536", &msg)); tor_end_capture_bugs_(); -#endif +#endif /* !defined(HAVE_RUST) */ done: tor_end_capture_bugs_(); @@ -459,7 +459,7 @@ test_protover_supported_protocols(void *arg) tt_assert(protocol_list_supports_protocol(supported_protocols, PRT_LINKAUTH, PROTOVER_LINKAUTH_V1)); -#endif +#endif /* defined(HAVE_WORKING_TOR_TLS_GET_TLSSECRETS) */ /* Latest LinkAuth is not exposed in the headers. */ tt_assert(protocol_list_supports_protocol(supported_protocols, PRT_LINKAUTH, diff --git a/src/test/test_rng.c b/src/test/test_rng.c index c749de112a..dcf08fff1d 100644 --- a/src/test/test_rng.c +++ b/src/test/test_rng.c @@ -46,7 +46,7 @@ main(int argc, char **argv) return 1; } } -#endif +#endif /* 0 */ crypto_fast_rng_t *rng = crypto_fast_rng_new(); while (1) { diff --git a/src/test/test_shared_random.c b/src/test/test_shared_random.c index 9fb88b9bee..9c8703fa6f 100644 --- a/src/test/test_shared_random.c +++ b/src/test/test_shared_random.c @@ -1367,7 +1367,7 @@ test_utils_auth(void *arg) sr_state_set_current_srv(sr_state_get_current_srv()); sr_state_set_previous_srv(sr_state_get_previous_srv()); } -#endif +#endif /* 0 */ done: sr_state_free_all(); diff --git a/src/test/test_tortls.c b/src/test/test_tortls.c index 11e35be2fa..5f87434f6a 100644 --- a/src/test/test_tortls.c +++ b/src/test/test_tortls.c @@ -219,7 +219,7 @@ test_tortls_tor_tls_get_error(void *data) crypto_pk_free(key2); tor_tls_free(tls); } -#endif +#endif /* defined(ENABLE_OPENSSL) */ static void test_tortls_x509_cert_get_id_digests(void *ignored) @@ -336,7 +336,7 @@ test_tortls_server_got_renegotiate(void *ignored) done: tor_free(tls); } -#endif +#endif /* defined(ENABLE_OPENSSL) */ static void test_tortls_evaluate_ecgroup_for_tls(void *ignored) @@ -526,7 +526,7 @@ struct testcase_t tortls_tests[] = { LOCAL_TEST_CASE(get_forced_write_size, 0), LOCAL_TEST_CASE(used_v1_handshake, TT_FORK), LOCAL_TEST_CASE(server_got_renegotiate, 0), -#endif +#endif /* defined(ENABLE_OPENSSL) */ LOCAL_TEST_CASE(evaluate_ecgroup_for_tls, 0), LOCAL_TEST_CASE(double_init, TT_FORK), LOCAL_TEST_CASE(address, TT_FORK), diff --git a/src/test/test_tortls.h b/src/test/test_tortls.h index 1a8b117d0f..4567b9f6a0 100644 --- a/src/test/test_tortls.h +++ b/src/test/test_tortls.h @@ -10,4 +10,4 @@ extern const char *notCompletelyValidCertString; extern const char *validCertString; extern const char *caCertString; -#endif +#endif /* !defined(TEST_TORTLS_H) */ diff --git a/src/test/test_tortls_openssl.c b/src/test/test_tortls_openssl.c index 73041a871c..81e2d3aa4f 100644 --- a/src/test/test_tortls_openssl.c +++ b/src/test/test_tortls_openssl.c @@ -133,7 +133,7 @@ library_init(void) #else SSL_library_init(); SSL_load_error_strings(); -#endif +#endif /* defined(OPENSSL_1_1_API) */ } static void @@ -477,7 +477,7 @@ fake_x509_free(X509 *cert) tor_free(cert); } } -#endif +#endif /* !defined(OPENSSL_OPAQUE) */ static tor_x509_cert_t *fixed_x509_cert = NULL; static tor_x509_cert_t * diff --git a/src/test/test_util.c b/src/test/test_util.c index 79df2825be..2faadd4e19 100644 --- a/src/test/test_util.c +++ b/src/test/test_util.c @@ -6116,9 +6116,9 @@ test_util_log_mallinfo(void *arg) } else { tt_u64_op(mem1, OP_LT, mem2); } -#else +#else /* !(defined(HAVE_MALLINFO)) */ tt_skip(); -#endif +#endif /* defined(HAVE_MALLINFO) */ done: teardown_capture_of_logs(); tor_free(log1); @@ -6175,7 +6175,7 @@ test_util_map_anon_nofork(void *arg) tt_skip(); done: ; -#else +#else /* !(defined(_WIN32)) */ /* We have the right OS support. We're going to try marking the buffer as * either zero-on-fork or as drop-on-fork, whichever is supported. Then we * will fork and send a byte back to the parent process. This will either @@ -6236,7 +6236,7 @@ test_util_map_anon_nofork(void *arg) * implemented. */ tt_skip(); } -#endif +#endif /* !defined(NOINHERIT_CAN_FAIL) */ done: tor_munmap_anonymous(ptr, sz); @@ -6246,7 +6246,7 @@ test_util_map_anon_nofork(void *arg) if (pipefd[1] >= 0) { close(pipefd[1]); } -#endif +#endif /* defined(_WIN32) */ } #define UTIL_LEGACY(name) \ diff --git a/src/tools/tor-gencert.c b/src/tools/tor-gencert.c index 25113420df..ea96f41dbf 100644 --- a/src/tools/tor-gencert.c +++ b/src/tools/tor-gencert.c @@ -31,7 +31,7 @@ DISABLE_GCC_WARNING(redundant-decls) #include ENABLE_GCC_WARNING(redundant-decls) -#endif +#endif /* defined(ENABLE_OPENSSL) */ #include From 2926f49b24938fee8a3e33357687eb2831eb62e0 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 5 Jun 2019 09:34:14 -0400 Subject: [PATCH 1089/2557] Regenerate practracker excpetions file --- scripts/maint/practracker/exceptions.txt | 64 +++++++++++------------- 1 file changed, 28 insertions(+), 36 deletions(-) diff --git a/scripts/maint/practracker/exceptions.txt b/scripts/maint/practracker/exceptions.txt index c596482504..eb0625b8d3 100644 --- a/scripts/maint/practracker/exceptions.txt +++ b/scripts/maint/practracker/exceptions.txt @@ -29,13 +29,13 @@ # # Remember: It is better to fix the problem than to add a new exception! -problem file-size /src/app/config/config.c 8520 -problem include-count /src/app/config/config.c 88 +problem file-size /src/app/config/config.c 8518 +problem include-count /src/app/config/config.c 87 problem function-size /src/app/config/config.c:options_act_reversible() 296 -problem function-size /src/app/config/config.c:options_act() 590 +problem function-size /src/app/config/config.c:options_act() 589 problem function-size /src/app/config/config.c:resolve_my_address() 192 -problem function-size /src/app/config/config.c:options_validate() 1220 -problem function-size /src/app/config/config.c:options_init_from_torrc() 210 +problem function-size /src/app/config/config.c:options_validate() 1217 +problem function-size /src/app/config/config.c:options_init_from_torrc() 207 problem function-size /src/app/config/config.c:options_init_from_string() 173 problem function-size /src/app/config/config.c:options_init_logs() 146 problem function-size /src/app/config/config.c:parse_bridge_line() 104 @@ -53,8 +53,7 @@ problem function-size /src/app/main/main.c:tor_init() 137 problem function-size /src/app/main/main.c:sandbox_init_filter() 291 problem function-size /src/app/main/main.c:run_tor_main_loop() 105 problem function-size /src/app/main/ntmain.c:nt_service_install() 125 -problem include-count /src/app/main/shutdown.c 52 -problem file-size /src/core/mainloop/connection.c 5570 +problem file-size /src/core/mainloop/connection.c 5569 problem include-count /src/core/mainloop/connection.c 62 problem function-size /src/core/mainloop/connection.c:connection_free_minimal() 185 problem function-size /src/core/mainloop/connection.c:connection_listener_new() 328 @@ -67,18 +66,15 @@ problem function-size /src/core/mainloop/connection.c:connection_handle_read_imp problem function-size /src/core/mainloop/connection.c:connection_buf_read_from_socket() 181 problem function-size /src/core/mainloop/connection.c:connection_handle_write_impl() 241 problem function-size /src/core/mainloop/connection.c:assert_connection_ok() 143 -problem file-size /src/core/mainloop/mainloop.c 3051 -problem include-count /src/core/mainloop/mainloop.c 66 +problem include-count /src/core/mainloop/mainloop.c 63 problem function-size /src/core/mainloop/mainloop.c:conn_close_if_marked() 108 problem function-size /src/core/mainloop/mainloop.c:run_connection_housekeeping() 123 -problem function-size /src/core/mainloop/mainloop.c:CALLBACK() 116 problem file-size /src/core/or/channel.c 3487 problem function-size /src/core/or/channeltls.c:channel_tls_handle_var_cell() 160 problem function-size /src/core/or/channeltls.c:channel_tls_process_versions_cell() 170 problem function-size /src/core/or/channeltls.c:channel_tls_process_netinfo_cell() 214 problem function-size /src/core/or/channeltls.c:channel_tls_process_certs_cell() 246 problem function-size /src/core/or/channeltls.c:channel_tls_process_authenticate_cell() 202 -problem file-size /src/core/or/circuitbuild.c 3061 problem include-count /src/core/or/circuitbuild.c 54 problem function-size /src/core/or/circuitbuild.c:get_unique_circ_id_by_chan() 128 problem function-size /src/core/or/circuitbuild.c:circuit_extend() 147 @@ -91,9 +87,12 @@ problem function-size /src/core/or/circuitlist.c:circuit_about_to_free() 120 problem function-size /src/core/or/circuitlist.c:circuits_handle_oom() 117 problem function-size /src/core/or/circuitmux.c:circuitmux_set_policy() 110 problem function-size /src/core/or/circuitmux.c:circuitmux_attach_circuit() 114 +problem function-size /src/core/or/circuitpadding.c:circpad_machine_schedule_padding() 107 +problem function-size /src/core/or/circuitpadding_machines.c:circpad_machine_relay_hide_intro_circuits() 104 +problem function-size /src/core/or/circuitpadding_machines.c:circpad_machine_client_hide_rend_circuits() 112 problem function-size /src/core/or/circuitstats.c:circuit_build_times_parse_state() 124 problem file-size /src/core/or/circuituse.c 3162 -problem function-size /src/core/or/circuituse.c:circuit_is_acceptable() 133 +problem function-size /src/core/or/circuituse.c:circuit_is_acceptable() 132 problem function-size /src/core/or/circuituse.c:circuit_expire_building() 394 problem function-size /src/core/or/circuituse.c:circuit_log_ancient_one_hop_circuits() 126 problem function-size /src/core/or/circuituse.c:circuit_build_failed() 149 @@ -120,13 +119,13 @@ problem function-size /src/core/or/connection_or.c:connection_or_compute_authent problem file-size /src/core/or/policies.c 3249 problem function-size /src/core/or/policies.c:policy_summarize() 107 problem function-size /src/core/or/protover.c:protover_all_supported() 117 -problem file-size /src/core/or/relay.c 3250 +problem file-size /src/core/or/relay.c 3244 problem function-size /src/core/or/relay.c:circuit_receive_relay_cell() 127 -problem function-size /src/core/or/relay.c:relay_send_command_from_edge_() 116 +problem function-size /src/core/or/relay.c:relay_send_command_from_edge_() 112 problem function-size /src/core/or/relay.c:connection_ap_process_end_not_open() 194 problem function-size /src/core/or/relay.c:connection_edge_process_relay_cell_not_open() 139 -problem function-size /src/core/or/relay.c:connection_edge_process_relay_cell() 520 -problem function-size /src/core/or/relay.c:connection_edge_package_raw_inbuf() 132 +problem function-size /src/core/or/relay.c:connection_edge_process_relay_cell() 430 +problem function-size /src/core/or/relay.c:connection_edge_package_raw_inbuf() 129 problem function-size /src/core/or/relay.c:circuit_resume_edge_reading_helper() 148 problem function-size /src/core/or/scheduler_kist.c:kist_scheduler_run() 171 problem function-size /src/core/or/scheduler_vanilla.c:vanilla_scheduler_run() 109 @@ -142,24 +141,21 @@ problem function-size /src/feature/client/entrynodes.c:entry_guard_parse_from_st problem function-size /src/feature/client/transports.c:handle_proxy_line() 108 problem function-size /src/feature/client/transports.c:parse_method_line_helper() 112 problem function-size /src/feature/client/transports.c:create_managed_proxy_environment() 109 -problem function-size /src/feature/control/control.c:connection_control_process_inbuf() 144 -problem function-size /src/feature/control/control_auth.c:handle_control_authchallenge() 115 -problem function-size /src/feature/control/control_auth.c:handle_control_authenticate() 188 +problem function-size /src/feature/control/control.c:connection_control_process_inbuf() 136 +problem function-size /src/feature/control/control_auth.c:handle_control_authchallenge() 103 +problem function-size /src/feature/control/control_auth.c:handle_control_authenticate() 187 problem function-size /src/feature/control/control_cmd.c:handle_control_extendcircuit() 151 -problem function-size /src/feature/control/control_cmd.c:handle_control_hsfetch() 114 -problem function-size /src/feature/control/control_cmd.c:handle_control_hspost() 117 -problem function-size /src/feature/control/control_cmd.c:handle_control_add_onion() 293 +problem function-size /src/feature/control/control_cmd.c:handle_control_add_onion() 269 problem function-size /src/feature/control/control_cmd.c:add_onion_helper_keyarg() 125 -problem function-size /src/feature/control/control_cmd.c:handle_control_command() 104 problem function-size /src/feature/control/control_events.c:control_event_stream_status() 119 problem include-count /src/feature/control/control_getinfo.c 54 problem function-size /src/feature/control/control_getinfo.c:getinfo_helper_misc() 109 problem function-size /src/feature/control/control_getinfo.c:getinfo_helper_dir() 304 problem function-size /src/feature/control/control_getinfo.c:getinfo_helper_events() 236 -problem function-size /src/feature/dirauth/bwauth.c:dirserv_read_measured_bandwidths() 127 -problem file-size /src/feature/dirauth/dirvote.c 4728 +problem function-size /src/feature/dirauth/bwauth.c:dirserv_read_measured_bandwidths() 124 +problem file-size /src/feature/dirauth/dirvote.c 4726 problem include-count /src/feature/dirauth/dirvote.c 53 -problem function-size /src/feature/dirauth/dirvote.c:format_networkstatus_vote() 250 +problem function-size /src/feature/dirauth/dirvote.c:format_networkstatus_vote() 249 problem function-size /src/feature/dirauth/dirvote.c:networkstatus_compute_bw_weights_v10() 235 problem function-size /src/feature/dirauth/dirvote.c:networkstatus_compute_consensus() 962 problem function-size /src/feature/dirauth/dirvote.c:networkstatus_add_detached_signatures() 123 @@ -201,9 +197,8 @@ problem function-size /src/feature/hs/hs_cell.c:hs_cell_parse_introduce2() 154 problem function-size /src/feature/hs/hs_client.c:send_introduce1() 104 problem function-size /src/feature/hs/hs_client.c:hs_config_client_authorization() 108 problem function-size /src/feature/hs/hs_common.c:hs_get_responsible_hsdirs() 104 -problem function-size /src/feature/hs/hs_common.c:hs_get_extend_info_from_lspecs() 101 -problem function-size /src/feature/hs/hs_config.c:config_generic_service() 149 -problem function-size /src/feature/hs/hs_descriptor.c:desc_encode_v3() 108 +problem function-size /src/feature/hs/hs_config.c:config_generic_service() 140 +problem function-size /src/feature/hs/hs_descriptor.c:desc_encode_v3() 104 problem function-size /src/feature/hs/hs_descriptor.c:decrypt_desc_layer() 110 problem function-size /src/feature/hs/hs_descriptor.c:decode_introduction_point() 122 problem function-size /src/feature/hs/hs_descriptor.c:desc_decode_superencrypted_v3() 109 @@ -230,13 +225,13 @@ problem function-size /src/feature/nodelist/routerlist.c:update_extrainfo_downlo problem function-size /src/feature/relay/dns.c:dns_resolve_impl() 134 problem function-size /src/feature/relay/dns.c:configure_nameservers() 161 problem function-size /src/feature/relay/dns.c:evdns_callback() 109 -problem file-size /src/feature/relay/router.c 3412 +problem file-size /src/feature/relay/router.c 3407 problem include-count /src/feature/relay/router.c 56 problem function-size /src/feature/relay/router.c:init_keys() 252 problem function-size /src/feature/relay/router.c:get_my_declared_family() 114 problem function-size /src/feature/relay/router.c:router_build_fresh_unsigned_routerinfo() 136 -problem function-size /src/feature/relay/router.c:router_dump_router_to_string() 375 -problem function-size /src/feature/relay/router.c:extrainfo_dump_to_string() 207 +problem function-size /src/feature/relay/router.c:router_dump_router_to_string() 371 +problem function-size /src/feature/relay/router.c:extrainfo_dump_to_string() 206 problem function-size /src/feature/relay/routerkeys.c:load_ed_keys() 294 problem function-size /src/feature/rend/rendcache.c:rend_cache_store_v2_desc_as_client() 193 problem function-size /src/feature/rend/rendclient.c:rend_client_send_introduction() 220 @@ -287,8 +282,5 @@ problem function-size /src/lib/tls/tortls_openssl.c:tor_tls_context_new() 171 problem function-size /src/lib/tls/x509_nss.c:tor_tls_create_certificate_internal() 126 problem function-size /src/tools/tor-gencert.c:parse_commandline() 111 problem function-size /src/tools/tor-resolve.c:build_socks5_resolve_request() 104 -problem function-size /src/tools/tor-resolve.c:do_resolve() 175 +problem function-size /src/tools/tor-resolve.c:do_resolve() 174 problem function-size /src/tools/tor-resolve.c:main() 112 -problem function-size /src/core/or/circuitpadding.c:circpad_machine_schedule_padding() 107 -problem function-size /src/core/or/circuitpadding_machines.c:circpad_machine_relay_hide_intro_circuits() 104 -problem function-size /src/core/or/circuitpadding_machines.c:circpad_machine_client_hide_rend_circuits() 112 From 892a313b6a9edbcb1930454b424503c4cc2c791a Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 5 Jun 2019 09:35:45 -0400 Subject: [PATCH 1090/2557] Replace a missing end-of-comment string This happened when I went to fix long lines after running "make autostyle". --- src/lib/tls/tortls_openssl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/tls/tortls_openssl.c b/src/lib/tls/tortls_openssl.c index 04027104e1..86f0ac42cc 100644 --- a/src/lib/tls/tortls_openssl.c +++ b/src/lib/tls/tortls_openssl.c @@ -673,7 +673,7 @@ tor_tls_context_new(crypto_pk_t *identity, unsigned int key_lifetime, SSL_CTX_set_tmp_ecdh(result->ctx, ec_key); EC_KEY_free(ec_key); } -#endif /* defined(SSL_CTX_set1_groups_list) || ...) +#endif /* defined(SSL_CTX_set1_groups_list) || ...) */ SSL_CTX_set_verify(result->ctx, SSL_VERIFY_PEER, always_accept_verify_cb); /* let us realloc bufs that we're writing from */ From 7f341d64828d48ebaf4bfaa2720c614a25c8dbba Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 5 Jun 2019 09:43:43 -0400 Subject: [PATCH 1091/2557] minor changelog edits --- ChangeLog | 45 +++++++++++++++++++++++---------------------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/ChangeLog b/ChangeLog index 6387d6dbb1..ca2215f163 100644 --- a/ChangeLog +++ b/ChangeLog @@ -2,7 +2,7 @@ Changes in version 0.4.1.2-alpha - 2019-06-05 Tor 0.4.1.2-alpha resolves numerous bugs--some of them from the previous alpha, and some much older. It also contains minor testing improvements, and an improvement to the security of our authenticated - sendme implementation. + SENDME implementation. o Major bugfixes (bridges): - Consider our directory information to have changed when our list @@ -16,26 +16,27 @@ Changes in version 0.4.1.2-alpha - 2019-06-05 and delay fetching router descriptors for the new ones. Fixes part of bug 29875; bugfix on 0.3.0.1-alpha. - o Major bugfixes (Flow Control, SENDME): - - The decrement of the stream-level package window was done in a - log_debug() statement meaning that if the debug logs were not - enabled, the decrement would never happen and thus the window - would be out of sync with the other end point. Fixes bug 30628; - bugfix on 0.4.1.1-alpha. + o Major bugfixes (flow control, SENDME): + - Decrement the stream-level package window after packaging a cell. + Previously, it was done inside a log_debug() call, meaning that if + debug logs were not enabled, the decrement would never happen, and + thus the window would be out of sync with the other end point. + Fixes bug 30628; bugfix on 0.4.1.1-alpha. - o Major bugfixes (Onion service reachability): + o Major bugfixes (onion service reachability): - Properly clean up the introduction point map and associated state when circuits change purpose from onion service circuits to - pathbias, measurement, or other circuit types. This should fix - some instances of introduction point failure. Fixes bug 29034; - bugfix on 0.3.2.1-alpha. + pathbias, measurement, or other circuit types. This may fix some + instances of introduction point failure. Fixes bug 29034; bugfix + on 0.3.2.1-alpha. o Minor features (authenticated SENDME): - Ensure that there is enough randomness on every circuit to prevent - an attacker from successfully predicting what SENDME cells they - will need to send: at a random interval, if we have not send - randomness already, leave some extra space at the end of a cell - that we can fill with random bytes. Closes ticket 26846. + an attacker from successfully predicting the hashes they will need + to include in authenticated SENDME cells. At a random interval, if + we have not sent randomness already, we now leave some extra space + at the end of a cell that we can fill with random bytes. Closes + ticket 26846. o Minor features (continuous integration): - When running coverage builds on Travis, we now set @@ -71,9 +72,9 @@ Changes in version 0.4.1.2-alpha - 2019-06-05 on 0.4.1.1-alpha. o Minor bugfixes (developer tooling): - - Fix pre-push hook to refrain from rejecting fixup and squash - commits when pushing to non-upstream git remote. Fixes bug 30286; - bugfix on 0.4.0.1-alpha. + - Fix pre-push hook to allow fixup and squash commits when pushing + to non-upstream git remote. Fixes bug 30286; bugfix + on 0.4.0.1-alpha. o Minor bugfixes (directory authority): - Move the "bandwidth-file-headers" line in directory authority @@ -88,7 +89,8 @@ Changes in version 0.4.1.2-alpha - 2019-06-05 o Minor bugfixes (out-of-memory handler): - When purging the DNS cache because of an out-of-memory condition, try purging just the older entries at first. Previously, we would - purge the whole thing. Fixes bug 29617; bugfix on 0.3.5.1-alpha. + always purge the whole thing. Fixes bug 29617; bugfix + on 0.3.5.1-alpha. o Minor bugfixes (portability): - Avoid crashing in our tor_vasprintf() implementation on systems @@ -107,9 +109,8 @@ Changes in version 0.4.1.2-alpha - 2019-06-05 o Minor bugfixes (static analysis): - Fix several spurious Coverity warnings about the unit tests, to - lower our chances of missing any real warnings in the future. - Fixes bug 30150; bugfix on 0.3.5.1-alpha and various other - Tor versions. + lower our chances of missing real warnings in the future. Fixes + bug 30150; bugfix on 0.3.5.1-alpha and various other Tor versions. o Testing: - Specify torrc paths (with empty files) when launching tor in From 917e4e9eae8645e65ea93836cbd82890eb5d7872 Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Wed, 5 Jun 2019 18:19:23 +0300 Subject: [PATCH 1092/2557] Don't access rend data after a circuit has been marked for close. This can cause issues if the circuit was repurposed into a padding circuit instead of closing, since in that case we will wipe off the rend_data. --- src/feature/rend/rendclient.c | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/feature/rend/rendclient.c b/src/feature/rend/rendclient.c index f84d221b1a..5bdd4d453e 100644 --- a/src/feature/rend/rendclient.c +++ b/src/feature/rend/rendclient.c @@ -403,14 +403,23 @@ rend_client_introduction_acked(origin_circuit_t *circ, } else { log_info(LD_REND,"...Found no rend circ. Dropping on the floor."); } + /* Save the rend data digest to a temporary object so that we don't access + * it after we mark the circuit for close. */ + const uint8_t *rend_digest_tmp = NULL; + size_t digest_len; + uint8_t *cached_rend_digest = NULL; + rend_digest_tmp = rend_data_get_pk_digest(circ->rend_data, &digest_len); + cached_rend_digest = tor_malloc_zero(digest_len); + memcpy(cached_rend_digest, rend_digest_tmp, digest_len); + /* close the circuit: we won't need it anymore. */ circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_C_INTRODUCE_ACKED); circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_FINISHED); /* close any other intros launched in parallel */ - rend_client_close_other_intros(rend_data_get_pk_digest(circ->rend_data, - NULL)); + rend_client_close_other_intros(cached_rend_digest); + tor_free(cached_rend_digest); /* free the temporary digest */ } else { /* It's a NAK; the introduction point didn't relay our request. */ circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_C_INTRODUCING); From 27e067df4fd3148b59dd0377d1a7b111460a2b53 Mon Sep 17 00:00:00 2001 From: Neel Chauhan Date: Wed, 5 Jun 2019 12:50:01 -0400 Subject: [PATCH 1093/2557] Add missing newline after decode_intro_points() closing bracket --- src/feature/hs/hs_descriptor.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/feature/hs/hs_descriptor.c b/src/feature/hs/hs_descriptor.c index 52dc8690c3..a8796c0029 100644 --- a/src/feature/hs/hs_descriptor.c +++ b/src/feature/hs/hs_descriptor.c @@ -1968,6 +1968,7 @@ decode_intro_points(const hs_descriptor_t *desc, SMARTLIST_FOREACH(intro_points, char *, a, tor_free(a)); smartlist_free(intro_points); } + /* Return 1 iff the given base64 encoded signature in b64_sig from the encoded * descriptor in encoded_desc validates the descriptor content. */ STATIC int From a42131bf4823bed42563f30d4925462b2df02c27 Mon Sep 17 00:00:00 2001 From: Mike Perry Date: Wed, 5 Jun 2019 11:37:32 -0700 Subject: [PATCH 1094/2557] Revert "test: Add test_hs_circ.c for HS circuit testing" This reverts commit 41b94722e5c93ec06911f9c63296a65ce295c1ea. --- src/test/include.am | 1 - src/test/test.c | 1 - src/test/test.h | 1 - src/test/test_hs_circ.c | 70 ----------------------------------------- 4 files changed, 73 deletions(-) delete mode 100644 src/test/test_hs_circ.c diff --git a/src/test/include.am b/src/test/include.am index fdfe96c559..85f9c9f880 100644 --- a/src/test/include.am +++ b/src/test/include.am @@ -145,7 +145,6 @@ src_test_test_SOURCES += \ src/test/test_hs_common.c \ src/test/test_hs_config.c \ src/test/test_hs_cell.c \ - src/test/test_hs_circ.c \ src/test/test_hs_ntor.c \ src/test/test_hs_service.c \ src/test/test_hs_client.c \ diff --git a/src/test/test.c b/src/test/test.c index d7682b6619..cac98dd839 100644 --- a/src/test/test.c +++ b/src/test/test.c @@ -871,7 +871,6 @@ struct testgroup_t testgroups[] = { { "guardfraction/", guardfraction_tests }, { "hs_cache/", hs_cache }, { "hs_cell/", hs_cell_tests }, - { "hs_circ/", hs_circ_tests }, { "hs_client/", hs_client_tests }, { "hs_common/", hs_common_tests }, { "hs_config/", hs_config_tests }, diff --git a/src/test/test.h b/src/test/test.h index b108d795ea..167fd090ac 100644 --- a/src/test/test.h +++ b/src/test/test.h @@ -221,7 +221,6 @@ extern struct testcase_t guardfraction_tests[]; extern struct testcase_t handle_tests[]; extern struct testcase_t hs_cache[]; extern struct testcase_t hs_cell_tests[]; -extern struct testcase_t hs_circ_tests[]; extern struct testcase_t hs_client_tests[]; extern struct testcase_t hs_common_tests[]; extern struct testcase_t hs_config_tests[]; diff --git a/src/test/test_hs_circ.c b/src/test/test_hs_circ.c deleted file mode 100644 index af28af2573..0000000000 --- a/src/test/test_hs_circ.c +++ /dev/null @@ -1,70 +0,0 @@ -/* Copyright (c) 2017-2019, The Tor Project, Inc. */ -/* See LICENSE for licensing information */ - -/** - * \file test_hs_circ.c - * \brief Test hidden service circuit functionality. - */ - -#define CIRCUITLIST_PRIVATE - -#include "test/test.h" -#include "test/test_helpers.h" -#include "test/log_test_helpers.h" - -#include "core/or/circuitbuild.h" -#include "core/or/circuitlist.h" -#include "core/or/circuituse.h" -#include "core/or/origin_circuit_st.h" - -#include "feature/hs/hs_circuit.h" -#include "feature/hs/hs_circuitmap.h" - -static void -test_circuit_repurpose(void *arg) -{ - origin_circuit_t *intro_circ = NULL; - const origin_circuit_t *search; - ed25519_keypair_t kp; - - (void) arg; - - hs_init(); - - intro_circ = origin_circuit_init(CIRCUIT_PURPOSE_S_ESTABLISH_INTRO, 0); - tt_assert(intro_circ); - ed25519_keypair_generate(&kp, 0); - - /* Register circuit in global map and make sure it is actually there. */ - hs_circuitmap_register_intro_circ_v3_service_side(intro_circ, - &kp.pubkey); - tt_assert(TO_CIRCUIT(intro_circ)->hs_token); - search = hs_circuitmap_get_intro_circ_v3_service_side(&kp.pubkey); - tt_mem_op(search, OP_EQ, intro_circ, sizeof(origin_circuit_t)); - - /* Setup circuit HS ident. We don't care about the service pubkey. */ - intro_circ->hs_ident = hs_ident_circuit_new(&kp.pubkey, - HS_IDENT_CIRCUIT_INTRO); - tt_assert(intro_circ->hs_ident); - - /* Trigger a repurpose. State should be cleaned up. */ - hs_circ_repurpose(TO_CIRCUIT(intro_circ)); - - /* Removed from map. */ - search = hs_circuitmap_get_intro_circ_v3_service_side(&kp.pubkey); - tt_assert(!search); - /* HS identifier has been removed. */ - tt_assert(!intro_circ->hs_ident); - - done: - circuit_free_(TO_CIRCUIT(intro_circ)); - hs_free_all(); -} - -struct testcase_t hs_circ_tests[] = { - { "repurpose", test_circuit_repurpose, TT_FORK, - NULL, NULL }, - - END_OF_TESTCASES -}; - From 31c34f6524fecd6a104a8dac04d9330b5323e3c0 Mon Sep 17 00:00:00 2001 From: Mike Perry Date: Wed, 5 Jun 2019 11:38:01 -0700 Subject: [PATCH 1095/2557] Revert "hs: Implement a helper to repurpose a circuit" This reverts commit 3789f22bcbfbc6de415a838e4c4bfb2555c7d6c3. --- src/core/or/circuituse.c | 6 ------ src/feature/hs/hs_circuit.c | 31 ------------------------------- src/feature/hs/hs_circuit.h | 1 - src/feature/rend/rendcommon.c | 11 ----------- src/feature/rend/rendcommon.h | 2 -- 5 files changed, 51 deletions(-) diff --git a/src/core/or/circuituse.c b/src/core/or/circuituse.c index b3d5c6bb81..485c389054 100644 --- a/src/core/or/circuituse.c +++ b/src/core/or/circuituse.c @@ -3068,12 +3068,6 @@ circuit_change_purpose(circuit_t *circ, uint8_t new_purpose) if (circ->purpose == new_purpose) return; - /* Take specific actions if we are repurposing a hidden service circuit. */ - if (circuit_purpose_is_hidden_service(circ->purpose) && - !circuit_purpose_is_hidden_service(new_purpose)) { - hs_circ_repurpose(circ); - } - if (CIRCUIT_IS_ORIGIN(circ)) { char old_purpose_desc[80] = ""; diff --git a/src/feature/hs/hs_circuit.c b/src/feature/hs/hs_circuit.c index 79377eb731..a6e86c5ab3 100644 --- a/src/feature/hs/hs_circuit.c +++ b/src/feature/hs/hs_circuit.c @@ -25,7 +25,6 @@ #include "feature/nodelist/describe.h" #include "feature/nodelist/nodelist.h" #include "feature/rend/rendservice.h" -#include "feature/rend/rendcommon.h" #include "feature/stats/rephist.h" #include "lib/crypt_ops/crypto_dh.h" #include "lib/crypt_ops/crypto_rand.h" @@ -1193,33 +1192,3 @@ hs_circ_cleanup(circuit_t *circ) hs_circuitmap_remove_circuit(circ); } } - -/* The given circuit will be repurposed so take the appropriate actions. A - * cleanup from the HS maps and of all HS related structures is done. - * - * Once this function returns, the circuit can be safely repurposed. */ -void -hs_circ_repurpose(circuit_t *circ) -{ - origin_circuit_t *origin_circ; - - tor_assert(circ); - - /* Only repurposing an origin circuit is possible for HS. */ - if (!CIRCUIT_IS_ORIGIN(circ)) { - return; - } - origin_circ = TO_ORIGIN_CIRCUIT(circ); - - /* First, cleanup the circuit from the HS maps. */ - hs_circ_cleanup(circ); - - /* Depending on the version, different cleanup is done. */ - if (origin_circ->rend_data) { - /* v2. */ - rend_circ_cleanup(origin_circ); - } else if (origin_circ->hs_ident) { - /* v3. */ - hs_ident_circuit_free(origin_circ->hs_ident); - } -} diff --git a/src/feature/hs/hs_circuit.h b/src/feature/hs/hs_circuit.h index 0786f3ee45..b8d8b25add 100644 --- a/src/feature/hs/hs_circuit.h +++ b/src/feature/hs/hs_circuit.h @@ -16,7 +16,6 @@ /* Cleanup function when the circuit is closed or/and freed. */ void hs_circ_cleanup(circuit_t *circ); -void hs_circ_repurpose(circuit_t *circ); /* Circuit API. */ int hs_circ_service_intro_has_opened(hs_service_t *service, diff --git a/src/feature/rend/rendcommon.c b/src/feature/rend/rendcommon.c index 265ee368f1..777de2984c 100644 --- a/src/feature/rend/rendcommon.c +++ b/src/feature/rend/rendcommon.c @@ -1046,14 +1046,3 @@ rend_circuit_pk_digest_eq(const origin_circuit_t *ocirc, match: return 1; } - -/* Cleanup the given circuit of all HS v2 data structure. */ -void -rend_circ_cleanup(origin_circuit_t *circ) -{ - tor_assert(circ); - - /* Both fields are set to NULL with these. */ - crypto_pk_free(circ->intro_key); - rend_data_free(circ->rend_data); -} diff --git a/src/feature/rend/rendcommon.h b/src/feature/rend/rendcommon.h index c9a04846d7..f136863c7a 100644 --- a/src/feature/rend/rendcommon.h +++ b/src/feature/rend/rendcommon.h @@ -71,8 +71,6 @@ int rend_non_anonymous_mode_enabled(const or_options_t *options); void assert_circ_anonymity_ok(const origin_circuit_t *circ, const or_options_t *options); -void rend_circ_cleanup(origin_circuit_t *circ); - #ifdef RENDCOMMON_PRIVATE STATIC int From c525135dac354892a45ad3d2f6de9450d393f09f Mon Sep 17 00:00:00 2001 From: Mike Perry Date: Wed, 5 Jun 2019 11:50:44 -0700 Subject: [PATCH 1096/2557] Bug 29034: Cleanup hs circuitmap when purpose changes. Leave the other rend and hs_ident data around until circuit free, since code may still try to inspect it after marking the circuit for close. The circuitmap is the important thing to clean up, since repurposed intropoints must be removed from this map to ensure validity. --- changes/bug29034 | 5 +++++ src/core/or/circuituse.c | 6 ++++++ 2 files changed, 11 insertions(+) create mode 100644 changes/bug29034 diff --git a/changes/bug29034 b/changes/bug29034 new file mode 100644 index 0000000000..e7aa9af00b --- /dev/null +++ b/changes/bug29034 @@ -0,0 +1,5 @@ + o Major bugfixes (Onion service reachability): + - Properly clean up the introduction point map when circuits change purpose + from onion service circuits to pathbias, measurement, or other circuit types. + This should fix some service-side instances of introduction point failure. + Fixes bug 29034; bugfix on 0.3.2.1-alpha. diff --git a/src/core/or/circuituse.c b/src/core/or/circuituse.c index 485c389054..18b419e99d 100644 --- a/src/core/or/circuituse.c +++ b/src/core/or/circuituse.c @@ -3082,6 +3082,12 @@ circuit_change_purpose(circuit_t *circ, uint8_t new_purpose) circ->purpose, circuit_purpose_to_string(new_purpose), new_purpose); + + /* Take specific actions if we are repurposing a hidden service circuit. */ + if (circuit_purpose_is_hidden_service(circ->purpose) && + !circuit_purpose_is_hidden_service(new_purpose)) { + hs_circ_cleanup(circ); + } } old_purpose = circ->purpose; From e54ce03b4f439d7cdf9d0d0da3be11b72a1a83fd Mon Sep 17 00:00:00 2001 From: Mike Perry Date: Wed, 5 Jun 2019 12:33:39 -0700 Subject: [PATCH 1097/2557] More LOG_PROTOCOL_WARN. Make origin-side messages about padding negotiation failure into LOG_PROTOCOL_WARN. I'm not sure I like this either.. But the negotiation refusal case might happen naturally due to consensus drift, and is functionally no different than a corrupted cell. --- src/core/or/circuitpadding.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/core/or/circuitpadding.c b/src/core/or/circuitpadding.c index 84650a5a98..ebad6518b6 100644 --- a/src/core/or/circuitpadding.c +++ b/src/core/or/circuitpadding.c @@ -2805,7 +2805,7 @@ circpad_handle_padding_negotiate(circuit_t *circ, cell_t *cell) circpad_negotiate_t *negotiate; if (CIRCUIT_IS_ORIGIN(circ)) { - log_fn(LOG_WARN, LD_CIRC, + log_fn(LOG_PROTOCOL_WARN, LD_CIRC, "Padding negotiate cell unsupported at origin."); return -1; } @@ -2871,7 +2871,7 @@ circpad_handle_padding_negotiated(circuit_t *circ, cell_t *cell, /* Verify this came from the expected hop */ if (!circpad_padding_is_from_expected_hop(circ, layer_hint)) { - log_fn(LOG_WARN, LD_CIRC, + log_fn(LOG_PROTOCOL_WARN, LD_CIRC, "Padding negotiated cell from wrong hop!"); return -1; } @@ -2898,7 +2898,7 @@ circpad_handle_padding_negotiated(circuit_t *circ, cell_t *cell, // and be sad free_circ_machineinfos_with_machine_num(circ, negotiated->machine_type); TO_ORIGIN_CIRCUIT(circ)->padding_negotiation_failed = 1; - log_fn(LOG_INFO, LD_CIRC, + log_fn(LOG_PROTOCOL_WARN, LD_CIRC, "Middle node did not accept our padding request."); } From 4ceec4760c659122d900e56559b75157d1837ae2 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 5 Jun 2019 15:38:25 -0400 Subject: [PATCH 1098/2557] change some CI-substitute logic --- doc/HACKING/ReleasingTor.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/doc/HACKING/ReleasingTor.md b/doc/HACKING/ReleasingTor.md index 4c87a366cc..02e846a3b9 100644 --- a/doc/HACKING/ReleasingTor.md +++ b/doc/HACKING/ReleasingTor.md @@ -40,10 +40,14 @@ new Tor release: * clang scan-build. (See the script in ./scripts/test/scan_build.sh) * make test-network and make test-network-all (with - --enable-expensive-hardening) + --enable-fragile-hardening) * Running Tor yourself and making sure that it actually works for you. + * Running Tor under valgrind. (Our 'fragile hardening' doesn't cover + libevent and openssl, so using valgrind will sometimes find extra + memory leaks.) + === II. Write a changelog From 30f0e993c9021c1ad31d5a5134d150fabb359c32 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 5 Jun 2019 15:53:52 -0400 Subject: [PATCH 1099/2557] push release date to tomorrow: too many last minute surprises --- ChangeLog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index ca2215f163..d44ce316c7 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,4 +1,4 @@ -Changes in version 0.4.1.2-alpha - 2019-06-05 +Changes in version 0.4.1.2-alpha - 2019-06-06 Tor 0.4.1.2-alpha resolves numerous bugs--some of them from the previous alpha, and some much older. It also contains minor testing improvements, and an improvement to the security of our authenticated From b07b1a4f6d1aae3755db7ccfa9e7417214375f13 Mon Sep 17 00:00:00 2001 From: teor Date: Thu, 6 Jun 2019 13:42:30 +1000 Subject: [PATCH 1100/2557] practracker: accept 3 extra lines in router_parse_entry_from_string() practracker exception for bug 30781. --- scripts/maint/practracker/exceptions.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/maint/practracker/exceptions.txt b/scripts/maint/practracker/exceptions.txt index eb0625b8d3..db9bf34e25 100644 --- a/scripts/maint/practracker/exceptions.txt +++ b/scripts/maint/practracker/exceptions.txt @@ -189,7 +189,7 @@ problem function-size /src/feature/dirparse/ns_parse.c:networkstatus_verify_bw_w problem function-size /src/feature/dirparse/ns_parse.c:networkstatus_parse_vote_from_string() 638 problem function-size /src/feature/dirparse/parsecommon.c:tokenize_string() 103 problem function-size /src/feature/dirparse/parsecommon.c:get_next_token() 159 -problem function-size /src/feature/dirparse/routerparse.c:router_parse_entry_from_string() 554 +problem function-size /src/feature/dirparse/routerparse.c:router_parse_entry_from_string() 557 problem function-size /src/feature/dirparse/routerparse.c:extrainfo_parse_entry_from_string() 210 problem function-size /src/feature/hibernate/hibernate.c:accounting_parse_options() 109 problem function-size /src/feature/hs/hs_cell.c:hs_cell_build_establish_intro() 115 From a4ea335a6906eb4f8f58b5cf458cf290d322d10f Mon Sep 17 00:00:00 2001 From: teor Date: Thu, 6 Jun 2019 08:45:57 +1000 Subject: [PATCH 1101/2557] dirauth: Fix some comments in the router status processing code. Fixes comments in dirserv_router_get_status() and was_router_added_t. Preparation for 30780 and 16564. --- src/feature/dirauth/process_descs.c | 15 ++++++++++----- src/feature/nodelist/routerlist.c | 13 +++++++------ src/feature/nodelist/routerlist.h | 4 ++-- 3 files changed, 19 insertions(+), 13 deletions(-) diff --git a/src/feature/dirauth/process_descs.c b/src/feature/dirauth/process_descs.c index 656922233e..17936add5f 100644 --- a/src/feature/dirauth/process_descs.c +++ b/src/feature/dirauth/process_descs.c @@ -216,9 +216,14 @@ dirserv_load_fingerprint_file(void) #define DISABLE_DISABLING_ED25519 -/** Check whether router has a nickname/identity key combination that - * we recognize from the fingerprint list, or an IP we automatically act on - * according to our configuration. Return the appropriate router status. +/** Check whether router has: + * - a nickname/identity key combination that we recognize from the fingerprint + * list, + * - an IP we automatically act on according to our configuration, + * - an appropriate version, and + * - matching pinned keys. + * + * Return the appropriate router status. * * If the status is 'FP_REJECT' and msg is provided, set * *msg to an explanation of why. */ @@ -236,7 +241,7 @@ dirserv_router_get_status(const routerinfo_t *router, const char **msg, return FP_REJECT; } - /* Check for the more usual versions to reject a router first. */ + /* Check for the more common reasons to reject a router first. */ const uint32_t r = dirserv_get_status_impl(d, router->nickname, router->addr, router->or_port, router->platform, msg, severity); @@ -535,7 +540,7 @@ dirserv_add_multiple_descriptors(const char *desc, size_t desclen, int general = purpose == ROUTER_PURPOSE_GENERAL; tor_assert(msg); - r=ROUTER_ADDED_SUCCESSFULLY; /*Least severe return value. */ + r=ROUTER_ADDED_SUCCESSFULLY; /* Least severe return value. */ if (!string_is_utf8_no_bom(desc, desclen)) { *msg = "descriptor(s) or extrainfo(s) not valid UTF-8 or had BOM."; diff --git a/src/feature/nodelist/routerlist.c b/src/feature/nodelist/routerlist.c index 5788347a0e..5948445c96 100644 --- a/src/feature/nodelist/routerlist.c +++ b/src/feature/nodelist/routerlist.c @@ -1459,12 +1459,13 @@ router_descriptor_is_older_than,(const routerinfo_t *router, int seconds)) } /** Add router to the routerlist, if we don't already have it. Replace - * older entries (if any) with the same key. Note: Callers should not hold - * their pointers to router if this function fails; router - * will either be inserted into the routerlist or freed. Similarly, even - * if this call succeeds, they should not hold their pointers to - * router after subsequent calls with other routerinfo's -- they - * might cause the original routerinfo to get freed. + * older entries (if any) with the same key. + * + * Note: Callers should not hold their pointers to router if this + * function fails; router will either be inserted into the routerlist or + * freed. Similarly, even if this call succeeds, they should not hold their + * pointers to router after subsequent calls with other routerinfo's -- + * they might cause the original routerinfo to get freed. * * Returns the status for the operation. Might set *msg if it wants * the poster of the router to know something. diff --git a/src/feature/nodelist/routerlist.h b/src/feature/nodelist/routerlist.h index 5771ebb1ab..d7f44cb807 100644 --- a/src/feature/nodelist/routerlist.h +++ b/src/feature/nodelist/routerlist.h @@ -37,8 +37,8 @@ typedef enum was_router_added_t { ROUTER_WAS_NOT_WANTED = -6, /* Router descriptor was rejected because it was older than * OLD_ROUTER_DESC_MAX_AGE. */ - ROUTER_WAS_TOO_OLD = -7, /* note contrast with 'NOT_NEW' */ - /* DOCDOC */ + ROUTER_WAS_TOO_OLD = -7, /* note contrast with 'ROUTER_IS_ALREADY_KNOWN' */ + /* Some certs on this router are expired. */ ROUTER_CERTS_EXPIRED = -8 } was_router_added_t; From 19bf5806adb80e513bb2707a1686216225fef420 Mon Sep 17 00:00:00 2001 From: teor Date: Thu, 6 Jun 2019 08:52:13 +1000 Subject: [PATCH 1102/2557] dirauth: Return a distinct status when formatting annotations fails Adds ROUTER_AUTHDIR_BUG_ANNOTATIONS to was_router_added_t. The out-of-order numbering is deliberate: it will be fixed by later commits for 16564. Fixes bug 30780; bugfix on 0.2.0.8-alpha. --- changes/bug30780 | 3 +++ src/feature/dirauth/process_descs.c | 4 +--- src/feature/nodelist/routerlist.h | 5 ++++- 3 files changed, 8 insertions(+), 4 deletions(-) create mode 100644 changes/bug30780 diff --git a/changes/bug30780 b/changes/bug30780 new file mode 100644 index 0000000000..5731d201a2 --- /dev/null +++ b/changes/bug30780 @@ -0,0 +1,3 @@ + o Minor bugfixes (directory authorities): + - Return a distinct status when formatting annotations fails. + Fixes bug 30780; bugfix on 0.2.0.8-alpha. diff --git a/src/feature/dirauth/process_descs.c b/src/feature/dirauth/process_descs.c index 17936add5f..a68d155651 100644 --- a/src/feature/dirauth/process_descs.c +++ b/src/feature/dirauth/process_descs.c @@ -556,9 +556,7 @@ dirserv_add_multiple_descriptors(const char *desc, size_t desclen, !general ? router_purpose_to_string(purpose) : "", !general ? "\n" : "")<0) { *msg = "Couldn't format annotations"; - /* XXX Not cool: we return -1 below, but (was_router_added_t)-1 is - * ROUTER_BAD_EI, which isn't what's gone wrong here. :( */ - return -1; + return ROUTER_AUTHDIR_BUG_ANNOTATIONS; } s = desc; diff --git a/src/feature/nodelist/routerlist.h b/src/feature/nodelist/routerlist.h index d7f44cb807..dc9203e015 100644 --- a/src/feature/nodelist/routerlist.h +++ b/src/feature/nodelist/routerlist.h @@ -39,7 +39,10 @@ typedef enum was_router_added_t { * OLD_ROUTER_DESC_MAX_AGE. */ ROUTER_WAS_TOO_OLD = -7, /* note contrast with 'ROUTER_IS_ALREADY_KNOWN' */ /* Some certs on this router are expired. */ - ROUTER_CERTS_EXPIRED = -8 + ROUTER_CERTS_EXPIRED = -8, + /* We couldn't format the annotations for this router. This is a directory + * authority bug. */ + ROUTER_AUTHDIR_BUG_ANNOTATIONS = -10 } was_router_added_t; /** How long do we avoid using a directory server after it's given us a 503? */ From 6be9d3aed88bf73cf10f06edb99f876fd8eeb1d9 Mon Sep 17 00:00:00 2001 From: teor Date: Thu, 6 Jun 2019 18:24:17 +1000 Subject: [PATCH 1103/2557] practracker: accept one extra line in routerlist.c practracker exception for 30780. --- scripts/maint/practracker/exceptions.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/maint/practracker/exceptions.txt b/scripts/maint/practracker/exceptions.txt index eb0625b8d3..4f723c7c1e 100644 --- a/scripts/maint/practracker/exceptions.txt +++ b/scripts/maint/practracker/exceptions.txt @@ -216,7 +216,7 @@ problem function-size /src/feature/nodelist/node_select.c:router_pick_directory_ problem function-size /src/feature/nodelist/node_select.c:compute_weighted_bandwidths() 206 problem function-size /src/feature/nodelist/node_select.c:router_pick_trusteddirserver_impl() 114 problem function-size /src/feature/nodelist/nodelist.c:compute_frac_paths_available() 193 -problem file-size /src/feature/nodelist/routerlist.c 3238 +problem file-size /src/feature/nodelist/routerlist.c 3239 problem function-size /src/feature/nodelist/routerlist.c:router_rebuild_store() 148 problem function-size /src/feature/nodelist/routerlist.c:router_add_to_routerlist() 169 problem function-size /src/feature/nodelist/routerlist.c:routerlist_remove_old_routers() 121 From ecc5feff386890ad34378c70a20cbbdd0d338225 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 6 Jun 2019 08:28:34 -0400 Subject: [PATCH 1104/2557] bump to 0.4.1.2-alpha-dev --- configure.ac | 4 ++-- contrib/win32build/tor-mingw.nsi.in | 2 +- src/win32/orconfig.h | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/configure.ac b/configure.ac index f068d16462..05b932973f 100644 --- a/configure.ac +++ b/configure.ac @@ -4,7 +4,7 @@ dnl Copyright (c) 2007-2019, The Tor Project, Inc. dnl See LICENSE for licensing information AC_PREREQ([2.63]) -AC_INIT([tor],[0.4.1.2-alpha]) +AC_INIT([tor],[0.4.1.2-alpha-dev]) AC_CONFIG_SRCDIR([src/app/main/tor_main.c]) AC_CONFIG_MACRO_DIR([m4]) @@ -14,7 +14,7 @@ AC_CONFIG_MACRO_DIR([m4]) # version number changes. Tor uses it to make sure that it # only shuts down for missing "required protocols" when those protocols # are listed as required by a consensus after this date. -AC_DEFINE(APPROX_RELEASE_DATE, ["2019-06-05"], # for 0.4.1.2-alpha +AC_DEFINE(APPROX_RELEASE_DATE, ["2019-06-06"], # for 0.4.1.2-alpha-dev [Approximate date when this software was released. (Updated when the version changes.)]) # "foreign" means we don't follow GNU package layout standards diff --git a/contrib/win32build/tor-mingw.nsi.in b/contrib/win32build/tor-mingw.nsi.in index 99783c40f2..700744d95c 100644 --- a/contrib/win32build/tor-mingw.nsi.in +++ b/contrib/win32build/tor-mingw.nsi.in @@ -8,7 +8,7 @@ !include "LogicLib.nsh" !include "FileFunc.nsh" !insertmacro GetParameters -!define VERSION "0.4.1.2-alpha" +!define VERSION "0.4.1.2-alpha-dev" !define INSTALLER "tor-${VERSION}-win32.exe" !define WEBSITE "https://www.torproject.org/" !define LICENSE "LICENSE" diff --git a/src/win32/orconfig.h b/src/win32/orconfig.h index a3daa98de0..768c6210c2 100644 --- a/src/win32/orconfig.h +++ b/src/win32/orconfig.h @@ -218,7 +218,7 @@ #define USING_TWOS_COMPLEMENT /* Version number of package */ -#define VERSION "0.4.1.2-alpha" +#define VERSION "0.4.1.2-alpha-dev" From c46e99c43c4ee032127f2229070e5c21c64d19be Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 7 Jun 2019 13:52:03 -0400 Subject: [PATCH 1105/2557] Tolerate net-unreachable failures in util/socketpair_ersatz This can happen when we have no network stack configured. Fixes bug 30804; bugfix on 0.2.5.1-alpha. --- changes/bug30804 | 4 ++++ src/test/test_util.c | 5 +++++ 2 files changed, 9 insertions(+) create mode 100644 changes/bug30804 diff --git a/changes/bug30804 b/changes/bug30804 new file mode 100644 index 0000000000..ba4a3e8b8c --- /dev/null +++ b/changes/bug30804 @@ -0,0 +1,4 @@ + o Minor bugfixes (testing): + - Teach the util/socketpair_ersatz test to work correctly when we + have no network stack configured. Fixes bug 30804; bugfix on + 0.2.5.1-alpha. diff --git a/src/test/test_util.c b/src/test/test_util.c index 2faadd4e19..41ecbfd388 100644 --- a/src/test/test_util.c +++ b/src/test/test_util.c @@ -5399,6 +5399,11 @@ test_util_socketpair(void *arg) tt_skip(); } #endif /* defined(__FreeBSD__) */ + if (ersatz && socketpair_result == -ENETUNREACH) { + /* We can also fail with -ENETUNREACH if we have no network stack at + * all. */ + tt_skip(); + } tt_int_op(0, OP_EQ, socketpair_result); tt_assert(SOCKET_OK(fds[0])); From b9041e8a631beda9762fcd0627fa91045086410d Mon Sep 17 00:00:00 2001 From: teor Date: Mon, 10 Jun 2019 20:56:40 +1000 Subject: [PATCH 1106/2557] test: fix a typo in test_rebind.sh Closes 30821. --- src/test/test_rebind.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/test_rebind.sh b/src/test/test_rebind.sh index a8f07c7c1e..1d54dd7f33 100755 --- a/src/test/test_rebind.sh +++ b/src/test/test_rebind.sh @@ -7,7 +7,7 @@ if test "$UNAME_OS" = 'CYGWIN' || \ test "$UNAME_OS" = 'MSYS' || \ test "$UNAME_OS" = 'MINGW'; then if test "$APPVEYOR" = 'True'; then - echo "This test is disabled on Windows CI, as it requires firewall examptions. Skipping." >&2 + echo "This test is disabled on Windows CI, as it requires firewall exemptions. Skipping." >&2 exit 77 fi fi From adc7b50eae6e5cdcfaccdb09f7faf5ace33934d7 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 10 Jun 2019 08:47:33 -0400 Subject: [PATCH 1107/2557] Bump master to 0.4.2.0-alpha-dev --- configure.ac | 4 ++-- contrib/win32build/tor-mingw.nsi.in | 2 +- src/win32/orconfig.h | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/configure.ac b/configure.ac index 05b932973f..b9060a62a8 100644 --- a/configure.ac +++ b/configure.ac @@ -4,7 +4,7 @@ dnl Copyright (c) 2007-2019, The Tor Project, Inc. dnl See LICENSE for licensing information AC_PREREQ([2.63]) -AC_INIT([tor],[0.4.1.2-alpha-dev]) +AC_INIT([tor],[0.4.2.0-alpha-dev]) AC_CONFIG_SRCDIR([src/app/main/tor_main.c]) AC_CONFIG_MACRO_DIR([m4]) @@ -14,7 +14,7 @@ AC_CONFIG_MACRO_DIR([m4]) # version number changes. Tor uses it to make sure that it # only shuts down for missing "required protocols" when those protocols # are listed as required by a consensus after this date. -AC_DEFINE(APPROX_RELEASE_DATE, ["2019-06-06"], # for 0.4.1.2-alpha-dev +AC_DEFINE(APPROX_RELEASE_DATE, ["2019-06-10"], # for 0.4.2.0-alpha-dev [Approximate date when this software was released. (Updated when the version changes.)]) # "foreign" means we don't follow GNU package layout standards diff --git a/contrib/win32build/tor-mingw.nsi.in b/contrib/win32build/tor-mingw.nsi.in index 700744d95c..1c96b18682 100644 --- a/contrib/win32build/tor-mingw.nsi.in +++ b/contrib/win32build/tor-mingw.nsi.in @@ -8,7 +8,7 @@ !include "LogicLib.nsh" !include "FileFunc.nsh" !insertmacro GetParameters -!define VERSION "0.4.1.2-alpha-dev" +!define VERSION "0.4.2.0-alpha-dev" !define INSTALLER "tor-${VERSION}-win32.exe" !define WEBSITE "https://www.torproject.org/" !define LICENSE "LICENSE" diff --git a/src/win32/orconfig.h b/src/win32/orconfig.h index 768c6210c2..25bfb9ada5 100644 --- a/src/win32/orconfig.h +++ b/src/win32/orconfig.h @@ -218,7 +218,7 @@ #define USING_TWOS_COMPLEMENT /* Version number of package */ -#define VERSION "0.4.1.2-alpha-dev" +#define VERSION "0.4.2.0-alpha-dev" From 46da530431d340cde22192d9df475d85b5d9fab7 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 10 Jun 2019 08:54:22 -0400 Subject: [PATCH 1108/2557] Update git scripts for new versions {maint,release}-0.4.1 are now real {maint,release}-0.3.4 are now EOL Closes ticket 30822 --- scripts/git/git-merge-forward.sh | 18 +++++++++--------- scripts/git/git-pull-all.sh | 14 +++++++------- scripts/git/git-push-all.sh | 2 +- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/scripts/git/git-merge-forward.sh b/scripts/git/git-merge-forward.sh index 67af7e98bf..98ad07c9cc 100755 --- a/scripts/git/git-merge-forward.sh +++ b/scripts/git/git-merge-forward.sh @@ -38,15 +38,15 @@ TOR_WKT_NAME="tor-wkt" # # First set of arrays are the maint-* branch and then the release-* branch. # New arrays need to be in the WORKTREE= array else they aren't considered. -MAINT_034=( "maint-0.3.4" "maint-0.2.9" "$GIT_PATH/$TOR_WKT_NAME/maint-0.3.4" ) -MAINT_035=( "maint-0.3.5" "maint-0.3.4" "$GIT_PATH/$TOR_WKT_NAME/maint-0.3.5" ) +MAINT_035=( "maint-0.3.5" "maint-0.2.9" "$GIT_PATH/$TOR_WKT_NAME/maint-0.3.5" ) MAINT_040=( "maint-0.4.0" "maint-0.3.5" "$GIT_PATH/$TOR_WKT_NAME/maint-0.4.0" ) -MAINT_MASTER=( "master" "maint-0.4.0" "$GIT_PATH/$TOR_MASTER_NAME" ) +MAINT_041=( "maint-0.4.1" "maint-0.4.0" "$GIT_PATH/$TOR_WKT_NAME/maint-0.4.1" ) +MAINT_MASTER=( "master" "maint-0.4.1" "$GIT_PATH/$TOR_MASTER_NAME" ) RELEASE_029=( "release-0.2.9" "maint-0.2.9" "$GIT_PATH/$TOR_WKT_NAME/release-0.2.9" ) -RELEASE_034=( "release-0.3.4" "maint-0.3.4" "$GIT_PATH/$TOR_WKT_NAME/release-0.3.4" ) RELEASE_035=( "release-0.3.5" "maint-0.3.5" "$GIT_PATH/$TOR_WKT_NAME/release-0.3.5" ) RELEASE_040=( "release-0.4.0" "maint-0.4.0" "$GIT_PATH/$TOR_WKT_NAME/release-0.4.0" ) +RELEASE_041=( "release-0.4.1" "maint-0.4.1" "$GIT_PATH/$TOR_WKT_NAME/release-0.4.1" ) # The master branch path has to be the main repository thus contains the # origin that will be used to fetch the updates. All the worktrees are created @@ -55,14 +55,14 @@ ORIGIN_PATH="$GIT_PATH/$TOR_MASTER_NAME" # SC2034 -- shellcheck thinks that these are unused. We know better. ACTUALLY_THESE_ARE_USED=< Date: Mon, 10 Jun 2019 08:56:26 -0400 Subject: [PATCH 1109/2557] push-all: Use TOR_UPSTREAM_REMOTE_NAME to override UPSTREAM_BRANCH We already allow this override in our pre-push hook, so let's allow it here too. (I call my upstream branches "origin") --- scripts/git/git-push-all.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/git/git-push-all.sh b/scripts/git/git-push-all.sh index 9e1b39c10a..0701b27b59 100755 --- a/scripts/git/git-push-all.sh +++ b/scripts/git/git-push-all.sh @@ -1,9 +1,9 @@ #!/bin/bash # The remote upstream branch on which git.torproject.org/tor.git points to. -UPSTREAM_BRANCH="upstream" +UPSTREAM_BRANCH=${TOR_UPSTREAM_REMOTE_NAME:-"upstream"} -git push $UPSTREAM_BRANCH \ +git push "$UPSTREAM_BRANCH" \ master \ {release,maint}-0.4.1 \ {release,maint}-0.4.0 \ From 0635170cf14c3b622ee9fdf67ea81f4e564dd39d Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 10 Jun 2019 09:05:26 -0400 Subject: [PATCH 1110/2557] Add an environment variable to set GIT_PATH in git scripts --- scripts/git/git-merge-forward.sh | 2 +- scripts/git/git-pull-all.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/git/git-merge-forward.sh b/scripts/git/git-merge-forward.sh index 98ad07c9cc..15af6f3dba 100755 --- a/scripts/git/git-merge-forward.sh +++ b/scripts/git/git-merge-forward.sh @@ -14,7 +14,7 @@ # ... which means that the tor worktrees are in /home//git/tor-wkt # Where are all those git repositories? -GIT_PATH="FULL_PATH_TO_GIT_REPOSITORY_DIRECTORY" +GIT_PATH=${TOR_FULL_GIT_PATH:-"FULL_PATH_TO_GIT_REPOSITORY_DIRECTORY"} # The tor master git repository directory from which all the worktree have # been created. TOR_MASTER_NAME="tor" diff --git a/scripts/git/git-pull-all.sh b/scripts/git/git-pull-all.sh index ec8511d275..6fe7e59812 100755 --- a/scripts/git/git-pull-all.sh +++ b/scripts/git/git-pull-all.sh @@ -14,7 +14,7 @@ # ... which means that the tor worktrees are in /home//git/tor-wkt # Where are all those git repositories? -GIT_PATH="FULL_PATH_TO_GIT_REPOSITORY_DIRECTORY" +GIT_PATH=${TOR_FULL_GIT_PATH:-"FULL_PATH_TO_GIT_REPOSITORY_DIRECTORY"} # The tor master git repository directory from which all the worktree have # been created. TOR_MASTER_NAME="tor" From 973800b847844cfaacf48658d02fe3ada77cbcf6 Mon Sep 17 00:00:00 2001 From: teor Date: Tue, 11 Jun 2019 14:29:10 +1000 Subject: [PATCH 1111/2557] scripts/git: Stop hard-coding the bash path in the git scripts Some OSes don't have bash in /usr/bin, others have an ancient bash at this path. Fixes bug 30840; bugfix on 0.4.0.1-alpha. --- changes/bug30840 | 4 ++++ scripts/git/git-merge-forward.sh | 2 +- scripts/git/git-pull-all.sh | 2 +- scripts/git/git-push-all.sh | 2 +- scripts/git/pre-commit.git-hook | 2 +- scripts/git/pre-push.git-hook | 2 +- 6 files changed, 9 insertions(+), 5 deletions(-) create mode 100644 changes/bug30840 diff --git a/changes/bug30840 b/changes/bug30840 new file mode 100644 index 0000000000..562b0fbd93 --- /dev/null +++ b/changes/bug30840 @@ -0,0 +1,4 @@ + o Minor bugfixes (git scripts): + - Stop hard-coding the bash path in the git scripts. Some OSes don't + have bash in /usr/bin, others have an ancient bash at this path. + Fixes bug 30840; bugfix on 0.4.0.1-alpha. diff --git a/scripts/git/git-merge-forward.sh b/scripts/git/git-merge-forward.sh index 15af6f3dba..c9ec55ac6f 100755 --- a/scripts/git/git-merge-forward.sh +++ b/scripts/git/git-merge-forward.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash ############################## # Configuration (change me!) # diff --git a/scripts/git/git-pull-all.sh b/scripts/git/git-pull-all.sh index 6fe7e59812..e5ba96a059 100755 --- a/scripts/git/git-pull-all.sh +++ b/scripts/git/git-pull-all.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash ################################## # User configuration (change me) # diff --git a/scripts/git/git-push-all.sh b/scripts/git/git-push-all.sh index 0701b27b59..2030a600ff 100755 --- a/scripts/git/git-push-all.sh +++ b/scripts/git/git-push-all.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # The remote upstream branch on which git.torproject.org/tor.git points to. UPSTREAM_BRANCH=${TOR_UPSTREAM_REMOTE_NAME:-"upstream"} diff --git a/scripts/git/pre-commit.git-hook b/scripts/git/pre-commit.git-hook index b285776c04..2a29837198 100755 --- a/scripts/git/pre-commit.git-hook +++ b/scripts/git/pre-commit.git-hook @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # # To install this script, copy it to .git/hooks/pre-commit in local copy of # tor git repo and make sure it has permission to execute. diff --git a/scripts/git/pre-push.git-hook b/scripts/git/pre-push.git-hook index c9e72a4d43..51b0c896c8 100755 --- a/scripts/git/pre-push.git-hook +++ b/scripts/git/pre-push.git-hook @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # git pre-push hook script to: # 1) prevent "fixup!" and "squash!" commit from ending up in master, release-* From 396134188f21b3f8bdfe35171f14c6b70446ea3e Mon Sep 17 00:00:00 2001 From: teor Date: Tue, 11 Jun 2019 14:34:44 +1000 Subject: [PATCH 1112/2557] Stop hard-coding env vars in the git scripts Set the env vars: * TOR_MASTER_NAME to override the tor master branch name, and * TOR_WKT_NAME to override the worktree path Fixes bug 30841; bugfix on 0.4.0.1-alpha. --- changes/bug30841 | 3 +++ scripts/git/git-merge-forward.sh | 4 ++-- scripts/git/git-pull-all.sh | 4 ++-- 3 files changed, 7 insertions(+), 4 deletions(-) create mode 100644 changes/bug30841 diff --git a/changes/bug30841 b/changes/bug30841 new file mode 100644 index 0000000000..c6d1c51469 --- /dev/null +++ b/changes/bug30841 @@ -0,0 +1,3 @@ + o Minor bugfixes (git scripts): + - Stop hard-coding the tor master branch name and worktree path in the + git scripts. Fixes bug 30841; bugfix on 0.4.0.1-alpha. diff --git a/scripts/git/git-merge-forward.sh b/scripts/git/git-merge-forward.sh index c9ec55ac6f..ba29983284 100755 --- a/scripts/git/git-merge-forward.sh +++ b/scripts/git/git-merge-forward.sh @@ -17,9 +17,9 @@ GIT_PATH=${TOR_FULL_GIT_PATH:-"FULL_PATH_TO_GIT_REPOSITORY_DIRECTORY"} # The tor master git repository directory from which all the worktree have # been created. -TOR_MASTER_NAME="tor" +TOR_MASTER_NAME=${TOR_MASTER_NAME:-"tor"} # The worktrees location (directory). -TOR_WKT_NAME="tor-wkt" +TOR_WKT_NAME=${TOR_WKT_NAME:-"tor-wkt"} ######################### # End of configuration. # diff --git a/scripts/git/git-pull-all.sh b/scripts/git/git-pull-all.sh index e5ba96a059..8eb42c7c18 100755 --- a/scripts/git/git-pull-all.sh +++ b/scripts/git/git-pull-all.sh @@ -17,9 +17,9 @@ GIT_PATH=${TOR_FULL_GIT_PATH:-"FULL_PATH_TO_GIT_REPOSITORY_DIRECTORY"} # The tor master git repository directory from which all the worktree have # been created. -TOR_MASTER_NAME="tor" +TOR_MASTER_NAME=${TOR_MASTER_NAME:-"tor"} # The worktrees location (directory). -TOR_WKT_NAME="tor-wkt" +TOR_WKT_NAME=${TOR_WKT_NAME:-"tor-wkt"} ######################### # End of configuration. # From 024d65e14e062803523fb8ee1dd775ef5d2e96fa Mon Sep 17 00:00:00 2001 From: Xiaoyin Liu Date: Fri, 7 Jun 2019 17:13:49 +0800 Subject: [PATCH 1113/2557] Free a string buffer in nt_service_install() The string buffer "command" is not freed if the specified account name doesn't exist. This patch fixes this bug. --- src/app/main/ntmain.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/app/main/ntmain.c b/src/app/main/ntmain.c index f00b712702..a2de5bb87e 100644 --- a/src/app/main/ntmain.c +++ b/src/app/main/ntmain.c @@ -608,6 +608,7 @@ nt_service_install(int argc, char **argv) &sidUse) == 0) { /* XXXX For some reason, the above test segfaults. Fix that. */ printf("User \"%s\" doesn't seem to exist.\n", user_acct); + tor_free(command); return -1; } else { printf("Will try to install service as user \"%s\".\n", user_acct); From bff42c86cd3e054ee14885f7eb4896340828f7be Mon Sep 17 00:00:00 2001 From: teor Date: Tue, 11 Jun 2019 15:15:19 +1000 Subject: [PATCH 1114/2557] changes: file for 30799 Note that this memory leak is in unreachable code. --- changes/bug30799 | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 changes/bug30799 diff --git a/changes/bug30799 b/changes/bug30799 new file mode 100644 index 0000000000..b10420a953 --- /dev/null +++ b/changes/bug30799 @@ -0,0 +1,4 @@ + o Minor bugfixes (memory management): + - Stop leaking a small amount of memory in nt_service_install(), in + unreachable code. Fixes bug 30799; bugfix on 0.2.0.7-alpha. + Patch by Xiaoyin Liu. From 7cf9d54e6d7a08f169a27f7d76731e61ebe63fb0 Mon Sep 17 00:00:00 2001 From: David Goulet Date: Wed, 29 May 2019 11:34:07 -0400 Subject: [PATCH 1115/2557] token-bucket: Implement a single counter object Closes #30687. Signed-off-by: David Goulet --- changes/ticket30687 | 3 + src/lib/evloop/token_bucket.c | 52 ++++++++++++ src/lib/evloop/token_bucket.h | 29 +++++++ src/test/include.am | 1 + src/test/test.c | 1 + src/test/test.h | 1 + src/test/test_token_bucket.c | 152 ++++++++++++++++++++++++++++++++++ 7 files changed, 239 insertions(+) create mode 100644 changes/ticket30687 create mode 100644 src/test/test_token_bucket.c diff --git a/changes/ticket30687 b/changes/ticket30687 new file mode 100644 index 0000000000..c3124eb64b --- /dev/null +++ b/changes/ticket30687 @@ -0,0 +1,3 @@ + o Minor feature (token bucket): + - Implement a generic token bucket that uses a single counter. This will be + useful for the anti-DoS onion service work. Closes ticket 30687. diff --git a/src/lib/evloop/token_bucket.c b/src/lib/evloop/token_bucket.c index ee6d631e3b..ec62d1b018 100644 --- a/src/lib/evloop/token_bucket.c +++ b/src/lib/evloop/token_bucket.c @@ -256,3 +256,55 @@ token_bucket_rw_dec(token_bucket_rw_t *bucket, flags |= TB_WRITE; return flags; } + +/** Initialize a token bucket in bucket, set up to allow rate + * per second, with a maximum burst of burst. The bucket is created + * such that now_ts is the current timestamp. The bucket starts out + * full. */ +void +token_bucket_ctr_init(token_bucket_ctr_t *bucket, uint32_t rate, + uint32_t burst, uint32_t now_ts) +{ + memset(bucket, 0, sizeof(token_bucket_ctr_t)); + token_bucket_ctr_adjust(bucket, rate, burst); + token_bucket_ctr_reset(bucket, now_ts); +} + +/** Change the configured rate and burst of the given token bucket object in + * bucket. */ +void +token_bucket_ctr_adjust(token_bucket_ctr_t *bucket, uint32_t rate, + uint32_t burst) +{ + token_bucket_cfg_init(&bucket->cfg, rate, burst); + token_bucket_raw_adjust(&bucket->counter, &bucket->cfg); +} + +/** Reset bucket to be full, as of timestamp now_ts. */ +void +token_bucket_ctr_reset(token_bucket_ctr_t *bucket, uint32_t now_ts) +{ + token_bucket_raw_reset(&bucket->counter, &bucket->cfg); + bucket->last_refilled_at_timestamp = now_ts; +} + +/** Refill bucket as appropriate, given that the current timestamp is + * now_ts. */ +void +token_bucket_ctr_refill(token_bucket_ctr_t *bucket, uint32_t now_ts) +{ + const uint32_t elapsed_ticks = + (now_ts - bucket->last_refilled_at_timestamp); + if (elapsed_ticks > UINT32_MAX-(300*1000)) { + /* Either about 48 days have passed since the last refill, or the + * monotonic clock has somehow moved backwards. (We're looking at you, + * Windows.). We accept up to a 5 minute jump backwards as + * "unremarkable". + */ + return; + } + + token_bucket_raw_refill_steps(&bucket->counter, &bucket->cfg, + elapsed_ticks); + bucket->last_refilled_at_timestamp = now_ts; +} diff --git a/src/lib/evloop/token_bucket.h b/src/lib/evloop/token_bucket.h index 9398d2baa3..35b4246b12 100644 --- a/src/lib/evloop/token_bucket.h +++ b/src/lib/evloop/token_bucket.h @@ -103,6 +103,35 @@ token_bucket_rw_get_write(const token_bucket_rw_t *bucket) return token_bucket_raw_get(&bucket->write_bucket); } +/** + * A specialized bucket containing a single counter. + */ + +typedef struct token_bucket_ctr_t { + token_bucket_cfg_t cfg; + token_bucket_raw_t counter; + uint32_t last_refilled_at_timestamp; +} token_bucket_ctr_t; + +void token_bucket_ctr_init(token_bucket_ctr_t *bucket, uint32_t rate, + uint32_t burst, uint32_t now_ts); +void token_bucket_ctr_adjust(token_bucket_ctr_t *bucket, uint32_t rate, + uint32_t burst); +void token_bucket_ctr_reset(token_bucket_ctr_t *bucket, uint32_t now_ts); +void token_bucket_ctr_refill(token_bucket_ctr_t *bucket, uint32_t now_ts); + +static inline bool +token_bucket_ctr_dec(token_bucket_ctr_t *bucket, ssize_t n) +{ + return token_bucket_raw_dec(&bucket->counter, n); +} + +static inline size_t +token_bucket_ctr_get(const token_bucket_ctr_t *bucket) +{ + return token_bucket_raw_get(&bucket->counter); +} + #ifdef TOKEN_BUCKET_PRIVATE /* To avoid making the rates too small, we consider units of "steps", diff --git a/src/test/include.am b/src/test/include.am index 85f9c9f880..624bca66d9 100644 --- a/src/test/include.am +++ b/src/test/include.am @@ -193,6 +193,7 @@ src_test_test_SOURCES += \ src/test/test_status.c \ src/test/test_storagedir.c \ src/test/test_threads.c \ + src/test/test_token_bucket.c \ src/test/test_tortls.c \ src/test/test_util.c \ src/test/test_util_format.c \ diff --git a/src/test/test.c b/src/test/test.c index cac98dd839..cc08531702 100644 --- a/src/test/test.c +++ b/src/test/test.c @@ -916,6 +916,7 @@ struct testgroup_t testgroups[] = { { "socks/", socks_tests }, { "status/" , status_tests }, { "storagedir/", storagedir_tests }, + { "token_bucket/", token_bucket_tests }, { "tortls/", tortls_tests }, #ifndef ENABLE_NSS { "tortls/openssl/", tortls_openssl_tests }, diff --git a/src/test/test.h b/src/test/test.h index 167fd090ac..85e8b07ff7 100644 --- a/src/test/test.h +++ b/src/test/test.h @@ -272,6 +272,7 @@ extern struct testcase_t sr_tests[]; extern struct testcase_t status_tests[]; extern struct testcase_t storagedir_tests[]; extern struct testcase_t thread_tests[]; +extern struct testcase_t token_bucket_tests[]; extern struct testcase_t tortls_openssl_tests[]; extern struct testcase_t tortls_tests[]; extern struct testcase_t util_format_tests[]; diff --git a/src/test/test_token_bucket.c b/src/test/test_token_bucket.c new file mode 100644 index 0000000000..d3ce591388 --- /dev/null +++ b/src/test/test_token_bucket.c @@ -0,0 +1,152 @@ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file test_bwmgt.c + * \brief tests for bandwidth management / token bucket functions + */ + +#define TOKEN_BUCKET_PRIVATE + +#include "core/or/or.h" +#include "test/test.h" + +#include "lib/evloop/token_bucket.h" + +// an imaginary time, in timestamp units. Chosen so it will roll over. +static const uint32_t START_TS = UINT32_MAX - 1000; +static const uint32_t RATE = 10; +static const uint32_t BURST = 50; + +static void +test_token_bucket_ctr_init(void *arg) +{ + (void) arg; + token_bucket_ctr_t tb; + + token_bucket_ctr_init(&tb, RATE, BURST, START_TS); + tt_uint_op(tb.cfg.rate, OP_EQ, RATE); + tt_uint_op(tb.cfg.burst, OP_EQ, BURST); + tt_uint_op(tb.last_refilled_at_timestamp, OP_EQ, START_TS); + tt_int_op(tb.counter.bucket, OP_EQ, BURST); + + done: + ; +} + +static void +test_token_bucket_ctr_adjust(void *arg) +{ + (void) arg; + token_bucket_ctr_t tb; + + token_bucket_ctr_init(&tb, RATE, BURST, START_TS); + + /* Increase burst. */ + token_bucket_ctr_adjust(&tb, RATE, BURST * 2); + tt_uint_op(tb.cfg.rate, OP_EQ, RATE); + tt_uint_op(tb.counter.bucket, OP_EQ, BURST); + tt_uint_op(tb.cfg.burst, OP_EQ, BURST * 2); + + /* Decrease burst but still above bucket value. */ + token_bucket_ctr_adjust(&tb, RATE, BURST + 10); + tt_uint_op(tb.cfg.rate, OP_EQ, RATE); + tt_uint_op(tb.counter.bucket, OP_EQ, BURST); + tt_uint_op(tb.cfg.burst, OP_EQ, BURST + 10); + + /* Decrease burst below bucket value. */ + token_bucket_ctr_adjust(&tb, RATE, BURST - 1); + tt_uint_op(tb.cfg.rate, OP_EQ, RATE); + tt_uint_op(tb.counter.bucket, OP_EQ, BURST - 1); + tt_uint_op(tb.cfg.burst, OP_EQ, BURST - 1); + + /* Change rate. */ + token_bucket_ctr_adjust(&tb, RATE * 2, BURST); + tt_uint_op(tb.cfg.rate, OP_EQ, RATE * 2); + tt_uint_op(tb.counter.bucket, OP_EQ, BURST - 1); + tt_uint_op(tb.cfg.burst, OP_EQ, BURST); + + done: + ; +} + +static void +test_token_bucket_ctr_dec(void *arg) +{ + (void) arg; + token_bucket_ctr_t tb; + + token_bucket_ctr_init(&tb, RATE, BURST, START_TS); + + /* Simple decrement by one. */ + tt_uint_op(0, OP_EQ, token_bucket_ctr_dec(&tb, 1)); + tt_uint_op(tb.counter.bucket, OP_EQ, BURST - 1); + + /* Down to 0. Becomes empty. */ + tt_uint_op(true, OP_EQ, token_bucket_ctr_dec(&tb, BURST - 1)); + tt_uint_op(tb.counter.bucket, OP_EQ, 0); + + /* Reset and try to underflow. */ + token_bucket_ctr_init(&tb, RATE, BURST, START_TS); + tt_uint_op(true, OP_EQ, token_bucket_ctr_dec(&tb, BURST + 1)); + tt_int_op(tb.counter.bucket, OP_EQ, -1); + + /* Keep underflowing shouldn't flag the bucket as empty. */ + tt_uint_op(false, OP_EQ, token_bucket_ctr_dec(&tb, BURST)); + tt_int_op(tb.counter.bucket, OP_EQ, (int32_t) ((BURST + 1) * -1)); + + done: + ; +} + +static void +test_token_bucket_ctr_refill(void *arg) +{ + (void) arg; + token_bucket_ctr_t tb; + + token_bucket_ctr_init(&tb, RATE, BURST, START_TS); + + /* Reduce of half the bucket and let a single second go before refill. */ + token_bucket_ctr_dec(&tb, BURST / 2); + tt_int_op(tb.counter.bucket, OP_EQ, BURST / 2); + token_bucket_ctr_refill(&tb, START_TS + 1); + tt_int_op(tb.counter.bucket, OP_EQ, (BURST / 2) + RATE); + tt_int_op(tb.last_refilled_at_timestamp, OP_EQ, START_TS + 1); + + /* No time change, nothing should move. */ + token_bucket_ctr_refill(&tb, START_TS + 1); + tt_int_op(tb.counter.bucket, OP_EQ, (BURST / 2) + RATE); + tt_int_op(tb.last_refilled_at_timestamp, OP_EQ, START_TS + 1); + + /* Add 99 seconds, bucket should be back to a full BURST. */ + token_bucket_ctr_refill(&tb, START_TS + 99); + tt_int_op(tb.counter.bucket, OP_EQ, BURST); + tt_int_op(tb.last_refilled_at_timestamp, OP_EQ, START_TS + 99); + + /* Empty bucket at once. */ + token_bucket_ctr_dec(&tb, BURST); + tt_int_op(tb.counter.bucket, OP_EQ, 0); + /* On second passes. */ + token_bucket_ctr_refill(&tb, START_TS + 100); + tt_int_op(tb.last_refilled_at_timestamp, OP_EQ, START_TS + 100); + tt_int_op(tb.counter.bucket, OP_EQ, RATE); + /* A second second passes. */ + token_bucket_ctr_refill(&tb, START_TS + 101); + tt_int_op(tb.last_refilled_at_timestamp, OP_EQ, START_TS + 101); + tt_int_op(tb.counter.bucket, OP_EQ, RATE * 2); + + done: + ; +} + +#define TOKEN_BUCKET(name) \ + { #name, test_token_bucket_ ## name , 0, NULL, NULL } + +struct testcase_t token_bucket_tests[] = { + TOKEN_BUCKET(ctr_init), + TOKEN_BUCKET(ctr_adjust), + TOKEN_BUCKET(ctr_dec), + TOKEN_BUCKET(ctr_refill), + END_OF_TESTCASES +}; From c1359b32a43b09be96e0388c12b75a9deda17e4f Mon Sep 17 00:00:00 2001 From: David Goulet Date: Wed, 5 Jun 2019 09:57:52 -0400 Subject: [PATCH 1116/2557] trunnel: Rename sendme.trunnel to sendme_cell.trunnel This is to avoid having two sendme.{c|h} in the repository since the subsystem is implemented in src/core/or/sendme.{c|h}. Fixes #30769 Signed-off-by: David Goulet --- changes/ticket30769 | 4 ++++ src/core/or/sendme.c | 2 +- src/ext/trunnel/trunnel-impl.h | 2 +- src/ext/trunnel/trunnel.c | 2 +- src/ext/trunnel/trunnel.h | 2 +- src/trunnel/channelpadding_negotiation.c | 2 +- src/trunnel/channelpadding_negotiation.h | 2 +- src/trunnel/circpad_negotiation.c | 2 +- src/trunnel/circpad_negotiation.h | 2 +- src/trunnel/ed25519_cert.c | 2 +- src/trunnel/ed25519_cert.h | 2 +- src/trunnel/hs/cell_common.c | 2 +- src/trunnel/hs/cell_common.h | 2 +- src/trunnel/hs/cell_establish_intro.c | 2 +- src/trunnel/hs/cell_establish_intro.h | 2 +- src/trunnel/hs/cell_introduce1.c | 2 +- src/trunnel/hs/cell_introduce1.h | 2 +- src/trunnel/hs/cell_rendezvous.c | 2 +- src/trunnel/hs/cell_rendezvous.h | 2 +- src/trunnel/include.am | 6 +++--- src/trunnel/link_handshake.c | 2 +- src/trunnel/link_handshake.h | 2 +- src/trunnel/netinfo.c | 2 +- src/trunnel/netinfo.h | 2 +- src/trunnel/pwbox.c | 2 +- src/trunnel/pwbox.h | 2 +- src/trunnel/{sendme.c => sendme_cell.c} | 8 ++++---- src/trunnel/{sendme.h => sendme_cell.h} | 6 +++--- src/trunnel/{sendme.trunnel => sendme_cell.trunnel} | 0 src/trunnel/socks5.c | 2 +- src/trunnel/socks5.h | 2 +- 31 files changed, 40 insertions(+), 36 deletions(-) create mode 100644 changes/ticket30769 rename src/trunnel/{sendme.c => sendme_cell.c} (97%) rename src/trunnel/{sendme.h => sendme_cell.h} (97%) rename src/trunnel/{sendme.trunnel => sendme_cell.trunnel} (100%) diff --git a/changes/ticket30769 b/changes/ticket30769 new file mode 100644 index 0000000000..74f63a1465 --- /dev/null +++ b/changes/ticket30769 @@ -0,0 +1,4 @@ + o Minor bugfixes (sendme, code structure): + - Rename the trunnel SENDME file definition from sendme.trunnel to + sendme_cell.trunnel to avoid having twice sendme.{c|h} in the repository. + Fixes bug 30769; bugfix on 0.4.1.1-alpha. diff --git a/src/core/or/sendme.c b/src/core/or/sendme.c index 47ac95f3cf..0757ce3d52 100644 --- a/src/core/or/sendme.c +++ b/src/core/or/sendme.c @@ -23,7 +23,7 @@ #include "core/or/sendme.h" #include "feature/nodelist/networkstatus.h" #include "lib/ctime/di_ops.h" -#include "trunnel/sendme.h" +#include "trunnel/sendme_cell.h" /* Return the minimum version given by the consensus (if any) that should be * used when emitting a SENDME cell. */ diff --git a/src/ext/trunnel/trunnel-impl.h b/src/ext/trunnel/trunnel-impl.h index 15d1c8633e..52afa9ccd4 100644 --- a/src/ext/trunnel/trunnel-impl.h +++ b/src/ext/trunnel/trunnel-impl.h @@ -1,4 +1,4 @@ -/* trunnel-impl.h -- copied from Trunnel v1.5.2 +/* trunnel-impl.h -- copied from Trunnel v1.5.3 * https://gitweb.torproject.org/trunnel.git * You probably shouldn't edit this file. */ diff --git a/src/ext/trunnel/trunnel.c b/src/ext/trunnel/trunnel.c index 3ae3fe02c8..01a55c5bec 100644 --- a/src/ext/trunnel/trunnel.c +++ b/src/ext/trunnel/trunnel.c @@ -1,4 +1,4 @@ -/* trunnel.c -- copied from Trunnel v1.5.2 +/* trunnel.c -- copied from Trunnel v1.5.3 * https://gitweb.torproject.org/trunnel.git * You probably shouldn't edit this file. */ diff --git a/src/ext/trunnel/trunnel.h b/src/ext/trunnel/trunnel.h index 9b708437b8..87c75f4ec3 100644 --- a/src/ext/trunnel/trunnel.h +++ b/src/ext/trunnel/trunnel.h @@ -1,4 +1,4 @@ -/* trunnel.h -- copied from Trunnel v1.5.2 +/* trunnel.h -- copied from Trunnel v1.5.3 * https://gitweb.torproject.org/trunnel.git * You probably shouldn't edit this file. */ diff --git a/src/trunnel/channelpadding_negotiation.c b/src/trunnel/channelpadding_negotiation.c index 59e6b38384..d96496e90c 100644 --- a/src/trunnel/channelpadding_negotiation.c +++ b/src/trunnel/channelpadding_negotiation.c @@ -1,4 +1,4 @@ -/* channelpadding_negotiation.c -- generated by Trunnel v1.5.2. +/* channelpadding_negotiation.c -- generated by Trunnel v1.5.3. * https://gitweb.torproject.org/trunnel.git * You probably shouldn't edit this file. */ diff --git a/src/trunnel/channelpadding_negotiation.h b/src/trunnel/channelpadding_negotiation.h index fcfc232fea..3f96174f68 100644 --- a/src/trunnel/channelpadding_negotiation.h +++ b/src/trunnel/channelpadding_negotiation.h @@ -1,4 +1,4 @@ -/* channelpadding_negotiation.h -- generated by Trunnel v1.5.2. +/* channelpadding_negotiation.h -- generated by Trunnel v1.5.3. * https://gitweb.torproject.org/trunnel.git * You probably shouldn't edit this file. */ diff --git a/src/trunnel/circpad_negotiation.c b/src/trunnel/circpad_negotiation.c index 236be06ada..547818f2ec 100644 --- a/src/trunnel/circpad_negotiation.c +++ b/src/trunnel/circpad_negotiation.c @@ -1,4 +1,4 @@ -/* circpad_negotiation.c -- generated by Trunnel v1.5.2. +/* circpad_negotiation.c -- generated by Trunnel v1.5.3. * https://gitweb.torproject.org/trunnel.git * You probably shouldn't edit this file. */ diff --git a/src/trunnel/circpad_negotiation.h b/src/trunnel/circpad_negotiation.h index d09080dc16..ba9155019e 100644 --- a/src/trunnel/circpad_negotiation.h +++ b/src/trunnel/circpad_negotiation.h @@ -1,4 +1,4 @@ -/* circpad_negotiation.h -- generated by Trunnel v1.5.2. +/* circpad_negotiation.h -- generated by Trunnel v1.5.3. * https://gitweb.torproject.org/trunnel.git * You probably shouldn't edit this file. */ diff --git a/src/trunnel/ed25519_cert.c b/src/trunnel/ed25519_cert.c index 1276c7a505..86b79ef9b6 100644 --- a/src/trunnel/ed25519_cert.c +++ b/src/trunnel/ed25519_cert.c @@ -1,4 +1,4 @@ -/* ed25519_cert.c -- generated by Trunnel v1.5.2. +/* ed25519_cert.c -- generated by Trunnel v1.5.3. * https://gitweb.torproject.org/trunnel.git * You probably shouldn't edit this file. */ diff --git a/src/trunnel/ed25519_cert.h b/src/trunnel/ed25519_cert.h index e086c6fced..bd91ce1055 100644 --- a/src/trunnel/ed25519_cert.h +++ b/src/trunnel/ed25519_cert.h @@ -1,4 +1,4 @@ -/* ed25519_cert.h -- generated by Trunnel v1.5.2. +/* ed25519_cert.h -- generated by Trunnel v1.5.3. * https://gitweb.torproject.org/trunnel.git * You probably shouldn't edit this file. */ diff --git a/src/trunnel/hs/cell_common.c b/src/trunnel/hs/cell_common.c index af223560c1..830af5c78b 100644 --- a/src/trunnel/hs/cell_common.c +++ b/src/trunnel/hs/cell_common.c @@ -1,4 +1,4 @@ -/* cell_common.c -- generated by Trunnel v1.5.2. +/* cell_common.c -- generated by Trunnel v1.5.3. * https://gitweb.torproject.org/trunnel.git * You probably shouldn't edit this file. */ diff --git a/src/trunnel/hs/cell_common.h b/src/trunnel/hs/cell_common.h index e08eedfdb3..c84d17d8e5 100644 --- a/src/trunnel/hs/cell_common.h +++ b/src/trunnel/hs/cell_common.h @@ -1,4 +1,4 @@ -/* cell_common.h -- generated by Trunnel v1.5.2. +/* cell_common.h -- generated by Trunnel v1.5.3. * https://gitweb.torproject.org/trunnel.git * You probably shouldn't edit this file. */ diff --git a/src/trunnel/hs/cell_establish_intro.c b/src/trunnel/hs/cell_establish_intro.c index ae3b7b1bc8..99ceadbda4 100644 --- a/src/trunnel/hs/cell_establish_intro.c +++ b/src/trunnel/hs/cell_establish_intro.c @@ -1,4 +1,4 @@ -/* cell_establish_intro.c -- generated by Trunnel v1.5.2. +/* cell_establish_intro.c -- generated by Trunnel v1.5.3. * https://gitweb.torproject.org/trunnel.git * You probably shouldn't edit this file. */ diff --git a/src/trunnel/hs/cell_establish_intro.h b/src/trunnel/hs/cell_establish_intro.h index ccaef5488c..1908645aa6 100644 --- a/src/trunnel/hs/cell_establish_intro.h +++ b/src/trunnel/hs/cell_establish_intro.h @@ -1,4 +1,4 @@ -/* cell_establish_intro.h -- generated by Trunnel v1.5.2. +/* cell_establish_intro.h -- generated by Trunnel v1.5.3. * https://gitweb.torproject.org/trunnel.git * You probably shouldn't edit this file. */ diff --git a/src/trunnel/hs/cell_introduce1.c b/src/trunnel/hs/cell_introduce1.c index 53b3d299f2..016c9fa8d6 100644 --- a/src/trunnel/hs/cell_introduce1.c +++ b/src/trunnel/hs/cell_introduce1.c @@ -1,4 +1,4 @@ -/* cell_introduce1.c -- generated by Trunnel v1.5.2. +/* cell_introduce1.c -- generated by Trunnel v1.5.3. * https://gitweb.torproject.org/trunnel.git * You probably shouldn't edit this file. */ diff --git a/src/trunnel/hs/cell_introduce1.h b/src/trunnel/hs/cell_introduce1.h index 986a531ca7..8dabff3cb5 100644 --- a/src/trunnel/hs/cell_introduce1.h +++ b/src/trunnel/hs/cell_introduce1.h @@ -1,4 +1,4 @@ -/* cell_introduce1.h -- generated by Trunnel v1.5.2. +/* cell_introduce1.h -- generated by Trunnel v1.5.3. * https://gitweb.torproject.org/trunnel.git * You probably shouldn't edit this file. */ diff --git a/src/trunnel/hs/cell_rendezvous.c b/src/trunnel/hs/cell_rendezvous.c index 53cb609138..1204e93cfc 100644 --- a/src/trunnel/hs/cell_rendezvous.c +++ b/src/trunnel/hs/cell_rendezvous.c @@ -1,4 +1,4 @@ -/* cell_rendezvous.c -- generated by Trunnel v1.5.2. +/* cell_rendezvous.c -- generated by Trunnel v1.5.3. * https://gitweb.torproject.org/trunnel.git * You probably shouldn't edit this file. */ diff --git a/src/trunnel/hs/cell_rendezvous.h b/src/trunnel/hs/cell_rendezvous.h index 39e14da25b..5a8c2ff52a 100644 --- a/src/trunnel/hs/cell_rendezvous.h +++ b/src/trunnel/hs/cell_rendezvous.h @@ -1,4 +1,4 @@ -/* cell_rendezvous.h -- generated by Trunnel v1.5.2. +/* cell_rendezvous.h -- generated by Trunnel v1.5.3. * https://gitweb.torproject.org/trunnel.git * You probably shouldn't edit this file. */ diff --git a/src/trunnel/include.am b/src/trunnel/include.am index ce15570b15..6c3a5ff06b 100644 --- a/src/trunnel/include.am +++ b/src/trunnel/include.am @@ -11,7 +11,7 @@ TRUNNELINPUTS = \ src/trunnel/link_handshake.trunnel \ src/trunnel/pwbox.trunnel \ src/trunnel/channelpadding_negotiation.trunnel \ - src/trunnel/sendme.trunnel \ + src/trunnel/sendme_cell.trunnel \ src/trunnel/socks5.trunnel \ src/trunnel/circpad_negotiation.trunnel @@ -25,7 +25,7 @@ TRUNNELSOURCES = \ src/trunnel/hs/cell_introduce1.c \ src/trunnel/hs/cell_rendezvous.c \ src/trunnel/channelpadding_negotiation.c \ - src/trunnel/sendme.c \ + src/trunnel/sendme_cell.c \ src/trunnel/socks5.c \ src/trunnel/netinfo.c \ src/trunnel/circpad_negotiation.c @@ -42,7 +42,7 @@ TRUNNELHEADERS = \ src/trunnel/hs/cell_introduce1.h \ src/trunnel/hs/cell_rendezvous.h \ src/trunnel/channelpadding_negotiation.h \ - src/trunnel/sendme.h \ + src/trunnel/sendme_cell.h \ src/trunnel/socks5.h \ src/trunnel/netinfo.h \ src/trunnel/circpad_negotiation.h diff --git a/src/trunnel/link_handshake.c b/src/trunnel/link_handshake.c index 03ead31c62..76db4b0e29 100644 --- a/src/trunnel/link_handshake.c +++ b/src/trunnel/link_handshake.c @@ -1,4 +1,4 @@ -/* link_handshake.c -- generated by Trunnel v1.5.2. +/* link_handshake.c -- generated by Trunnel v1.5.3. * https://gitweb.torproject.org/trunnel.git * You probably shouldn't edit this file. */ diff --git a/src/trunnel/link_handshake.h b/src/trunnel/link_handshake.h index 6a23483adc..0c7ac36b1b 100644 --- a/src/trunnel/link_handshake.h +++ b/src/trunnel/link_handshake.h @@ -1,4 +1,4 @@ -/* link_handshake.h -- generated by Trunnel v1.5.2. +/* link_handshake.h -- generated by Trunnel v1.5.3. * https://gitweb.torproject.org/trunnel.git * You probably shouldn't edit this file. */ diff --git a/src/trunnel/netinfo.c b/src/trunnel/netinfo.c index 5d815b9b12..d7d0cddc89 100644 --- a/src/trunnel/netinfo.c +++ b/src/trunnel/netinfo.c @@ -1,4 +1,4 @@ -/* netinfo.c -- generated by Trunnel v1.5.2. +/* netinfo.c -- generated by Trunnel v1.5.3. * https://gitweb.torproject.org/trunnel.git * You probably shouldn't edit this file. */ diff --git a/src/trunnel/netinfo.h b/src/trunnel/netinfo.h index ac46e603ba..37c2ae3c2d 100644 --- a/src/trunnel/netinfo.h +++ b/src/trunnel/netinfo.h @@ -1,4 +1,4 @@ -/* netinfo.h -- generated by Trunnel v1.5.2. +/* netinfo.h -- generated by Trunnel v1.5.3. * https://gitweb.torproject.org/trunnel.git * You probably shouldn't edit this file. */ diff --git a/src/trunnel/pwbox.c b/src/trunnel/pwbox.c index c356515d36..c159a5e687 100644 --- a/src/trunnel/pwbox.c +++ b/src/trunnel/pwbox.c @@ -1,4 +1,4 @@ -/* pwbox.c -- generated by Trunnel v1.5.2. +/* pwbox.c -- generated by Trunnel v1.5.3. * https://gitweb.torproject.org/trunnel.git * You probably shouldn't edit this file. */ diff --git a/src/trunnel/pwbox.h b/src/trunnel/pwbox.h index a9a421408a..36d595f4ef 100644 --- a/src/trunnel/pwbox.h +++ b/src/trunnel/pwbox.h @@ -1,4 +1,4 @@ -/* pwbox.h -- generated by Trunnel v1.5.2. +/* pwbox.h -- generated by Trunnel v1.5.3. * https://gitweb.torproject.org/trunnel.git * You probably shouldn't edit this file. */ diff --git a/src/trunnel/sendme.c b/src/trunnel/sendme_cell.c similarity index 97% rename from src/trunnel/sendme.c rename to src/trunnel/sendme_cell.c index 262b915234..b9f8fe967f 100644 --- a/src/trunnel/sendme.c +++ b/src/trunnel/sendme_cell.c @@ -1,11 +1,11 @@ -/* sendme.c -- generated by Trunnel v1.5.2. +/* sendme_cell.c -- generated by Trunnel v1.5.3. * https://gitweb.torproject.org/trunnel.git * You probably shouldn't edit this file. */ #include #include "trunnel-impl.h" -#include "sendme.h" +#include "sendme_cell.h" #define TRUNNEL_SET_ERROR_CODE(obj) \ do { \ @@ -15,8 +15,8 @@ #if defined(__COVERITY__) || defined(__clang_analyzer__) /* If we're running a static analysis tool, we don't want it to complain * that some of our remaining-bytes checks are dead-code. */ -int sendme_deadcode_dummy__ = 0; -#define OR_DEADCODE_DUMMY || sendme_deadcode_dummy__ +int sendmecell_deadcode_dummy__ = 0; +#define OR_DEADCODE_DUMMY || sendmecell_deadcode_dummy__ #else #define OR_DEADCODE_DUMMY #endif diff --git a/src/trunnel/sendme.h b/src/trunnel/sendme_cell.h similarity index 97% rename from src/trunnel/sendme.h rename to src/trunnel/sendme_cell.h index f3c3dd78c4..45efb9f10d 100644 --- a/src/trunnel/sendme.h +++ b/src/trunnel/sendme_cell.h @@ -1,9 +1,9 @@ -/* sendme.h -- generated by Trunnel v1.5.2. +/* sendme_cell.h -- generated by Trunnel v1.5.3. * https://gitweb.torproject.org/trunnel.git * You probably shouldn't edit this file. */ -#ifndef TRUNNEL_SENDME_H -#define TRUNNEL_SENDME_H +#ifndef TRUNNEL_SENDME_CELL_H +#define TRUNNEL_SENDME_CELL_H #include #include "trunnel.h" diff --git a/src/trunnel/sendme.trunnel b/src/trunnel/sendme_cell.trunnel similarity index 100% rename from src/trunnel/sendme.trunnel rename to src/trunnel/sendme_cell.trunnel diff --git a/src/trunnel/socks5.c b/src/trunnel/socks5.c index 057a52b042..f32862e353 100644 --- a/src/trunnel/socks5.c +++ b/src/trunnel/socks5.c @@ -1,4 +1,4 @@ -/* socks5.c -- generated by Trunnel v1.5.2. +/* socks5.c -- generated by Trunnel v1.5.3. * https://gitweb.torproject.org/trunnel.git * You probably shouldn't edit this file. */ diff --git a/src/trunnel/socks5.h b/src/trunnel/socks5.h index d3bea152e7..23ac64faba 100644 --- a/src/trunnel/socks5.h +++ b/src/trunnel/socks5.h @@ -1,4 +1,4 @@ -/* socks5.h -- generated by Trunnel v1.5.2. +/* socks5.h -- generated by Trunnel v1.5.3. * https://gitweb.torproject.org/trunnel.git * You probably shouldn't edit this file. */ From a15ec8bf84db2dcadcc292b9672fbd6d35ffdf77 Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Tue, 11 Jun 2019 14:28:38 +0300 Subject: [PATCH 1117/2557] circpad: some more logging changes. - Add an info log when receiving a STOP command. - Keep warning if we receive padding from a wrong hop. --- src/core/or/circuitpadding.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/core/or/circuitpadding.c b/src/core/or/circuitpadding.c index ebad6518b6..7521fd3394 100644 --- a/src/core/or/circuitpadding.c +++ b/src/core/or/circuitpadding.c @@ -2821,6 +2821,8 @@ circpad_handle_padding_negotiate(circuit_t *circ, cell_t *cell) /* Free the machine corresponding to this machine type */ if (free_circ_machineinfos_with_machine_num(circ, negotiate->machine_type)) { + log_info(LD_CIRC, "Received STOP command for machine %u", + negotiate->machine_type); goto done; } log_fn(LOG_PROTOCOL_WARN, LD_CIRC, @@ -2871,7 +2873,7 @@ circpad_handle_padding_negotiated(circuit_t *circ, cell_t *cell, /* Verify this came from the expected hop */ if (!circpad_padding_is_from_expected_hop(circ, layer_hint)) { - log_fn(LOG_PROTOCOL_WARN, LD_CIRC, + log_fn(LOG_WARN, LD_CIRC, "Padding negotiated cell from wrong hop!"); return -1; } From 93ddc51cbd325ef3d5cf3a5b9948ff65c09fc5aa Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 29 May 2019 09:21:45 -0400 Subject: [PATCH 1118/2557] Give a more useful failure messgae when we fail to minherit(). Part of ticket 30686. --- src/lib/crypt_ops/crypto_rand_fast.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/lib/crypt_ops/crypto_rand_fast.c b/src/lib/crypt_ops/crypto_rand_fast.c index b71ade81bd..d2ce1a542f 100644 --- a/src/lib/crypt_ops/crypto_rand_fast.c +++ b/src/lib/crypt_ops/crypto_rand_fast.c @@ -181,7 +181,11 @@ crypto_fast_rng_new_from_seed(const uint8_t *seed) #else /* We decided above that noinherit would always do _something_. Assert here * that we were correct. */ - tor_assert(inherit != INHERIT_RES_KEEP); + tor_assertf(inherit != INHERIT_RES_KEEP, + "We failed to create a non-inheritable memory region, even " + "though we believed such a failure to be impossible! This is " + "probably a bug in Tor support for your platform; please report " + "it."); #endif return result; } From cad0de35bd49064f5712f32a2b6e41ffe6e822e6 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 29 May 2019 09:22:18 -0400 Subject: [PATCH 1119/2557] Give a compile warning when we don't have any flags for minherit(). Part of ticket 30686. --- src/lib/malloc/map_anon.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/lib/malloc/map_anon.c b/src/lib/malloc/map_anon.c index f4fda00bff..6afea713df 100644 --- a/src/lib/malloc/map_anon.c +++ b/src/lib/malloc/map_anon.c @@ -70,6 +70,11 @@ #endif +#if defined(HAVE_MINHERIT) && !defined(FLAG_ZERO) && !defined(FLAG_NOINHERIT) +#warn "minherit() is defined, but we couldn't find the right flag for it." +#warn "This is probably a bug in Tor's support for this platform." +#endif + /** * Helper: try to prevent the sz bytes at mem from being swapped * to disk. Return 0 on success or if the facility is not available on this From 5068ccab0b1849e836729c43a2ca891139cf107b Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 29 May 2019 09:23:13 -0400 Subject: [PATCH 1120/2557] Add a changes file for ticket 30686. --- changes/ticket30686 | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 changes/ticket30686 diff --git a/changes/ticket30686 b/changes/ticket30686 new file mode 100644 index 0000000000..36473c1a02 --- /dev/null +++ b/changes/ticket30686 @@ -0,0 +1,5 @@ + o Minor features (logging): + - Give a more useful assertion failure message if we think we have + minherit() but we fail to make a region non-inheritable. Give a + compile-time warning if our support for minherit() is + incomplete. Closes ticket 30686. From a8c0f4ddfe3f0a63bd499959c8d921346aa9766e Mon Sep 17 00:00:00 2001 From: Taylor Yu Date: Fri, 8 Mar 2019 09:41:43 -0600 Subject: [PATCH 1121/2557] Rework orconn tracking to use pubsub Part of ticket 29976. --- src/app/main/main.c | 1 + src/core/or/connection_or.c | 28 ++++----- src/core/or/orconn_event.c | 96 +++++++++++++++++------------ src/core/or/orconn_event.h | 31 +++------- src/feature/control/btrack.c | 8 +++ src/feature/control/btrack_orconn.c | 49 +++++++-------- src/feature/control/btrack_orconn.h | 3 + src/test/test_btrack.c | 65 +++++++++++-------- src/test/test_controller_events.c | 51 ++++++++------- src/test/test_extorport.c | 2 +- src/test/test_helpers.c | 54 ++++++++++++++++ src/test/test_helpers.h | 6 ++ 12 files changed, 245 insertions(+), 149 deletions(-) diff --git a/src/app/main/main.c b/src/app/main/main.c index 6e325f0b10..54f2afaa01 100644 --- a/src/app/main/main.c +++ b/src/app/main/main.c @@ -1256,6 +1256,7 @@ pubsub_connect(void) /* XXXX For each pubsub channel, its delivery strategy should be set at * this XXXX point, using tor_mainloop_set_delivery_strategy(). */ + tor_mainloop_set_delivery_strategy("orconn", DELIV_IMMEDIATE); } } diff --git a/src/core/or/connection_or.c b/src/core/or/connection_or.c index 830e09fd54..4c93351e31 100644 --- a/src/core/or/connection_or.c +++ b/src/core/or/connection_or.c @@ -414,13 +414,12 @@ void connection_or_event_status(or_connection_t *conn, or_conn_status_event_t tp, int reason) { - orconn_event_msg_t msg; + orconn_status_msg_t *msg = tor_malloc(sizeof(*msg)); - msg.type = ORCONN_MSGTYPE_STATUS; - msg.u.status.gid = conn->base_.global_identifier; - msg.u.status.status = tp; - msg.u.status.reason = reason; - orconn_event_publish(&msg); + msg->gid = conn->base_.global_identifier; + msg->status = tp; + msg->reason = reason; + orconn_status_publish(msg); control_event_or_conn_status(conn, tp, reason); } @@ -433,26 +432,25 @@ connection_or_event_status(or_connection_t *conn, or_conn_status_event_t tp, static void connection_or_state_publish(const or_connection_t *conn, uint8_t state) { - orconn_event_msg_t msg; + orconn_state_msg_t *msg = tor_malloc(sizeof(*msg)); - msg.type = ORCONN_MSGTYPE_STATE; - msg.u.state.gid = conn->base_.global_identifier; + msg->gid = conn->base_.global_identifier; if (conn->is_pt) { /* Do extra decoding because conn->proxy_type indicates the proxy * protocol that tor uses to talk with the transport plugin, * instead of PROXY_PLUGGABLE. */ tor_assert_nonfatal(conn->proxy_type != PROXY_NONE); - msg.u.state.proxy_type = PROXY_PLUGGABLE; + msg->proxy_type = PROXY_PLUGGABLE; } else { - msg.u.state.proxy_type = conn->proxy_type; + msg->proxy_type = conn->proxy_type; } - msg.u.state.state = state; + msg->state = state; if (conn->chan) { - msg.u.state.chan = TLS_CHAN_TO_BASE(conn->chan)->global_identifier; + msg->chan = TLS_CHAN_TO_BASE(conn->chan)->global_identifier; } else { - msg.u.state.chan = 0; + msg->chan = 0; } - orconn_event_publish(&msg); + orconn_state_publish(msg); } /** Call this to change or_connection_t states, so the owning channel_tls_t can diff --git a/src/core/or/orconn_event.c b/src/core/or/orconn_event.c index 9fb34bd1ff..86f112fc09 100644 --- a/src/core/or/orconn_event.c +++ b/src/core/or/orconn_event.c @@ -17,65 +17,83 @@ **/ #include "core/or/or.h" +#include "lib/pubsub/pubsub.h" #include "lib/subsys/subsys.h" #define ORCONN_EVENT_PRIVATE #include "core/or/orconn_event.h" #include "core/or/orconn_event_sys.h" -/** List of subscribers */ -static smartlist_t *orconn_event_rcvrs; +DECLARE_PUBLISH(orconn_state); +DECLARE_PUBLISH(orconn_status); -/** Initialize subscriber list */ -static int -orconn_event_init(void) +static void +orconn_event_free(msg_aux_data_t u) { - orconn_event_rcvrs = smartlist_new(); + tor_free_(u.ptr); +} + +static char * +orconn_state_fmt(msg_aux_data_t u) +{ + orconn_state_msg_t *msg = (orconn_state_msg_t *)u.ptr; + char *s = NULL; + + tor_asprintf(&s, "", + msg->gid, msg->chan, msg->proxy_type, msg->state); + return s; +} + +static char * +orconn_status_fmt(msg_aux_data_t u) +{ + orconn_status_msg_t *msg = (orconn_status_msg_t *)u.ptr; + char *s = NULL; + + tor_asprintf(&s, "", + msg->gid, msg->status, msg->reason); + return s; +} + +static dispatch_typefns_t orconn_state_fns = { + .free_fn = orconn_event_free, + .fmt_fn = orconn_state_fmt, +}; + +static dispatch_typefns_t orconn_status_fns = { + .free_fn = orconn_event_free, + .fmt_fn = orconn_status_fmt, +}; + +static int +orconn_add_pubsub(struct pubsub_connector_t *connector) +{ + if (DISPATCH_REGISTER_TYPE(connector, orconn_state, &orconn_state_fns)) + return -1; + if (DISPATCH_REGISTER_TYPE(connector, orconn_status, &orconn_status_fns)) + return -1; + if (DISPATCH_ADD_PUB(connector, orconn, orconn_state) != 0) + return -1; + if (DISPATCH_ADD_PUB(connector, orconn, orconn_status) != 0) + return -1; return 0; } -/** Free subscriber list */ -static void -orconn_event_fini(void) +void +orconn_state_publish(orconn_state_msg_t *msg) { - smartlist_free(orconn_event_rcvrs); + PUBLISH(orconn_state, msg); } -/** - * Subscribe to messages about OR connection events - * - * Register a callback function to receive messages about ORCONNs. - * The publisher calls this function synchronously. - **/ void -orconn_event_subscribe(orconn_event_rcvr_t fn) +orconn_status_publish(orconn_status_msg_t *msg) { - tor_assert(fn); - /* Don't duplicate subscriptions. */ - if (smartlist_contains(orconn_event_rcvrs, fn)) - return; - - smartlist_add(orconn_event_rcvrs, fn); -} - -/** - * Publish a message about OR connection events - * - * This calls the subscriber receiver function synchronously. - **/ -void -orconn_event_publish(const orconn_event_msg_t *msg) -{ - SMARTLIST_FOREACH_BEGIN(orconn_event_rcvrs, orconn_event_rcvr_t, fn) { - tor_assert(fn); - (*fn)(msg); - } SMARTLIST_FOREACH_END(fn); + PUBLISH(orconn_status, msg); } const subsys_fns_t sys_orconn_event = { .name = "orconn_event", .supported = true, .level = -33, - .initialize = orconn_event_init, - .shutdown = orconn_event_fini, + .add_pubsub = orconn_add_pubsub, }; diff --git a/src/core/or/orconn_event.h b/src/core/or/orconn_event.h index 80289d53e6..a3b37d2c3d 100644 --- a/src/core/or/orconn_event.h +++ b/src/core/or/orconn_event.h @@ -16,6 +16,8 @@ #ifndef TOR_ORCONN_EVENT_H #define TOR_ORCONN_EVENT_H +#include "lib/pubsub/pubsub.h" + /** * @name States of OR connections * @@ -62,12 +64,6 @@ typedef enum or_conn_status_event_t { OR_CONN_EVENT_NEW = 4, } or_conn_status_event_t; -/** Discriminant values for orconn event message */ -typedef enum orconn_msgtype_t { - ORCONN_MSGTYPE_STATE, - ORCONN_MSGTYPE_STATUS, -} orconn_msgtype_t; - /** * Message for orconn state update * @@ -83,6 +79,8 @@ typedef struct orconn_state_msg_t { uint8_t state; /**< new connection state */ } orconn_state_msg_t; +DECLARE_MESSAGE(orconn_state, orconn_state, orconn_state_msg_t *); + /** * Message for orconn status event * @@ -95,26 +93,11 @@ typedef struct orconn_status_msg_t { int reason; /**< reason */ } orconn_status_msg_t; -/** Discriminated union for the actual message */ -typedef struct orconn_event_msg_t { - int type; - union { - orconn_state_msg_t state; - orconn_status_msg_t status; - } u; -} orconn_event_msg_t; - -/** - * Receiver function pointer for OR subscribers - * - * This function gets called synchronously by the publisher. - **/ -typedef void (*orconn_event_rcvr_t)(const orconn_event_msg_t *); - -void orconn_event_subscribe(orconn_event_rcvr_t); +DECLARE_MESSAGE(orconn_status, orconn_status, orconn_status_msg_t *); #ifdef ORCONN_EVENT_PRIVATE -void orconn_event_publish(const orconn_event_msg_t *); +void orconn_state_publish(orconn_state_msg_t *); +void orconn_status_publish(orconn_status_msg_t *); #endif #endif /* defined(TOR_ORCONN_EVENT_H) */ diff --git a/src/feature/control/btrack.c b/src/feature/control/btrack.c index d3d12cb2b7..3a6ae07881 100644 --- a/src/feature/control/btrack.c +++ b/src/feature/control/btrack.c @@ -24,6 +24,7 @@ #include "feature/control/btrack_circuit.h" #include "feature/control/btrack_orconn.h" #include "feature/control/btrack_sys.h" +#include "lib/pubsub/pubsub.h" #include "lib/subsys/subsys.h" static int @@ -44,10 +45,17 @@ btrack_fini(void) btrack_circ_fini(); } +static int +btrack_add_pubsub(pubsub_connector_t *connector) +{ + return btrack_orconn_add_pubsub(connector); +} + const subsys_fns_t sys_btrack = { .name = "btrack", .supported = true, .level = -30, .initialize = btrack_init, .shutdown = btrack_fini, + .add_pubsub = btrack_add_pubsub, }; diff --git a/src/feature/control/btrack_orconn.c b/src/feature/control/btrack_orconn.c index 93ebe8d9cc..cbeb7b4ff1 100644 --- a/src/feature/control/btrack_orconn.c +++ b/src/feature/control/btrack_orconn.c @@ -45,6 +45,10 @@ #include "feature/control/btrack_orconn_cevent.h" #include "feature/control/btrack_orconn_maps.h" #include "lib/log/log.h" +#include "lib/pubsub/pubsub.h" + +DECLARE_SUBSCRIBE(orconn_state, bto_state_rcvr); +DECLARE_SUBSCRIBE(orconn_status, bto_status_rcvr); /** Pair of a best ORCONN GID and with its state */ typedef struct bto_best_t { @@ -110,16 +114,17 @@ bto_reset_bests(void) * message comes from code in connection_or.c. **/ static void -bto_state_rcvr(const orconn_state_msg_t *msg) +bto_state_rcvr(const msg_t *msg, const orconn_state_msg_t *arg) { bt_orconn_t *bto; - bto = bto_find_or_new(msg->gid, msg->chan); + (void)msg; + bto = bto_find_or_new(arg->gid, arg->chan); log_debug(LD_BTRACK, "ORCONN gid=%"PRIu64" chan=%"PRIu64 " proxy_type=%d state=%d", - msg->gid, msg->chan, msg->proxy_type, msg->state); - bto->proxy_type = msg->proxy_type; - bto->state = msg->state; + arg->gid, arg->chan, arg->proxy_type, arg->state); + bto->proxy_type = arg->proxy_type; + bto->state = arg->state; if (bto->is_orig) bto_update_bests(bto); } @@ -130,33 +135,20 @@ bto_state_rcvr(const orconn_state_msg_t *msg) * control.c. **/ static void -bto_status_rcvr(const orconn_status_msg_t *msg) +bto_status_rcvr(const msg_t *msg, const orconn_status_msg_t *arg) { - switch (msg->status) { + (void)msg; + switch (arg->status) { case OR_CONN_EVENT_FAILED: case OR_CONN_EVENT_CLOSED: log_info(LD_BTRACK, "ORCONN DELETE gid=%"PRIu64" status=%d reason=%d", - msg->gid, msg->status, msg->reason); - return bto_delete(msg->gid); + arg->gid, arg->status, arg->reason); + return bto_delete(arg->gid); default: break; } } -/** Dispatch to individual ORCONN message handlers */ -static void -bto_event_rcvr(const orconn_event_msg_t *msg) -{ - switch (msg->type) { - case ORCONN_MSGTYPE_STATE: - return bto_state_rcvr(&msg->u.state); - case ORCONN_MSGTYPE_STATUS: - return bto_status_rcvr(&msg->u.status); - default: - tor_assert(false); - } -} - /** * Create or update a cached ORCONN state for a newly launched * connection, including whether it's launched by an origin circuit @@ -190,12 +182,21 @@ int btrack_orconn_init(void) { bto_init_maps(); - orconn_event_subscribe(bto_event_rcvr); ocirc_event_subscribe(bto_chan_rcvr); return 0; } +int +btrack_orconn_add_pubsub(pubsub_connector_t *connector) +{ + if (DISPATCH_ADD_SUB(connector, orconn, orconn_state)) + return -1; + if (DISPATCH_ADD_SUB(connector, orconn, orconn_status)) + return -1; + return 0; +} + /** Clear the hash maps and reset the "best" states */ void btrack_orconn_fini(void) diff --git a/src/feature/control/btrack_orconn.h b/src/feature/control/btrack_orconn.h index 6ab4892a78..fed9a58eb0 100644 --- a/src/feature/control/btrack_orconn.h +++ b/src/feature/control/btrack_orconn.h @@ -9,6 +9,8 @@ #ifndef TOR_BTRACK_ORCONN_H #define TOR_BTRACK_ORCONN_H +#include "lib/pubsub/pubsub.h" + #ifdef BTRACK_ORCONN_PRIVATE #include "ht.h" @@ -33,6 +35,7 @@ typedef struct bt_orconn_t { #endif /* defined(BTRACK_ORCONN_PRIVATE) */ int btrack_orconn_init(void); +int btrack_orconn_add_pubsub(pubsub_connector_t *); void btrack_orconn_fini(void); #endif /* defined(TOR_BTRACK_ORCONN_H) */ diff --git a/src/test/test_btrack.c b/src/test/test_btrack.c index 48486fb5a1..fef1da4842 100644 --- a/src/test/test_btrack.c +++ b/src/test/test_btrack.c @@ -4,6 +4,7 @@ #include "core/or/or.h" #include "test/test.h" +#include "test_helpers.h" #include "test/log_test_helpers.h" #define OCIRC_EVENT_PRIVATE @@ -11,21 +12,38 @@ #include "core/or/ocirc_event.h" #include "core/or/orconn_event.h" +static void +send_state(const orconn_state_msg_t *msg_in) +{ + orconn_state_msg_t *msg = tor_malloc(sizeof(*msg)); + + *msg = *msg_in; + orconn_state_publish(msg); +} + +static void +send_status(const orconn_status_msg_t *msg_in) +{ + orconn_status_msg_t *msg = tor_malloc(sizeof(*msg)); + + *msg = *msg_in; + orconn_status_publish(msg); +} + static void test_btrack_launch(void *arg) { - orconn_event_msg_t conn; + orconn_state_msg_t conn; ocirc_event_msg_t circ; (void)arg; - conn.type = ORCONN_MSGTYPE_STATE; - conn.u.state.gid = 1; - conn.u.state.chan = 1; - conn.u.state.proxy_type = PROXY_NONE; - conn.u.state.state = OR_CONN_STATE_CONNECTING; + conn.gid = 1; + conn.chan = 1; + conn.proxy_type = PROXY_NONE; + conn.state = OR_CONN_STATE_CONNECTING; setup_full_capture_of_logs(LOG_DEBUG); - orconn_event_publish(&conn); + send_state(&conn); expect_log_msg_containing("ORCONN gid=1 chan=1 proxy_type=0 state=1"); expect_no_log_msg_containing("ORCONN BEST_"); teardown_capture_of_logs(); @@ -40,11 +58,11 @@ test_btrack_launch(void *arg) expect_log_msg_containing("ORCONN BEST_ANY state -1->1 gid=1"); teardown_capture_of_logs(); - conn.u.state.gid = 2; - conn.u.state.chan = 2; + conn.gid = 2; + conn.chan = 2; setup_full_capture_of_logs(LOG_DEBUG); - orconn_event_publish(&conn); + send_state(&conn); expect_log_msg_containing("ORCONN gid=2 chan=2 proxy_type=0 state=1"); expect_no_log_msg_containing("ORCONN BEST_"); teardown_capture_of_logs(); @@ -65,27 +83,26 @@ test_btrack_launch(void *arg) static void test_btrack_delete(void *arg) { - orconn_event_msg_t conn; + orconn_state_msg_t state; + orconn_status_msg_t status; (void)arg; - conn.type = ORCONN_MSGTYPE_STATE; - conn.u.state.gid = 1; - conn.u.state.chan = 1; - conn.u.state.proxy_type = PROXY_NONE; - conn.u.state.state = OR_CONN_STATE_CONNECTING; + state.gid = 1; + state.chan = 1; + state.proxy_type = PROXY_NONE; + state.state = OR_CONN_STATE_CONNECTING; setup_full_capture_of_logs(LOG_DEBUG); - orconn_event_publish(&conn); + send_state(&state); expect_log_msg_containing("ORCONN gid=1 chan=1 proxy_type=0"); teardown_capture_of_logs(); - conn.type = ORCONN_MSGTYPE_STATUS; - conn.u.status.gid = 1; - conn.u.status.status = OR_CONN_EVENT_CLOSED; - conn.u.status.reason = 0; + status.gid = 1; + status.status = OR_CONN_EVENT_CLOSED; + status.reason = 0; setup_full_capture_of_logs(LOG_DEBUG); - orconn_event_publish(&conn); + send_status(&status); expect_log_msg_containing("ORCONN DELETE gid=1 status=3 reason=0"); teardown_capture_of_logs(); @@ -94,7 +111,7 @@ test_btrack_delete(void *arg) } struct testcase_t btrack_tests[] = { - { "launch", test_btrack_launch, TT_FORK, 0, NULL }, - { "delete", test_btrack_delete, TT_FORK, 0, NULL }, + { "launch", test_btrack_launch, TT_FORK, &helper_pubsub_setup, NULL }, + { "delete", test_btrack_delete, TT_FORK, &helper_pubsub_setup, NULL }, END_OF_TESTCASES }; diff --git a/src/test/test_controller_events.c b/src/test/test_controller_events.c index 910aacace3..14fe4fd661 100644 --- a/src/test/test_controller_events.c +++ b/src/test/test_controller_events.c @@ -7,6 +7,7 @@ #define CONTROL_EVENTS_PRIVATE #define OCIRC_EVENT_PRIVATE #define ORCONN_EVENT_PRIVATE +#include "app/main/subsysmgr.h" #include "core/or/or.h" #include "core/or/channel.h" #include "core/or/channeltls.h" @@ -16,6 +17,7 @@ #include "core/mainloop/connection.h" #include "feature/control/control_events.h" #include "test/test.h" +#include "test/test_helpers.h" #include "core/or/or_circuit_st.h" #include "core/or/origin_circuit_st.h" @@ -394,20 +396,22 @@ test_cntev_dirboot_defer_orconn(void *arg) } static void -setup_orconn_state(orconn_event_msg_t *msg, uint64_t gid, uint64_t chan, +setup_orconn_state(orconn_state_msg_t *msg, uint64_t gid, uint64_t chan, int proxy_type) { - msg->type = ORCONN_MSGTYPE_STATE; - msg->u.state.gid = gid; - msg->u.state.chan = chan; - msg->u.state.proxy_type = proxy_type; + msg->gid = gid; + msg->chan = chan; + msg->proxy_type = proxy_type; } static void -send_orconn_state(orconn_event_msg_t *msg, uint8_t state) +send_orconn_state(const orconn_state_msg_t *msg_in, uint8_t state) { - msg->u.state.state = state; - orconn_event_publish(msg); + orconn_state_msg_t *msg = tor_malloc(sizeof(*msg)); + + *msg = *msg_in; + msg->state = state; + orconn_state_publish(msg); } static void @@ -425,7 +429,7 @@ send_ocirc_chan(uint32_t gid, uint64_t chan, bool onehop) static void test_cntev_orconn_state(void *arg) { - orconn_event_msg_t conn; + orconn_state_msg_t conn; (void)arg; MOCK(queue_control_event_string, mock_queue_control_event_string); @@ -442,8 +446,8 @@ test_cntev_orconn_state(void *arg) send_orconn_state(&conn, OR_CONN_STATE_OPEN); assert_bootmsg("15 TAG=handshake_done"); - conn.u.state.gid = 2; - conn.u.state.chan = 2; + conn.gid = 2; + conn.chan = 2; send_orconn_state(&conn, OR_CONN_STATE_CONNECTING); /* It doesn't know it's an origin circuit yet */ assert_bootmsg("15 TAG=handshake_done"); @@ -464,7 +468,7 @@ test_cntev_orconn_state(void *arg) static void test_cntev_orconn_state_pt(void *arg) { - orconn_event_msg_t conn; + orconn_state_msg_t conn; (void)arg; MOCK(queue_control_event_string, mock_queue_control_event_string); @@ -484,8 +488,8 @@ test_cntev_orconn_state_pt(void *arg) assert_bootmsg("15 TAG=handshake_done"); send_ocirc_chan(2, 2, false); - conn.u.state.gid = 2; - conn.u.state.chan = 2; + conn.gid = 2; + conn.chan = 2; send_orconn_state(&conn, OR_CONN_STATE_CONNECTING); assert_bootmsg("76 TAG=ap_conn_pt"); send_orconn_state(&conn, OR_CONN_STATE_PROXY_HANDSHAKING); @@ -499,7 +503,7 @@ test_cntev_orconn_state_pt(void *arg) static void test_cntev_orconn_state_proxy(void *arg) { - orconn_event_msg_t conn; + orconn_state_msg_t conn; (void)arg; MOCK(queue_control_event_string, mock_queue_control_event_string); @@ -519,8 +523,8 @@ test_cntev_orconn_state_proxy(void *arg) assert_bootmsg("15 TAG=handshake_done"); send_ocirc_chan(2, 2, false); - conn.u.state.gid = 2; - conn.u.state.chan = 2; + conn.gid = 2; + conn.chan = 2; send_orconn_state(&conn, OR_CONN_STATE_CONNECTING); assert_bootmsg("78 TAG=ap_conn_proxy"); send_orconn_state(&conn, OR_CONN_STATE_PROXY_HANDSHAKING); @@ -534,15 +538,18 @@ test_cntev_orconn_state_proxy(void *arg) #define TEST(name, flags) \ { #name, test_cntev_ ## name, flags, 0, NULL } +#define T_PUBSUB(name, setup) \ + { #name, test_cntev_ ## name, TT_FORK, &helper_pubsub_setup, NULL } + struct testcase_t controller_event_tests[] = { TEST(sum_up_cell_stats, TT_FORK), TEST(append_cell_stats, TT_FORK), TEST(format_cell_stats, TT_FORK), TEST(event_mask, TT_FORK), - TEST(dirboot_defer_desc, TT_FORK), - TEST(dirboot_defer_orconn, TT_FORK), - TEST(orconn_state, TT_FORK), - TEST(orconn_state_pt, TT_FORK), - TEST(orconn_state_proxy, TT_FORK), + T_PUBSUB(dirboot_defer_desc, TT_FORK), + T_PUBSUB(dirboot_defer_orconn, TT_FORK), + T_PUBSUB(orconn_state, TT_FORK), + T_PUBSUB(orconn_state_pt, TT_FORK), + T_PUBSUB(orconn_state_proxy, TT_FORK), END_OF_TESTCASES }; diff --git a/src/test/test_extorport.c b/src/test/test_extorport.c index 38aca90266..cb53a4e662 100644 --- a/src/test/test_extorport.c +++ b/src/test/test_extorport.c @@ -587,6 +587,6 @@ struct testcase_t extorport_tests[] = { { "cookie_auth", test_ext_or_cookie_auth, TT_FORK, NULL, NULL }, { "cookie_auth_testvec", test_ext_or_cookie_auth_testvec, TT_FORK, NULL, NULL }, - { "handshake", test_ext_or_handshake, TT_FORK, NULL, NULL }, + { "handshake", test_ext_or_handshake, TT_FORK, &helper_pubsub_setup, NULL }, END_OF_TESTCASES }; diff --git a/src/test/test_helpers.c b/src/test/test_helpers.c index 489c257761..e856dc6cca 100644 --- a/src/test/test_helpers.c +++ b/src/test/test_helpers.c @@ -17,12 +17,17 @@ #include "lib/buf/buffers.h" #include "app/config/config.h" #include "app/config/confparse.h" +#include "app/main/subsysmgr.h" #include "core/mainloop/connection.h" #include "lib/crypt_ops/crypto_rand.h" #include "core/mainloop/mainloop.h" #include "feature/nodelist/nodelist.h" #include "core/or/relay.h" #include "feature/nodelist/routerlist.h" +#include "lib/dispatch/dispatch.h" +#include "lib/dispatch/dispatch_naming.h" +#include "lib/pubsub/pubsub_build.h" +#include "lib/pubsub/pubsub_connect.h" #include "lib/encoding/confline.h" #include "lib/net/resolve.h" @@ -303,3 +308,52 @@ helper_parse_options(const char *conf) } return opt; } + +/** + * Dispatch alertfn callback: flush all messages right now. Implements + * DELIV_IMMEDIATE. + **/ +static void +alertfn_immediate(dispatch_t *d, channel_id_t chan, void *arg) +{ + (void) arg; + dispatch_flush(d, chan, INT_MAX); +} + +/** + * Setup helper for tests that need pubsub active + * + * Does not hook up mainloop events. Does set immediate delivery for + * all channels. + */ +void * +helper_setup_pubsub(const struct testcase_t *testcase) +{ + dispatch_t *dispatcher = NULL; + pubsub_builder_t *builder = pubsub_builder_new(); + channel_id_t chan = get_channel_id("orconn"); + + (void)testcase; + (void)subsystems_add_pubsub(builder); + dispatcher = pubsub_builder_finalize(builder, NULL); + tor_assert(dispatcher); + dispatch_set_alert_fn(dispatcher, chan, alertfn_immediate, NULL); + return dispatcher; +} + +/** + * Cleanup helper for tests that need pubsub active + */ +int +helper_cleanup_pubsub(const struct testcase_t *testcase, void *dispatcher_) +{ + dispatch_t *dispatcher = dispatcher_; + + (void)testcase; + dispatch_free(dispatcher); + return 1; +} + +const struct testcase_setup_t helper_pubsub_setup = { + helper_setup_pubsub, helper_cleanup_pubsub +}; diff --git a/src/test/test_helpers.h b/src/test/test_helpers.h index 9e376a563d..d82072bb34 100644 --- a/src/test/test_helpers.h +++ b/src/test/test_helpers.h @@ -7,6 +7,7 @@ #define BUFFERS_PRIVATE #include "core/or/or.h" +#include "tinytest.h" const char *get_yesterday_date_str(void); @@ -31,5 +32,10 @@ or_options_t *helper_parse_options(const char *conf); extern const char TEST_DESCRIPTORS[]; +void *helper_setup_pubsub(const struct testcase_t *); +int helper_cleanup_pubsub(const struct testcase_t *, void *); + +extern const struct testcase_setup_t helper_pubsub_setup; + #endif /* !defined(TOR_TEST_HELPERS_H) */ From 0bce0c339d5f4c6ddeb90c21502a9716fb6cd17a Mon Sep 17 00:00:00 2001 From: Taylor Yu Date: Thu, 28 Mar 2019 17:45:49 -0500 Subject: [PATCH 1122/2557] Rework origin circuit tracking to use pubsub Part of ticket 29976. --- src/app/main/main.c | 1 + src/core/or/circuitbuild.c | 11 ++- src/core/or/circuitlist.c | 33 ++++---- src/core/or/ocirc_event.c | 114 +++++++++++++++++++-------- src/core/or/ocirc_event.h | 35 +++----- src/feature/control/btrack.c | 9 ++- src/feature/control/btrack_circuit.c | 52 ++++++------ src/feature/control/btrack_circuit.h | 3 + src/feature/control/btrack_orconn.c | 19 +++-- src/test/test_btrack.c | 24 ++++-- src/test/test_circuitstats.c | 2 +- src/test/test_controller_events.c | 11 ++- src/test/test_helpers.c | 2 + 13 files changed, 179 insertions(+), 137 deletions(-) diff --git a/src/app/main/main.c b/src/app/main/main.c index 54f2afaa01..04a0cec19d 100644 --- a/src/app/main/main.c +++ b/src/app/main/main.c @@ -1257,6 +1257,7 @@ pubsub_connect(void) * this XXXX point, using tor_mainloop_set_delivery_strategy(). */ tor_mainloop_set_delivery_strategy("orconn", DELIV_IMMEDIATE); + tor_mainloop_set_delivery_strategy("ocirc", DELIV_IMMEDIATE); } } diff --git a/src/core/or/circuitbuild.c b/src/core/or/circuitbuild.c index 3a4e729429..ff809c01cf 100644 --- a/src/core/or/circuitbuild.c +++ b/src/core/or/circuitbuild.c @@ -522,14 +522,13 @@ origin_circuit_get_guard_state(origin_circuit_t *circ) static void circuit_chan_publish(const origin_circuit_t *circ, const channel_t *chan) { - ocirc_event_msg_t msg; + ocirc_chan_msg_t *msg = tor_malloc(sizeof(*msg)); - msg.type = OCIRC_MSGTYPE_CHAN; - msg.u.chan.gid = circ->global_identifier; - msg.u.chan.chan = chan->global_identifier; - msg.u.chan.onehop = circ->build_state->onehop_tunnel; + msg->gid = circ->global_identifier; + msg->chan = chan->global_identifier; + msg->onehop = circ->build_state->onehop_tunnel; - ocirc_event_publish(&msg); + ocirc_chan_publish(msg); } /** Start establishing the first hop of our circuit. Figure out what diff --git a/src/core/or/circuitlist.c b/src/core/or/circuitlist.c index 72952a8a52..55f04a77b9 100644 --- a/src/core/or/circuitlist.c +++ b/src/core/or/circuitlist.c @@ -496,17 +496,16 @@ int circuit_event_status(origin_circuit_t *circ, circuit_status_event_t tp, int reason_code) { - ocirc_event_msg_t msg; + ocirc_cevent_msg_t *msg = tor_malloc(sizeof(*msg)); tor_assert(circ); - msg.type = OCIRC_MSGTYPE_CEVENT; - msg.u.cevent.gid = circ->global_identifier; - msg.u.cevent.evtype = tp; - msg.u.cevent.reason = reason_code; - msg.u.cevent.onehop = circ->build_state->onehop_tunnel; + msg->gid = circ->global_identifier; + msg->evtype = tp; + msg->reason = reason_code; + msg->onehop = circ->build_state->onehop_tunnel; - ocirc_event_publish(&msg); + ocirc_cevent_publish(msg); return control_event_circuit_status(circ, tp, reason_code); } @@ -514,26 +513,25 @@ circuit_event_status(origin_circuit_t *circ, circuit_status_event_t tp, * Helper function to publish a state change message * * circuit_set_state() calls this to notify subscribers about a change - * of the state of an origin circuit. + * of the state of an origin circuit. @a circ must be an origin + * circuit. **/ static void circuit_state_publish(const circuit_t *circ) { - ocirc_event_msg_t msg; + ocirc_state_msg_t *msg = tor_malloc(sizeof(*msg)); const origin_circuit_t *ocirc; - if (!CIRCUIT_IS_ORIGIN(circ)) - return; + tor_assert(CIRCUIT_IS_ORIGIN(circ)); ocirc = CONST_TO_ORIGIN_CIRCUIT(circ); /* Only inbound OR circuits can be in this state, not origin circuits. */ tor_assert(circ->state != CIRCUIT_STATE_ONIONSKIN_PENDING); - msg.type = OCIRC_MSGTYPE_STATE; - msg.u.state.gid = ocirc->global_identifier; - msg.u.state.state = circ->state; - msg.u.state.onehop = ocirc->build_state->onehop_tunnel; + msg->gid = ocirc->global_identifier; + msg->state = circ->state; + msg->onehop = ocirc->build_state->onehop_tunnel; - ocirc_event_publish(&msg); + ocirc_state_publish(msg); } /** Change the state of circ to state, adding it to or removing @@ -565,7 +563,8 @@ circuit_set_state(circuit_t *circ, uint8_t state) if (state == CIRCUIT_STATE_GUARD_WAIT || state == CIRCUIT_STATE_OPEN) tor_assert(!circ->n_chan_create_cell); circ->state = state; - circuit_state_publish(circ); + if (CIRCUIT_IS_ORIGIN(circ)) + circuit_state_publish(circ); } /** Append to out all circuits in state CHAN_WAIT waiting for diff --git a/src/core/or/ocirc_event.c b/src/core/or/ocirc_event.c index 4a6fc748c9..3cb9147134 100644 --- a/src/core/or/ocirc_event.c +++ b/src/core/or/ocirc_event.c @@ -26,59 +26,103 @@ #include "core/or/origin_circuit_st.h" #include "lib/subsys/subsys.h" -/** List of subscribers */ -static smartlist_t *ocirc_event_rcvrs; +DECLARE_PUBLISH(ocirc_state); +DECLARE_PUBLISH(ocirc_chan); +DECLARE_PUBLISH(ocirc_cevent); -/** Initialize subscriber list */ -static int -ocirc_event_init(void) +static void +ocirc_event_free(msg_aux_data_t u) { - ocirc_event_rcvrs = smartlist_new(); + tor_free_(u.ptr); +} + +static char * +ocirc_state_fmt(msg_aux_data_t u) +{ + ocirc_state_msg_t *msg = (ocirc_state_msg_t *)u.ptr; + char *s = NULL; + + tor_asprintf(&s, "", + msg->gid, msg->state, msg->onehop); + return s; +} + +static char * +ocirc_chan_fmt(msg_aux_data_t u) +{ + ocirc_chan_msg_t *msg = (ocirc_chan_msg_t *)u.ptr; + char *s = NULL; + + tor_asprintf(&s, "", + msg->gid, msg->chan, msg->onehop); + return s; +} + +static char * +ocirc_cevent_fmt(msg_aux_data_t u) +{ + ocirc_cevent_msg_t *msg = (ocirc_cevent_msg_t *)u.ptr; + char *s = NULL; + + tor_asprintf(&s, "", + msg->gid, msg->evtype, msg->reason, msg->onehop); + return s; +} + +static dispatch_typefns_t ocirc_state_fns = { + .free_fn = ocirc_event_free, + .fmt_fn = ocirc_state_fmt, +}; + +static dispatch_typefns_t ocirc_chan_fns = { + .free_fn = ocirc_event_free, + .fmt_fn = ocirc_chan_fmt, +}; + +static dispatch_typefns_t ocirc_cevent_fns = { + .free_fn = ocirc_event_free, + .fmt_fn = ocirc_cevent_fmt, +}; + +static int +ocirc_add_pubsub(struct pubsub_connector_t *connector) +{ + if (DISPATCH_REGISTER_TYPE(connector, ocirc_state, ô_state_fns)) + return -1; + if (DISPATCH_REGISTER_TYPE(connector, ocirc_chan, ô_chan_fns)) + return -1; + if (DISPATCH_REGISTER_TYPE(connector, ocirc_cevent, ô_cevent_fns)) + return -1; + if (DISPATCH_ADD_PUB(connector, ocirc, ocirc_state)) + return -1; + if (DISPATCH_ADD_PUB(connector, ocirc, ocirc_chan)) + return -1; + if (DISPATCH_ADD_PUB(connector, ocirc, ocirc_cevent)) + return -1; return 0; } -/** Free subscriber list */ -static void -ocirc_event_fini(void) +void +ocirc_state_publish(ocirc_state_msg_t *msg) { - smartlist_free(ocirc_event_rcvrs); + PUBLISH(ocirc_state, msg); } -/** - * Subscribe to messages about origin circuit events - * - * Register a callback function to receive messages about origin - * circuits. The publisher calls this function synchronously. - **/ void -ocirc_event_subscribe(ocirc_event_rcvr_t fn) +ocirc_chan_publish(ocirc_chan_msg_t *msg) { - tor_assert(fn); - /* Don't duplicate subscriptions. */ - if (smartlist_contains(ocirc_event_rcvrs, fn)) - return; - - smartlist_add(ocirc_event_rcvrs, fn); + PUBLISH(ocirc_chan, msg); } -/** - * Publish a message about OR connection events - * - * This calls the subscriber receiver function synchronously. - **/ void -ocirc_event_publish(const ocirc_event_msg_t *msg) +ocirc_cevent_publish(ocirc_cevent_msg_t *msg) { - SMARTLIST_FOREACH_BEGIN(ocirc_event_rcvrs, ocirc_event_rcvr_t, fn) { - tor_assert(fn); - (*fn)(msg); - } SMARTLIST_FOREACH_END(fn); + PUBLISH(ocirc_cevent, msg); } const subsys_fns_t sys_ocirc_event = { .name = "ocirc_event", .supported = true, .level = -32, - .initialize = ocirc_event_init, - .shutdown = ocirc_event_fini, + .add_pubsub = ocirc_add_pubsub, }; diff --git a/src/core/or/ocirc_event.h b/src/core/or/ocirc_event.h index 0b125c2898..ad6d69ffbe 100644 --- a/src/core/or/ocirc_event.h +++ b/src/core/or/ocirc_event.h @@ -12,6 +12,7 @@ #include #include "lib/cc/torint.h" +#include "lib/pubsub/pubsub.h" /** Used to indicate the type of a circuit event passed to the controller. * The various types are defined in control-spec.txt */ @@ -30,6 +31,8 @@ typedef struct ocirc_state_msg_t { bool onehop; /**< one-hop circuit? */ } ocirc_state_msg_t; +DECLARE_MESSAGE(ocirc_state, ocirc_state, ocirc_state_msg_t *); + /** * Message when a channel gets associated to a circuit. * @@ -44,6 +47,8 @@ typedef struct ocirc_chan_msg_t { bool onehop; /**< one-hop circuit? */ } ocirc_chan_msg_t; +DECLARE_MESSAGE(ocirc_chan, ocirc_chan, ocirc_chan_msg_t *); + /** * Message for origin circuit status event * @@ -56,34 +61,12 @@ typedef struct ocirc_cevent_msg_t { bool onehop; /**< one-hop circuit? */ } ocirc_cevent_msg_t; -/** Discriminant values for origin circuit event message */ -typedef enum ocirc_msgtype_t { - OCIRC_MSGTYPE_STATE, - OCIRC_MSGTYPE_CHAN, - OCIRC_MSGTYPE_CEVENT, -} ocirc_msgtype_t; - -/** Discriminated union for the actual message */ -typedef struct ocirc_event_msg_t { - int type; - union { - ocirc_state_msg_t state; - ocirc_chan_msg_t chan; - ocirc_cevent_msg_t cevent; - } u; -} ocirc_event_msg_t; - -/** - * Receiver function pointer for origin circuit subscribers - * - * This function gets called synchronously by the publisher. - **/ -typedef void (*ocirc_event_rcvr_t)(const ocirc_event_msg_t *); - -void ocirc_event_subscribe(ocirc_event_rcvr_t fn); +DECLARE_MESSAGE(ocirc_cevent, ocirc_cevent, ocirc_cevent_msg_t *); #ifdef OCIRC_EVENT_PRIVATE -void ocirc_event_publish(const ocirc_event_msg_t *msg); +void ocirc_state_publish(ocirc_state_msg_t *msg); +void ocirc_chan_publish(ocirc_chan_msg_t *msg); +void ocirc_cevent_publish(ocirc_cevent_msg_t *msg); #endif #endif /* defined(TOR_OCIRC_EVENT_STATE_H) */ diff --git a/src/feature/control/btrack.c b/src/feature/control/btrack.c index 3a6ae07881..3ce97dc855 100644 --- a/src/feature/control/btrack.c +++ b/src/feature/control/btrack.c @@ -32,8 +32,6 @@ btrack_init(void) { if (btrack_orconn_init()) return -1; - if (btrack_circ_init()) - return -1; return 0; } @@ -48,7 +46,12 @@ btrack_fini(void) static int btrack_add_pubsub(pubsub_connector_t *connector) { - return btrack_orconn_add_pubsub(connector); + if (btrack_orconn_add_pubsub(connector)) + return -1; + if (btrack_circ_add_pubsub(connector)) + return -1; + + return 0; } const subsys_fns_t sys_btrack = { diff --git a/src/feature/control/btrack_circuit.c b/src/feature/control/btrack_circuit.c index dcee9e460e..2980c77ddc 100644 --- a/src/feature/control/btrack_circuit.c +++ b/src/feature/control/btrack_circuit.c @@ -109,51 +109,53 @@ btc_update_evtype(const ocirc_cevent_msg_t *msg, btc_best_t *best, return false; } +DECLARE_SUBSCRIBE(ocirc_state, btc_state_rcvr); +DECLARE_SUBSCRIBE(ocirc_cevent, btc_cevent_rcvr); +DECLARE_SUBSCRIBE(ocirc_chan, btc_chan_rcvr); + static void -btc_state_rcvr(const ocirc_state_msg_t *msg) +btc_state_rcvr(const msg_t *msg, const ocirc_state_msg_t *arg) { + (void)msg; log_debug(LD_BTRACK, "CIRC gid=%"PRIu32" state=%d onehop=%d", - msg->gid, msg->state, msg->onehop); + arg->gid, arg->state, arg->onehop); - btc_update_state(msg, &best_any_state, "ANY"); - if (msg->onehop) + btc_update_state(arg, &best_any_state, "ANY"); + if (arg->onehop) return; - btc_update_state(msg, &best_ap_state, "AP"); + btc_update_state(arg, &best_ap_state, "AP"); } static void -btc_cevent_rcvr(const ocirc_cevent_msg_t *msg) +btc_cevent_rcvr(const msg_t *msg, const ocirc_cevent_msg_t *arg) { + (void)msg; log_debug(LD_BTRACK, "CIRC gid=%"PRIu32" evtype=%d reason=%d onehop=%d", - msg->gid, msg->evtype, msg->reason, msg->onehop); + arg->gid, arg->evtype, arg->reason, arg->onehop); - btc_update_evtype(msg, &best_any_evtype, "ANY"); - if (msg->onehop) + btc_update_evtype(arg, &best_any_evtype, "ANY"); + if (arg->onehop) return; - btc_update_evtype(msg, &best_ap_evtype, "AP"); + btc_update_evtype(arg, &best_ap_evtype, "AP"); } static void -btc_event_rcvr(const ocirc_event_msg_t *msg) +btc_chan_rcvr(const msg_t *msg, const ocirc_chan_msg_t *arg) { - switch (msg->type) { - case OCIRC_MSGTYPE_STATE: - return btc_state_rcvr(&msg->u.state); - case OCIRC_MSGTYPE_CHAN: - log_debug(LD_BTRACK, "CIRC gid=%"PRIu32" chan=%"PRIu64" onehop=%d", - msg->u.chan.gid, msg->u.chan.chan, msg->u.chan.onehop); - break; - case OCIRC_MSGTYPE_CEVENT: - return btc_cevent_rcvr(&msg->u.cevent); - default: - break; - } + (void)msg; + log_debug(LD_BTRACK, "CIRC gid=%"PRIu32" chan=%"PRIu64" onehop=%d", + arg->gid, arg->chan, arg->onehop); } int -btrack_circ_init(void) +btrack_circ_add_pubsub(pubsub_connector_t *connector) { - ocirc_event_subscribe(btc_event_rcvr); + if (DISPATCH_ADD_SUB(connector, ocirc, ocirc_chan)) + return -1; + if (DISPATCH_ADD_SUB(connector, ocirc, ocirc_cevent)) + return -1; + if (DISPATCH_ADD_SUB(connector, ocirc, ocirc_state)) + return -1; return 0; } diff --git a/src/feature/control/btrack_circuit.h b/src/feature/control/btrack_circuit.h index c40822f1f1..b2ae6484f0 100644 --- a/src/feature/control/btrack_circuit.h +++ b/src/feature/control/btrack_circuit.h @@ -9,7 +9,10 @@ #ifndef TOR_BTRACK_CIRCUIT_H #define TOR_BTRACK_CIRCUIT_H +#include "lib/pubsub/pubsub.h" + int btrack_circ_init(void); void btrack_circ_fini(void); +int btrack_circ_add_pubsub(pubsub_connector_t *); #endif /* defined(TOR_BTRACK_CIRCUIT_H) */ diff --git a/src/feature/control/btrack_orconn.c b/src/feature/control/btrack_orconn.c index cbeb7b4ff1..922b542a0c 100644 --- a/src/feature/control/btrack_orconn.c +++ b/src/feature/control/btrack_orconn.c @@ -49,6 +49,7 @@ DECLARE_SUBSCRIBE(orconn_state, bto_state_rcvr); DECLARE_SUBSCRIBE(orconn_status, bto_status_rcvr); +DECLARE_SUBSCRIBE(ocirc_chan, bto_chan_rcvr); /** Pair of a best ORCONN GID and with its state */ typedef struct bto_best_t { @@ -155,21 +156,18 @@ bto_status_rcvr(const msg_t *msg, const orconn_status_msg_t *arg) * and whether it's a one-hop circuit. **/ static void -bto_chan_rcvr(const ocirc_event_msg_t *msg) +bto_chan_rcvr(const msg_t *msg, const ocirc_chan_msg_t *arg) { bt_orconn_t *bto; - /* Ignore other kinds of origin circuit events; we don't need them */ - if (msg->type != OCIRC_MSGTYPE_CHAN) - return; - - bto = bto_find_or_new(0, msg->u.chan.chan); - if (!bto->is_orig || (bto->is_onehop && !msg->u.chan.onehop)) { + (void)msg; + bto = bto_find_or_new(0, arg->chan); + if (!bto->is_orig || (bto->is_onehop && !arg->onehop)) { log_debug(LD_BTRACK, "ORCONN LAUNCH chan=%"PRIu64" onehop=%d", - msg->u.chan.chan, msg->u.chan.onehop); + arg->chan, arg->onehop); } bto->is_orig = true; - if (!msg->u.chan.onehop) + if (!arg->onehop) bto->is_onehop = false; bto_update_bests(bto); } @@ -182,7 +180,6 @@ int btrack_orconn_init(void) { bto_init_maps(); - ocirc_event_subscribe(bto_chan_rcvr); return 0; } @@ -194,6 +191,8 @@ btrack_orconn_add_pubsub(pubsub_connector_t *connector) return -1; if (DISPATCH_ADD_SUB(connector, orconn, orconn_status)) return -1; + if (DISPATCH_ADD_SUB(connector, ocirc, ocirc_chan)) + return -1; return 0; } diff --git a/src/test/test_btrack.c b/src/test/test_btrack.c index fef1da4842..9e5d0d0723 100644 --- a/src/test/test_btrack.c +++ b/src/test/test_btrack.c @@ -30,11 +30,20 @@ send_status(const orconn_status_msg_t *msg_in) orconn_status_publish(msg); } +static void +send_chan(const ocirc_chan_msg_t *msg_in) +{ + ocirc_chan_msg_t *msg = tor_malloc(sizeof(*msg)); + + *msg = *msg_in; + ocirc_chan_publish(msg); +} + static void test_btrack_launch(void *arg) { orconn_state_msg_t conn; - ocirc_event_msg_t circ; + ocirc_chan_msg_t circ; (void)arg; conn.gid = 1; @@ -48,12 +57,11 @@ test_btrack_launch(void *arg) expect_no_log_msg_containing("ORCONN BEST_"); teardown_capture_of_logs(); - circ.type = OCIRC_MSGTYPE_CHAN; - circ.u.chan.chan = 1; - circ.u.chan.onehop = true; + circ.chan = 1; + circ.onehop = true; setup_full_capture_of_logs(LOG_DEBUG); - ocirc_event_publish(&circ); + send_chan(&circ); expect_log_msg_containing("ORCONN LAUNCH chan=1 onehop=1"); expect_log_msg_containing("ORCONN BEST_ANY state -1->1 gid=1"); teardown_capture_of_logs(); @@ -67,11 +75,11 @@ test_btrack_launch(void *arg) expect_no_log_msg_containing("ORCONN BEST_"); teardown_capture_of_logs(); - circ.u.chan.chan = 2; - circ.u.chan.onehop = false; + circ.chan = 2; + circ.onehop = false; setup_full_capture_of_logs(LOG_DEBUG); - ocirc_event_publish(&circ); + send_chan(&circ); expect_log_msg_containing("ORCONN LAUNCH chan=2 onehop=0"); expect_log_msg_containing("ORCONN BEST_AP state -1->1 gid=2"); teardown_capture_of_logs(); diff --git a/src/test/test_circuitstats.c b/src/test/test_circuitstats.c index 2a09622f09..9bfaabeb2f 100644 --- a/src/test/test_circuitstats.c +++ b/src/test/test_circuitstats.c @@ -197,7 +197,7 @@ test_circuitstats_hoplen(void *arg) } #define TEST_CIRCUITSTATS(name, flags) \ - { #name, test_##name, (flags), NULL, NULL } + { #name, test_##name, (flags), &helper_pubsub_setup, NULL } struct testcase_t circuitstats_tests[] = { TEST_CIRCUITSTATS(circuitstats_hoplen, TT_FORK), diff --git a/src/test/test_controller_events.c b/src/test/test_controller_events.c index 14fe4fd661..a8967bba50 100644 --- a/src/test/test_controller_events.c +++ b/src/test/test_controller_events.c @@ -417,13 +417,12 @@ send_orconn_state(const orconn_state_msg_t *msg_in, uint8_t state) static void send_ocirc_chan(uint32_t gid, uint64_t chan, bool onehop) { - ocirc_event_msg_t msg; + ocirc_chan_msg_t *msg = tor_malloc(sizeof(*msg)); - msg.type = OCIRC_MSGTYPE_CHAN; - msg.u.chan.gid = gid; - msg.u.chan.chan = chan; - msg.u.chan.onehop = onehop; - ocirc_event_publish(&msg); + msg->gid = gid; + msg->chan = chan; + msg->onehop = onehop; + ocirc_chan_publish(msg); } static void diff --git a/src/test/test_helpers.c b/src/test/test_helpers.c index e856dc6cca..b4389f2d17 100644 --- a/src/test/test_helpers.c +++ b/src/test/test_helpers.c @@ -338,6 +338,8 @@ helper_setup_pubsub(const struct testcase_t *testcase) dispatcher = pubsub_builder_finalize(builder, NULL); tor_assert(dispatcher); dispatch_set_alert_fn(dispatcher, chan, alertfn_immediate, NULL); + chan = get_channel_id("ocirc"); + dispatch_set_alert_fn(dispatcher, chan, alertfn_immediate, NULL); return dispatcher; } From 5f5f6bb8fb68d171a39eb1e5c6e6649087ec551d Mon Sep 17 00:00:00 2001 From: Taylor Yu Date: Wed, 22 May 2019 16:33:04 -0500 Subject: [PATCH 1123/2557] Add changes file for 29976 --- changes/ticket29976 | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 changes/ticket29976 diff --git a/changes/ticket29976 b/changes/ticket29976 new file mode 100644 index 0000000000..9991bfb1ff --- /dev/null +++ b/changes/ticket29976 @@ -0,0 +1,3 @@ + o Code simplification and refactoring: + - Rework bootstrap tracking to use the new publish-subscribe + subsystem. Closes ticket 29976. From 16e71c7fb0f62ad9e9768987c32f397eef3247fb Mon Sep 17 00:00:00 2001 From: teor Date: Thu, 13 Jun 2019 18:23:50 +1000 Subject: [PATCH 1124/2557] practracker: accept an extra line in nt_service_install() Part of 30799. --- scripts/maint/practracker/exceptions.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/maint/practracker/exceptions.txt b/scripts/maint/practracker/exceptions.txt index eb0625b8d3..023f3feb2f 100644 --- a/scripts/maint/practracker/exceptions.txt +++ b/scripts/maint/practracker/exceptions.txt @@ -52,7 +52,7 @@ problem function-size /src/app/main/main.c:dumpstats() 102 problem function-size /src/app/main/main.c:tor_init() 137 problem function-size /src/app/main/main.c:sandbox_init_filter() 291 problem function-size /src/app/main/main.c:run_tor_main_loop() 105 -problem function-size /src/app/main/ntmain.c:nt_service_install() 125 +problem function-size /src/app/main/ntmain.c:nt_service_install() 126 problem file-size /src/core/mainloop/connection.c 5569 problem include-count /src/core/mainloop/connection.c 62 problem function-size /src/core/mainloop/connection.c:connection_free_minimal() 185 From e218da17229adcec3c0a4ce774c0963bfcf1f11e Mon Sep 17 00:00:00 2001 From: teor Date: Fri, 14 Jun 2019 11:44:26 +1000 Subject: [PATCH 1125/2557] make: Improve the documentation for test-network-all "make test-network-all" shows the warnings from each test-network.sh run on the console, so developers see new warnings early. Improve the documentation for this feature, and rename a Makefile variable so the code is self-documenting. Fixes bug 30455; bugfix on 0.3.0.4-rc. --- Makefile.am | 19 +++++++++++-------- changes/bug30455 | 5 +++++ 2 files changed, 16 insertions(+), 8 deletions(-) create mode 100644 changes/bug30455 diff --git a/Makefile.am b/Makefile.am index 7a0d40d6a5..333fcf2e0a 100644 --- a/Makefile.am +++ b/Makefile.am @@ -185,7 +185,7 @@ TEST_CFLAGS= TEST_CPPFLAGS=-DTOR_UNIT_TESTS @TOR_MODULES_ALL_ENABLED@ TEST_NETWORK_FLAGS=--hs-multi-client 1 endif -TEST_NETWORK_WARNING_FLAGS=--quiet --only-warnings +TEST_NETWORK_SHOW_WARNINGS_FOR_LAST_RUN_FLAGS=--quiet --only-warnings if LIBFUZZER_ENABLED TEST_CFLAGS += -fsanitize-coverage=trace-pc-guard,trace-cmp,trace-div @@ -250,12 +250,15 @@ test-network: need-chutney-path $(TESTING_TOR_BINARY) src/tools/tor-gencert $(top_srcdir)/src/test/test-network.sh $(TEST_NETWORK_FLAGS) # Run all available tests using automake's test-driver -# only run IPv6 tests if we can ping6 ::1 (localhost) -# only run IPv6 tests if we can ping ::1 (localhost) -# some IPv6 tests will fail without an IPv6 DNS server (see #16971 and #17011) -# only run mixed tests if we have a tor-stable binary -# Try the syntax for BSD ping6, Linux ping6, and Linux ping -6, -# because they're incompatible +# - only run IPv6 tests if we can ping6 or ping -6 ::1 (localhost) +# we try the syntax for BSD ping6, Linux ping6, and Linux ping -6, +# because they're incompatible +# - some IPv6 tests may fail without an IPv6 DNS server +# (see #16971 and #17011) +# - only run mixed tests if we have a tor-stable binary +# - show tor warnings on the console after each network run +# (otherwise, warnings go to the logs, and people don't see them unless +# there is a network failure) test-network-all: need-chutney-path test-driver $(TESTING_TOR_BINARY) src/tools/tor-gencert mkdir -p $(TEST_NETWORK_ALL_LOG_DIR) rm -f $(TEST_NETWORK_ALL_LOG_DIR)/*.log $(TEST_NETWORK_ALL_LOG_DIR)/*.trs @@ -279,7 +282,7 @@ test-network-all: need-chutney-path test-driver $(TESTING_TOR_BINARY) src/tools/ done; \ for f in $$flavors; do \ $(SHELL) $(top_srcdir)/test-driver --test-name $$f --log-file $(TEST_NETWORK_ALL_LOG_DIR)/$$f.log --trs-file $(TEST_NETWORK_ALL_LOG_DIR)/$$f.trs $(TEST_NETWORK_ALL_DRIVER_FLAGS) $(top_srcdir)/src/test/test-network.sh --flavor $$f $(TEST_NETWORK_FLAGS); \ - $(top_srcdir)/src/test/test-network.sh $(TEST_NETWORK_WARNING_FLAGS); \ + $(top_srcdir)/src/test/test-network.sh $(TEST_NETWORK_SHOW_WARNINGS_FOR_LAST_RUN_FLAGS); \ done; \ echo "Log and result files are available in $(TEST_NETWORK_ALL_LOG_DIR)."; \ ! grep -q FAIL $(TEST_NETWORK_ALL_LOG_DIR)/*.trs diff --git a/changes/bug30455 b/changes/bug30455 new file mode 100644 index 0000000000..aecbde5a33 --- /dev/null +++ b/changes/bug30455 @@ -0,0 +1,5 @@ + o Minor bugfixes (chutney, makefiles, documentation): + - "make test-network-all" shows the warnings from each test-network.sh + run on the console, so developers see new warnings early. Improve the + documentation for this feature, and rename a Makefile variable so the + code is self-documenting. Fixes bug 30455; bugfix on 0.3.0.4-rc. From 990b434c4f4aa5e9c410bb083c1b98cead92bb59 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 7 Jun 2019 20:03:26 -0400 Subject: [PATCH 1126/2557] Make evloop into a subsystem. Note that the event base object is _not_ created from the initialize function, since it is configuration-dependent. This will wait until configuration is integrated into subsystems. Closes ticket 30806. --- changes/ticket30806 | 3 ++ src/app/main/main.c | 4 --- src/app/main/shutdown.c | 2 -- src/app/main/subsystem_list.c | 3 ++ src/lib/evloop/.may_include | 1 + src/lib/evloop/evloop_sys.c | 49 +++++++++++++++++++++++++++++++++ src/lib/evloop/evloop_sys.h | 17 ++++++++++++ src/lib/evloop/include.am | 2 ++ src/test/test_channelpadding.c | 7 ----- src/test/test_compat_libevent.c | 2 -- 10 files changed, 75 insertions(+), 15 deletions(-) create mode 100644 changes/ticket30806 create mode 100644 src/lib/evloop/evloop_sys.c create mode 100644 src/lib/evloop/evloop_sys.h diff --git a/changes/ticket30806 b/changes/ticket30806 new file mode 100644 index 0000000000..4f09ea2af3 --- /dev/null +++ b/changes/ticket30806 @@ -0,0 +1,3 @@ + o Code simplification and refactoring: + - Use the subsystems mechanism to manage the main event loop code. + Closes ticket 30806. diff --git a/src/app/main/main.c b/src/app/main/main.c index 6e325f0b10..17b0000d98 100644 --- a/src/app/main/main.c +++ b/src/app/main/main.c @@ -653,10 +653,6 @@ tor_init(int argc, char *argv[]) return -1; } - if (tor_init_libevent_rng() < 0) { - log_warn(LD_NET, "Problem initializing libevent RNG."); - } - /* Scan/clean unparseable descriptors; after reading config */ routerparse_init(); diff --git a/src/app/main/shutdown.c b/src/app/main/shutdown.c index cc0091a9ab..93d6351d1b 100644 --- a/src/app/main/shutdown.c +++ b/src/app/main/shutdown.c @@ -160,8 +160,6 @@ tor_free_all(int postfork) subsystems_shutdown(); - tor_libevent_free_all(); - /* Stuff in util.c and address.c*/ if (!postfork) { esc_router_info(NULL); diff --git a/src/app/main/subsystem_list.c b/src/app/main/subsystem_list.c index f595796232..95d96f78d2 100644 --- a/src/app/main/subsystem_list.c +++ b/src/app/main/subsystem_list.c @@ -25,6 +25,7 @@ #include "lib/time/time_sys.h" #include "lib/tls/tortls_sys.h" #include "lib/wallclock/wallclock_sys.h" +#include "lib/evloop/evloop_sys.h" #include "feature/dirauth/dirauth_sys.h" @@ -50,6 +51,8 @@ const subsys_fns_t *tor_subsystems[] = { &sys_ocirc_event, /* -32 */ &sys_btrack, /* -30 */ + &sys_evloop, /* -20 */ + &sys_mainloop, /* 5 */ &sys_or, /* 20 */ diff --git a/src/lib/evloop/.may_include b/src/lib/evloop/.may_include index 273de7bb94..54aa75fbff 100644 --- a/src/lib/evloop/.may_include +++ b/src/lib/evloop/.may_include @@ -8,6 +8,7 @@ lib/log/*.h lib/malloc/*.h lib/net/*.h lib/string/*.h +lib/subsys/*.h lib/testsupport/*.h lib/thread/*.h lib/time/*.h diff --git a/src/lib/evloop/evloop_sys.c b/src/lib/evloop/evloop_sys.c new file mode 100644 index 0000000000..56641a3175 --- /dev/null +++ b/src/lib/evloop/evloop_sys.c @@ -0,0 +1,49 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file evloop_sys.c + * @brief Subsystem definition for the event loop module + **/ + +#include "orconfig.h" +#include "lib/subsys/subsys.h" +#include "lib/evloop/compat_libevent.h" +#include "lib/evloop/evloop_sys.h" +#include "lib/log/log.h" + +static int +subsys_evloop_initialize(void) +{ + if (tor_init_libevent_rng() < 0) { + log_warn(LD_NET, "Problem initializing libevent RNG."); + return -1; + } + return 0; +} + +static void +subsys_evloop_postfork(void) +{ +#ifdef TOR_UNIT_TESTS + tor_libevent_postfork(); +#endif +} + +static void +subsys_evloop_shutdown(void) +{ + tor_libevent_free_all(); +} + +const struct subsys_fns_t sys_evloop = { + .name = "evloop", + .supported = true, + .level = -20, + .initialize = subsys_evloop_initialize, + .shutdown = subsys_evloop_shutdown, + .postfork = subsys_evloop_postfork, +}; diff --git a/src/lib/evloop/evloop_sys.h b/src/lib/evloop/evloop_sys.h new file mode 100644 index 0000000000..e6155c25b0 --- /dev/null +++ b/src/lib/evloop/evloop_sys.h @@ -0,0 +1,17 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file evloop_sys.h + * @brief Declare subsystem object for the event loop module. + **/ + +#ifndef TOR_LIB_EVLOOP_EVLOOP_SYS_H +#define TOR_LIB_EVLOOP_EVLOOP_SYS_H + +extern const struct subsys_fns_t sys_evloop; + +#endif /* !defined(TOR_LIB_EVLOOP_EVLOOP_SYS_H) */ diff --git a/src/lib/evloop/include.am b/src/lib/evloop/include.am index 6595b3a34b..41cd2f45c5 100644 --- a/src/lib/evloop/include.am +++ b/src/lib/evloop/include.am @@ -8,6 +8,7 @@ endif # ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_evloop_a_SOURCES = \ src/lib/evloop/compat_libevent.c \ + src/lib/evloop/evloop_sys.c \ src/lib/evloop/procmon.c \ src/lib/evloop/timers.c \ src/lib/evloop/token_bucket.c \ @@ -21,6 +22,7 @@ src_lib_libtor_evloop_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) # ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/evloop/compat_libevent.h \ + src/lib/evloop/evloop_sys.h \ src/lib/evloop/procmon.h \ src/lib/evloop/timers.h \ src/lib/evloop/token_bucket.h \ diff --git a/src/test/test_channelpadding.c b/src/test/test_channelpadding.c index 5d012e462b..885246628e 100644 --- a/src/test/test_channelpadding.c +++ b/src/test/test_channelpadding.c @@ -289,8 +289,6 @@ test_channelpadding_timers(void *arg) channel_t *chans[CHANNELS_TO_TEST]; (void)arg; - tor_libevent_postfork(); - if (!connection_array) connection_array = smartlist_new(); @@ -393,7 +391,6 @@ test_channelpadding_killonehop(void *arg) channelpadding_decision_t decision; int64_t new_time; (void)arg; - tor_libevent_postfork(); routerstatus_t *relay = tor_malloc_zero(sizeof(routerstatus_t)); monotime_init(); @@ -502,8 +499,6 @@ test_channelpadding_consensus(void *arg) int64_t new_time; (void)arg; - tor_libevent_postfork(); - /* * Params tested: * nf_pad_before_usage @@ -898,8 +893,6 @@ test_channelpadding_decide_to_pad_channel(void *arg) connection_array = smartlist_new(); (void)arg; - tor_libevent_postfork(); - monotime_init(); monotime_enable_test_mocking(); monotime_set_mock_time_nsec(1); diff --git a/src/test/test_compat_libevent.c b/src/test/test_compat_libevent.c index 5d625483da..ecd97e3474 100644 --- a/src/test/test_compat_libevent.c +++ b/src/test/test_compat_libevent.c @@ -151,8 +151,6 @@ test_compat_libevent_postloop_events(void *arg) mainloop_event_t *a = NULL, *b = NULL; periodic_timer_t *timed = NULL; - tor_libevent_postfork(); - /* If postloop events don't work, then these events will activate one * another ad infinitum and, and the periodic event will never occur. */ b = mainloop_event_postloop_new(activate_event_cb, &a); From ac5e44d9ce9d07c05b4ba0cb21f058a6322692ec Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 12 Jun 2019 13:49:47 -0400 Subject: [PATCH 1127/2557] Renaming: CONFIG_TYPE_UINT -> CONFIG_TYPE_POSINT This name has been a historical source of confusion, since "uint" usually suggests "unsigned int" to people, when the real type is "nonnegative int". --- src/app/config/config.c | 30 +++++++++++------------ src/app/config/confparse.c | 6 ++--- src/app/config/statefile.c | 14 +++++------ src/feature/dirauth/shared_random_state.c | 2 +- 4 files changed, 26 insertions(+), 26 deletions(-) diff --git a/src/app/config/config.c b/src/app/config/config.c index 6bcf511565..847355dada 100644 --- a/src/app/config/config.c +++ b/src/app/config/config.c @@ -317,7 +317,7 @@ static config_var_t option_vars_[] = { OBSOLETE("AuthDirRejectUnlisted"), OBSOLETE("AuthDirListBadDirs"), V(AuthDirListBadExits, BOOL, "0"), - V(AuthDirMaxServersPerAddr, UINT, "2"), + V(AuthDirMaxServersPerAddr, POSINT, "2"), OBSOLETE("AuthDirMaxServersPerAuthAddr"), V(AuthDirHasIPv6Connectivity, BOOL, "0"), VAR("AuthoritativeDirectory", BOOL, AuthoritativeDir, "0"), @@ -352,7 +352,7 @@ static config_var_t option_vars_[] = { V(ClientUseIPv6, BOOL, "0"), V(ClientUseIPv4, BOOL, "1"), V(ConsensusParams, STRING, NULL), - V(ConnLimit, UINT, "1000"), + V(ConnLimit, POSINT, "1000"), V(ConnDirectionStatistics, BOOL, "0"), V(ConstrainedSockets, BOOL, "0"), V(ConstrainedSockSize, MEMUNIT, "8192"), @@ -402,14 +402,14 @@ static config_var_t option_vars_[] = { V(DormantCanceledByStartup, BOOL, "0"), /* DoS circuit creation options. */ V(DoSCircuitCreationEnabled, AUTOBOOL, "auto"), - V(DoSCircuitCreationMinConnections, UINT, "0"), - V(DoSCircuitCreationRate, UINT, "0"), - V(DoSCircuitCreationBurst, UINT, "0"), + V(DoSCircuitCreationMinConnections, POSINT, "0"), + V(DoSCircuitCreationRate, POSINT, "0"), + V(DoSCircuitCreationBurst, POSINT, "0"), V(DoSCircuitCreationDefenseType, INT, "0"), V(DoSCircuitCreationDefenseTimePeriod, INTERVAL, "0"), /* DoS connection options. */ V(DoSConnectionEnabled, AUTOBOOL, "auto"), - V(DoSConnectionMaxConcurrentCount, UINT, "0"), + V(DoSConnectionMaxConcurrentCount, POSINT, "0"), V(DoSConnectionDefenseType, INT, "0"), /* DoS single hop client options. */ V(DoSRefuseSingleHopClientRendezvous, AUTOBOOL, "auto"), @@ -522,7 +522,7 @@ static config_var_t option_vars_[] = { VAR("MapAddress", LINELIST, AddressMap, NULL), V(MaxAdvertisedBandwidth, MEMUNIT, "1 GB"), V(MaxCircuitDirtiness, INTERVAL, "10 minutes"), - V(MaxClientCircuitsPending, UINT, "32"), + V(MaxClientCircuitsPending, POSINT, "32"), V(MaxConsensusAgeForDiffs, INTERVAL, "0 seconds"), VAR("MaxMemInQueues", MEMUNIT, MaxMemInQueues_raw, "0"), OBSOLETE("MaxOnionsPending"), @@ -539,10 +539,10 @@ static config_var_t option_vars_[] = { OBSOLETE("WarnUnsafeSocks"), VAR("NodeFamily", LINELIST, NodeFamilies, NULL), V(NoExec, BOOL, "0"), - V(NumCPUs, UINT, "0"), - V(NumDirectoryGuards, UINT, "0"), - V(NumEntryGuards, UINT, "0"), - V(NumPrimaryGuards, UINT, "0"), + V(NumCPUs, POSINT, "0"), + V(NumDirectoryGuards, POSINT, "0"), + V(NumEntryGuards, POSINT, "0"), + V(NumPrimaryGuards, POSINT, "0"), V(OfflineMasterKey, BOOL, "0"), OBSOLETE("ORListenAddress"), VPORT(ORPort), @@ -666,7 +666,7 @@ static config_var_t option_vars_[] = { V(V3AuthVotingInterval, INTERVAL, "1 hour"), V(V3AuthVoteDelay, INTERVAL, "5 minutes"), V(V3AuthDistDelay, INTERVAL, "5 minutes"), - V(V3AuthNIntervalsValid, UINT, "3"), + V(V3AuthNIntervalsValid, POSINT, "3"), V(V3AuthUseLegacyKey, BOOL, "0"), V(V3BandwidthsFile, FILENAME, NULL), V(GuardfractionFile, FILENAME, NULL), @@ -715,7 +715,7 @@ static config_var_t option_vars_[] = { * blocked), but we also don't want to fail if only some mirrors are * blackholed. Clients will try 3 directories simultaneously. * (Relays never use simultaneous connections.) */ - V(ClientBootstrapConsensusMaxInProgressTries, UINT, "3"), + V(ClientBootstrapConsensusMaxInProgressTries, POSINT, "3"), /* When a client has any running bridges, check each bridge occasionally, * whether or not that bridge is actually up. */ V(TestingBridgeDownloadInitialDelay, CSV_INTERVAL,"10800"), @@ -749,7 +749,7 @@ static const config_var_t testing_tor_network_defaults[] = { V(DirAllowPrivateAddresses, BOOL, "1"), V(EnforceDistinctSubnets, BOOL, "0"), V(AssumeReachable, BOOL, "1"), - V(AuthDirMaxServersPerAddr, UINT, "0"), + V(AuthDirMaxServersPerAddr, POSINT, "0"), V(ClientBootstrapConsensusAuthorityDownloadInitialDelay, CSV_INTERVAL, "0"), V(ClientBootstrapConsensusFallbackDownloadInitialDelay, CSV_INTERVAL, "0"), V(ClientBootstrapConsensusAuthorityOnlyDownloadInitialDelay, CSV_INTERVAL, @@ -8177,7 +8177,7 @@ getinfo_helper_config(control_connection_t *conn, switch (var->type) { case CONFIG_TYPE_STRING: type = "String"; break; case CONFIG_TYPE_FILENAME: type = "Filename"; break; - case CONFIG_TYPE_UINT: type = "Integer"; break; + case CONFIG_TYPE_POSINT: type = "Integer"; break; case CONFIG_TYPE_UINT64: type = "Integer"; break; case CONFIG_TYPE_INT: type = "SignedInteger"; break; case CONFIG_TYPE_PORT: type = "Port"; break; diff --git a/src/app/config/confparse.c b/src/app/config/confparse.c index 8681f648da..cf83df728a 100644 --- a/src/app/config/confparse.c +++ b/src/app/config/confparse.c @@ -181,7 +181,7 @@ config_assign_value(const config_format_t *fmt, void *options, } /* fall through */ case CONFIG_TYPE_INT: - case CONFIG_TYPE_UINT: + case CONFIG_TYPE_POSINT: i = (int)tor_parse_long(c->value, 10, var->type==CONFIG_TYPE_INT ? INT_MIN : 0, var->type==CONFIG_TYPE_PORT ? 65535 : INT_MAX, @@ -580,7 +580,7 @@ config_get_assigned_option(const config_format_t *fmt, const void *options, case CONFIG_TYPE_CSV_INTERVAL: case CONFIG_TYPE_INTERVAL: case CONFIG_TYPE_MSEC_INTERVAL: - case CONFIG_TYPE_UINT: + case CONFIG_TYPE_POSINT: case CONFIG_TYPE_INT: /* This means every or_options_t uint or bool element * needs to be an int. Not, say, a uint16_t or char. */ @@ -786,7 +786,7 @@ config_clear(const config_format_t *fmt, void *options, case CONFIG_TYPE_CSV_INTERVAL: case CONFIG_TYPE_INTERVAL: case CONFIG_TYPE_MSEC_INTERVAL: - case CONFIG_TYPE_UINT: + case CONFIG_TYPE_POSINT: case CONFIG_TYPE_INT: case CONFIG_TYPE_PORT: case CONFIG_TYPE_BOOL: diff --git a/src/app/config/statefile.c b/src/app/config/statefile.c index fdfd68b244..c6c5ec14f5 100644 --- a/src/app/config/statefile.c +++ b/src/app/config/statefile.c @@ -105,19 +105,19 @@ static config_var_t state_vars_[] = { V(HidServRevCounter, LINELIST, NULL), V(BWHistoryReadEnds, ISOTIME, NULL), - V(BWHistoryReadInterval, UINT, "900"), + V(BWHistoryReadInterval, POSINT, "900"), V(BWHistoryReadValues, CSV, ""), V(BWHistoryReadMaxima, CSV, ""), V(BWHistoryWriteEnds, ISOTIME, NULL), - V(BWHistoryWriteInterval, UINT, "900"), + V(BWHistoryWriteInterval, POSINT, "900"), V(BWHistoryWriteValues, CSV, ""), V(BWHistoryWriteMaxima, CSV, ""), V(BWHistoryDirReadEnds, ISOTIME, NULL), - V(BWHistoryDirReadInterval, UINT, "900"), + V(BWHistoryDirReadInterval, POSINT, "900"), V(BWHistoryDirReadValues, CSV, ""), V(BWHistoryDirReadMaxima, CSV, ""), V(BWHistoryDirWriteEnds, ISOTIME, NULL), - V(BWHistoryDirWriteInterval, UINT, "900"), + V(BWHistoryDirWriteInterval, POSINT, "900"), V(BWHistoryDirWriteValues, CSV, ""), V(BWHistoryDirWriteMaxima, CSV, ""), @@ -128,12 +128,12 @@ static config_var_t state_vars_[] = { V(LastRotatedOnionKey, ISOTIME, NULL), V(LastWritten, ISOTIME, NULL), - V(TotalBuildTimes, UINT, NULL), - V(CircuitBuildAbandonedCount, UINT, "0"), + V(TotalBuildTimes, POSINT, NULL), + V(CircuitBuildAbandonedCount, POSINT, "0"), VAR("CircuitBuildTimeBin", LINELIST_S, BuildtimeHistogram, NULL), VAR("BuildtimeHistogram", LINELIST_V, BuildtimeHistogram, NULL), - V(MinutesSinceUserActivity, UINT, NULL), + V(MinutesSinceUserActivity, POSINT, NULL), V(Dormant, AUTOBOOL, "auto"), END_OF_CONFIG_VARS diff --git a/src/feature/dirauth/shared_random_state.c b/src/feature/dirauth/shared_random_state.c index b669e3836e..b2c7acba1a 100644 --- a/src/feature/dirauth/shared_random_state.c +++ b/src/feature/dirauth/shared_random_state.c @@ -68,7 +68,7 @@ static void disk_state_free_cb(void *); /* Array of variables that are saved to disk as a persistent state. */ static config_var_t state_vars[] = { - V(Version, UINT, "0"), + V(Version, POSINT, "0"), V(TorVersion, STRING, NULL), V(ValidAfter, ISOTIME, NULL), V(ValidUntil, ISOTIME, NULL), From fe9d15cf4b392bd6daef47f3aab97adeeebe7402 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 12 Jun 2019 16:33:40 -0400 Subject: [PATCH 1128/2557] Remove the PORT configuration type: nothing uses it. All of our port configurations now use an extended format. --- src/app/config/config.c | 1 - src/app/config/confparse.c | 16 +--------------- src/app/config/confparse.h | 8 +++----- 3 files changed, 4 insertions(+), 21 deletions(-) diff --git a/src/app/config/config.c b/src/app/config/config.c index 847355dada..7908007051 100644 --- a/src/app/config/config.c +++ b/src/app/config/config.c @@ -8180,7 +8180,6 @@ getinfo_helper_config(control_connection_t *conn, case CONFIG_TYPE_POSINT: type = "Integer"; break; case CONFIG_TYPE_UINT64: type = "Integer"; break; case CONFIG_TYPE_INT: type = "SignedInteger"; break; - case CONFIG_TYPE_PORT: type = "Port"; break; case CONFIG_TYPE_INTERVAL: type = "TimeInterval"; break; case CONFIG_TYPE_MSEC_INTERVAL: type = "TimeMsecInterval"; break; case CONFIG_TYPE_MEMUNIT: type = "DataSize"; break; diff --git a/src/app/config/confparse.c b/src/app/config/confparse.c index cf83df728a..14d9a368d6 100644 --- a/src/app/config/confparse.c +++ b/src/app/config/confparse.c @@ -174,17 +174,11 @@ config_assign_value(const config_format_t *fmt, void *options, switch (var->type) { - case CONFIG_TYPE_PORT: - if (!strcasecmp(c->value, "auto")) { - *(int *)lvalue = CFG_AUTO_PORT; - break; - } - /* fall through */ case CONFIG_TYPE_INT: case CONFIG_TYPE_POSINT: i = (int)tor_parse_long(c->value, 10, var->type==CONFIG_TYPE_INT ? INT_MIN : 0, - var->type==CONFIG_TYPE_PORT ? 65535 : INT_MAX, + INT_MAX, &ok, NULL); if (!ok) { tor_asprintf(msg, @@ -570,13 +564,6 @@ config_get_assigned_option(const config_format_t *fmt, const void *options, } escape_val = 0; /* Can't need escape. */ break; - case CONFIG_TYPE_PORT: - if (*(int*)value == CFG_AUTO_PORT) { - result->value = tor_strdup("auto"); - escape_val = 0; - break; - } - /* fall through */ case CONFIG_TYPE_CSV_INTERVAL: case CONFIG_TYPE_INTERVAL: case CONFIG_TYPE_MSEC_INTERVAL: @@ -788,7 +775,6 @@ config_clear(const config_format_t *fmt, void *options, case CONFIG_TYPE_MSEC_INTERVAL: case CONFIG_TYPE_POSINT: case CONFIG_TYPE_INT: - case CONFIG_TYPE_PORT: case CONFIG_TYPE_BOOL: *(int*)lvalue = 0; break; diff --git a/src/app/config/confparse.h b/src/app/config/confparse.h index 57f1ec1762..2c923eb6b9 100644 --- a/src/app/config/confparse.h +++ b/src/app/config/confparse.h @@ -17,11 +17,9 @@ typedef enum config_type_t { CONFIG_TYPE_STRING = 0, /**< An arbitrary string. */ CONFIG_TYPE_FILENAME, /**< A filename: some prefixes get expanded. */ - CONFIG_TYPE_UINT, /**< A non-negative integer less than MAX_INT */ + CONFIG_TYPE_POSINT, /**< A non-negative integer less than MAX_INT */ CONFIG_TYPE_INT, /**< Any integer. */ CONFIG_TYPE_UINT64, /**< A value in range 0..UINT64_MAX */ - CONFIG_TYPE_PORT, /**< A port from 1...65535, 0 for "not set", or - * "auto". */ CONFIG_TYPE_INTERVAL, /**< A number of seconds, with optional units*/ CONFIG_TYPE_MSEC_INTERVAL,/**< A number of milliseconds, with optional * units */ @@ -57,8 +55,8 @@ typedef enum config_type_t { typedef union { char **STRING; char **FILENAME; - int *UINT; /* yes, really: Even though the confparse type is called - * "UINT", it still uses the C int type -- it just enforces that + int *POSINT; /* yes, really: Even though the confparse type is called + * "POSINT", it still uses the C int type -- it just enforces that * the values are in range [0,INT_MAX]. */ uint64_t *UINT64; From 26436fb1b64bd35c1342b71df0284b992d770856 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Sat, 15 Jun 2019 11:24:43 -0400 Subject: [PATCH 1129/2557] Add more unit tests for confparse.c, so we can refactor. This set of tests gets the line coverage to 100%. --- src/app/config/confparse.c | 21 +- src/app/config/confparse.h | 5 + src/test/include.am | 1 + src/test/test.c | 1 + src/test/test.h | 1 + src/test/test_confparse.c | 842 +++++++++++++++++++++++++++++++++++++ 6 files changed, 867 insertions(+), 4 deletions(-) create mode 100644 src/test/test_confparse.c diff --git a/src/app/config/confparse.c b/src/app/config/confparse.c index 14d9a368d6..e4bb6e89bd 100644 --- a/src/app/config/confparse.c +++ b/src/app/config/confparse.c @@ -21,6 +21,7 @@ * specified, and a linked list of key-value pairs. */ +#define CONFPARSE_PRIVATE #include "core/or/or.h" #include "app/config/confparse.h" #include "feature/nodelist/routerset.h" @@ -87,7 +88,7 @@ const char * config_find_deprecation(const config_format_t *fmt, const char *key) { if (BUG(fmt == NULL) || BUG(key == NULL)) - return NULL; + return NULL; // LCOV_EXCL_LINE if (fmt->deprecations == NULL) return NULL; @@ -352,9 +353,11 @@ config_assign_value(const config_format_t *fmt, void *options, tor_asprintf(msg, "You may not provide a value for virtual option '%s'", c->key); return -1; + // LCOV_EXCL_START default: - tor_assert(0); + tor_assert_unreached(); break; + // LCOV_EXCL_STOP } return 0; } @@ -455,7 +458,9 @@ config_assign_line(const config_format_t *fmt, void *options, } return 0; } else if (c->command == CONFIG_LINE_CLEAR && !clear_first) { - config_reset(fmt, options, var, use_defaults); + // XXXX This is unreachable, since a CLEAR line always has an + // XXXX empty value. + config_reset(fmt, options, var, use_defaults); // LCOV_EXCL_LINE } if (options_seen && (var->type != CONFIG_TYPE_LINELIST && @@ -477,7 +482,7 @@ config_assign_line(const config_format_t *fmt, void *options, /** Restore the option named key in options to its default value. * Called from config_assign(). */ -static void +STATIC void config_reset_line(const config_format_t *fmt, void *options, const char *key, int use_defaults) { @@ -625,12 +630,14 @@ config_get_assigned_option(const config_format_t *fmt, const void *options, tor_free(result); result = config_lines_dup(*(const config_line_t**)value); break; + // LCOV_EXCL_START default: tor_free(result->key); tor_free(result); log_warn(LD_BUG,"Unknown type %d for known key '%s'", var->type, key); return NULL; + // LCOV_EXCL_STOP } if (escape_val) { @@ -829,8 +836,10 @@ config_reset(const config_format_t *fmt, void *options, c->key = tor_strdup(var->name); c->value = tor_strdup(var->initvalue); if (config_assign_value(fmt, options, c, &msg) < 0) { + // LCOV_EXCL_START log_warn(LD_BUG, "Failed to assign default: %s", msg); tor_free(msg); /* if this happens it's a bug */ + // LCOV_EXCL_STOP } config_free_lines(c); } @@ -896,10 +905,12 @@ config_dup(const config_format_t *fmt, const void *old) if (line) { char *msg = NULL; if (config_assign(fmt, newopts, line, 0, &msg) < 0) { + // LCOV_EXCL_START log_err(LD_BUG, "config_get_assigned_option() generated " "something we couldn't config_assign(): %s", msg); tor_free(msg); tor_assert(0); + // LCOV_EXCL_STOP } } config_free_lines(line); @@ -948,9 +959,11 @@ config_dump(const config_format_t *fmt, const void *default_options, /* XXX use a 1 here so we don't add a new log line while dumping */ if (default_options == NULL) { if (fmt->validate_fn(NULL, defaults_tmp, defaults_tmp, 1, &msg) < 0) { + // LCOV_EXCL_START log_err(LD_BUG, "Failed to validate default config: %s", msg); tor_free(msg); tor_assert(0); + // LCOV_EXCL_STOP } } diff --git a/src/app/config/confparse.h b/src/app/config/confparse.h index 2c923eb6b9..2112abf715 100644 --- a/src/app/config/confparse.h +++ b/src/app/config/confparse.h @@ -228,4 +228,9 @@ void warn_deprecated_option(const char *what, const char *why); #define CFG_EQ_LINELIST(a,b,opt) config_lines_eq((a)->opt, (b)->opt) #define CFG_EQ_ROUTERSET(a,b,opt) routerset_equal((a)->opt, (b)->opt) +#ifdef CONFPARSE_PRIVATE +STATIC void config_reset_line(const config_format_t *fmt, void *options, + const char *key, int use_defaults); +#endif + #endif /* !defined(TOR_CONFPARSE_H) */ diff --git a/src/test/include.am b/src/test/include.am index 624bca66d9..0ec4d96ad4 100644 --- a/src/test/include.am +++ b/src/test/include.am @@ -120,6 +120,7 @@ src_test_test_SOURCES += \ src/test/test_circuitstats.c \ src/test/test_compat_libevent.c \ src/test/test_config.c \ + src/test/test_confparse.c \ src/test/test_connection.c \ src/test/test_conscache.c \ src/test/test_consdiff.c \ diff --git a/src/test/test.c b/src/test/test.c index cc08531702..266b7454a3 100644 --- a/src/test/test.c +++ b/src/test/test.c @@ -840,6 +840,7 @@ struct testgroup_t testgroups[] = { { "circuituse/", circuituse_tests }, { "compat/libevent/", compat_libevent_tests }, { "config/", config_tests }, + { "config/parse/", confparse_tests }, { "connection/", connection_tests }, { "conscache/", conscache_tests }, { "consdiff/", consdiff_tests }, diff --git a/src/test/test.h b/src/test/test.h index 85e8b07ff7..322716a9ab 100644 --- a/src/test/test.h +++ b/src/test/test.h @@ -197,6 +197,7 @@ extern struct testcase_t circuitstats_tests[]; extern struct testcase_t circuituse_tests[]; extern struct testcase_t compat_libevent_tests[]; extern struct testcase_t config_tests[]; +extern struct testcase_t confparse_tests[]; extern struct testcase_t connection_tests[]; extern struct testcase_t conscache_tests[]; extern struct testcase_t consdiff_tests[]; diff --git a/src/test/test_confparse.c b/src/test/test_confparse.c new file mode 100644 index 0000000000..89a6eb5265 --- /dev/null +++ b/src/test/test_confparse.c @@ -0,0 +1,842 @@ +/* Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/* + * Tests for confparse.c module that we use to parse various + * configuration/state file types. + */ + +#define CONFPARSE_PRIVATE +#include "orconfig.h" + +#include "core/or/or.h" +#include "lib/encoding/confline.h" +#include "feature/nodelist/routerset.h" +#include "app/config/confparse.h" +#include "test/test.h" +#include "test/log_test_helpers.h" + +typedef struct test_struct_t { + uint32_t magic; + char *s; + char *fn; + int pos; + int i; + int deprecated_int; + uint64_t u64; + int interval; + int msec_interval; + uint64_t mem; + double dbl; + int boolean; + int autobool; + time_t time; + smartlist_t *csv; + int csv_interval; + config_line_t *lines; + config_line_t *mixed_lines; + routerset_t *routerset; + int hidden_int; + config_line_t *mixed_hidden_lines; + + config_line_t *extra_lines; +} test_struct_t; + +static test_struct_t test_struct_t_dummy; + +#define VAR(name,conftype,member,initvalue) \ + { name, CONFIG_TYPE_##conftype, offsetof(test_struct_t, member), \ + initvalue CONF_TEST_MEMBERS(test_struct_t, conftype, member) } + +#define V(name,conftype,initvalue) \ + VAR( #name, conftype, name, initvalue ) + +#define OBSOLETE(name) \ + { name, CONFIG_TYPE_OBSOLETE, 0, NULL, {.INT=NULL} } + +static config_var_t test_vars[] = { + V(s, STRING, "hello"), + V(fn, FILENAME, NULL), + V(pos, POSINT, NULL), + V(i, INT, "-10"), + V(deprecated_int, INT, "3"), + V(u64, UINT64, NULL), + V(interval, INTERVAL, "10 seconds"), + V(msec_interval, MSEC_INTERVAL, "150 msec"), + V(mem, MEMUNIT, "10 MB"), + V(dbl, DOUBLE, NULL), + V(boolean, BOOL, "0"), + V(autobool, AUTOBOOL, "auto"), + V(time, ISOTIME, NULL), + V(csv, CSV, NULL), + V(csv_interval, CSV_INTERVAL, "5 seconds"), + V(lines, LINELIST, NULL), + VAR("MixedLines", LINELIST_V, mixed_lines, NULL), + VAR("LineTypeA", LINELIST_S, mixed_lines, NULL), + VAR("LineTypeB", LINELIST_S, mixed_lines, NULL), + OBSOLETE("obsolete"), + V(routerset, ROUTERSET, NULL), + VAR("__HiddenInt", POSINT, hidden_int, "0"), + VAR("MixedHiddenLines", LINELIST_V, mixed_hidden_lines, NULL), + VAR("__HiddenLineA", LINELIST_S, mixed_hidden_lines, NULL), + VAR("VisibleLineB", LINELIST_S, mixed_hidden_lines, NULL), + + END_OF_CONFIG_VARS, +}; + +static config_abbrev_t test_abbrevs[] = { + { "uint", "pos", 0, 0 }, + { "float", "dbl", 0, 1 }, + { NULL, NULL, 0, 0 } +}; + +static config_deprecation_t test_deprecation_notes[] = { + { "deprecated_int", "This integer is deprecated." }, + { NULL, NULL } +}; + +static int +test_validate_cb(void *old_options, void *options, void *default_options, + int from_setconf, char **msg) +{ + (void)old_options; + (void)default_options; + (void)from_setconf; + (void)msg; + test_struct_t *ts = options; + + if (ts->i == 0xbad) { + *msg = tor_strdup("bad value for i"); + return -1; + } + return 0; +} + +static void test_free_cb(void *options); + +#define TEST_MAGIC 0x1337 + +static config_format_t test_fmt = { + sizeof(test_struct_t), + TEST_MAGIC, + offsetof(test_struct_t, magic), + test_abbrevs, + test_deprecation_notes, + test_vars, + test_validate_cb, + test_free_cb, + NULL, +}; + +static void +test_free_cb(void *options) +{ + if (!options) + return; + + config_free(&test_fmt, options); +} + +/* Make sure that config_init sets everything to the right defaults. */ +static void +test_confparse_init(void *arg) +{ + (void)arg; + test_struct_t *tst = config_new(&test_fmt); + config_init(&test_fmt, tst); + + // Make sure that options are initialized right. */ + tt_uint_op(tst->magic, OP_EQ, TEST_MAGIC); + tt_str_op(tst->s, OP_EQ, "hello"); + tt_ptr_op(tst->fn, OP_EQ, NULL); + tt_int_op(tst->pos, OP_EQ, 0); + tt_int_op(tst->i, OP_EQ, -10); + tt_int_op(tst->deprecated_int, OP_EQ, 3); + tt_u64_op(tst->u64, OP_EQ, 0); + tt_int_op(tst->interval, OP_EQ, 10); + tt_int_op(tst->msec_interval, OP_EQ, 150); + tt_u64_op(tst->mem, OP_EQ, 10 * 1024 * 1024); + tt_double_op(tst->dbl, OP_LT, .0000000001); + tt_double_op(tst->dbl, OP_GT, -0.0000000001); + tt_int_op(tst->boolean, OP_EQ, 0); + tt_int_op(tst->autobool, OP_EQ, -1); + tt_i64_op(tst->time, OP_EQ, 0); + tt_ptr_op(tst->csv, OP_EQ, NULL); + tt_int_op(tst->csv_interval, OP_EQ, 5); + tt_ptr_op(tst->lines, OP_EQ, NULL); + tt_ptr_op(tst->mixed_lines, OP_EQ, NULL); + tt_int_op(tst->hidden_int, OP_EQ, 0); + + done: + config_free(&test_fmt, tst); +} + +static const char simple_settings[] = + "s this is a \n" + "fn /simple/test of the\n" + "uint 77\n" // this is an abbrev + "i 3\n" + "u64 1000000000000 \n" + "interval 5 minutes \n" + "msec_interval 5 minutes \n" + "mem 10\n" + "dbl 6.060842\n" + "BOOLEAN 1\n" + "aUtObOOl 0\n" + "time 2019-06-14 13:58:51\n" + "csv configuration, parsing , system \n" + "csv_interval 10 seconds, 5 seconds, 10 hours\n" + "lines hello\n" + "LINES world\n" + "linetypea i d\n" + "linetypeb i c\n" + "routerset $FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF\n" + "__hiddenint 11\n" + "__hiddenlineA XYZ\n" + "visiblelineB ABC\n"; + +/* Return a configuration object set up from simple_settings above. */ +static test_struct_t * +get_simple_config(void) +{ + test_struct_t *result = NULL; + test_struct_t *tst = config_new(&test_fmt); + config_line_t *lines = NULL; + char *msg = NULL; + + config_init(&test_fmt, tst); + + int r = config_get_lines(simple_settings, &lines, 0); + tt_int_op(r, OP_EQ, 0); + r = config_assign(&test_fmt, tst, lines, 0, &msg); + tt_int_op(r, OP_EQ, 0); + tt_ptr_op(msg, OP_EQ, NULL); + + result = tst; + tst = NULL; // prevent free + done: + tor_free(msg); + config_free_lines(lines); + config_free(&test_fmt, tst); + return result; +} + +/* Make sure that config_assign can parse things. */ +static void +test_confparse_assign_simple(void *arg) +{ + (void)arg; + test_struct_t *tst = get_simple_config(); + + tt_str_op(tst->s, OP_EQ, "this is a"); + tt_str_op(tst->fn, OP_EQ, "/simple/test of the"); + tt_int_op(tst->pos, OP_EQ, 77); + tt_int_op(tst->i, OP_EQ, 3); + tt_int_op(tst->deprecated_int, OP_EQ, 3); + tt_u64_op(tst->u64, OP_EQ, UINT64_C(1000000000000)); + tt_int_op(tst->interval, OP_EQ, 5 * 60); + tt_int_op(tst->msec_interval, OP_EQ, 5 * 60 * 1000); + tt_u64_op(tst->mem, OP_EQ, 10); + tt_double_op(tst->dbl, OP_LT, 6.060843); + tt_double_op(tst->dbl, OP_GT, 6.060841); + tt_int_op(tst->boolean, OP_EQ, 1); + tt_int_op(tst->autobool, OP_EQ, 0); + tt_i64_op(tst->time, OP_EQ, 1560520731); + tt_ptr_op(tst->csv, OP_NE, NULL); + tt_int_op(smartlist_len(tst->csv), OP_EQ, 3); + tt_str_op(smartlist_get(tst->csv, 0), OP_EQ, "configuration"); + tt_str_op(smartlist_get(tst->csv, 1), OP_EQ, "parsing"); + tt_str_op(smartlist_get(tst->csv, 2), OP_EQ, "system"); + tt_int_op(tst->csv_interval, OP_EQ, 10); + tt_int_op(tst->hidden_int, OP_EQ, 11); + + tt_assert(tst->lines); + tt_str_op(tst->lines->key, OP_EQ, "lines"); + tt_str_op(tst->lines->value, OP_EQ, "hello"); + tt_assert(tst->lines->next); + tt_str_op(tst->lines->next->key, OP_EQ, "lines"); + tt_str_op(tst->lines->next->value, OP_EQ, "world"); + tt_assert(!tst->lines->next->next); + + tt_assert(tst->mixed_lines); + tt_str_op(tst->mixed_lines->key, OP_EQ, "LineTypeA"); + tt_str_op(tst->mixed_lines->value, OP_EQ, "i d"); + tt_assert(tst->mixed_lines->next); + tt_str_op(tst->mixed_lines->next->key, OP_EQ, "LineTypeB"); + tt_str_op(tst->mixed_lines->next->value, OP_EQ, "i c"); + tt_assert(!tst->mixed_lines->next->next); + + tt_assert(tst->mixed_hidden_lines); + tt_str_op(tst->mixed_hidden_lines->key, OP_EQ, "__HiddenLineA"); + tt_str_op(tst->mixed_hidden_lines->value, OP_EQ, "XYZ"); + tt_assert(tst->mixed_hidden_lines->next); + tt_str_op(tst->mixed_hidden_lines->next->key, OP_EQ, "VisibleLineB"); + tt_str_op(tst->mixed_hidden_lines->next->value, OP_EQ, "ABC"); + tt_assert(!tst->mixed_hidden_lines->next->next); + + done: + config_free(&test_fmt, tst); +} + +/* Try to assign to an obsolete option, and make sure we get a warning. */ +static void +test_confparse_assign_obsolete(void *arg) +{ + (void)arg; + test_struct_t *tst = config_new(&test_fmt); + config_line_t *lines = NULL; + char *msg = NULL; + + config_init(&test_fmt, tst); + + int r = config_get_lines("obsolete option here", + &lines, 0); + tt_int_op(r, OP_EQ, 0); + setup_capture_of_logs(LOG_WARN); + r = config_assign(&test_fmt, tst, lines, 0, &msg); + tt_int_op(r, OP_EQ, 0); + tt_ptr_op(msg, OP_EQ, NULL); + expect_single_log_msg_containing("Skipping obsolete configuration option"); + + done: + teardown_capture_of_logs(); + config_free(&test_fmt, tst); + config_free_lines(lines); + tor_free(msg); +} + +/* Try to assign to an deprecated option, and make sure we get a warning + * but the assignment works anyway. */ +static void +test_confparse_assign_deprecated(void *arg) +{ + (void)arg; + test_struct_t *tst = config_new(&test_fmt); + config_line_t *lines = NULL; + char *msg = NULL; + + config_init(&test_fmt, tst); + + int r = config_get_lines("deprecated_int 7", + &lines, 0); + tt_int_op(r, OP_EQ, 0); + setup_capture_of_logs(LOG_WARN); + r = config_assign(&test_fmt, tst, lines, CAL_WARN_DEPRECATIONS, &msg); + tt_int_op(r, OP_EQ, 0); + tt_ptr_op(msg, OP_EQ, NULL); + expect_single_log_msg_containing("This integer is deprecated."); + + tt_int_op(tst->deprecated_int, OP_EQ, 7); + + done: + teardown_capture_of_logs(); + config_free(&test_fmt, tst); + config_free_lines(lines); + tor_free(msg); +} + +/* Try to re-assign an option name that has been depreacted in favor of + * another. */ +static void +test_confparse_assign_replaced(void *arg) +{ + (void)arg; + test_struct_t *tst = config_new(&test_fmt); + config_line_t *lines = NULL; + char *msg = NULL; + + config_init(&test_fmt, tst); + + int r = config_get_lines("float 1000\n", &lines, 0); + tt_int_op(r, OP_EQ, 0); + setup_capture_of_logs(LOG_WARN); + r = config_assign(&test_fmt, tst, lines, CAL_WARN_DEPRECATIONS, &msg); + tt_int_op(r, OP_EQ, 0); + tt_ptr_op(msg, OP_EQ, NULL); + expect_single_log_msg_containing("use 'dbl' instead."); + + tt_double_op(tst->dbl, OP_GT, 999.999); + tt_double_op(tst->dbl, OP_LT, 1000.001); + + done: + teardown_capture_of_logs(); + config_free(&test_fmt, tst); + config_free_lines(lines); + tor_free(msg); +} + +/* Try to set a linelist value with no option. */ +static void +test_confparse_assign_emptystring(void *arg) +{ + (void)arg; + test_struct_t *tst = config_new(&test_fmt); + config_line_t *lines = NULL; + char *msg = NULL; + + config_init(&test_fmt, tst); + + int r = config_get_lines("lines\n", &lines, 0); + tt_int_op(r, OP_EQ, 0); + setup_capture_of_logs(LOG_WARN); + r = config_assign(&test_fmt, tst, lines, 0, &msg); + tt_int_op(r, OP_EQ, 0); + tt_ptr_op(msg, OP_EQ, NULL); + expect_single_log_msg_containing("has no value"); + + done: + teardown_capture_of_logs(); + config_free(&test_fmt, tst); + config_free_lines(lines); + tor_free(msg); +} + +/* Try to set a the same option twice; make sure we get a warning. */ +static void +test_confparse_assign_twice(void *arg) +{ + (void)arg; + test_struct_t *tst = config_new(&test_fmt); + config_line_t *lines = NULL; + char *msg = NULL; + + config_init(&test_fmt, tst); + + int r = config_get_lines("pos 10\n" + "pos 99\n", &lines, 0); + tt_int_op(r, OP_EQ, 0); + setup_capture_of_logs(LOG_WARN); + r = config_assign(&test_fmt, tst, lines, 0, &msg); + tt_int_op(r, OP_EQ, 0); + tt_ptr_op(msg, OP_EQ, NULL); + expect_single_log_msg_containing("used more than once"); + + done: + teardown_capture_of_logs(); + config_free(&test_fmt, tst); + config_free_lines(lines); + tor_free(msg); +} + +typedef struct badval_test_t { + const char *cfg; + const char *expect_msg; +} badval_test_t; + +/* Try to set an option and make sure that we get a failure and an expected + * warning. */ +static void +test_confparse_assign_badval(void *arg) +{ + const badval_test_t *bt = arg; + test_struct_t *tst = config_new(&test_fmt); + config_line_t *lines = NULL; + char *msg = NULL; + + config_init(&test_fmt, tst); + + int r = config_get_lines(bt->cfg, &lines, 0); + tt_int_op(r, OP_EQ, 0); + setup_capture_of_logs(LOG_WARN); + r = config_assign(&test_fmt, tst, lines, 0, &msg); + tt_int_op(r, OP_LT, 0); + tt_ptr_op(msg, OP_NE, NULL); + if (! strstr(msg, bt->expect_msg)) { + TT_DIE(("'%s' did not contain '%s'" , msg, bt->expect_msg)); + } + + done: + teardown_capture_of_logs(); + config_free(&test_fmt, tst); + config_free_lines(lines); + tor_free(msg); +} + +/* Various arguments for badval test. + * + * Note that the expected warnings here are _very_ truncated, since we + * are writing these tests before a refactoring that we expect will + * change them. + */ +static const badval_test_t bv_notint = { "pos X\n", "malformed" }; +static const badval_test_t bv_negint = { "pos -10\n", "out of bounds" }; +static const badval_test_t bv_badu64 = { "u64 u64\n", "malformed" }; +static const badval_test_t bv_badcsvi1 = + { "csv_interval 10 wl\n", "malformed" }; +static const badval_test_t bv_badcsvi2 = + { "csv_interval cl,10\n", "malformed" }; +static const badval_test_t bv_nonoption = { "fnord 10\n", "Unknown option" }; +static const badval_test_t bv_badmem = { "mem 3 trits\n", "malformed" }; +static const badval_test_t bv_badbool = { "boolean 7\n", "expects 0 or 1" }; +static const badval_test_t bv_badabool = + { "autobool 7\n", "expects 0, 1, or 'auto'" }; +static const badval_test_t bv_badtime = { "time lunchtime\n", "Invalid time" }; +static const badval_test_t bv_virt = { "MixedLines 7\n", "virtual option" }; +static const badval_test_t bv_rs = { "Routerset 2.2.2.2.2\n", "Invalid" }; + +/* Try config_dump(), and make sure it behaves correctly */ +static void +test_confparse_dump(void *arg) +{ + (void)arg; + test_struct_t *tst = get_simple_config(); + char *dumped = NULL; + + /* Minimal version. */ + dumped = config_dump(&test_fmt, NULL, tst, 1, 0); + tt_str_op(dumped, OP_EQ, + "s this is a\n" + "fn /simple/test of the\n" + "pos 77\n" + "i 3\n" + "u64 1000000000000\n" + "interval 300\n" + "msec_interval 300000\n" + "mem 10\n" + "dbl 6.060842\n" + "boolean 1\n" + "autobool 0\n" + "time 2019-06-14 13:58:51\n" + "csv configuration,parsing,system\n" + "csv_interval 10\n" + "lines hello\n" + "lines world\n" + "LineTypeA i d\n" + "LineTypeB i c\n" + "routerset $FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF\n" + "VisibleLineB ABC\n"); + + /* Maximal */ + tor_free(dumped); + dumped = config_dump(&test_fmt, NULL, tst, 0, 0); + tt_str_op(dumped, OP_EQ, + "s this is a\n" + "fn /simple/test of the\n" + "pos 77\n" + "i 3\n" + "deprecated_int 3\n" + "u64 1000000000000\n" + "interval 300\n" + "msec_interval 300000\n" + "mem 10\n" + "dbl 6.060842\n" + "boolean 1\n" + "autobool 0\n" + "time 2019-06-14 13:58:51\n" + "csv configuration,parsing,system\n" + "csv_interval 10\n" + "lines hello\n" + "lines world\n" + "LineTypeA i d\n" + "LineTypeB i c\n" + "routerset $FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF\n" + "VisibleLineB ABC\n"); + + /* commented */ + tor_free(dumped); + dumped = config_dump(&test_fmt, NULL, tst, 0, 1); + tt_str_op(dumped, OP_EQ, + "s this is a\n" + "fn /simple/test of the\n" + "pos 77\n" + "i 3\n" + "# deprecated_int 3\n" + "u64 1000000000000\n" + "interval 300\n" + "msec_interval 300000\n" + "mem 10\n" + "dbl 6.060842\n" + "boolean 1\n" + "autobool 0\n" + "time 2019-06-14 13:58:51\n" + "csv configuration,parsing,system\n" + "csv_interval 10\n" + "lines hello\n" + "lines world\n" + "LineTypeA i d\n" + "LineTypeB i c\n" + "routerset $FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF\n" + "VisibleLineB ABC\n"); + + done: + config_free(&test_fmt, tst); + tor_free(dumped); +} + +/* Try confparse_reset_line(), and make sure it behaves correctly */ +static void +test_confparse_reset(void *arg) +{ + (void)arg; + test_struct_t *tst = get_simple_config(); + + config_reset_line(&test_fmt, tst, "interval", 0); + tt_int_op(tst->interval, OP_EQ, 0); + + config_reset_line(&test_fmt, tst, "interval", 1); + tt_int_op(tst->interval, OP_EQ, 10); + + done: + config_free(&test_fmt, tst); +} + +/* Try setting options a second time on a config object, and make sure + * it behaves correctly. */ +static void +test_confparse_reassign(void *arg) +{ + (void)arg; + test_struct_t *tst = get_simple_config(); + config_line_t *lines = NULL; + char *msg = NULL, *rs = NULL; + + int r = config_get_lines( + "s eleven\n" + "i 12\n" + "lines 13\n" + "csv 14,15\n" + "routerset 127.0.0.1\n", + &lines, 0); + r = config_assign(&test_fmt, tst,lines, 0, &msg); + tt_int_op(r, OP_EQ, 0); + tt_ptr_op(msg, OP_EQ, NULL); + + tt_str_op(tst->s, OP_EQ, "eleven"); + tt_str_op(tst->fn, OP_EQ, "/simple/test of the"); // unchanged + tt_int_op(tst->pos, OP_EQ, 77); // unchanged + tt_int_op(tst->i, OP_EQ, 12); + tt_ptr_op(tst->lines, OP_NE, NULL); + tt_str_op(tst->lines->key, OP_EQ, "lines"); + tt_str_op(tst->lines->value, OP_EQ, "13"); + tt_ptr_op(tst->lines->next, OP_EQ, NULL); + tt_int_op(smartlist_len(tst->csv), OP_EQ, 2); + tt_str_op(smartlist_get(tst->csv, 0), OP_EQ, "14"); + tt_str_op(smartlist_get(tst->csv, 1), OP_EQ, "15"); + + rs = routerset_to_string(tst->routerset); + tt_str_op(rs, OP_EQ, "127.0.0.1"); + + // Try again with the CLEAR_FIRST and USE_DEFAULTS flags + r = config_assign(&test_fmt, tst, lines, + CAL_CLEAR_FIRST|CAL_USE_DEFAULTS, &msg); + tt_int_op(r, OP_EQ, 0); + + tt_ptr_op(msg, OP_EQ, NULL); + tt_str_op(tst->s, OP_EQ, "eleven"); + // tt_ptr_op(tst->fn, OP_EQ, NULL); //XXXX why is this not cleared? + // tt_int_op(tst->pos, OP_EQ, 0); //XXXX why is this not cleared? + tt_int_op(tst->i, OP_EQ, 12); + + done: + config_free(&test_fmt, tst); + config_free_lines(lines); + tor_free(msg); + tor_free(rs); +} + +/* Try setting options a second time on a config object, using the +foo + * linelist-extending syntax. */ +static void +test_confparse_reassign_extend(void *arg) +{ + (void)arg; + test_struct_t *tst = get_simple_config(); + config_line_t *lines = NULL; + char *msg = NULL; + + int r = config_get_lines( + "+lines 13\n", + &lines, 1); // allow extended format. + tt_int_op(r, OP_EQ, 0); + r = config_assign(&test_fmt, tst,lines, 0, &msg); + tt_int_op(r, OP_EQ, 0); + tt_ptr_op(msg, OP_EQ, NULL); + + tt_assert(tst->lines); + tt_str_op(tst->lines->key, OP_EQ, "lines"); + tt_str_op(tst->lines->value, OP_EQ, "hello"); + tt_assert(tst->lines->next); + tt_str_op(tst->lines->next->key, OP_EQ, "lines"); + tt_str_op(tst->lines->next->value, OP_EQ, "world"); + tt_assert(tst->lines->next->next); + tt_str_op(tst->lines->next->next->key, OP_EQ, "lines"); + tt_str_op(tst->lines->next->next->value, OP_EQ, "13"); + tt_assert(tst->lines->next->next->next == NULL); + config_free_lines(lines); + + r = config_get_lines( + "/lines\n", + &lines, 1); // allow extended format. + tt_int_op(r, OP_EQ, 0); + r = config_assign(&test_fmt, tst, lines, 0, &msg); + tt_int_op(r, OP_EQ, 0); + tt_ptr_op(msg, OP_EQ, NULL); + tt_assert(tst->lines == NULL); + config_free_lines(lines); + + config_free(&test_fmt, tst); + tst = get_simple_config(); + r = config_get_lines( + "/lines away!\n", + &lines, 1); // allow extended format. + tt_int_op(r, OP_EQ, 0); + r = config_assign(&test_fmt, tst, lines, 0, &msg); + tt_int_op(r, OP_EQ, 0); + tt_ptr_op(msg, OP_EQ, NULL); + tt_assert(tst->lines == NULL); + + done: + config_free(&test_fmt, tst); + config_free_lines(lines); + tor_free(msg); +} + +/* Test out confparse_get_assigned(). */ +static void +test_confparse_get_assigned(void *arg) +{ + (void)arg; + test_struct_t *tst = get_simple_config(); + config_line_t *lines = NULL; + + lines = config_get_assigned_option(&test_fmt, tst, "I", 1); + tt_assert(lines); + tt_str_op(lines->key, OP_EQ, "i"); + tt_str_op(lines->value, OP_EQ, "3"); + tt_assert(lines->next == NULL); + config_free_lines(lines); + + lines = config_get_assigned_option(&test_fmt, tst, "s", 1); + tt_assert(lines); + tt_str_op(lines->key, OP_EQ, "s"); + tt_str_op(lines->value, OP_EQ, "this is a"); + tt_assert(lines->next == NULL); + config_free_lines(lines); + + lines = config_get_assigned_option(&test_fmt, tst, "obsolete", 1); + tt_assert(!lines); + + lines = config_get_assigned_option(&test_fmt, tst, "nonesuch", 1); + tt_assert(!lines); + + lines = config_get_assigned_option(&test_fmt, tst, "mixedlines", 1); + tt_assert(lines); + tt_str_op(lines->key, OP_EQ, "LineTypeA"); + tt_str_op(lines->value, OP_EQ, "i d"); + tt_assert(lines->next); + tt_str_op(lines->next->key, OP_EQ, "LineTypeB"); + tt_str_op(lines->next->value, OP_EQ, "i c"); + tt_assert(lines->next->next == NULL); + config_free_lines(lines); + + lines = config_get_assigned_option(&test_fmt, tst, "linetypeb", 1); + tt_assert(lines); + tt_str_op(lines->key, OP_EQ, "LineTypeB"); + tt_str_op(lines->value, OP_EQ, "i c"); + tt_assert(lines->next == NULL); + config_free_lines(lines); + + tor_free(tst->s); + tst->s = tor_strdup("Hello\nWorld"); + lines = config_get_assigned_option(&test_fmt, tst, "s", 1); + tt_assert(lines); + tt_str_op(lines->key, OP_EQ, "s"); + tt_str_op(lines->value, OP_EQ, "\"Hello\\nWorld\""); + tt_assert(lines->next == NULL); + config_free_lines(lines); + + done: + config_free(&test_fmt, tst); + config_free_lines(lines); +} + +/* Another variant, which accepts and stores unrecognized lines.*/ +#define ETEST_MAGIC 13371337 + +static config_var_t extra = VAR("__extra", LINELIST, extra_lines, NULL); + +static config_format_t etest_fmt = { + sizeof(test_struct_t), + ETEST_MAGIC, + offsetof(test_struct_t, magic), + test_abbrevs, + test_deprecation_notes, + test_vars, + test_validate_cb, + test_free_cb, + &extra, +}; + +/* Try out the feature where we can store unrecognized lines and dump them + * again. (State files use this.) */ +static void +test_confparse_extra_lines(void *arg) +{ + (void)arg; + test_struct_t *tst = config_new(&etest_fmt); + config_line_t *lines = NULL; + char *msg = NULL, *dump = NULL; + + config_init(&etest_fmt, tst); + + int r = config_get_lines( + "unknotty addita\n" + "pos 99\n" + "wombat knish\n", &lines, 0); + tt_int_op(r, OP_EQ, 0); + r = config_assign(&etest_fmt, tst, lines, 0, &msg); + tt_int_op(r, OP_EQ, 0); + tt_ptr_op(msg, OP_EQ, NULL); + + tt_assert(tst->extra_lines); + + dump = config_dump(&etest_fmt, NULL, tst, 1, 0); + tt_str_op(dump, OP_EQ, + "pos 99\n" + "unknotty addita\n" + "wombat knish\n"); + + done: + tor_free(msg); + tor_free(dump); + config_free_lines(lines); + config_free(&etest_fmt, tst); +} + +#define CONFPARSE_TEST(name, flags) \ + { #name, test_confparse_ ## name, flags, NULL, NULL } + +#define BADVAL_TEST(name) \ + { "badval_" #name, test_confparse_assign_badval, 0, \ + &passthrough_setup, (void*)&bv_ ## name } + +struct testcase_t confparse_tests[] = { + CONFPARSE_TEST(init, 0), + CONFPARSE_TEST(assign_simple, 0), + CONFPARSE_TEST(assign_obsolete, 0), + CONFPARSE_TEST(assign_deprecated, 0), + CONFPARSE_TEST(assign_replaced, 0), + CONFPARSE_TEST(assign_emptystring, 0), + CONFPARSE_TEST(assign_twice, 0), + BADVAL_TEST(notint), + BADVAL_TEST(negint), + BADVAL_TEST(badu64), + BADVAL_TEST(badcsvi1), + BADVAL_TEST(badcsvi2), + BADVAL_TEST(nonoption), + BADVAL_TEST(badmem), + BADVAL_TEST(badbool), + BADVAL_TEST(badabool), + BADVAL_TEST(badtime), + BADVAL_TEST(virt), + BADVAL_TEST(rs), + CONFPARSE_TEST(dump, 0), + CONFPARSE_TEST(reset, 0), + CONFPARSE_TEST(reassign, 0), + CONFPARSE_TEST(reassign_extend, 0), + CONFPARSE_TEST(get_assigned, 0), + CONFPARSE_TEST(extra_lines, 0), + END_OF_TESTCASES +}; From 4613159c61549a4901690da5ecd5101eb6a20736 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Sat, 15 Jun 2019 16:19:25 -0400 Subject: [PATCH 1130/2557] Changes file for ticket 30893 (confparse testing) --- changes/ticket30893 | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 changes/ticket30893 diff --git a/changes/ticket30893 b/changes/ticket30893 new file mode 100644 index 0000000000..e1b56d2182 --- /dev/null +++ b/changes/ticket30893 @@ -0,0 +1,3 @@ + o Minor features (testing)P: + - Improve test coverage for our existing configuration parsing and + management API. Closes ticket 30893. From 66a15013fa5abf3c4d3345ca29c22ad13d45e22f Mon Sep 17 00:00:00 2001 From: rl1987 Date: Sun, 16 Jun 2019 20:18:30 +0300 Subject: [PATCH 1131/2557] Allow excluding documentation from the build --- changes/ticket19381 | 4 ++++ configure.ac | 8 ++++++++ doc/include.am | 25 ++++++++++++++++++++----- 3 files changed, 32 insertions(+), 5 deletions(-) create mode 100644 changes/ticket19381 diff --git a/changes/ticket19381 b/changes/ticket19381 new file mode 100644 index 0000000000..ee51e2a3e2 --- /dev/null +++ b/changes/ticket19381 @@ -0,0 +1,4 @@ + o Minor features (build system): + - Add --disable-manpage and --disable-html-manual options to configure + script. This will enable shortening build times by not building + documentation. Resolves issue 19381. diff --git a/configure.ac b/configure.ac index 75ca03d7d8..9ec123f51e 100644 --- a/configure.ac +++ b/configure.ac @@ -105,6 +105,12 @@ if test "$enable_memory_sentinels" = "no"; then [Defined if we're turning off memory safety code to look for bugs]) fi +AC_ARG_ENABLE(manpage, + AS_HELP_STRING(--disable-manpage, [Disable manpage generation.])) + +AC_ARG_ENABLE(html-manual, + AS_HELP_STRING(--disable-html-manual, [Disable HTML documentation.])) + AC_ARG_ENABLE(asciidoc, AS_HELP_STRING(--disable-asciidoc, [don't use asciidoc (disables building of manpages)]), [case "${enableval}" in @@ -299,6 +305,8 @@ AC_PATH_PROG([ASCIIDOC], [asciidoc], none) AC_PATH_PROGS([A2X], [a2x a2x.py], none) AM_CONDITIONAL(USE_ASCIIDOC, test "x$asciidoc" = "xtrue") +AM_CONDITIONAL(BUILD_MANPAGE, [test "x$enable_manpage" != "xno"]) +AM_CONDITIONAL(BUILD_HTML_DOCS, [test "x$enable_html_manual" != "xno"]) AM_PROG_CC_C_O AC_PROG_CC_C99 diff --git a/doc/include.am b/doc/include.am index 0a123aae11..a9d3fa1c98 100644 --- a/doc/include.am +++ b/doc/include.am @@ -15,17 +15,32 @@ all_mans = doc/tor doc/tor-gencert doc/tor-resolve doc/torify doc/tor-print-ed-signing-cert if USE_ASCIIDOC -nodist_man1_MANS = $(all_mans:=.1) -doc_DATA = $(all_mans:=.html) -html_in = $(all_mans:=.html.in) -man_in = $(all_mans:=.1.in) txt_in = $(all_mans:=.1.txt) + +if BUILD_HTML_DOCS +html_in = $(all_mans:=.html.in) +doc_DATA = $(all_mans:=.html) else html_in = +doc_DATA = +endif + +if BUILD_MANPAGE +nodist_man1_MANS = $(all_mans:=.1) +man_in = $(all_mans:=.1.in) +else +nodist_man1_MANS = +man_in = +endif + +else + +html_in = +doc_DATA = man_in = txt_in = nodist_man1_MANS = -doc_DATA = + endif EXTRA_DIST+= doc/asciidoc-helper.sh \ From 19dbd431c52994fc14bf7cb2e1a1a0dc686c2599 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Sun, 9 Jun 2019 18:20:36 +0300 Subject: [PATCH 1132/2557] Shellcheck all the scripts --- Makefile.am | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/Makefile.am b/Makefile.am index 7a0d40d6a5..df30e0b59a 100644 --- a/Makefile.am +++ b/Makefile.am @@ -224,10 +224,19 @@ test: all shellcheck: # Only use shellcheck if it is present if command -v shellcheck; then \ - find $(top_srcdir)/scripts/ -name "*.sh" -exec shellcheck {} +; \ + find "$(top_srcdir)" -name "*.sh" -not -path "$(top_srcdir)/src/ext/*" -exec shellcheck {} +; \ if [ -d "$(top_srcdir)/scripts/test" ]; then \ shellcheck $(top_srcdir)/scripts/test/cov-diff $(top_srcdir)/scripts/test/coverage; \ fi; \ + if [ -e "$(top_srcdir)/contrib/dirauth-tools/nagios-check-tor-authority-cert" ]; then \ + shellcheck "$(top_srcdir)/contrib/dirauth-tools/nagios-check-tor-authority-cert"; \ + fi; \ + if [ -e "$(top_srcdir)/contrib/client-tools/torify" ]; then \ + shellcheck "$(top_srcdir)/contrib/client-tools/torify"; \ + fi; \ + if [ -d "$(top_srcdir)/scripts/git" ]; then \ + shellcheck $(top_srcdir)/scripts/git/*.git-hook; \ + fi; \ fi check-local: check-spaces check-changes check-includes check-best-practices shellcheck From e8da65ee91658d7223739828a3e070e8e33a17ac Mon Sep 17 00:00:00 2001 From: rl1987 Date: Sun, 9 Jun 2019 18:21:14 +0300 Subject: [PATCH 1133/2557] Fix SC2034 in pre-push.git-hook --- scripts/git/pre-push.git-hook | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/git/pre-push.git-hook b/scripts/git/pre-push.git-hook index c9e72a4d43..e2a636e6d7 100755 --- a/scripts/git/pre-push.git-hook +++ b/scripts/git/pre-push.git-hook @@ -35,7 +35,6 @@ if [ -e scripts/maint/practracker/practracker.py ]; then fi remote="$1" -remote_loc="$2" remote_name=$(git remote --verbose | grep "$2" | awk '{print $1}' | head -n 1) From 86478be50f4364025310d2846d16d09c15962387 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Sun, 9 Jun 2019 18:23:05 +0300 Subject: [PATCH 1134/2557] Add changes file --- changes/ticket29533 | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 changes/ticket29533 diff --git a/changes/ticket29533 b/changes/ticket29533 new file mode 100644 index 0000000000..27ef681218 --- /dev/null +++ b/changes/ticket29533 @@ -0,0 +1,3 @@ + o Testing: + - Run shellcheck for all non-third-party shell scripts that are shipped + with Tor. Closes ticket 29533. From 6a0763cd665e50174df5a7ea6ae0a27db7c9945f Mon Sep 17 00:00:00 2001 From: David Goulet Date: Tue, 18 Jun 2019 13:32:45 -0400 Subject: [PATCH 1135/2557] guard: Ignore marked for close circuit when changing state to open When we consider all circuits in "waiting for guard" state to be promoted to an "open" state, we were considering all circuits, even the one marked for close. This ultiamtely triggers a "circuit_has_opened()" called on the circuit that is marked for close which then leads to possible undesirable behaviors within a subsystem. For instance, the HS subsystem would be unable to find the authentication key of the introduction point circuit leading to a BUG() warning and a duplicate mark for close on the circuit. This commit also adds a unit test to make sure we never select marked for close circuits when upgrading its guard state from waiting for guard to open. Fixes #30871 Signed-off-by: David Goulet --- changes/ticket30871 | 6 +++++ src/feature/client/entrynodes.c | 4 +++ src/test/test_circuitbuild.c | 47 +++++++++++++++++++++++++++++++++ 3 files changed, 57 insertions(+) create mode 100644 changes/ticket30871 diff --git a/changes/ticket30871 b/changes/ticket30871 new file mode 100644 index 0000000000..81c076bb02 --- /dev/null +++ b/changes/ticket30871 @@ -0,0 +1,6 @@ + o Major bugfixes (circuit build, guard): + - When considering upgrading circuits from "waiting for guard" to "open", + always ignore the ones that are mark for close. Else, we can end up in + the situation where a subsystem is notified of that circuit opening but + still marked for close leading to undesirable behavior. Fixes bug 30871; + bugfix on 0.3.0.1-alpha. diff --git a/src/feature/client/entrynodes.c b/src/feature/client/entrynodes.c index 4afcee2021..54a9238d8f 100644 --- a/src/feature/client/entrynodes.c +++ b/src/feature/client/entrynodes.c @@ -2611,6 +2611,10 @@ entry_guards_upgrade_waiting_circuits(guard_selection_t *gs, entry_guard_t *guard = entry_guard_handle_get(state->guard); if (!guard || guard->in_selection != gs) continue; + if (TO_CIRCUIT(circ)->marked_for_close) { + /* Don't consider any marked for close circuits. */ + continue; + } smartlist_add(all_circuits, circ); } SMARTLIST_FOREACH_END(circ); diff --git a/src/test/test_circuitbuild.c b/src/test/test_circuitbuild.c index 47218a559a..0c23091594 100644 --- a/src/test/test_circuitbuild.c +++ b/src/test/test_circuitbuild.c @@ -4,6 +4,8 @@ /* See LICENSE for licensing information */ #define CIRCUITBUILD_PRIVATE +#define CIRCUITLIST_PRIVATE +#define ENTRYNODES_PRIVATE #include "core/or/or.h" #include "test/test.h" @@ -13,7 +15,11 @@ #include "core/or/circuitbuild.h" #include "core/or/circuitlist.h" +#include "core/or/cpath_build_state_st.h" #include "core/or/extend_info_st.h" +#include "core/or/origin_circuit_st.h" + +#include "feature/client/entrynodes.h" /* Dummy nodes smartlist for testing */ static smartlist_t dummy_nodes; @@ -126,10 +132,51 @@ test_new_route_len_unhandled_exit(void *arg) UNMOCK(count_acceptable_nodes); } +static void +test_upgrade_from_guard_wait(void *arg) +{ + circuit_t *circ = NULL; + origin_circuit_t *orig_circ = NULL; + entry_guard_t *guard = NULL; + smartlist_t *list = NULL; + + (void) arg; + + circ = dummy_origin_circuit_new(0); + orig_circ = TO_ORIGIN_CIRCUIT(circ); + tt_assert(orig_circ); + + orig_circ->build_state = tor_malloc_zero(sizeof(cpath_build_state_t)); + + circuit_set_state(circ, CIRCUIT_STATE_GUARD_WAIT); + + /* Put it in guard wait state. */ + guard = tor_malloc_zero(sizeof(*guard)); + guard->in_selection = get_guard_selection_info(); + + orig_circ->guard_state = + circuit_guard_state_new(guard, GUARD_CIRC_STATE_WAITING_FOR_BETTER_GUARD, + NULL); + + /* Mark the circuit for close. */ + circuit_mark_for_close(circ, END_CIRC_REASON_TORPROTOCOL); + tt_int_op(circ->marked_for_close, OP_NE, 0); + + /* We shouldn't pick the mark for close circuit. */ + list = circuit_find_circuits_to_upgrade_from_guard_wait(); + tt_assert(!list); + + done: + circuit_free(circ); + entry_guard_free_(guard); +} + struct testcase_t circuitbuild_tests[] = { { "noexit", test_new_route_len_noexit, 0, NULL, NULL }, { "safe_exit", test_new_route_len_safe_exit, 0, NULL, NULL }, { "unsafe_exit", test_new_route_len_unsafe_exit, 0, NULL, NULL }, { "unhandled_exit", test_new_route_len_unhandled_exit, 0, NULL, NULL }, + { "upgrade_from_guard_wait", test_upgrade_from_guard_wait, TT_FORK, + NULL, NULL }, END_OF_TESTCASES }; From 05b6f73f12374e40e38bb1b35f3df593aa287610 Mon Sep 17 00:00:00 2001 From: David Goulet Date: Tue, 18 Jun 2019 13:33:10 -0400 Subject: [PATCH 1136/2557] Make the great grand practracker happy... Signed-off-by: David Goulet --- scripts/maint/practracker/exceptions.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/maint/practracker/exceptions.txt b/scripts/maint/practracker/exceptions.txt index db9bf34e25..9073f2f1dd 100644 --- a/scripts/maint/practracker/exceptions.txt +++ b/scripts/maint/practracker/exceptions.txt @@ -135,8 +135,8 @@ problem function-size /src/feature/client/addressmap.c:addressmap_rewrite() 112 problem function-size /src/feature/client/bridges.c:rewrite_node_address_for_bridge() 126 problem function-size /src/feature/client/circpathbias.c:pathbias_measure_close_rate() 108 problem function-size /src/feature/client/dnsserv.c:evdns_server_callback() 153 -problem file-size /src/feature/client/entrynodes.c 3820 -problem function-size /src/feature/client/entrynodes.c:entry_guards_upgrade_waiting_circuits() 153 +problem file-size /src/feature/client/entrynodes.c 3824 +problem function-size /src/feature/client/entrynodes.c:entry_guards_upgrade_waiting_circuits() 157 problem function-size /src/feature/client/entrynodes.c:entry_guard_parse_from_state() 246 problem function-size /src/feature/client/transports.c:handle_proxy_line() 108 problem function-size /src/feature/client/transports.c:parse_method_line_helper() 112 From 16a0b7ed6779bf72a8a471c558a5e09084921d8b Mon Sep 17 00:00:00 2001 From: David Goulet Date: Tue, 18 Jun 2019 13:32:45 -0400 Subject: [PATCH 1137/2557] guard: Ignore marked for close circuit when changing state to open When we consider all circuits in "waiting for guard" state to be promoted to an "open" state, we were considering all circuits, even the one marked for close. This ultiamtely triggers a "circuit_has_opened()" called on the circuit that is marked for close which then leads to possible undesirable behaviors within a subsystem. For instance, the HS subsystem would be unable to find the authentication key of the introduction point circuit leading to a BUG() warning and a duplicate mark for close on the circuit. This commit also adds a unit test to make sure we never select marked for close circuits when upgrading its guard state from waiting for guard to open. Fixes #30871 Signed-off-by: David Goulet --- changes/ticket30871 | 6 +++++ src/feature/client/entrynodes.c | 4 +++ src/test/test_circuitbuild.c | 47 +++++++++++++++++++++++++++++++++ 3 files changed, 57 insertions(+) create mode 100644 changes/ticket30871 diff --git a/changes/ticket30871 b/changes/ticket30871 new file mode 100644 index 0000000000..81c076bb02 --- /dev/null +++ b/changes/ticket30871 @@ -0,0 +1,6 @@ + o Major bugfixes (circuit build, guard): + - When considering upgrading circuits from "waiting for guard" to "open", + always ignore the ones that are mark for close. Else, we can end up in + the situation where a subsystem is notified of that circuit opening but + still marked for close leading to undesirable behavior. Fixes bug 30871; + bugfix on 0.3.0.1-alpha. diff --git a/src/feature/client/entrynodes.c b/src/feature/client/entrynodes.c index 4afcee2021..54a9238d8f 100644 --- a/src/feature/client/entrynodes.c +++ b/src/feature/client/entrynodes.c @@ -2611,6 +2611,10 @@ entry_guards_upgrade_waiting_circuits(guard_selection_t *gs, entry_guard_t *guard = entry_guard_handle_get(state->guard); if (!guard || guard->in_selection != gs) continue; + if (TO_CIRCUIT(circ)->marked_for_close) { + /* Don't consider any marked for close circuits. */ + continue; + } smartlist_add(all_circuits, circ); } SMARTLIST_FOREACH_END(circ); diff --git a/src/test/test_circuitbuild.c b/src/test/test_circuitbuild.c index 47218a559a..196d8cd355 100644 --- a/src/test/test_circuitbuild.c +++ b/src/test/test_circuitbuild.c @@ -4,6 +4,8 @@ /* See LICENSE for licensing information */ #define CIRCUITBUILD_PRIVATE +#define CIRCUITLIST_PRIVATE +#define ENTRYNODES_PRIVATE #include "core/or/or.h" #include "test/test.h" @@ -13,7 +15,11 @@ #include "core/or/circuitbuild.h" #include "core/or/circuitlist.h" +#include "core/or/cpath_build_state_st.h" #include "core/or/extend_info_st.h" +#include "core/or/origin_circuit_st.h" + +#include "feature/client/entrynodes.h" /* Dummy nodes smartlist for testing */ static smartlist_t dummy_nodes; @@ -126,10 +132,51 @@ test_new_route_len_unhandled_exit(void *arg) UNMOCK(count_acceptable_nodes); } +static void +test_upgrade_from_guard_wait(void *arg) +{ + circuit_t *circ = NULL; + origin_circuit_t *orig_circ = NULL; + entry_guard_t *guard = NULL; + smartlist_t *list = NULL; + + (void) arg; + + circ = dummy_origin_circuit_new(0); + orig_circ = TO_ORIGIN_CIRCUIT(circ); + tt_assert(orig_circ); + + orig_circ->build_state = tor_malloc_zero(sizeof(cpath_build_state_t)); + + circuit_set_state(circ, CIRCUIT_STATE_GUARD_WAIT); + + /* Put it in guard wait state. */ + guard = tor_malloc_zero(sizeof(*guard)); + guard->in_selection = get_guard_selection_info(); + + orig_circ->guard_state = + circuit_guard_state_new(guard, GUARD_CIRC_STATE_WAITING_FOR_BETTER_GUARD, + NULL); + + /* Mark the circuit for close. */ + circuit_mark_for_close(circ, END_CIRC_REASON_TORPROTOCOL); + tt_int_op(circ->marked_for_close, OP_NE, 0); + + /* We shouldn't pick the mark for close circuit. */ + list = circuit_find_circuits_to_upgrade_from_guard_wait(); + tt_assert(!list); + + done: + circuit_free(circ); + entry_guard_free_(guard); +} + struct testcase_t circuitbuild_tests[] = { { "noexit", test_new_route_len_noexit, 0, NULL, NULL }, { "safe_exit", test_new_route_len_safe_exit, 0, NULL, NULL }, { "unsafe_exit", test_new_route_len_unsafe_exit, 0, NULL, NULL }, { "unhandled_exit", test_new_route_len_unhandled_exit, 0, NULL, NULL }, + { "upgrade_from_guard_wait", test_upgrade_from_guard_wait, TT_FORK, + &helper_pubsub_setup, NULL }, END_OF_TESTCASES }; From 9644f3462a4c069a914beb04fda73820bcf29779 Mon Sep 17 00:00:00 2001 From: David Goulet Date: Tue, 18 Jun 2019 13:33:10 -0400 Subject: [PATCH 1138/2557] Make the great grand practracker happy... Signed-off-by: David Goulet --- scripts/maint/practracker/exceptions.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/maint/practracker/exceptions.txt b/scripts/maint/practracker/exceptions.txt index 815d0ed097..613f046a21 100644 --- a/scripts/maint/practracker/exceptions.txt +++ b/scripts/maint/practracker/exceptions.txt @@ -135,8 +135,8 @@ problem function-size /src/feature/client/addressmap.c:addressmap_rewrite() 112 problem function-size /src/feature/client/bridges.c:rewrite_node_address_for_bridge() 126 problem function-size /src/feature/client/circpathbias.c:pathbias_measure_close_rate() 108 problem function-size /src/feature/client/dnsserv.c:evdns_server_callback() 153 -problem file-size /src/feature/client/entrynodes.c 3820 -problem function-size /src/feature/client/entrynodes.c:entry_guards_upgrade_waiting_circuits() 153 +problem file-size /src/feature/client/entrynodes.c 3824 +problem function-size /src/feature/client/entrynodes.c:entry_guards_upgrade_waiting_circuits() 157 problem function-size /src/feature/client/entrynodes.c:entry_guard_parse_from_state() 246 problem function-size /src/feature/client/transports.c:handle_proxy_line() 108 problem function-size /src/feature/client/transports.c:parse_method_line_helper() 112 From 91c7d395cf7c62f8ccf3261132cb5d71972f79ae Mon Sep 17 00:00:00 2001 From: David Goulet Date: Wed, 19 Jun 2019 07:50:02 -0400 Subject: [PATCH 1139/2557] changes: Fix typo in changes/ticket30893 Signed-off-by: David Goulet --- changes/ticket30893 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changes/ticket30893 b/changes/ticket30893 index e1b56d2182..638b99a9f7 100644 --- a/changes/ticket30893 +++ b/changes/ticket30893 @@ -1,3 +1,3 @@ - o Minor features (testing)P: + o Minor features (testing): - Improve test coverage for our existing configuration parsing and management API. Closes ticket 30893. From 87511766873fd22ebeff1bc9dcfa78773a890083 Mon Sep 17 00:00:00 2001 From: David Goulet Date: Wed, 19 Jun 2019 09:22:07 -0400 Subject: [PATCH 1140/2557] hs-v3: Close intro circuits when cleaning client cache Fixes #30921 Signed-off-by: David Goulet --- changes/ticket30921 | 5 +++ src/feature/hs/hs_cache.c | 5 +++ src/test/test_hs_client.c | 89 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 99 insertions(+) create mode 100644 changes/ticket30921 diff --git a/changes/ticket30921 b/changes/ticket30921 new file mode 100644 index 0000000000..50ec570ffe --- /dev/null +++ b/changes/ticket30921 @@ -0,0 +1,5 @@ + o Minor bugfixes (onion service v3): + - When purging the client descriptor cache, always also close any + introduction point circuits associated with it. This avoids picking those + when connecting to them later while not having the descriptor to complete + the introduction. Fixes bug 30921; bugfix on 0.3.2.1-alpha. diff --git a/src/feature/hs/hs_cache.c b/src/feature/hs/hs_cache.c index 05f9940ae6..9817113b23 100644 --- a/src/feature/hs/hs_cache.c +++ b/src/feature/hs/hs_cache.c @@ -710,6 +710,11 @@ cache_clean_v3_as_client(time_t now) MAP_DEL_CURRENT(key); entry_size = cache_get_client_entry_size(entry); bytes_removed += entry_size; + /* We just removed an old descriptor. We need to close all intro circuits + * so we don't have leftovers that can be selected while lacking a + * descriptor. We leave the rendezvous circuits opened because they could + * be in use. */ + hs_client_close_intro_circuits_from_desc(entry->desc); /* Entry is not in the cache anymore, destroy it. */ cache_client_desc_free(entry); /* Update our OOM. We didn't use the remove() function because we are in diff --git a/src/test/test_hs_client.c b/src/test/test_hs_client.c index 0d25a98bb3..fb497d52a1 100644 --- a/src/test/test_hs_client.c +++ b/src/test/test_hs_client.c @@ -37,6 +37,7 @@ #include "feature/hs/hs_config.h" #include "feature/hs/hs_ident.h" #include "feature/hs/hs_cache.h" +#include "feature/rend/rendcache.h" #include "core/or/circuitlist.h" #include "core/or/circuitbuild.h" #include "core/mainloop/connection.h" @@ -1007,6 +1008,92 @@ test_close_intro_circuits_new_desc(void *arg) UNMOCK(networkstatus_get_live_consensus); } +static void +test_close_intro_circuits_cache_clean(void *arg) +{ + int ret; + ed25519_keypair_t service_kp; + circuit_t *circ = NULL; + origin_circuit_t *ocirc = NULL; + hs_descriptor_t *desc1 = NULL; + + (void) arg; + + hs_init(); + rend_cache_init(); + + /* This is needed because of the client cache expiration timestamp is based + * on having a consensus. See cached_client_descriptor_has_expired(). */ + MOCK(networkstatus_get_live_consensus, + mock_networkstatus_get_live_consensus); + + /* Set consensus time */ + parse_rfc1123_time("Sat, 26 Oct 1985 13:00:00 UTC", + &mock_ns.valid_after); + parse_rfc1123_time("Sat, 26 Oct 1985 14:00:00 UTC", + &mock_ns.fresh_until); + parse_rfc1123_time("Sat, 26 Oct 1985 16:00:00 UTC", + &mock_ns.valid_until); + + /* Generate service keypair */ + tt_int_op(0, OP_EQ, ed25519_keypair_generate(&service_kp, 0)); + + /* Create and add to the global list a dummy client introduction circuits. + * We'll then make sure the hs_ident is attached to a dummy descriptor. */ + circ = dummy_origin_circuit_new(0); + tt_assert(circ); + circ->purpose = CIRCUIT_PURPOSE_C_INTRODUCING; + ocirc = TO_ORIGIN_CIRCUIT(circ); + + /* Build the first descriptor and cache it. */ + { + char *encoded; + desc1 = hs_helper_build_hs_desc_with_ip(&service_kp); + tt_assert(desc1); + ret = hs_desc_encode_descriptor(desc1, &service_kp, NULL, &encoded); + tt_int_op(ret, OP_EQ, 0); + tt_assert(encoded); + + /* Store it */ + ret = hs_cache_store_as_client(encoded, &service_kp.pubkey); + tt_int_op(ret, OP_EQ, 0); + tor_free(encoded); + tt_assert(hs_cache_lookup_as_client(&service_kp.pubkey)); + } + + /* We'll pick one introduction point and associate it with the circuit. */ + { + const hs_desc_intro_point_t *ip = + smartlist_get(desc1->encrypted_data.intro_points, 0); + tt_assert(ip); + ocirc->hs_ident = hs_ident_circuit_new(&service_kp.pubkey, + HS_IDENT_CIRCUIT_INTRO); + ed25519_pubkey_copy(ô->hs_ident->intro_auth_pk, + &ip->auth_key_cert->signed_key); + } + + /* Before we are about to clean up the intro circuits, make sure it is + * actually there. */ + tt_assert(circuit_get_next_intro_circ(NULL, true)); + + /* Cleanup the client cache. The ns valid after time is what decides if the + * descriptor has expired so put it in the future enough (72h) so we are + * sure to always expire. */ + mock_ns.valid_after = approx_time() + (72 * 24 * 60 * 60); + hs_cache_clean_as_client(0); + + /* Once stored, our intro circuit should be closed because it is related to + * an old introduction point that doesn't exists anymore. */ + tt_assert(!circuit_get_next_intro_circ(NULL, true)); + + done: + circuit_free(circ); + hs_descriptor_free(desc1); + hs_free_all(); + rend_cache_free_all(); + UNMOCK(networkstatus_get_live_consensus); +} + struct testcase_t hs_client_tests[] = { { "e2e_rend_circuit_setup_legacy", test_e2e_rend_circuit_setup_legacy, TT_FORK, NULL, NULL }, @@ -1026,6 +1113,8 @@ struct testcase_t hs_client_tests[] = { TT_FORK, NULL, NULL }, { "close_intro_circuits_new_desc", test_close_intro_circuits_new_desc, TT_FORK, NULL, NULL }, + { "close_intro_circuits_cache_clean", test_close_intro_circuits_cache_clean, + TT_FORK, NULL, NULL }, END_OF_TESTCASES }; From f2b1eb1f052c99e0be096b98888e9854cf57a64c Mon Sep 17 00:00:00 2001 From: David Goulet Date: Wed, 19 Jun 2019 11:09:14 -0400 Subject: [PATCH 1141/2557] hs: Disallow single hop client circuit when introducing This will effectively also deny any bridge to be used as a single hop to the introduction point since bridge do not authenticate like clients. Fixes #24963 Signed-off-by: David Goulet --- changes/ticket24963 | 5 +++++ src/feature/hs/hs_intropoint.c | 9 +++++++++ 2 files changed, 14 insertions(+) create mode 100644 changes/ticket24963 diff --git a/changes/ticket24963 b/changes/ticket24963 new file mode 100644 index 0000000000..50adcfaaf4 --- /dev/null +++ b/changes/ticket24963 @@ -0,0 +1,5 @@ + o Minor feature (onion service): + - Disallow single hop clients to introduce directly at the introduction + point. We've removed Tor2web a while back and rendezvous are blocked at + the relays. This is to remove load off the network from spammy clients. + Close ticket 24963. diff --git a/src/feature/hs/hs_intropoint.c b/src/feature/hs/hs_intropoint.c index 9333060e7e..447f73b602 100644 --- a/src/feature/hs/hs_intropoint.c +++ b/src/feature/hs/hs_intropoint.c @@ -10,6 +10,7 @@ #include "core/or/or.h" #include "app/config/config.h" +#include "core/or/channel.h" #include "core/or/circuitlist.h" #include "core/or/circuituse.h" #include "core/or/relay.h" @@ -546,6 +547,14 @@ circuit_is_suitable_for_introduce1(const or_circuit_t *circ) return 0; } + /* Disallow single hop client circuit. */ + if (channel_is_client(circ->p_chan)) { + log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, + "Single hop client was rejected while trying to introduce. " + "Closing circuit."); + return 0; + } + return 1; } From 4ecd09cfb7c5d43450d481807f60feb0aa98bccf Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 19 Jun 2019 14:29:08 -0400 Subject: [PATCH 1142/2557] remove practracker from check-local (0.4.1 only) --- Makefile.am | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile.am b/Makefile.am index 7a0d40d6a5..3f3de34ceb 100644 --- a/Makefile.am +++ b/Makefile.am @@ -230,7 +230,7 @@ shellcheck: fi; \ fi -check-local: check-spaces check-changes check-includes check-best-practices shellcheck +check-local: check-spaces check-changes check-includes shellcheck need-chutney-path: @if test ! -d "$$CHUTNEY_PATH"; then \ From f9de9052e1e63401bb0d824eeb3cf0677b6dfd7d Mon Sep 17 00:00:00 2001 From: rl1987 Date: Sun, 23 Jun 2019 13:31:47 +0300 Subject: [PATCH 1143/2557] On Travis, put random data in ~/.torrc --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index 5d5d95f6b3..1901d950a4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -191,6 +191,9 @@ install: - python --version ## run stem tests if they are enabled. - if [[ "$TEST_STEM" != "" ]]; then pushd stem; python -c "from stem import stem; print(stem.__version__);"; git log -1; popd; fi + ## We don't want Tor tests to depends on default configuration file at + ## ~/.torrc. So we put some random bytes in there, to make sure it doesn't. + - dd ibs=1 count=1024 if=/dev/urandom > ~/.torrc script: # Skip test_rebind on macOS From 1d504a408d129bf561138360a5d727b37c0e29a9 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Sun, 23 Jun 2019 13:38:49 +0300 Subject: [PATCH 1144/2557] Add changes file --- changes/ticket30102 | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 changes/ticket30102 diff --git a/changes/ticket30102 b/changes/ticket30102 new file mode 100644 index 0000000000..c8b1148da3 --- /dev/null +++ b/changes/ticket30102 @@ -0,0 +1,4 @@ + o Minor features (continuous integration): + - When running CI builds on Travis, put some random data in ~/.torrc, + to make sure no tests are dependent on default Tor configuration. + Resolves issue 30102. From a52e00b5b315101ccb84c0ed981b6f34c4c52def Mon Sep 17 00:00:00 2001 From: rl1987 Date: Sun, 23 Jun 2019 14:00:43 +0300 Subject: [PATCH 1145/2557] Fix shellcheck warning SC2034 in test_rebind.sh. Bugfix on be0a4be276c945e4e90b43ce8f784b5b75bef122 (not in any Tor release). --- src/test/test_rebind.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/test/test_rebind.sh b/src/test/test_rebind.sh index e0d8394d38..d6d9d86668 100755 --- a/src/test/test_rebind.sh +++ b/src/test/test_rebind.sh @@ -12,8 +12,6 @@ if test "$UNAME_OS" = 'CYGWIN' || \ fi fi -exitcode=0 - tmpdir= clean () { if [ -n "$tmpdir" ] && [ -d "$tmpdir" ]; then From 84f0f15066b10581c673525fe3c41d32511437e1 Mon Sep 17 00:00:00 2001 From: rl1987 Date: Sun, 23 Jun 2019 14:12:05 +0300 Subject: [PATCH 1146/2557] Improve comment in .travis.yml --- .travis.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1901d950a4..fe34fac255 100644 --- a/.travis.yml +++ b/.travis.yml @@ -191,8 +191,9 @@ install: - python --version ## run stem tests if they are enabled. - if [[ "$TEST_STEM" != "" ]]; then pushd stem; python -c "from stem import stem; print(stem.__version__);"; git log -1; popd; fi - ## We don't want Tor tests to depends on default configuration file at - ## ~/.torrc. So we put some random bytes in there, to make sure it doesn't. + ## We don't want Tor tests to depend on default configuration file at + ## ~/.torrc. So we put some random bytes in there, to make sure we get build + ## failures in case Tor is reading it during CI jobs. - dd ibs=1 count=1024 if=/dev/urandom > ~/.torrc script: From 8356cc5b514a5d6a0170927d7c3730e2e1164298 Mon Sep 17 00:00:00 2001 From: teor Date: Mon, 24 Jun 2019 19:44:24 +1000 Subject: [PATCH 1147/2557] stats: Always publish pluggable transports in extra info documents Always publish bridge pluggable transport information in the extra info descriptor, even if ExtraInfoStatistics is 0. This information is needed by BridgeDB. Fixes bug 30956; bugfix on 0.4.1.1-alpha. --- changes/bug30956 | 4 ++++ doc/tor.1.txt | 6 ++++-- src/feature/relay/router.c | 17 +++++++++++------ 3 files changed, 19 insertions(+), 8 deletions(-) create mode 100644 changes/bug30956 diff --git a/changes/bug30956 b/changes/bug30956 new file mode 100644 index 0000000000..8f52a81de3 --- /dev/null +++ b/changes/bug30956 @@ -0,0 +1,4 @@ + o Minor bugfixes (pluggable transports): + - Always publish bridge pluggable transport information in the extra info + descriptor, even if ExtraInfoStatistics is 0. This information is + needed by BridgeDB. Fixes bug 30956; bugfix on 0.4.1.1-alpha. diff --git a/doc/tor.1.txt b/doc/tor.1.txt index 4bd365c774..95d56c6dbd 100644 --- a/doc/tor.1.txt +++ b/doc/tor.1.txt @@ -2451,8 +2451,10 @@ is non-zero): [[ExtraInfoStatistics]] **ExtraInfoStatistics** **0**|**1**:: When this option is enabled, Tor includes previously gathered statistics in its extra-info documents that it uploads to the directory authorities. - Disabling this option also disables bandwidth usage statistics, GeoIPFile - hashes, and ServerTransportPlugin lists in the extra-info file. + Disabling this option also removes bandwidth usage statistics, and + GeoIPFile and GeoIPv6File hashes from the extra-info file. Bridge + ServerTransportPlugin lines are always includes in the extra-info file, + because they are required by BridgeDB. (Default: 1) [[ExtendAllowPrivateAddresses]] **ExtendAllowPrivateAddresses** **0**|**1**:: diff --git a/src/feature/relay/router.c b/src/feature/relay/router.c index e0daf34db2..25bb1835c2 100644 --- a/src/feature/relay/router.c +++ b/src/feature/relay/router.c @@ -3175,6 +3175,15 @@ extrainfo_dump_to_string(char **s_out, extrainfo_t *extrainfo, published); smartlist_add(chunks, pre); + /* Add information about the pluggable transports we support, even if we + * are not publishing statistics. This information is needed by BridgeDB + * to distribute bridges. */ + if (options->ServerTransportPlugin) { + char *pluggable_transports = pt_get_extra_info_descriptor_string(); + if (pluggable_transports) + smartlist_add(chunks, pluggable_transports); + } + if (options->ExtraInfoStatistics && write_stats_to_extrainfo) { log_info(LD_GENERAL, "Adding stats to extra-info descriptor."); /* Bandwidth usage stats don't have their own option */ @@ -3182,6 +3191,7 @@ extrainfo_dump_to_string(char **s_out, extrainfo_t *extrainfo, contents = rep_hist_get_bandwidth_lines(); smartlist_add(chunks, contents); } + /* geoip hashes aren't useful unless we are publishing other stats */ if (geoip_is_loaded(AF_INET)) smartlist_add_asprintf(chunks, "geoip-db-digest %s\n", geoip_db_digest(AF_INET)); @@ -3223,12 +3233,7 @@ extrainfo_dump_to_string(char **s_out, extrainfo_t *extrainfo, if (contents) smartlist_add(chunks, contents); } - /* Add information about the pluggable transports we support. */ - if (options->ServerTransportPlugin) { - char *pluggable_transports = pt_get_extra_info_descriptor_string(); - if (pluggable_transports) - smartlist_add(chunks, pluggable_transports); - } + /* bridge statistics */ if (should_record_bridge_info(options)) { const char *bridge_stats = geoip_get_bridge_stats_extrainfo(now); if (bridge_stats) { From 45be44ed9c1b3ddbe2ad4caaebc3ae0076bdbd07 Mon Sep 17 00:00:00 2001 From: teor Date: Mon, 24 Jun 2019 20:32:38 +1000 Subject: [PATCH 1148/2557] stats: Split extrainfo_dump_to_string() into smaller functions. Closes ticket 30956. --- changes/ticket30956_refactor | 3 + scripts/maint/practracker/exceptions.txt | 3 +- src/feature/relay/router.c | 186 +++++++++++++++++------ 3 files changed, 146 insertions(+), 46 deletions(-) create mode 100644 changes/ticket30956_refactor diff --git a/changes/ticket30956_refactor b/changes/ticket30956_refactor new file mode 100644 index 0000000000..81151c6cc9 --- /dev/null +++ b/changes/ticket30956_refactor @@ -0,0 +1,3 @@ + o Code simplification and refactoring: + - Split extrainfo_dump_to_string() into smaller functions. + Closes ticket 30956. diff --git a/scripts/maint/practracker/exceptions.txt b/scripts/maint/practracker/exceptions.txt index 815d0ed097..4db452b897 100644 --- a/scripts/maint/practracker/exceptions.txt +++ b/scripts/maint/practracker/exceptions.txt @@ -225,13 +225,12 @@ problem function-size /src/feature/nodelist/routerlist.c:update_extrainfo_downlo problem function-size /src/feature/relay/dns.c:dns_resolve_impl() 134 problem function-size /src/feature/relay/dns.c:configure_nameservers() 161 problem function-size /src/feature/relay/dns.c:evdns_callback() 109 -problem file-size /src/feature/relay/router.c 3407 +problem file-size /src/feature/relay/router.c 3510 problem include-count /src/feature/relay/router.c 56 problem function-size /src/feature/relay/router.c:init_keys() 252 problem function-size /src/feature/relay/router.c:get_my_declared_family() 114 problem function-size /src/feature/relay/router.c:router_build_fresh_unsigned_routerinfo() 136 problem function-size /src/feature/relay/router.c:router_dump_router_to_string() 371 -problem function-size /src/feature/relay/router.c:extrainfo_dump_to_string() 206 problem function-size /src/feature/relay/routerkeys.c:load_ed_keys() 294 problem function-size /src/feature/rend/rendcache.c:rend_cache_store_v2_desc_as_client() 193 problem function-size /src/feature/rend/rendclient.c:rend_client_send_introduction() 220 diff --git a/src/feature/relay/router.c b/src/feature/relay/router.c index 25bb1835c2..6b33265294 100644 --- a/src/feature/relay/router.c +++ b/src/feature/relay/router.c @@ -3113,33 +3113,22 @@ load_stats_file(const char *filename, const char *end_line, time_t now, return r; } -/** Write the contents of extrainfo, to * *s_out, signing them - * with ident_key. - * - * If ExtraInfoStatistics is 1, also write aggregated statistics and related - * configuration data before signing. Most statistics also have an option that - * enables or disables that particular statistic. - * - * Return 0 on success, negative on failure. */ -int -extrainfo_dump_to_string(char **s_out, extrainfo_t *extrainfo, - crypto_pk_t *ident_key, - const ed25519_keypair_t *signing_keypair) +/** Add header strings to chunks, based on the extrainfo object extrainfo, + * and ed25519 keypair signing_keypair, if emit_ed_sigs is true. + * Helper for extrainfo_dump_to_string(). + * Returns 0 on success, negative on failure. */ +static int +extrainfo_dump_to_string_header_helper( + smartlist_t *chunks, + const extrainfo_t *extrainfo, + const ed25519_keypair_t *signing_keypair, + int emit_ed_sigs) { - const or_options_t *options = get_options(); char identity[HEX_DIGEST_LEN+1]; char published[ISO_TIME_LEN+1]; - char digest[DIGEST_LEN]; - int result; - static int write_stats_to_extrainfo = 1; - char sig[DIROBJ_MAX_SIG_LEN+1]; - char *s = NULL, *pre, *contents, *cp, *s_dup = NULL; - time_t now = time(NULL); - smartlist_t *chunks = smartlist_new(); - extrainfo_t *ei_tmp = NULL; - const int emit_ed_sigs = signing_keypair && - extrainfo->cache_info.signing_key_cert; char *ed_cert_line = NULL; + char *pre = NULL; + int rv = -1; base16_encode(identity, sizeof(identity), extrainfo->cache_info.identity_digest, DIGEST_LEN); @@ -3175,6 +3164,29 @@ extrainfo_dump_to_string(char **s_out, extrainfo_t *extrainfo, published); smartlist_add(chunks, pre); + rv = 0; + goto done; + + err: + rv = -1; + + done: + tor_free(ed_cert_line); + return rv; +} + +/** Add pluggable transport and statistics strings to chunks, skipping + * statistics if write_stats_to_extrainfo is false. + * Helper for extrainfo_dump_to_string(). + * Can not fail. */ +static void +extrainfo_dump_to_string_stats_helper(smartlist_t *chunks, + int write_stats_to_extrainfo) +{ + const or_options_t *options = get_options(); + char *contents = NULL; + time_t now = time(NULL); + /* Add information about the pluggable transports we support, even if we * are not publishing statistics. This information is needed by BridgeDB * to distribute bridges. */ @@ -3241,21 +3253,113 @@ extrainfo_dump_to_string(char **s_out, extrainfo_t *extrainfo, } } } +} + +/** Add an ed25519 signature of chunks to chunks, using the ed25519 keypair + * signing_keypair. + * Helper for extrainfo_dump_to_string(). + * Returns 0 on success, negative on failure. */ +static int +extrainfo_dump_to_string_ed_sig_helper( + smartlist_t *chunks, + const ed25519_keypair_t *signing_keypair) +{ + char sha256_digest[DIGEST256_LEN]; + ed25519_signature_t ed_sig; + char buf[ED25519_SIG_BASE64_LEN+1]; + int rv = -1; + + smartlist_add_strdup(chunks, "router-sig-ed25519 "); + crypto_digest_smartlist_prefix(sha256_digest, DIGEST256_LEN, + ED_DESC_SIGNATURE_PREFIX, + chunks, "", DIGEST_SHA256); + if (ed25519_sign(&ed_sig, (const uint8_t*)sha256_digest, DIGEST256_LEN, + signing_keypair) < 0) + goto err; + ed25519_signature_to_base64(buf, &ed_sig); + + smartlist_add_asprintf(chunks, "%s\n", buf); + + rv = 0; + goto done; + + err: + rv = -1; + + done: + return rv; +} + +/** Add an RSA signature of extrainfo_string to chunks, using the RSA key + * ident_key. + * Helper for extrainfo_dump_to_string(). + * Returns 0 on success, negative on failure. */ +static int +extrainfo_dump_to_string_rsa_sig_helper(smartlist_t *chunks, + crypto_pk_t *ident_key, + const char *extrainfo_string) +{ + char sig[DIROBJ_MAX_SIG_LEN+1]; + char digest[DIGEST_LEN]; + int rv = -1; + + memset(sig, 0, sizeof(sig)); + if (router_get_extrainfo_hash(extrainfo_string, strlen(extrainfo_string), + digest) < 0 || + router_append_dirobj_signature(sig, sizeof(sig), digest, DIGEST_LEN, + ident_key) < 0) { + log_warn(LD_BUG, "Could not append signature to extra-info " + "descriptor."); + goto err; + } + smartlist_add_strdup(chunks, sig); + + rv = 0; + goto done; + + err: + rv = -1; + + done: + return rv; +} + +/** Write the contents of extrainfo, to * *s_out, signing them + * with ident_key. + * + * If ExtraInfoStatistics is 1, also write aggregated statistics and related + * configuration data before signing. Most statistics also have an option that + * enables or disables that particular statistic. + * + * Always write pluggable transport lines. + * + * Return 0 on success, negative on failure. */ +int +extrainfo_dump_to_string(char **s_out, extrainfo_t *extrainfo, + crypto_pk_t *ident_key, + const ed25519_keypair_t *signing_keypair) +{ + int result; + static int write_stats_to_extrainfo = 1; + char *s = NULL, *cp, *s_dup = NULL; + smartlist_t *chunks = smartlist_new(); + extrainfo_t *ei_tmp = NULL; + const int emit_ed_sigs = signing_keypair && + extrainfo->cache_info.signing_key_cert; + int rv = 0; + + rv = extrainfo_dump_to_string_header_helper(chunks, extrainfo, + signing_keypair, + emit_ed_sigs); + if (rv < 0) + goto err; + + extrainfo_dump_to_string_stats_helper(chunks, write_stats_to_extrainfo); if (emit_ed_sigs) { - char sha256_digest[DIGEST256_LEN]; - smartlist_add_strdup(chunks, "router-sig-ed25519 "); - crypto_digest_smartlist_prefix(sha256_digest, DIGEST256_LEN, - ED_DESC_SIGNATURE_PREFIX, - chunks, "", DIGEST_SHA256); - ed25519_signature_t ed_sig; - char buf[ED25519_SIG_BASE64_LEN+1]; - if (ed25519_sign(&ed_sig, (const uint8_t*)sha256_digest, DIGEST256_LEN, - signing_keypair) < 0) + rv = extrainfo_dump_to_string_ed_sig_helper(chunks, signing_keypair); + if (rv < 0) goto err; - ed25519_signature_to_base64(buf, &ed_sig); - - smartlist_add_asprintf(chunks, "%s\n", buf); } smartlist_add_strdup(chunks, "router-signature\n"); @@ -3285,15 +3389,10 @@ extrainfo_dump_to_string(char **s_out, extrainfo_t *extrainfo, } } - memset(sig, 0, sizeof(sig)); - if (router_get_extrainfo_hash(s, strlen(s), digest) < 0 || - router_append_dirobj_signature(sig, sizeof(sig), digest, DIGEST_LEN, - ident_key) < 0) { - log_warn(LD_BUG, "Could not append signature to extra-info " - "descriptor."); + rv = extrainfo_dump_to_string_rsa_sig_helper(chunks, ident_key, s); + if (rv < 0) goto err; - } - smartlist_add_strdup(chunks, sig); + tor_free(s); s = smartlist_join_strings(chunks, "", 0, NULL); @@ -3329,7 +3428,6 @@ extrainfo_dump_to_string(char **s_out, extrainfo_t *extrainfo, SMARTLIST_FOREACH(chunks, char *, chunk, tor_free(chunk)); smartlist_free(chunks); tor_free(s_dup); - tor_free(ed_cert_line); extrainfo_free(ei_tmp); return result; From 5beb32d3d9a9640e515e6369d3a908e93b8895ef Mon Sep 17 00:00:00 2001 From: teor Date: Mon, 24 Jun 2019 21:19:49 +1000 Subject: [PATCH 1149/2557] stats: Stop removing the ed25519 signature if the extra info file is too big If the signature data was removed, but the keyword was kept, this could result in an unparseable extra info file. Fixes bug 30958; bugfix on 0.2.7.2-alpha. --- changes/bug30958 | 5 +++++ src/or/router.c | 12 +++++++----- 2 files changed, 12 insertions(+), 5 deletions(-) create mode 100644 changes/bug30958 diff --git a/changes/bug30958 b/changes/bug30958 new file mode 100644 index 0000000000..374c8e46f7 --- /dev/null +++ b/changes/bug30958 @@ -0,0 +1,5 @@ + o Minor bugfixes (statistics): + - Stop removing the ed25519 signature if the extra info file is too big. + If the signature data was removed, but the keyword was kept, this could + result in an unparseable extra info file. Fixes bug 30958; + bugfix on 0.2.7.2-alpha. diff --git a/src/or/router.c b/src/or/router.c index c416474226..3fce45115c 100644 --- a/src/or/router.c +++ b/src/or/router.c @@ -3252,11 +3252,13 @@ extrainfo_dump_to_string(char **s_out, extrainfo_t *extrainfo, while (strlen(s) > MAX_EXTRAINFO_UPLOAD_SIZE - DIROBJ_MAX_SIG_LEN) { /* So long as there are at least two chunks (one for the initial * extra-info line and one for the router-signature), we can keep removing - * things. */ - if (smartlist_len(chunks) > 2) { - /* We remove the next-to-last element (remember, len-1 is the last - element), since we need to keep the router-signature element. */ - int idx = smartlist_len(chunks) - 2; + * things. If emit_ed_sigs is true, we also keep 2 additional chunks at the + * end for the ed25519 signature. */ + const int required_chunks = emit_ed_sigs ? 4 : 2; + if (smartlist_len(chunks) > required_chunks) { + /* We remove the next-to-last or 4th-last element (remember, len-1 is the + * last element), since we need to keep the router-signature elements. */ + int idx = smartlist_len(chunks) - required_chunks; char *e = smartlist_get(chunks, idx); smartlist_del_keeporder(chunks, idx); log_warn(LD_GENERAL, "We just generated an extra-info descriptor " From aab5f42ae01cb33ec32ff02f1468bb0360d341e0 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 24 Jun 2019 09:32:46 -0400 Subject: [PATCH 1150/2557] bump to 0.4.1.3-alpha --- configure.ac | 4 ++-- contrib/win32build/tor-mingw.nsi.in | 2 +- src/win32/orconfig.h | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/configure.ac b/configure.ac index 05b932973f..107f676bdb 100644 --- a/configure.ac +++ b/configure.ac @@ -4,7 +4,7 @@ dnl Copyright (c) 2007-2019, The Tor Project, Inc. dnl See LICENSE for licensing information AC_PREREQ([2.63]) -AC_INIT([tor],[0.4.1.2-alpha-dev]) +AC_INIT([tor],[0.4.1.3-alpha]) AC_CONFIG_SRCDIR([src/app/main/tor_main.c]) AC_CONFIG_MACRO_DIR([m4]) @@ -14,7 +14,7 @@ AC_CONFIG_MACRO_DIR([m4]) # version number changes. Tor uses it to make sure that it # only shuts down for missing "required protocols" when those protocols # are listed as required by a consensus after this date. -AC_DEFINE(APPROX_RELEASE_DATE, ["2019-06-06"], # for 0.4.1.2-alpha-dev +AC_DEFINE(APPROX_RELEASE_DATE, ["2019-06-24"], # for 0.4.1.3-alpha [Approximate date when this software was released. (Updated when the version changes.)]) # "foreign" means we don't follow GNU package layout standards diff --git a/contrib/win32build/tor-mingw.nsi.in b/contrib/win32build/tor-mingw.nsi.in index 700744d95c..da4aea718e 100644 --- a/contrib/win32build/tor-mingw.nsi.in +++ b/contrib/win32build/tor-mingw.nsi.in @@ -8,7 +8,7 @@ !include "LogicLib.nsh" !include "FileFunc.nsh" !insertmacro GetParameters -!define VERSION "0.4.1.2-alpha-dev" +!define VERSION "0.4.1.3-alpha" !define INSTALLER "tor-${VERSION}-win32.exe" !define WEBSITE "https://www.torproject.org/" !define LICENSE "LICENSE" diff --git a/src/win32/orconfig.h b/src/win32/orconfig.h index 768c6210c2..a90a212b42 100644 --- a/src/win32/orconfig.h +++ b/src/win32/orconfig.h @@ -218,7 +218,7 @@ #define USING_TWOS_COMPLEMENT /* Version number of package */ -#define VERSION "0.4.1.2-alpha-dev" +#define VERSION "0.4.1.3-alpha" From a87700633c91f98d3cef16bc7e4809a7ab6b4c2c Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 24 Jun 2019 13:45:05 -0400 Subject: [PATCH 1151/2557] Don't try to shellcheck src/rust/registry Fixes bug 30963; bug not in any released Tor. --- Makefile.am | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile.am b/Makefile.am index df30e0b59a..67ef77f876 100644 --- a/Makefile.am +++ b/Makefile.am @@ -224,7 +224,7 @@ test: all shellcheck: # Only use shellcheck if it is present if command -v shellcheck; then \ - find "$(top_srcdir)" -name "*.sh" -not -path "$(top_srcdir)/src/ext/*" -exec shellcheck {} +; \ + find "$(top_srcdir)" -name "*.sh" -not -path "$(top_srcdir)/src/ext/*" -not -path "$(top_srcdir)/src/rust/registry/*" -exec shellcheck {} +; \ if [ -d "$(top_srcdir)/scripts/test" ]; then \ shellcheck $(top_srcdir)/scripts/test/cov-diff $(top_srcdir)/scripts/test/coverage; \ fi; \ From 4bac6b6e7de55138927d075433754ba305bb2479 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 24 Jun 2019 13:49:19 -0400 Subject: [PATCH 1152/2557] Remove the unused "exitcode" variable from test_rebind.sh Fixes a shellcheck issue; closes ticket 30964. No user-visible change. --- src/test/test_rebind.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/test/test_rebind.sh b/src/test/test_rebind.sh index e0d8394d38..d6d9d86668 100755 --- a/src/test/test_rebind.sh +++ b/src/test/test_rebind.sh @@ -12,8 +12,6 @@ if test "$UNAME_OS" = 'CYGWIN' || \ fi fi -exitcode=0 - tmpdir= clean () { if [ -n "$tmpdir" ] && [ -d "$tmpdir" ]; then From 81d16d8d0cd2789c2d20b350c59bc3a283dda432 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 12 Jun 2019 16:37:21 -0400 Subject: [PATCH 1153/2557] checkSpace.pl: Allow 'bool' before a space and an open-paren We need this so we can declare function pointers returning bool without upsetting our style checker. :/ --- scripts/maint/checkSpace.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/maint/checkSpace.pl b/scripts/maint/checkSpace.pl index 433ae62807..9c9b68ff9d 100755 --- a/scripts/maint/checkSpace.pl +++ b/scripts/maint/checkSpace.pl @@ -177,7 +177,7 @@ for my $fn (@ARGV) { $1 ne "elsif" and $1 ne "WINAPI" and $2 ne "WINAPI" and $1 ne "void" and $1 ne "__attribute__" and $1 ne "op" and $1 ne "size_t" and $1 ne "double" and $1 ne "uint64_t" and - $1 ne "workqueue_reply_t") { + $1 ne "workqueue_reply_t" and $1 ne "bool") { msg " fn ():$fn:$.\n"; } } From 246599abb4dbfdb2f8ac073fc5b2059f602e8ed7 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 12 Jun 2019 10:18:30 -0400 Subject: [PATCH 1154/2557] Start moving types that will be used for config vars to lib/conf This will be a lower-level module than anything that actually sets or handles configuration variables. Part of 30864. --- src/app/config/confparse.h | 65 +---------------------------- src/include.am | 1 + src/lib/conf/.may_include | 2 + src/lib/conf/conftypes.h | 83 ++++++++++++++++++++++++++++++++++++++ src/lib/conf/include.am | 4 ++ 5 files changed, 91 insertions(+), 64 deletions(-) create mode 100644 src/lib/conf/.may_include create mode 100644 src/lib/conf/conftypes.h create mode 100644 src/lib/conf/include.am diff --git a/src/app/config/confparse.h b/src/app/config/confparse.h index 2112abf715..bd06a4a0d0 100644 --- a/src/app/config/confparse.h +++ b/src/app/config/confparse.h @@ -13,70 +13,7 @@ #ifndef TOR_CONFPARSE_H #define TOR_CONFPARSE_H -/** Enumeration of types which option values can take */ -typedef enum config_type_t { - CONFIG_TYPE_STRING = 0, /**< An arbitrary string. */ - CONFIG_TYPE_FILENAME, /**< A filename: some prefixes get expanded. */ - CONFIG_TYPE_POSINT, /**< A non-negative integer less than MAX_INT */ - CONFIG_TYPE_INT, /**< Any integer. */ - CONFIG_TYPE_UINT64, /**< A value in range 0..UINT64_MAX */ - CONFIG_TYPE_INTERVAL, /**< A number of seconds, with optional units*/ - CONFIG_TYPE_MSEC_INTERVAL,/**< A number of milliseconds, with optional - * units */ - CONFIG_TYPE_MEMUNIT, /**< A number of bytes, with optional units*/ - CONFIG_TYPE_DOUBLE, /**< A floating-point value */ - CONFIG_TYPE_BOOL, /**< A boolean value, expressed as 0 or 1. */ - CONFIG_TYPE_AUTOBOOL, /**< A boolean+auto value, expressed 0 for false, - * 1 for true, and -1 for auto */ - CONFIG_TYPE_ISOTIME, /**< An ISO-formatted time relative to UTC. */ - CONFIG_TYPE_CSV, /**< A list of strings, separated by commas and - * optional whitespace. */ - CONFIG_TYPE_CSV_INTERVAL, /**< A list of strings, separated by commas and - * optional whitespace, representing intervals in - * seconds, with optional units. We allow - * multiple values here for legacy reasons, but - * ignore every value after the first. */ - CONFIG_TYPE_LINELIST, /**< Uninterpreted config lines */ - CONFIG_TYPE_LINELIST_S, /**< Uninterpreted, context-sensitive config lines, - * mixed with other keywords. */ - CONFIG_TYPE_LINELIST_V, /**< Catch-all "virtual" option to summarize - * context-sensitive config lines when fetching. - */ - CONFIG_TYPE_ROUTERSET, /**< A list of router names, addrs, and fps, - * parsed into a routerset_t. */ - CONFIG_TYPE_OBSOLETE, /**< Obsolete (ignored) option. */ -} config_type_t; - -#ifdef TOR_UNIT_TESTS -/** - * Union used when building in test mode typechecking the members of a type - * used with confparse.c. See CONF_CHECK_VAR_TYPE for a description of how - * it is used. */ -typedef union { - char **STRING; - char **FILENAME; - int *POSINT; /* yes, really: Even though the confparse type is called - * "POSINT", it still uses the C int type -- it just enforces that - * the values are in range [0,INT_MAX]. - */ - uint64_t *UINT64; - int *INT; - int *PORT; - int *INTERVAL; - int *MSEC_INTERVAL; - uint64_t *MEMUNIT; - double *DOUBLE; - int *BOOL; - int *AUTOBOOL; - time_t *ISOTIME; - smartlist_t **CSV; - int *CSV_INTERVAL; - struct config_line_t **LINELIST; - struct config_line_t **LINELIST_S; - struct config_line_t **LINELIST_V; - routerset_t **ROUTERSET; -} confparse_dummy_values_t; -#endif /* defined(TOR_UNIT_TESTS) */ +#include "lib/conf/conftypes.h" /** An abbreviation for a configuration option allowed on the command line. */ typedef struct config_abbrev_t { diff --git a/src/include.am b/src/include.am index 77c126ba45..ab1983b3cd 100644 --- a/src/include.am +++ b/src/include.am @@ -5,6 +5,7 @@ include src/lib/err/include.am include src/lib/cc/include.am include src/lib/ctime/include.am include src/lib/compress/include.am +include src/lib/conf/include.am include src/lib/container/include.am include src/lib/crypt_ops/include.am include src/lib/defs/include.am diff --git a/src/lib/conf/.may_include b/src/lib/conf/.may_include new file mode 100644 index 0000000000..4285c3dcb8 --- /dev/null +++ b/src/lib/conf/.may_include @@ -0,0 +1,2 @@ +orconfig.h +lib/cc/*.h diff --git a/src/lib/conf/conftypes.h b/src/lib/conf/conftypes.h new file mode 100644 index 0000000000..eb4eb1245f --- /dev/null +++ b/src/lib/conf/conftypes.h @@ -0,0 +1,83 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file conftypes.h + * @brief Types used to specify configurable options. + **/ + +#ifndef TOR_SRC_LIB_CONF_CONFTYPES_H +#define TOR_SRC_LIB_CONF_CONFTYPES_H + +#include "lib/cc/torint.h" + +/** Enumeration of types which option values can take */ +typedef enum config_type_t { + CONFIG_TYPE_STRING = 0, /**< An arbitrary string. */ + CONFIG_TYPE_FILENAME, /**< A filename: some prefixes get expanded. */ + CONFIG_TYPE_POSINT, /**< A non-negative integer less than MAX_INT */ + CONFIG_TYPE_INT, /**< Any integer. */ + CONFIG_TYPE_UINT64, /**< A value in range 0..UINT64_MAX */ + CONFIG_TYPE_INTERVAL, /**< A number of seconds, with optional units*/ + CONFIG_TYPE_MSEC_INTERVAL,/**< A number of milliseconds, with optional + * units */ + CONFIG_TYPE_MEMUNIT, /**< A number of bytes, with optional units*/ + CONFIG_TYPE_DOUBLE, /**< A floating-point value */ + CONFIG_TYPE_BOOL, /**< A boolean value, expressed as 0 or 1. */ + CONFIG_TYPE_AUTOBOOL, /**< A boolean+auto value, expressed 0 for false, + * 1 for true, and -1 for auto */ + CONFIG_TYPE_ISOTIME, /**< An ISO-formatted time relative to UTC. */ + CONFIG_TYPE_CSV, /**< A list of strings, separated by commas and + * optional whitespace. */ + CONFIG_TYPE_CSV_INTERVAL, /**< A list of strings, separated by commas and + * optional whitespace, representing intervals in + * seconds, with optional units. We allow + * multiple values here for legacy reasons, but + * ignore every value after the first. */ + CONFIG_TYPE_LINELIST, /**< Uninterpreted config lines */ + CONFIG_TYPE_LINELIST_S, /**< Uninterpreted, context-sensitive config lines, + * mixed with other keywords. */ + CONFIG_TYPE_LINELIST_V, /**< Catch-all "virtual" option to summarize + * context-sensitive config lines when fetching. + */ + // XXXX this doesn't belong at this level of abstraction. + CONFIG_TYPE_ROUTERSET, /**< A list of router names, addrs, and fps, + * parsed into a routerset_t. */ + CONFIG_TYPE_OBSOLETE, /**< Obsolete (ignored) option. */ +} config_type_t; + +#ifdef TOR_UNIT_TESTS +/** + * Union used when building in test mode typechecking the members of a type + * used with confparse.c. See CONF_CHECK_VAR_TYPE for a description of how + * it is used. */ +typedef union { + char **STRING; + char **FILENAME; + int *POSINT; /* yes, really: Even though the confparse type is called + * "POSINT", it still uses the C int type -- it just enforces + * that the values are in range [0,INT_MAX]. + */ + uint64_t *UINT64; + int *INT; + int *INTERVAL; + int *MSEC_INTERVAL; + uint64_t *MEMUNIT; + double *DOUBLE; + int *BOOL; + int *AUTOBOOL; + time_t *ISOTIME; + struct smartlist_t **CSV; + int *CSV_INTERVAL; + struct config_line_t **LINELIST; + struct config_line_t **LINELIST_S; + struct config_line_t **LINELIST_V; + // XXXX this doesn't belong at this level of abstraction. + struct routerset_t **ROUTERSET; +} confparse_dummy_values_t; +#endif /* defined(TOR_UNIT_TESTS) */ + +#endif /* !defined(TOR_SRC_LIB_CONF_CONFTYPES_H) */ diff --git a/src/lib/conf/include.am b/src/lib/conf/include.am new file mode 100644 index 0000000000..25355697d2 --- /dev/null +++ b/src/lib/conf/include.am @@ -0,0 +1,4 @@ + +# ADD_C_FILE: INSERT HEADERS HERE. +noinst_HEADERS += \ + src/lib/conf/conftypes.h From 458da8a80d356a0c8015f025828cd1523838ecac Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 13 Jun 2019 11:59:17 -0400 Subject: [PATCH 1155/2557] Move unit-parsing code to src/lib/confmgt lib/confmgt is at a higher level than lib/conf, since it needs to call down to logging and similar modules. --- .gitignore | 2 + Makefile.am | 2 + src/app/config/confparse.c | 194 +-------------------------------- src/include.am | 1 + src/lib/confmgt/.may_include | 7 ++ src/lib/confmgt/include.am | 18 ++++ src/lib/confmgt/unitparse.c | 202 +++++++++++++++++++++++++++++++++++ src/lib/confmgt/unitparse.h | 34 ++++++ 8 files changed, 267 insertions(+), 193 deletions(-) create mode 100644 src/lib/confmgt/.may_include create mode 100644 src/lib/confmgt/include.am create mode 100644 src/lib/confmgt/unitparse.c create mode 100644 src/lib/confmgt/unitparse.h diff --git a/.gitignore b/.gitignore index 0865f981b1..77610b3193 100644 --- a/.gitignore +++ b/.gitignore @@ -162,6 +162,8 @@ uptime-*.json /src/lib/libtor-buf-testing.a /src/lib/libtor-compress.a /src/lib/libtor-compress-testing.a +/src/lib/libtor-confmgt.a +/src/lib/libtor-confmgt-testing.a /src/lib/libtor-container.a /src/lib/libtor-container-testing.a /src/lib/libtor-crypt-ops.a diff --git a/Makefile.am b/Makefile.am index df30e0b59a..0083608ce9 100644 --- a/Makefile.am +++ b/Makefile.am @@ -54,6 +54,7 @@ TOR_UTIL_LIBS = \ src/lib/libtor-math.a \ src/lib/libtor-meminfo.a \ src/lib/libtor-osinfo.a \ + src/lib/libtor-confmgt.a \ src/lib/libtor-log.a \ src/lib/libtor-lock.a \ src/lib/libtor-fdio.a \ @@ -88,6 +89,7 @@ TOR_UTIL_TESTING_LIBS = \ src/lib/libtor-meminfo-testing.a \ src/lib/libtor-osinfo-testing.a \ src/lib/libtor-term-testing.a \ + src/lib/libtor-confmgt-testing.a \ src/lib/libtor-log-testing.a \ src/lib/libtor-lock-testing.a \ src/lib/libtor-fdio-testing.a \ diff --git a/src/app/config/confparse.c b/src/app/config/confparse.c index 3bf14b378d..296e7c2a39 100644 --- a/src/app/config/confparse.c +++ b/src/app/config/confparse.c @@ -26,12 +26,10 @@ #include "app/config/confparse.h" #include "feature/nodelist/routerset.h" +#include "lib/confmgt/unitparse.h" #include "lib/container/bitarray.h" #include "lib/encoding/confline.h" -static uint64_t config_parse_memunit(const char *s, int *ok); -static int config_parse_msec_interval(const char *s, int *ok); -static int config_parse_interval(const char *s, int *ok); static void config_reset(const config_format_t *fmt, void *options, const config_var_t *var, int use_defaults); @@ -1014,193 +1012,3 @@ config_dump(const config_format_t *fmt, const void *default_options, } return result; } - -/** Mapping from a unit name to a multiplier for converting that unit into a - * base unit. Used by config_parse_unit. */ -struct unit_table_t { - const char *unit; /**< The name of the unit */ - uint64_t multiplier; /**< How many of the base unit appear in this unit */ -}; - -/** Table to map the names of memory units to the number of bytes they - * contain. */ -static struct unit_table_t memory_units[] = { - { "", 1 }, - { "b", 1<< 0 }, - { "byte", 1<< 0 }, - { "bytes", 1<< 0 }, - { "kb", 1<<10 }, - { "kbyte", 1<<10 }, - { "kbytes", 1<<10 }, - { "kilobyte", 1<<10 }, - { "kilobytes", 1<<10 }, - { "kilobits", 1<<7 }, - { "kilobit", 1<<7 }, - { "kbits", 1<<7 }, - { "kbit", 1<<7 }, - { "m", 1<<20 }, - { "mb", 1<<20 }, - { "mbyte", 1<<20 }, - { "mbytes", 1<<20 }, - { "megabyte", 1<<20 }, - { "megabytes", 1<<20 }, - { "megabits", 1<<17 }, - { "megabit", 1<<17 }, - { "mbits", 1<<17 }, - { "mbit", 1<<17 }, - { "gb", 1<<30 }, - { "gbyte", 1<<30 }, - { "gbytes", 1<<30 }, - { "gigabyte", 1<<30 }, - { "gigabytes", 1<<30 }, - { "gigabits", 1<<27 }, - { "gigabit", 1<<27 }, - { "gbits", 1<<27 }, - { "gbit", 1<<27 }, - { "tb", UINT64_C(1)<<40 }, - { "tbyte", UINT64_C(1)<<40 }, - { "tbytes", UINT64_C(1)<<40 }, - { "terabyte", UINT64_C(1)<<40 }, - { "terabytes", UINT64_C(1)<<40 }, - { "terabits", UINT64_C(1)<<37 }, - { "terabit", UINT64_C(1)<<37 }, - { "tbits", UINT64_C(1)<<37 }, - { "tbit", UINT64_C(1)<<37 }, - { NULL, 0 }, -}; - -/** Table to map the names of time units to the number of seconds they - * contain. */ -static struct unit_table_t time_units[] = { - { "", 1 }, - { "second", 1 }, - { "seconds", 1 }, - { "minute", 60 }, - { "minutes", 60 }, - { "hour", 60*60 }, - { "hours", 60*60 }, - { "day", 24*60*60 }, - { "days", 24*60*60 }, - { "week", 7*24*60*60 }, - { "weeks", 7*24*60*60 }, - { "month", 2629728, }, /* about 30.437 days */ - { "months", 2629728, }, - { NULL, 0 }, -}; - -/** Table to map the names of time units to the number of milliseconds - * they contain. */ -static struct unit_table_t time_msec_units[] = { - { "", 1 }, - { "msec", 1 }, - { "millisecond", 1 }, - { "milliseconds", 1 }, - { "second", 1000 }, - { "seconds", 1000 }, - { "minute", 60*1000 }, - { "minutes", 60*1000 }, - { "hour", 60*60*1000 }, - { "hours", 60*60*1000 }, - { "day", 24*60*60*1000 }, - { "days", 24*60*60*1000 }, - { "week", 7*24*60*60*1000 }, - { "weeks", 7*24*60*60*1000 }, - { NULL, 0 }, -}; - -/** Parse a string val containing a number, zero or more - * spaces, and an optional unit string. If the unit appears in the - * table u, then multiply the number by the unit multiplier. - * On success, set *ok to 1 and return this product. - * Otherwise, set *ok to 0. - */ -static uint64_t -config_parse_units(const char *val, struct unit_table_t *u, int *ok) -{ - uint64_t v = 0; - double d = 0; - int use_float = 0; - char *cp; - - tor_assert(ok); - - v = tor_parse_uint64(val, 10, 0, UINT64_MAX, ok, &cp); - if (!*ok || (cp && *cp == '.')) { - d = tor_parse_double(val, 0, (double)UINT64_MAX, ok, &cp); - if (!*ok) - goto done; - use_float = 1; - } - - if (!cp) { - *ok = 1; - v = use_float ? ((uint64_t)d) : v; - goto done; - } - - cp = (char*) eat_whitespace(cp); - - for ( ;u->unit;++u) { - if (!strcasecmp(u->unit, cp)) { - if (use_float) - v = (uint64_t)(u->multiplier * d); - else - v *= u->multiplier; - *ok = 1; - goto done; - } - } - log_warn(LD_CONFIG, "Unknown unit '%s'.", cp); - *ok = 0; - done: - - if (*ok) - return v; - else - return 0; -} - -/** Parse a string in the format "number unit", where unit is a unit of - * information (byte, KB, M, etc). On success, set *ok to true - * and return the number of bytes specified. Otherwise, set - * *ok to false and return 0. */ -static uint64_t -config_parse_memunit(const char *s, int *ok) -{ - uint64_t u = config_parse_units(s, memory_units, ok); - return u; -} - -/** Parse a string in the format "number unit", where unit is a unit of - * time in milliseconds. On success, set *ok to true and return - * the number of milliseconds in the provided interval. Otherwise, set - * *ok to 0 and return -1. */ -static int -config_parse_msec_interval(const char *s, int *ok) -{ - uint64_t r; - r = config_parse_units(s, time_msec_units, ok); - if (r > INT_MAX) { - log_warn(LD_CONFIG, "Msec interval '%s' is too long", s); - *ok = 0; - return -1; - } - return (int)r; -} - -/** Parse a string in the format "number unit", where unit is a unit of time. - * On success, set *ok to true and return the number of seconds in - * the provided interval. Otherwise, set *ok to 0 and return -1. - */ -static int -config_parse_interval(const char *s, int *ok) -{ - uint64_t r; - r = config_parse_units(s, time_units, ok); - if (r > INT_MAX) { - log_warn(LD_CONFIG, "Interval '%s' is too long", s); - *ok = 0; - return -1; - } - return (int)r; -} diff --git a/src/include.am b/src/include.am index ab1983b3cd..065bdc31cb 100644 --- a/src/include.am +++ b/src/include.am @@ -6,6 +6,7 @@ include src/lib/cc/include.am include src/lib/ctime/include.am include src/lib/compress/include.am include src/lib/conf/include.am +include src/lib/confmgt/include.am include src/lib/container/include.am include src/lib/crypt_ops/include.am include src/lib/defs/include.am diff --git a/src/lib/confmgt/.may_include b/src/lib/confmgt/.may_include new file mode 100644 index 0000000000..640acf3770 --- /dev/null +++ b/src/lib/confmgt/.may_include @@ -0,0 +1,7 @@ +orconfig.h +lib/cc/*.h +lib/conf/*.h +lib/confmgt/*.h +lib/log/*.h +lib/malloc/*.h +lib/string/*.h diff --git a/src/lib/confmgt/include.am b/src/lib/confmgt/include.am new file mode 100644 index 0000000000..729e6147ff --- /dev/null +++ b/src/lib/confmgt/include.am @@ -0,0 +1,18 @@ +noinst_LIBRARIES += src/lib/libtor-confmgt.a + +if UNITTESTS_ENABLED +noinst_LIBRARIES += src/lib/libtor-confmgt-testing.a +endif + +# ADD_C_FILE: INSERT SOURCES HERE. +src_lib_libtor_confmgt_a_SOURCES = \ + src/lib/confmgt/unitparse.c + +src_lib_libtor_confmgt_testing_a_SOURCES = \ + $(src_lib_libtor_confmgt_a_SOURCES) +src_lib_libtor_confmgt_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) +src_lib_libtor_confmgt_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) + +# ADD_C_FILE: INSERT HEADERS HERE. +noinst_HEADERS += \ + src/lib/confmgt/unitparse.h diff --git a/src/lib/confmgt/unitparse.c b/src/lib/confmgt/unitparse.c new file mode 100644 index 0000000000..906195259d --- /dev/null +++ b/src/lib/confmgt/unitparse.c @@ -0,0 +1,202 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file unitparse.c + * @brief Functions for parsing values with units from a configuration file. + **/ + +#include "orconfig.h" +#include "lib/confmgt/unitparse.h" +#include "lib/log/log.h" +#include "lib/log/util_bug.h" +#include "lib/string/parse_int.h" +#include "lib/string/util_string.h" + +#include + +/** Table to map the names of memory units to the number of bytes they + * contain. */ +const struct unit_table_t memory_units[] = { + { "", 1 }, + { "b", 1<< 0 }, + { "byte", 1<< 0 }, + { "bytes", 1<< 0 }, + { "kb", 1<<10 }, + { "kbyte", 1<<10 }, + { "kbytes", 1<<10 }, + { "kilobyte", 1<<10 }, + { "kilobytes", 1<<10 }, + { "kilobits", 1<<7 }, + { "kilobit", 1<<7 }, + { "kbits", 1<<7 }, + { "kbit", 1<<7 }, + { "m", 1<<20 }, + { "mb", 1<<20 }, + { "mbyte", 1<<20 }, + { "mbytes", 1<<20 }, + { "megabyte", 1<<20 }, + { "megabytes", 1<<20 }, + { "megabits", 1<<17 }, + { "megabit", 1<<17 }, + { "mbits", 1<<17 }, + { "mbit", 1<<17 }, + { "gb", 1<<30 }, + { "gbyte", 1<<30 }, + { "gbytes", 1<<30 }, + { "gigabyte", 1<<30 }, + { "gigabytes", 1<<30 }, + { "gigabits", 1<<27 }, + { "gigabit", 1<<27 }, + { "gbits", 1<<27 }, + { "gbit", 1<<27 }, + { "tb", UINT64_C(1)<<40 }, + { "tbyte", UINT64_C(1)<<40 }, + { "tbytes", UINT64_C(1)<<40 }, + { "terabyte", UINT64_C(1)<<40 }, + { "terabytes", UINT64_C(1)<<40 }, + { "terabits", UINT64_C(1)<<37 }, + { "terabit", UINT64_C(1)<<37 }, + { "tbits", UINT64_C(1)<<37 }, + { "tbit", UINT64_C(1)<<37 }, + { NULL, 0 }, +}; + +/** Table to map the names of time units to the number of seconds they + * contain. */ +const struct unit_table_t time_units[] = { + { "", 1 }, + { "second", 1 }, + { "seconds", 1 }, + { "minute", 60 }, + { "minutes", 60 }, + { "hour", 60*60 }, + { "hours", 60*60 }, + { "day", 24*60*60 }, + { "days", 24*60*60 }, + { "week", 7*24*60*60 }, + { "weeks", 7*24*60*60 }, + { "month", 2629728, }, /* about 30.437 days */ + { "months", 2629728, }, + { NULL, 0 }, +}; + +/** Table to map the names of time units to the number of milliseconds + * they contain. */ +const struct unit_table_t time_msec_units[] = { + { "", 1 }, + { "msec", 1 }, + { "millisecond", 1 }, + { "milliseconds", 1 }, + { "second", 1000 }, + { "seconds", 1000 }, + { "minute", 60*1000 }, + { "minutes", 60*1000 }, + { "hour", 60*60*1000 }, + { "hours", 60*60*1000 }, + { "day", 24*60*60*1000 }, + { "days", 24*60*60*1000 }, + { "week", 7*24*60*60*1000 }, + { "weeks", 7*24*60*60*1000 }, + { NULL, 0 }, +}; + +/** Parse a string val containing a number, zero or more + * spaces, and an optional unit string. If the unit appears in the + * table u, then multiply the number by the unit multiplier. + * On success, set *ok to 1 and return this product. + * Otherwise, set *ok to 0. + */ +uint64_t +config_parse_units(const char *val, const unit_table_t *u, int *ok) +{ + uint64_t v = 0; + double d = 0; + int use_float = 0; + char *cp; + + tor_assert(ok); + + v = tor_parse_uint64(val, 10, 0, UINT64_MAX, ok, &cp); + if (!*ok || (cp && *cp == '.')) { + d = tor_parse_double(val, 0, (double)UINT64_MAX, ok, &cp); + if (!*ok) + goto done; + use_float = 1; + } + + if (!cp) { + *ok = 1; + v = use_float ? ((uint64_t)d) : v; + goto done; + } + + cp = (char*) eat_whitespace(cp); + + for ( ;u->unit;++u) { + if (!strcasecmp(u->unit, cp)) { + if (use_float) + v = (uint64_t)(u->multiplier * d); + else + v *= u->multiplier; + *ok = 1; + goto done; + } + } + log_warn(LD_CONFIG, "Unknown unit '%s'.", cp); + *ok = 0; + done: + + if (*ok) + return v; + else + return 0; +} + +/** Parse a string in the format "number unit", where unit is a unit of + * information (byte, KB, M, etc). On success, set *ok to true + * and return the number of bytes specified. Otherwise, set + * *ok to false and return 0. */ +uint64_t +config_parse_memunit(const char *s, int *ok) +{ + uint64_t u = config_parse_units(s, memory_units, ok); + return u; +} + +/** Parse a string in the format "number unit", where unit is a unit of + * time in milliseconds. On success, set *ok to true and return + * the number of milliseconds in the provided interval. Otherwise, set + * *ok to 0 and return -1. */ +int +config_parse_msec_interval(const char *s, int *ok) +{ + uint64_t r; + r = config_parse_units(s, time_msec_units, ok); + if (r > INT_MAX) { + log_warn(LD_CONFIG, "Msec interval '%s' is too long", s); + *ok = 0; + return -1; + } + return (int)r; +} + +/** Parse a string in the format "number unit", where unit is a unit of time. + * On success, set *ok to true and return the number of seconds in + * the provided interval. Otherwise, set *ok to 0 and return -1. + */ +int +config_parse_interval(const char *s, int *ok) +{ + uint64_t r; + r = config_parse_units(s, time_units, ok); + if (r > INT_MAX) { + log_warn(LD_CONFIG, "Interval '%s' is too long", s); + *ok = 0; + return -1; + } + return (int)r; +} diff --git a/src/lib/confmgt/unitparse.h b/src/lib/confmgt/unitparse.h new file mode 100644 index 0000000000..216361a7d4 --- /dev/null +++ b/src/lib/confmgt/unitparse.h @@ -0,0 +1,34 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file unitparse.h + * @brief Header for lib/confmgt/unitparse.c + **/ + +#ifndef TOR_LIB_CONFMGT_UNITPARSE_H +#define TOR_LIB_CONFMGT_UNITPARSE_H + +#include + +/** Mapping from a unit name to a multiplier for converting that unit into a + * base unit. Used by config_parse_unit. */ +typedef struct unit_table_t { + const char *unit; /**< The name of the unit */ + uint64_t multiplier; /**< How many of the base unit appear in this unit */ +} unit_table_t; + +extern const unit_table_t memory_units[]; +extern const unit_table_t time_units[]; +extern const struct unit_table_t time_msec_units[]; + +uint64_t config_parse_units(const char *val, const unit_table_t *u, int *ok); + +uint64_t config_parse_memunit(const char *s, int *ok); +int config_parse_msec_interval(const char *s, int *ok); +int config_parse_interval(const char *s, int *ok); + +#endif /* !defined(TOR_LIB_CONFMGT_UNITPARSE_H) */ From 5a2ab886baaa125fe715acca8f7daf35031855aa Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 13 Jun 2019 15:19:26 -0400 Subject: [PATCH 1156/2557] Add a function to append an existing line to a config line list. We had an existing function to do this, but it took a pair of strings rather than a line. --- src/lib/encoding/confline.c | 15 ++++++++++++--- src/lib/encoding/confline.h | 3 ++- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/src/lib/encoding/confline.c b/src/lib/encoding/confline.c index fdb575e03f..b36e83dcc6 100644 --- a/src/lib/encoding/confline.c +++ b/src/lib/encoding/confline.c @@ -33,14 +33,23 @@ config_line_append(config_line_t **lst, const char *key, const char *val) { - tor_assert(lst); - config_line_t *newline; newline = tor_malloc_zero(sizeof(config_line_t)); newline->key = tor_strdup(key); newline->value = tor_strdup(val); newline->next = NULL; + + config_line_append_line(lst, newline); +} + +/** Helper: append newline to the end of lst. */ +void +config_line_append_line(config_line_t **lst, + config_line_t *newline) +{ + tor_assert(lst); + while (*lst) lst = &((*lst)->next); @@ -256,7 +265,7 @@ config_lines_dup_and_filter(const config_line_t *inp, /** Return true iff a and b contain identical keys and values in identical * order. */ int -config_lines_eq(config_line_t *a, config_line_t *b) +config_lines_eq(const config_line_t *a, const config_line_t *b) { while (a && b) { if (strcasecmp(a->key, b->key) || strcmp(a->value, b->value)) diff --git a/src/lib/encoding/confline.h b/src/lib/encoding/confline.h index 56ea36bf61..7d0b9ce5fc 100644 --- a/src/lib/encoding/confline.h +++ b/src/lib/encoding/confline.h @@ -41,6 +41,7 @@ typedef struct config_line_t { void config_line_append(config_line_t **lst, const char *key, const char *val); +void config_line_append_line(config_line_t **lst, config_line_t *newline); void config_line_prepend(config_line_t **lst, const char *key, const char *val); config_line_t *config_lines_dup(const config_line_t *inp); @@ -50,7 +51,7 @@ const config_line_t *config_line_find(const config_line_t *lines, const char *key); const config_line_t *config_line_find_case(const config_line_t *lines, const char *key); -int config_lines_eq(config_line_t *a, config_line_t *b); +int config_lines_eq(const config_line_t *a, const config_line_t *b); int config_count_key(const config_line_t *a, const char *key); void config_free_lines_(config_line_t *front); #define config_free_lines(front) \ From c60a85d22ab87fef5a7de2fee616ad112835177b Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 14 Jun 2019 08:42:24 -0400 Subject: [PATCH 1157/2557] Add a "typed_var" abstraction to implement lvalue access in C. Right now, this has been done at a high level by confparse.c, but it makes more sense to lower it. This API is radically un-typesafe as it stands; we'll be wrapping it in a safer API as we do #30914 and lower the struct manipulation code as well. Closes ticket 30864. --- Makefile.am | 4 +- changes/ticket30864 | 3 + scripts/maint/add_c_file.py | 6 +- src/app/config/confparse.c | 349 ++------------ src/lib/conf/conftypes.h | 16 + src/lib/confmgt/.may_include | 2 + src/lib/confmgt/include.am | 7 +- src/lib/confmgt/type_defs.c | 727 ++++++++++++++++++++++++++++++ src/lib/confmgt/type_defs.h | 17 + src/lib/confmgt/typedvar.c | 305 +++++++++++++ src/lib/confmgt/typedvar.h | 49 ++ src/lib/confmgt/var_type_def_st.h | 147 ++++++ src/test/test_confparse.c | 4 +- src/test/test_options.c | 12 +- 14 files changed, 1315 insertions(+), 333 deletions(-) create mode 100644 changes/ticket30864 create mode 100644 src/lib/confmgt/type_defs.c create mode 100644 src/lib/confmgt/type_defs.h create mode 100644 src/lib/confmgt/typedvar.c create mode 100644 src/lib/confmgt/typedvar.h create mode 100644 src/lib/confmgt/var_type_def_st.h diff --git a/Makefile.am b/Makefile.am index 0083608ce9..6445502933 100644 --- a/Makefile.am +++ b/Makefile.am @@ -41,6 +41,7 @@ TOR_UTIL_LIBS = \ src/lib/libtor-geoip.a \ src/lib/libtor-process.a \ src/lib/libtor-buf.a \ + src/lib/libtor-confmgt.a \ src/lib/libtor-pubsub.a \ src/lib/libtor-dispatch.a \ src/lib/libtor-time.a \ @@ -54,7 +55,6 @@ TOR_UTIL_LIBS = \ src/lib/libtor-math.a \ src/lib/libtor-meminfo.a \ src/lib/libtor-osinfo.a \ - src/lib/libtor-confmgt.a \ src/lib/libtor-log.a \ src/lib/libtor-lock.a \ src/lib/libtor-fdio.a \ @@ -75,6 +75,7 @@ TOR_UTIL_TESTING_LIBS = \ src/lib/libtor-geoip-testing.a \ src/lib/libtor-process-testing.a \ src/lib/libtor-buf-testing.a \ + src/lib/libtor-confmgt-testing.a \ src/lib/libtor-pubsub-testing.a \ src/lib/libtor-dispatch-testing.a \ src/lib/libtor-time-testing.a \ @@ -89,7 +90,6 @@ TOR_UTIL_TESTING_LIBS = \ src/lib/libtor-meminfo-testing.a \ src/lib/libtor-osinfo-testing.a \ src/lib/libtor-term-testing.a \ - src/lib/libtor-confmgt-testing.a \ src/lib/libtor-log-testing.a \ src/lib/libtor-lock-testing.a \ src/lib/libtor-fdio-testing.a \ diff --git a/changes/ticket30864 b/changes/ticket30864 new file mode 100644 index 0000000000..b8fb571300 --- /dev/null +++ b/changes/ticket30864 @@ -0,0 +1,3 @@ + o Code simplification and refactoring: + - Extract our variable manipulation code from confparse.c to a new + lower-level typedvar.h module. Closes ticket 30864. diff --git a/scripts/maint/add_c_file.py b/scripts/maint/add_c_file.py index 499415974f..adf7ce79bb 100755 --- a/scripts/maint/add_c_file.py +++ b/scripts/maint/add_c_file.py @@ -125,8 +125,8 @@ class AutomakeChunk: Y \ Z """ - self.prespace = "\t" - self.postspace = "\t\t" + prespace = "\t" + postspace = "\t\t" for lineno, line in enumerate(self.lines): m = re.match(r'(\s+)(\S+)(\s+)\\', line) if not m: @@ -135,7 +135,7 @@ class AutomakeChunk: if fname > member: self.insert_before(lineno, member, prespace, postspace) return - self.insert_at_end(member) + self.insert_at_end(member, prespace, postspace) def insert_before(self, lineno, member, prespace, postspace): self.lines.insert(lineno, diff --git a/src/app/config/confparse.c b/src/app/config/confparse.c index 296e7c2a39..bc2ab24e4f 100644 --- a/src/app/config/confparse.c +++ b/src/app/config/confparse.c @@ -30,6 +30,8 @@ #include "lib/container/bitarray.h" #include "lib/encoding/confline.h" +#include "lib/confmgt/typedvar.h" + static void config_reset(const config_format_t *fmt, void *options, const config_var_t *var, int use_defaults); @@ -160,7 +162,6 @@ static int config_assign_value(const config_format_t *fmt, void *options, config_line_t *c, char **msg) { - int i, ok; const config_var_t *var; void *lvalue; @@ -168,144 +169,14 @@ config_assign_value(const config_format_t *fmt, void *options, var = config_find_option(fmt, c->key); tor_assert(var); + tor_assert(!strcmp(c->key, var->name)); lvalue = STRUCT_VAR_P(options, var->var_offset); - switch (var->type) { + if (var->type == CONFIG_TYPE_ROUTERSET) { + // XXXX make the backend extensible so that we don't have to + // XXXX handle ROUTERSET specially. - case CONFIG_TYPE_INT: - case CONFIG_TYPE_POSINT: - i = (int)tor_parse_long(c->value, 10, - var->type==CONFIG_TYPE_INT ? INT_MIN : 0, - INT_MAX, - &ok, NULL); - if (!ok) { - tor_asprintf(msg, - "Int keyword '%s %s' is malformed or out of bounds.", - c->key, c->value); - return -1; - } - *(int *)lvalue = i; - break; - - case CONFIG_TYPE_UINT64: { - uint64_t u64 = tor_parse_uint64(c->value, 10, - 0, UINT64_MAX, &ok, NULL); - if (!ok) { - tor_asprintf(msg, - "uint64 keyword '%s %s' is malformed or out of bounds.", - c->key, c->value); - return -1; - } - *(uint64_t *)lvalue = u64; - break; - } - - case CONFIG_TYPE_CSV_INTERVAL: { - /* We used to have entire smartlists here. But now that all of our - * download schedules use exponential backoff, only the first part - * matters. */ - const char *comma = strchr(c->value, ','); - const char *val = c->value; - char *tmp = NULL; - if (comma) { - tmp = tor_strndup(c->value, comma - c->value); - val = tmp; - } - - i = config_parse_interval(val, &ok); - if (!ok) { - tor_asprintf(msg, - "Interval '%s %s' is malformed or out of bounds.", - c->key, c->value); - tor_free(tmp); - return -1; - } - *(int *)lvalue = i; - tor_free(tmp); - break; - } - - case CONFIG_TYPE_INTERVAL: { - i = config_parse_interval(c->value, &ok); - if (!ok) { - tor_asprintf(msg, - "Interval '%s %s' is malformed or out of bounds.", - c->key, c->value); - return -1; - } - *(int *)lvalue = i; - break; - } - - case CONFIG_TYPE_MSEC_INTERVAL: { - i = config_parse_msec_interval(c->value, &ok); - if (!ok) { - tor_asprintf(msg, - "Msec interval '%s %s' is malformed or out of bounds.", - c->key, c->value); - return -1; - } - *(int *)lvalue = i; - break; - } - - case CONFIG_TYPE_MEMUNIT: { - uint64_t u64 = config_parse_memunit(c->value, &ok); - if (!ok) { - tor_asprintf(msg, - "Value '%s %s' is malformed or out of bounds.", - c->key, c->value); - return -1; - } - *(uint64_t *)lvalue = u64; - break; - } - - case CONFIG_TYPE_BOOL: - i = (int)tor_parse_long(c->value, 10, 0, 1, &ok, NULL); - if (!ok) { - tor_asprintf(msg, - "Boolean '%s %s' expects 0 or 1.", - c->key, c->value); - return -1; - } - *(int *)lvalue = i; - break; - - case CONFIG_TYPE_AUTOBOOL: - if (!strcasecmp(c->value, "auto")) - *(int *)lvalue = -1; - else if (!strcmp(c->value, "0")) - *(int *)lvalue = 0; - else if (!strcmp(c->value, "1")) - *(int *)lvalue = 1; - else { - tor_asprintf(msg, "Boolean '%s %s' expects 0, 1, or 'auto'.", - c->key, c->value); - return -1; - } - break; - - case CONFIG_TYPE_STRING: - case CONFIG_TYPE_FILENAME: - tor_free(*(char **)lvalue); - *(char **)lvalue = tor_strdup(c->value); - break; - - case CONFIG_TYPE_DOUBLE: - *(double *)lvalue = atof(c->value); - break; - - case CONFIG_TYPE_ISOTIME: - if (parse_iso_time(c->value, (time_t *)lvalue)) { - tor_asprintf(msg, - "Invalid time '%s' for keyword '%s'", c->value, c->key); - return -1; - } - break; - - case CONFIG_TYPE_ROUTERSET: if (*(routerset_t**)lvalue) { routerset_free(*(routerset_t**)lvalue); } @@ -315,50 +186,10 @@ config_assign_value(const config_format_t *fmt, void *options, c->value, c->key); return -1; } - break; - - case CONFIG_TYPE_CSV: - if (*(smartlist_t**)lvalue) { - SMARTLIST_FOREACH(*(smartlist_t**)lvalue, char *, cp, tor_free(cp)); - smartlist_clear(*(smartlist_t**)lvalue); - } else { - *(smartlist_t**)lvalue = smartlist_new(); - } - - smartlist_split_string(*(smartlist_t**)lvalue, c->value, ",", - SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); - break; - - case CONFIG_TYPE_LINELIST: - case CONFIG_TYPE_LINELIST_S: - { - config_line_t *lastval = *(config_line_t**)lvalue; - if (lastval && lastval->fragile) { - if (c->command != CONFIG_LINE_APPEND) { - config_free_lines(lastval); - *(config_line_t**)lvalue = NULL; - } else { - lastval->fragile = 0; - } - } - - config_line_append((config_line_t**)lvalue, c->key, c->value); - } - break; - case CONFIG_TYPE_OBSOLETE: - log_warn(LD_CONFIG, "Skipping obsolete configuration option '%s'", c->key); - break; - case CONFIG_TYPE_LINELIST_V: - tor_asprintf(msg, - "You may not provide a value for virtual option '%s'", c->key); - return -1; - // LCOV_EXCL_START - default: - tor_assert_unreached(); - break; - // LCOV_EXCL_STOP + return 0; } - return 0; + + return typed_var_kvassign(lvalue, c, msg, var->type); } /** Mark every linelist in options "fragile", so that fresh assignments @@ -544,100 +375,15 @@ config_get_assigned_option(const config_format_t *fmt, const void *options, } value = STRUCT_VAR_P(options, var->var_offset); - result = tor_malloc_zero(sizeof(config_line_t)); - result->key = tor_strdup(var->name); - switch (var->type) - { - case CONFIG_TYPE_STRING: - case CONFIG_TYPE_FILENAME: - if (*(char**)value) { - result->value = tor_strdup(*(char**)value); - } else { - tor_free(result->key); - tor_free(result); - return NULL; - } - break; - case CONFIG_TYPE_ISOTIME: - if (*(time_t*)value) { - result->value = tor_malloc(ISO_TIME_LEN+1); - format_iso_time(result->value, *(time_t*)value); - } else { - tor_free(result->key); - tor_free(result); - } - escape_val = 0; /* Can't need escape. */ - break; - case CONFIG_TYPE_CSV_INTERVAL: - case CONFIG_TYPE_INTERVAL: - case CONFIG_TYPE_MSEC_INTERVAL: - case CONFIG_TYPE_POSINT: - case CONFIG_TYPE_INT: - /* This means every or_options_t uint or bool element - * needs to be an int. Not, say, a uint16_t or char. */ - tor_asprintf(&result->value, "%d", *(int*)value); - escape_val = 0; /* Can't need escape. */ - break; - case CONFIG_TYPE_UINT64: /* Fall through */ - case CONFIG_TYPE_MEMUNIT: - tor_asprintf(&result->value, "%"PRIu64, - (*(uint64_t*)value)); - escape_val = 0; /* Can't need escape. */ - break; - case CONFIG_TYPE_DOUBLE: - tor_asprintf(&result->value, "%f", *(double*)value); - escape_val = 0; /* Can't need escape. */ - break; - - case CONFIG_TYPE_AUTOBOOL: - if (*(int*)value == -1) { - result->value = tor_strdup("auto"); - escape_val = 0; - break; - } - /* fall through */ - case CONFIG_TYPE_BOOL: - result->value = tor_strdup(*(int*)value ? "1" : "0"); - escape_val = 0; /* Can't need escape. */ - break; - case CONFIG_TYPE_ROUTERSET: - result->value = routerset_to_string(*(routerset_t**)value); - break; - case CONFIG_TYPE_CSV: - if (*(smartlist_t**)value) - result->value = - smartlist_join_strings(*(smartlist_t**)value, ",", 0, NULL); - else - result->value = tor_strdup(""); - break; - case CONFIG_TYPE_OBSOLETE: - log_fn(LOG_INFO, LD_CONFIG, - "You asked me for the value of an obsolete config option '%s'.", - key); - tor_free(result->key); - tor_free(result); - return NULL; - case CONFIG_TYPE_LINELIST_S: - tor_free(result->key); - tor_free(result); - result = config_lines_dup_and_filter(*(const config_line_t **)value, - key); - break; - case CONFIG_TYPE_LINELIST: - case CONFIG_TYPE_LINELIST_V: - tor_free(result->key); - tor_free(result); - result = config_lines_dup(*(const config_line_t**)value); - break; - // LCOV_EXCL_START - default: - tor_free(result->key); - tor_free(result); - log_warn(LD_BUG,"Unknown type %d for known key '%s'", - var->type, key); - return NULL; - // LCOV_EXCL_STOP - } + if (var->type == CONFIG_TYPE_ROUTERSET) { + // XXXX make the backend extensible so that we don't have to + // XXXX handle ROUTERSET specially. + result = tor_malloc_zero(sizeof(config_line_t)); + result->key = tor_strdup(var->name); + result->value = routerset_to_string(*(routerset_t**)value); + } else { + result = typed_var_kvencode(var->name, value, var->type); + } if (escape_val) { config_line_t *line; @@ -765,56 +511,17 @@ config_clear(const config_format_t *fmt, void *options, { void *lvalue = STRUCT_VAR_P(options, var->var_offset); (void)fmt; /* unused */ - switch (var->type) { - case CONFIG_TYPE_STRING: - case CONFIG_TYPE_FILENAME: - tor_free(*(char**)lvalue); - break; - case CONFIG_TYPE_DOUBLE: - *(double*)lvalue = 0.0; - break; - case CONFIG_TYPE_ISOTIME: - *(time_t*)lvalue = 0; - break; - case CONFIG_TYPE_CSV_INTERVAL: - case CONFIG_TYPE_INTERVAL: - case CONFIG_TYPE_MSEC_INTERVAL: - case CONFIG_TYPE_POSINT: - case CONFIG_TYPE_INT: - case CONFIG_TYPE_BOOL: - *(int*)lvalue = 0; - break; - case CONFIG_TYPE_AUTOBOOL: - *(int*)lvalue = -1; - break; - case CONFIG_TYPE_UINT64: - case CONFIG_TYPE_MEMUNIT: - *(uint64_t*)lvalue = 0; - break; - case CONFIG_TYPE_ROUTERSET: - if (*(routerset_t**)lvalue) { - routerset_free(*(routerset_t**)lvalue); - *(routerset_t**)lvalue = NULL; - } - break; - case CONFIG_TYPE_CSV: - if (*(smartlist_t**)lvalue) { - SMARTLIST_FOREACH(*(smartlist_t **)lvalue, char *, cp, tor_free(cp)); - smartlist_free(*(smartlist_t **)lvalue); - *(smartlist_t **)lvalue = NULL; - } - break; - case CONFIG_TYPE_LINELIST: - case CONFIG_TYPE_LINELIST_S: - config_free_lines(*(config_line_t **)lvalue); - *(config_line_t **)lvalue = NULL; - break; - case CONFIG_TYPE_LINELIST_V: - /* handled by linelist_s. */ - break; - case CONFIG_TYPE_OBSOLETE: - break; + if (var->type == CONFIG_TYPE_ROUTERSET) { + // XXXX make the backend extensible so that we don't have to + // XXXX handle ROUTERSET specially. + if (*(routerset_t**)lvalue) { + routerset_free(*(routerset_t**)lvalue); + *(routerset_t**)lvalue = NULL; + } + return; } + + typed_var_free(lvalue, var->type); } /** Clear the option indexed by var in options. Then if diff --git a/src/lib/conf/conftypes.h b/src/lib/conf/conftypes.h index eb4eb1245f..b03234b62c 100644 --- a/src/lib/conf/conftypes.h +++ b/src/lib/conf/conftypes.h @@ -7,6 +7,22 @@ /** * @file conftypes.h * @brief Types used to specify configurable options. + * + * This header defines the types that different modules will use in order to + * declare their configuration and state variables, and tell the configuration + * management code about those variables. From the individual module's point + * of view, its configuration and state are simply data structures. + * + * For defining new variable types, see var_type_def_st.h. + * + * For the code that manipulates variables defined via this module, see + * lib/confmgt/, especially typedvar.h and (later) structvar.h. The + * configuration manager is responsible for encoding, decoding, and + * maintaining the configuration structures used by the various modules. + * + * STATUS NOTE: This is a work in process refactoring. It is not yet possible + * for modules to define their own variables, and much of the configuration + * management code is still in src/app/config/. **/ #ifndef TOR_SRC_LIB_CONF_CONFTYPES_H diff --git a/src/lib/confmgt/.may_include b/src/lib/confmgt/.may_include index 640acf3770..d85dbf6904 100644 --- a/src/lib/confmgt/.may_include +++ b/src/lib/confmgt/.may_include @@ -2,6 +2,8 @@ orconfig.h lib/cc/*.h lib/conf/*.h lib/confmgt/*.h +lib/container/*.h +lib/encoding/*.h lib/log/*.h lib/malloc/*.h lib/string/*.h diff --git a/src/lib/confmgt/include.am b/src/lib/confmgt/include.am index 729e6147ff..a2c7649957 100644 --- a/src/lib/confmgt/include.am +++ b/src/lib/confmgt/include.am @@ -6,6 +6,8 @@ endif # ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_confmgt_a_SOURCES = \ + src/lib/confmgt/type_defs.c \ + src/lib/confmgt/typedvar.c \ src/lib/confmgt/unitparse.c src_lib_libtor_confmgt_testing_a_SOURCES = \ @@ -15,4 +17,7 @@ src_lib_libtor_confmgt_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) # ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ - src/lib/confmgt/unitparse.h + src/lib/confmgt/type_defs.h \ + src/lib/confmgt/typedvar.h \ + src/lib/confmgt/unitparse.h \ + src/lib/confmgt/var_type_def_st.h diff --git a/src/lib/confmgt/type_defs.c b/src/lib/confmgt/type_defs.c new file mode 100644 index 0000000000..62b4c1019d --- /dev/null +++ b/src/lib/confmgt/type_defs.c @@ -0,0 +1,727 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file type_defs.c + * @brief Definitions for various low-level configuration types. + * + * This module creates a number of var_type_def_t objects, to be used by + * typedvar.c in manipulating variables. + * + * The types here are common types that can be implemented with Tor's + * low-level functionality. To define new types, see var_type_def_st.h. + **/ + +#include "orconfig.h" +#include "lib/conf/conftypes.h" +#include "lib/confmgt/typedvar.h" +#include "lib/confmgt/type_defs.h" +#include "lib/confmgt/unitparse.h" + +#include "lib/cc/compat_compiler.h" +#include "lib/conf/conftypes.h" +#include "lib/container/smartlist.h" +#include "lib/encoding/confline.h" +#include "lib/encoding/time_fmt.h" +#include "lib/log/escape.h" +#include "lib/log/log.h" +#include "lib/log/util_bug.h" +#include "lib/malloc/malloc.h" +#include "lib/string/parse_int.h" +#include "lib/string/printf.h" + +#include "lib/confmgt/var_type_def_st.h" + +#include +#include + +////// +// CONFIG_TYPE_STRING +// CONFIG_TYPE_FILENAME +// +// These two types are the same for now, but they have different names. +////// + +static int +string_parse(void *target, const char *value, char **errmsg, + const void *params) +{ + (void)params; + (void)errmsg; + char **p = (char**)target; + *p = tor_strdup(value); + return 0; +} + +static char * +string_encode(const void *value, const void *params) +{ + (void)params; + const char **p = (const char**)value; + return *p ? tor_strdup(*p) : NULL; +} + +static void +string_clear(void *value, const void *params) +{ + (void)params; + char **p = (char**)value; + tor_free(*p); // sets *p to NULL. +} + +static const var_type_fns_t string_fns = { + .parse = string_parse, + .encode = string_encode, + .clear = string_clear, +}; + +///// +// CONFIG_TYPE_INT +// CONFIG_TYPE_POSINT +// +// These types are implemented as int, possibly with a restricted range. +///// + +typedef struct int_type_params_t { + int minval; + int maxval; +} int_parse_params_t; + +static const int_parse_params_t INT_PARSE_UNRESTRICTED = { + .minval = INT_MIN, + .maxval = INT_MAX, +}; + +static const int_parse_params_t INT_PARSE_POSINT = { + .minval = 0, + .maxval = INT_MAX, +}; + +static int +int_parse(void *target, const char *value, char **errmsg, const void *params) +{ + const int_parse_params_t *pp; + if (params) { + pp = params; + } else { + pp = &INT_PARSE_UNRESTRICTED; + } + int *p = target; + int ok=0; + *p = (int)tor_parse_long(value, 10, pp->minval, pp->maxval, &ok, NULL); + if (!ok) { + tor_asprintf(errmsg, "Integer %s is malformed or out of bounds.", + value); + return -1; + } + return 0; +} + +static char * +int_encode(const void *value, const void *params) +{ + (void)params; + int v = *(int*)value; + char *result; + tor_asprintf(&result, "%d", v); + return result; +} + +static void +int_clear(void *value, const void *params) +{ + (void)params; + *(int*)value = 0; +} + +static bool +int_ok(const void *value, const void *params) +{ + const int_parse_params_t *pp = params; + if (pp) { + int v = *(int*)value; + return pp->minval <= v && v <= pp->maxval; + } else { + return true; + } +} + +static const var_type_fns_t int_fns = { + .parse = int_parse, + .encode = int_encode, + .clear = int_clear, + .ok = int_ok, +}; + +///// +// CONFIG_TYPE_UINT64 +// +// This type is an unrestricted u64. +///// + +static int +uint64_parse(void *target, const char *value, char **errmsg, + const void *params) +{ + (void)params; + (void)errmsg; + uint64_t *p = target; + int ok=0; + *p = tor_parse_uint64(value, 10, 0, UINT64_MAX, &ok, NULL); + if (!ok) { + tor_asprintf(errmsg, "Integer %s is malformed or out of bounds.", + value); + return -1; + } + return 0; +} + +static char * +uint64_encode(const void *value, const void *params) +{ + (void)params; + uint64_t v = *(uint64_t*)value; + char *result; + tor_asprintf(&result, "%"PRIu64, v); + return result; +} + +static void +uint64_clear(void *value, const void *params) +{ + (void)params; + *(uint64_t*)value = 0; +} + +static const var_type_fns_t uint64_fns = { + .parse = uint64_parse, + .encode = uint64_encode, + .clear = uint64_clear, +}; + +///// +// CONFIG_TYPE_INTERVAL +// CONFIG_TYPE_MSEC_INTERVAL +// CONFIG_TYPE_MEMUNIT +// +// These types are implemented using the config_parse_units() function. +// The intervals are stored as ints, whereas memory units are stored as +// uint64_ts. +///// + +static int +units_parse_u64(void *target, const char *value, char **errmsg, + const void *params) +{ + const unit_table_t *table = params; + tor_assert(table); + uint64_t *v = (uint64_t*)target; + int ok=1; + *v = config_parse_units(value, table, &ok); + if (!ok) { + *errmsg = tor_strdup("Provided value is malformed or out of bounds."); + return -1; + } + return 0; +} + +static int +units_parse_int(void *target, const char *value, char **errmsg, + const void *params) +{ + const unit_table_t *table = params; + tor_assert(table); + int *v = (int*)target; + int ok=1; + uint64_t u64 = config_parse_units(value, table, &ok); + if (!ok) { + *errmsg = tor_strdup("Provided value is malformed or out of bounds."); + return -1; + } + if (u64 > INT_MAX) { + tor_asprintf(errmsg, "Provided value %s is too large", value); + return -1; + } + *v = (int) u64; + return 0; +} + +static bool +units_ok_int(const void *value, const void *params) +{ + (void)params; + int v = *(int*)value; + return v >= 0; +} + +static const var_type_fns_t memunit_fns = { + .parse = units_parse_u64, + .encode = uint64_encode, // doesn't use params + .clear = uint64_clear, // doesn't use params +}; + +static const var_type_fns_t interval_fns = { + .parse = units_parse_int, + .encode = int_encode, // doesn't use params + .clear = int_clear, // doesn't use params, + .ok = units_ok_int // can't use int_ok, since that expects int params. +}; + +///// +// CONFIG_TYPE_DOUBLE +// +// This is a nice simple double. +///// + +static int +double_parse(void *target, const char *value, char **errmsg, + const void *params) +{ + (void)params; + (void)errmsg; + double *v = (double*)target; + // XXXX This is the preexisting behavior, but we should detect errors here. + *v = atof(value); + return 0; +} + +static char * +double_encode(const void *value, const void *params) +{ + (void)params; + double v = *(double*)value; + char *result; + tor_asprintf(&result, "%f", v); + return result; +} + +static void +double_clear(void *value, const void *params) +{ + (void)params; + double *v = (double *)value; + *v = 0.0; +} + +static const var_type_fns_t double_fns = { + .parse = double_parse, + .encode = double_encode, + .clear = double_clear, +}; + +///// +// CONFIG_TYPE_BOOL +// CONFIG_TYPE_AUTOBOOL +// +// These types are implemented as a case-insensitive string-to-integer +// mapping. +///// + +typedef struct enumeration_table_t { + const char *name; + int value; +} enumeration_table_t; + +static int +enum_parse(void *target, const char *value, char **errmsg, + const void *params) +{ + const enumeration_table_t *table = params; + int *p = (int *)target; + for (; table->name; ++table) { + if (!strcasecmp(value, table->name)) { + *p = table->value; + return 0; + } + } + tor_asprintf(errmsg, "Unrecognized value %s.", value); + return -1; +} + +static char * +enum_encode(const void *value, const void *params) +{ + int v = *(const int*)value; + const enumeration_table_t *table = params; + for (; table->name; ++table) { + if (v == table->value) + return tor_strdup(table->name); + } + return NULL; // error. +} + +static void +enum_clear(void *value, const void *params) +{ + int *p = (int*)value; + const enumeration_table_t *table = params; + tor_assert(table->name); + *p = table->value; +} + +static bool +enum_ok(const void *value, const void *params) +{ + int v = *(const int*)value; + const enumeration_table_t *table = params; + for (; table->name; ++table) { + if (v == table->value) + return true; + } + return false; +} + +static const enumeration_table_t enum_table_bool[] = { + { "0", 0 }, + { "1", 1 }, + { NULL, 0 }, +}; + +static const enumeration_table_t enum_table_autobool[] = { + { "0", 0 }, + { "1", 1 }, + { "auto", -1 }, + { NULL, 0 }, +}; + +static const var_type_fns_t enum_fns = { + .parse = enum_parse, + .encode = enum_encode, + .clear = enum_clear, + .ok = enum_ok, +}; + +///// +// CONFIG_TYPE_ISOTIME +// +// This is a time_t, encoded in ISO8601 format. +///// + +static int +time_parse(void *target, const char *value, char **errmsg, + const void *params) +{ + (void) params; + time_t *p = target; + if (parse_iso_time(value, p) < 0) { + tor_asprintf(errmsg, "Invalid time %s", escaped(value)); + return -1; + } + return 0; +} + +static char * +time_encode(const void *value, const void *params) +{ + (void)params; + time_t v = *(const time_t *)value; + char *result = tor_malloc(ISO_TIME_LEN+1); + format_iso_time(result, v); + return result; +} + +static void +time_clear(void *value, const void *params) +{ + (void)params; + time_t *t = value; + *t = 0; +} + +static const var_type_fns_t time_fns = { + .parse = time_parse, + .encode = time_encode, + .clear = time_clear, +}; + +///// +// CONFIG_TYPE_CSV +// +// This type is a comma-separated list of strings, stored in a smartlist_t. +// An empty list may be encoded either as an empty smartlist, or as NULL. +///// + +static int +csv_parse(void *target, const char *value, char **errmsg, + const void *params) +{ + (void)params; + (void)errmsg; + smartlist_t **sl = (smartlist_t**)target; + *sl = smartlist_new(); + smartlist_split_string(*sl, value, ",", + SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); + return 0; +} + +static char * +csv_encode(const void *value, const void *params) +{ + (void)params; + const smartlist_t *sl = *(const smartlist_t **)value; + if (! sl) + return tor_strdup(""); + + return smartlist_join_strings(*(smartlist_t**)value, ",", 0, NULL); +} + +static void +csv_clear(void *value, const void *params) +{ + (void)params; + smartlist_t **sl = (smartlist_t**)value; + if (!*sl) + return; + SMARTLIST_FOREACH(*sl, char *, cp, tor_free(cp)); + smartlist_free(*sl); // clears pointer. +} + +static const var_type_fns_t csv_fns = { + .parse = csv_parse, + .encode = csv_encode, + .clear = csv_clear, +}; + +///// +// CONFIG_TYPE_CSV_INTERVAL +// +// This type used to be a list of time intervals, used to determine a download +// schedule. Now, only the first interval counts: everything after the first +// comma is discarded. +///// + +static int +legacy_csv_interval_parse(void *target, const char *value, char **errmsg, + const void *params) +{ + (void)params; + /* We used to have entire smartlists here. But now that all of our + * download schedules use exponential backoff, only the first part + * matters. */ + const char *comma = strchr(value, ','); + const char *val = value; + char *tmp = NULL; + if (comma) { + tmp = tor_strndup(val, comma - val); + val = tmp; + } + + int rv = units_parse_int(target, val, errmsg, &time_units); + tor_free(tmp); + return rv; +} + +static const var_type_fns_t legacy_csv_interval_fns = { + .parse = legacy_csv_interval_parse, + .encode = int_encode, + .clear = int_clear, +}; + +///// +// CONFIG_TYPE_LINELIST +// CONFIG_TYPE_LINELIST_S +// CONFIG_TYPE_LINELIST_V +// +// A linelist is a raw config_line_t list. Order is preserved. +// +// The LINELIST type is used for homogeneous lists, where all the lines +// have the same key. +// +// The LINELIST_S and LINELIST_V types are used for the case where multiple +// lines of different keys are kept in a single list, to preserve their +// relative order. The unified list is stored as a "virtual" variable whose +// type is LINELIST_V; the individual sublists are treated as variables of +// type LINELIST_S. +// +// A linelist may be fragile or non-fragile. Assigning a line to a fragile +// linelist replaces the list with the line. If the line has the "APPEND" +// command set on it, or if the list is non-fragile, the line is appended. +// Either way, the new list is non-fragile. +///// + +static int +linelist_kv_parse(void *target, const struct config_line_t *line, + char **errmsg, const void *params) +{ + (void)params; + (void)errmsg; + config_line_t **lines = target; + + if (*lines && (*lines)->fragile) { + if (line->command == CONFIG_LINE_APPEND) { + (*lines)->fragile = 0; + } else { + config_free_lines(*lines); // sets it to NULL + } + } + + config_line_append(lines, line->key, line->value); + return 0; +} + +static int +linelist_kv_virt_noparse(void *target, const struct config_line_t *line, + char **errmsg, const void *params) +{ + (void)target; + (void)line; + (void)params; + *errmsg = tor_strdup("Cannot assign directly to virtual option."); + return -1; +} + +static struct config_line_t * +linelist_kv_encode(const char *key, const void *value, + const void *params) +{ + (void)key; + (void)params; + config_line_t *lines = *(config_line_t **)value; + return config_lines_dup(lines); +} + +static struct config_line_t * +linelist_s_kv_encode(const char *key, const void *value, + const void *params) +{ + (void)params; + config_line_t *lines = *(config_line_t **)value; + return config_lines_dup_and_filter(lines, key); +} + +static void +linelist_clear(void *target, const void *params) +{ + (void)params; + config_line_t **lines = target; + config_free_lines(*lines); // sets it to NULL +} + +static bool +linelist_eq(const void *a, const void *b, const void *params) +{ + (void)params; + const config_line_t *lines_a = *(const config_line_t **)a; + const config_line_t *lines_b = *(const config_line_t **)b; + return config_lines_eq(lines_a, lines_b); +} + +static int +linelist_copy(void *target, const void *value, const void *params) +{ + (void)params; + config_line_t **ptr = (config_line_t **)target; + const config_line_t *val = *(const config_line_t **)value; + config_free_lines(*ptr); + *ptr = config_lines_dup(val); + return 0; +} + +static const var_type_fns_t linelist_fns = { + .kv_parse = linelist_kv_parse, + .kv_encode = linelist_kv_encode, + .clear = linelist_clear, + .eq = linelist_eq, + .copy = linelist_copy, +}; + +static const var_type_fns_t linelist_v_fns = { + .kv_parse = linelist_kv_virt_noparse, + .kv_encode = linelist_kv_encode, + .clear = linelist_clear, + .eq = linelist_eq, + .copy = linelist_copy, +}; + +static const var_type_fns_t linelist_s_fns = { + .kv_parse = linelist_kv_parse, + .kv_encode = linelist_s_kv_encode, + .clear = linelist_clear, + .eq = linelist_eq, + .copy = linelist_copy, +}; + +///// +// CONFIG_TYPE_ROUTERSET +// +// XXXX This type is not implemented here, since routerset_t is not available +// XXXX to this module. +///// + +///// +// CONFIG_TYPE_OBSOLETE +// +// Used to indicate an obsolete option. +// +// XXXX This is not a type, and should be handled at a higher level of +// XXXX abstraction. +///// + +static int +ignore_parse(void *target, const char *value, char **errmsg, + const void *params) +{ + (void)target; + (void)value; + (void)errmsg; + (void)params; + // XXXX move this to a higher level, once such a level exists. + log_warn(LD_GENERAL, "Skipping obsolete configuration option."); + return 0; +} + +static char * +ignore_encode(const void *value, const void *params) +{ + (void)value; + (void)params; + return NULL; +} + +static const var_type_fns_t ignore_fns = { + .parse = ignore_parse, + .encode = ignore_encode, +}; + +/** + * Table mapping conf_type_t values to var_type_def_t objects. + **/ +static const var_type_def_t type_definitions_table[] = { + [CONFIG_TYPE_STRING] = { "String", &string_fns, NULL }, + [CONFIG_TYPE_FILENAME] = { "Filename", &string_fns, NULL }, + [CONFIG_TYPE_INT] = { "SignedInteger", &int_fns, &INT_PARSE_UNRESTRICTED }, + [CONFIG_TYPE_POSINT] = { "Integer", &int_fns, &INT_PARSE_POSINT }, + [CONFIG_TYPE_UINT64] = { "Integer", &uint64_fns, NULL, }, + [CONFIG_TYPE_MEMUNIT] = { "DataSize", &memunit_fns, &memory_units }, + [CONFIG_TYPE_INTERVAL] = { "TimeInterval", &interval_fns, &time_units }, + [CONFIG_TYPE_MSEC_INTERVAL] = { "TimeMsecInterval", &interval_fns, + &time_msec_units }, + [CONFIG_TYPE_DOUBLE] = { "Float", &double_fns, NULL }, + [CONFIG_TYPE_BOOL] = { "Boolean", &enum_fns, &enum_table_bool }, + [CONFIG_TYPE_AUTOBOOL] = { "Boolean+Auto", &enum_fns, &enum_table_autobool }, + [CONFIG_TYPE_ISOTIME] = { "Time", &time_fns, NULL }, + [CONFIG_TYPE_CSV] = { "CommaList", &csv_fns, NULL }, + [CONFIG_TYPE_CSV_INTERVAL] = { "TimeInterval", &legacy_csv_interval_fns, + NULL }, + [CONFIG_TYPE_LINELIST] = { "LineList", &linelist_fns, NULL }, + [CONFIG_TYPE_LINELIST_S] = { "Dependent", &linelist_s_fns, NULL }, + [CONFIG_TYPE_LINELIST_V] = { "Virtual", &linelist_v_fns, NULL }, + [CONFIG_TYPE_OBSOLETE] = { "Obsolete", &ignore_fns, NULL } +}; + +/** + * Return a pointer to the var_type_def_t object for the given + * config_type_t value, or NULL if no such type definition exists. + **/ +const var_type_def_t * +lookup_type_def(config_type_t type) +{ + int t = type; + tor_assert(t >= 0); + if (t >= (int)ARRAY_LENGTH(type_definitions_table)) + return NULL; + return &type_definitions_table[t]; +} diff --git a/src/lib/confmgt/type_defs.h b/src/lib/confmgt/type_defs.h new file mode 100644 index 0000000000..ecf040529e --- /dev/null +++ b/src/lib/confmgt/type_defs.h @@ -0,0 +1,17 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file type_defs.h + * @brief Header for lib/confmgt/type_defs.c + **/ + +#ifndef TOR_LIB_CONFMGT_TYPE_DEFS_H +#define TOR_LIB_CONFMGT_TYPE_DEFS_H + +const struct var_type_def_t *lookup_type_def(config_type_t type); + +#endif /* !defined(TOR_LIB_CONFMGT_TYPE_DEFS_H) */ diff --git a/src/lib/confmgt/typedvar.c b/src/lib/confmgt/typedvar.c new file mode 100644 index 0000000000..fc45c44481 --- /dev/null +++ b/src/lib/confmgt/typedvar.c @@ -0,0 +1,305 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file typedvar.c + * @brief Functions for accessing a pointer as an object of a given type. + * + * These functions represent a low-level API for accessing a typed variable. + * They are used in the configuration system to examine and set fields in + * configuration objects used by individual modules. + * + * Almost no code should call these directly. + **/ + +#include "orconfig.h" +#include "lib/conf/conftypes.h" +#include "lib/confmgt/type_defs.h" +#include "lib/confmgt/typedvar.h" +#include "lib/encoding/confline.h" +#include "lib/log/escape.h" +#include "lib/log/log.h" +#include "lib/log/util_bug.h" +#include "lib/malloc/malloc.h" +#include "lib/string/util_string.h" + +#include "lib/confmgt/var_type_def_st.h" + +#include +#include + +/** + * Try to parse a string in value that encodes an object of the type + * defined by def. + * + * On success, adjust the lvalue pointed to by target to hold that + * value, and return 0. On failure, set *errmsg to a newly allocated + * string holding an error message, and return -1. + **/ +int +typed_var_assign_ex(void *target, const char *value, char **errmsg, + const var_type_def_t *def) +{ + if (BUG(!def)) + return -1; + // clear old value if needed. + typed_var_free_ex(target, def); + + tor_assert(def->fns->parse); + return def->fns->parse(target, value, errmsg, def->params); +} + +/** + * Try to parse a single line from the head ofline that encodes an + * object of the type defined in def. On success and failure, behave as + * typed_var_assign_ex(). + * + * All types for which keys are significant should use this function. + * + * Note that although multiple lines may be provided in line, + * only the first one is handled by this function. + **/ +int +typed_var_kvassign_ex(void *target, const config_line_t *line, + char **errmsg, const var_type_def_t *def) +{ + if (BUG(!def)) + return -1; + + if (def->fns->kv_parse) { + // We do _not_ free the old value here, since linelist options + // sometimes have append semantics. + return def->fns->kv_parse(target, line, errmsg, def->params); + } + + return typed_var_assign_ex(target, line->value, errmsg, def); +} + +/** + * Release storage held by a variable in target of type defined by + * def, and set target to a reasonable default. + **/ +void +typed_var_free_ex(void *target, const var_type_def_t *def) +{ + if (BUG(!def)) + return; + if (def->fns->clear) { + def->fns->clear(target, def->params); + } +} + +/** + * Encode a value of type def pointed to by value, and return + * its result in a newly allocated string. The string may need to be escaped. + * + * Returns NULL if this option has a NULL value, or on internal error. + **/ +char * +typed_var_encode_ex(const void *value, const var_type_def_t *def) +{ + if (BUG(!def)) + return NULL; + tor_assert(def->fns->encode); + return def->fns->encode(value, def->params); +} + +/** + * As typed_var_encode_ex(), but returns a newly allocated config_line_t + * object. The provided key is used as the key of the lines, unless + * the type is one (line a linelist) that encodes its own keys. + * + * This function may return a list of multiple lines. + * + * Returns NULL if there are no lines to encode, or on internal error. + */ +config_line_t * +typed_var_kvencode_ex(const char *key, const void *value, + const var_type_def_t *def) +{ + if (BUG(!def)) + return NULL; + if (def->fns->kv_encode) { + return def->fns->kv_encode(key, value, def->params); + } + char *encoded_value = typed_var_encode_ex(value, def); + if (!encoded_value) + return NULL; + + config_line_t *result = tor_malloc_zero(sizeof(config_line_t)); + result->key = tor_strdup(key); + result->value = encoded_value; + return result; +} + +/** + * Set dest to contain the same value as src. Both types + * must be as defined by def. + * + * Return 0 on success, and -1 on failure. + **/ +int +typed_var_copy_ex(void *dest, const void *src, const var_type_def_t *def) +{ + if (BUG(!def)) + return -1; + if (def->fns->copy) { + // If we have been provided a copy fuction, use it. + return def->fns->copy(dest, src, def); + } + + // Otherwise, encode 'src' and parse the result into 'def'. + char *enc = typed_var_encode_ex(src, def); + if (!enc) { + typed_var_free_ex(dest, def); + return 0; + } + char *err = NULL; + int rv = typed_var_assign_ex(dest, enc, &err, def); + if (BUG(rv < 0)) { + log_warn(LD_BUG, "Encoded value %s was not parseable as a %s: %s", + escaped(enc), def->name, err?err:""); + } + tor_free(err); + tor_free(enc); + return rv; +} + +/** + * Return true if a and b are semantically equivalent. + * Both types must be as defined by def. + **/ +bool +typed_var_eq_ex(const void *a, const void *b, const var_type_def_t *def) +{ + if (BUG(!def)) + return false; + + if (def->fns->eq) { + // Use a provided eq function if we got one. + return def->fns->eq(a, b, def->params); + } + + // Otherwise, encode the values and compare them. + char *enc_a = typed_var_encode_ex(a, def); + char *enc_b = typed_var_encode_ex(b, def); + bool eq = !strcmp_opt(enc_a,enc_b); + tor_free(enc_a); + tor_free(enc_b); + return eq; +} + +/** + * Check whether value encodes a valid value according to the + * type definition in def. + */ +bool +typed_var_ok_ex(const void *value, const var_type_def_t *def) +{ + if (BUG(!def)) + return false; + + if (def->fns->ok) + return def->fns->ok(value, def->params); + + return true; +} + +/* ===== + * The functions below take a config_type_t instead of a var_type_def_t. + * I'd like to deprecate them eventually and use var_type_def_t everywhere, + * but for now they make migration easier. + * ===== */ + +/** + * As typed_var_assign_ex(), but look up the definition of the configuration + * type from a provided config_type_t enum. + */ +int +typed_var_assign(void *target, const char *value, char **errmsg, + config_type_t type) +{ + const var_type_def_t *def = lookup_type_def(type); + return typed_var_assign_ex(target, value, errmsg, def); +} + +/** + * As typed_var_kvassign_ex(), but look up the definition of the configuration + * type from a provided config_type_t enum. + */ +int +typed_var_kvassign(void *target, const config_line_t *line, char **errmsg, + config_type_t type) +{ + const var_type_def_t *def = lookup_type_def(type); + return typed_var_kvassign_ex(target, line, errmsg, def); +} + +/** + * As typed_var_free_ex(), but look up the definition of the configuration + * type from a provided config_type_t enum. + */ +void +typed_var_free(void *target, config_type_t type) +{ + const var_type_def_t *def = lookup_type_def(type); + return typed_var_free_ex(target, def); +} + +/** + * As typed_var_encode_ex(), but look up the definition of the configuration + * type from a provided config_type_t enum. + */ +char * +typed_var_encode(const void *value, config_type_t type) +{ + const var_type_def_t *def = lookup_type_def(type); + return typed_var_encode_ex(value, def); +} + +/** + * As typed_var_kvencode_ex(), but look up the definition of the configuration + * type from a provided config_type_t enum. + */ +config_line_t * +typed_var_kvencode(const char *key, const void *value, config_type_t type) +{ + const var_type_def_t *def = lookup_type_def(type); + return typed_var_kvencode_ex(key, value, def); +} + +/** + * As typed_var_copy_ex(), but look up the definition of the configuration type + * from a provided config_type_t enum. + */ +int +typed_var_copy(void *dest, const void *src, config_type_t type) +{ + const var_type_def_t *def = lookup_type_def(type); + return typed_var_copy_ex(dest, src, def); +} + +/** + * As typed_var_eq_ex(), but look up the definition of the configuration type + * from a provided config_type_t enum. + */ +bool +typed_var_eq(const void *a, const void *b, config_type_t type) +{ + const var_type_def_t *def = lookup_type_def(type); + return typed_var_eq_ex(a, b, def); +} + +/** + * As typed_var_ok_ex(), but look up the definition of the configuration type + * from a provided config_type_t enum. + */ +bool +typed_var_ok(const void *value, config_type_t type) +{ + const var_type_def_t *def = lookup_type_def(type); + return typed_var_ok_ex(value, def); +} diff --git a/src/lib/confmgt/typedvar.h b/src/lib/confmgt/typedvar.h new file mode 100644 index 0000000000..720ad54fc6 --- /dev/null +++ b/src/lib/confmgt/typedvar.h @@ -0,0 +1,49 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file typedvar.h + * @brief Header for lib/confmgt/typedvar.c + **/ + +#ifndef TOR_LIB_CONFMGT_TYPEDVAR_H +#define TOR_LIB_CONFMGT_TYPEDVAR_H + +#include + +enum config_type_t; +struct config_line_t; + +typedef struct var_type_fns_t var_type_fns_t; +typedef struct var_type_def_t var_type_def_t; + +int typed_var_assign(void *target, const char *value, char **errmsg, + enum config_type_t type); +void typed_var_free(void *target, enum config_type_t type); +char *typed_var_encode(const void *value, enum config_type_t type); +int typed_var_copy(void *dest, const void *src, enum config_type_t type); +bool typed_var_eq(const void *a, const void *b, enum config_type_t type); +bool typed_var_ok(const void *value, enum config_type_t type); + +int typed_var_kvassign(void *target, const struct config_line_t *line, + char **errmsg, enum config_type_t type); +struct config_line_t *typed_var_kvencode(const char *key, const void *value, + enum config_type_t type); + +int typed_var_assign_ex(void *target, const char *value, char **errmsg, + const var_type_def_t *def); +void typed_var_free_ex(void *target, const var_type_def_t *def); +char *typed_var_encode_ex(const void *value, const var_type_def_t *def); +int typed_var_copy_ex(void *dest, const void *src, const var_type_def_t *def); +bool typed_var_eq_ex(const void *a, const void *b, const var_type_def_t *def); +bool typed_var_ok_ex(const void *value, const var_type_def_t *def); + +int typed_var_kvassign_ex(void *target, const struct config_line_t *line, + char **errmsg, const var_type_def_t *def); +struct config_line_t *typed_var_kvencode_ex(const char *key, const void *value, + const var_type_def_t *def); + +#endif /* !defined(TOR_LIB_CONFMGT_TYPEDVAR_H) */ diff --git a/src/lib/confmgt/var_type_def_st.h b/src/lib/confmgt/var_type_def_st.h new file mode 100644 index 0000000000..c63ff66eff --- /dev/null +++ b/src/lib/confmgt/var_type_def_st.h @@ -0,0 +1,147 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file typedvar.h + * @brief Structure declarations for typedvar type definitions. + * + * This structure is used for defining new variable types. If you are not + * defining a new variable type for use by the configuration management + * system, you don't need this structure. + * + * For defining new variables, see the types in conftypes.h. + * + * For data-driven access to configuration variables, see the other members of + * lib/confmgt/. + * + * STATUS NOTE: It is not yet possible to actually define new variables + * outside of config.c, and many of the types that will eventually be used + * to do so are not yet moved. This will change as more of #29211 is + * completed. + **/ + +#ifndef TOR_LIB_CONFMGT_VAR_TYPE_DEF_ST_H +#define TOR_LIB_CONFMGT_VAR_TYPE_DEF_ST_H + +#include + +struct config_line_t; + +/** + * A structure full of functions pointers to implement a variable type. + * + * Every type MUST implement parse or kv_parse and encode or kv_encode; + * the other functions pointers MAY be NULL. + * + * All functions here take a params argument, whose value + * is determined by the type definition. Two types may have the + * same functions, but differ only in parameters. + **/ +struct var_type_fns_t { + /** + * Try to parse a string in value that encodes an object of this + * type. On success, adjust the lvalue pointed to by target to hold + * that value, and return 0. On failure, set *errmsg to a newly + * allocated string holding an error message, and return -1. + **/ + int (*parse)(void *target, const char *value, char **errmsg, + const void *params); + /** + * Try to parse a single line from the head ofline that encodes + * an object of this type. On success and failure, behave as in the parse() + * function. + * + * If this function is absent, it is implemented in terms of parse(). + * + * All types for which keys are significant should use this method. For + * example, a "linelist" type records the actual keys that are given + * for each line, and so should use this method. + * + * Note that although multiple lines may be provided in line, + * only the first one should be handled by this function. + **/ + int (*kv_parse)(void *target, const struct config_line_t *line, + char **errmsg, const void *params); + /** + * Encode a value pointed to by value and return its result + * in a newly allocated string. The string may need to be escaped. + * + * If this function is absent, it is implemented in terms of kv_encode(). + * + * Returns NULL if this option has a NULL value, or on internal error. + * + * Requirement: all strings generated by encode() should produce a + * semantically equivalent value when given to parse(). + **/ + char *(*encode)(const void *value, const void *params); + /** + * As encode(), but returns a newly allocated config_line_t object. The + * provided key is used as the key of the lines, unless the type is + * one that encodes its own keys. + * + * Unlike kv_parse(), this function will return a list of multiple lines, + * if value is such that it must be encoded by multiple lines. + * + * Returns NULL if there are no lines to encode, or on internal error. + * + * If this function is absent, it is implemented in terms of encode(). + **/ + struct config_line_t *(*kv_encode)(const char *key, const void *value, + const void *params); + /** + * Free all storage held in arg, and set arg to a default + * value -- usually zero or NULL. + * + * If this function is absent, the default implementation does nothing. + **/ + void (*clear)(void *arg, const void *params); + /** + * Return true if a and b hold the same value, and false + * otherwise. + * + * If this function is absent, it is implemented by encoding both a and + * b and comparing their encoded strings for equality. + **/ + bool (*eq)(const void *a, const void *b, const void *params); + /** + * Try to copy the value from value into target. + * On success return 0; on failure return -1. + * + * If this function is absent, it is implemented by encoding the value + * into a string, and then parsing it into the target. + **/ + int (*copy)(void *target, const void *value, const void *params); + /** + * Check whether value holds a valid value according to the + * rules of this type; return true if it does and false if it doesn't. + * + * The default implementation for this function assumes that all + * values are valid. + **/ + bool (*ok)(const void *value, const void *params); +}; + +/** + * A structure describing a type that can be manipulated with the typedvar_* + * functions. + **/ +struct var_type_def_t { + /** + * The name of this type. Should not include spaces. Used for + * debugging, log messages, and the controller API. */ + const char *name; + /** + * A function table for this type. + */ + const struct var_type_fns_t *fns; + /** + * A pointer to a value that should be passed as the 'params' argument when + * calling the functions in this type's function table. + */ + const void *params; +}; + +#endif /* !defined(TOR_LIB_CONFMGT_VAR_TYPE_DEF_ST_H) */ diff --git a/src/test/test_confparse.c b/src/test/test_confparse.c index 89a6eb5265..fef361ef65 100644 --- a/src/test/test_confparse.c +++ b/src/test/test_confparse.c @@ -469,9 +469,9 @@ static const badval_test_t bv_badcsvi2 = { "csv_interval cl,10\n", "malformed" }; static const badval_test_t bv_nonoption = { "fnord 10\n", "Unknown option" }; static const badval_test_t bv_badmem = { "mem 3 trits\n", "malformed" }; -static const badval_test_t bv_badbool = { "boolean 7\n", "expects 0 or 1" }; +static const badval_test_t bv_badbool = { "boolean 7\n", "Unrecognized value"}; static const badval_test_t bv_badabool = - { "autobool 7\n", "expects 0, 1, or 'auto'" }; + { "autobool 7\n", "Unrecognized value" }; static const badval_test_t bv_badtime = { "time lunchtime\n", "Invalid time" }; static const badval_test_t bv_virt = { "MixedLines 7\n", "virtual option" }; static const badval_test_t bv_rs = { "Routerset 2.2.2.2.2\n", "Invalid" }; diff --git a/src/test/test_options.c b/src/test/test_options.c index d693fe0568..b39cd4f1e4 100644 --- a/src/test/test_options.c +++ b/src/test/test_options.c @@ -258,13 +258,17 @@ test_options_validate(void *arg) WANT_ERR("BridgeRelay 1\nDirCache 0", "We're a bridge but DirCache is disabled.", PH_VALIDATE); + // XXXX We should replace this with a more full error message once #29211 + // XXXX is done. It is truncated for now because at the current stage + // XXXX of refactoring, we can't give a full error message like before. WANT_ERR_LOG("HeartbeatPeriod 21 snarks", - "Interval 'HeartbeatPeriod 21 snarks' is malformed or" - " out of bounds.", LOG_WARN, "Unknown unit 'snarks'.", + "malformed or out of bounds", LOG_WARN, + "Unknown unit 'snarks'.", PH_ASSIGN); + // XXXX As above. WANT_ERR_LOG("LogTimeGranularity 21 snarks", - "Msec interval 'LogTimeGranularity 21 snarks' is malformed or" - " out of bounds.", LOG_WARN, "Unknown unit 'snarks'.", + "malformed or out of bounds", LOG_WARN, + "Unknown unit 'snarks'.", PH_ASSIGN); OK("HeartbeatPeriod 1 hour", PH_VALIDATE); OK("LogTimeGranularity 100 milliseconds", PH_VALIDATE); From f0074372928829835adf6aa2dde9af712097c7c0 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 19 Jun 2019 08:14:59 -0400 Subject: [PATCH 1158/2557] Further clarify our clarification about the type of POSINT --- src/lib/conf/conftypes.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/lib/conf/conftypes.h b/src/lib/conf/conftypes.h index b03234b62c..e66ab3d5aa 100644 --- a/src/lib/conf/conftypes.h +++ b/src/lib/conf/conftypes.h @@ -73,10 +73,10 @@ typedef enum config_type_t { typedef union { char **STRING; char **FILENAME; - int *POSINT; /* yes, really: Even though the confparse type is called - * "POSINT", it still uses the C int type -- it just enforces - * that the values are in range [0,INT_MAX]. - */ + int *POSINT; /* yes, this is really an int, and not an unsigned int. For + * historical reasons, many configuration values are restricted + * to the range [0,INT_MAX], and stored in signed ints. + */ uint64_t *UINT64; int *INT; int *INTERVAL; From 705bda859ed8bbdd753d587b9814de4af3afee80 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 19 Jun 2019 08:15:13 -0400 Subject: [PATCH 1159/2557] Add unit tests for the unitparse.c module. --- src/lib/confmgt/unitparse.c | 6 +++- src/test/test_confparse.c | 62 +++++++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+), 1 deletion(-) diff --git a/src/lib/confmgt/unitparse.c b/src/lib/confmgt/unitparse.c index 906195259d..c3ed8285a4 100644 --- a/src/lib/confmgt/unitparse.c +++ b/src/lib/confmgt/unitparse.c @@ -128,10 +128,14 @@ config_parse_units(const char *val, const unit_table_t *u, int *ok) use_float = 1; } - if (!cp) { + if (BUG(!cp)) { + // cp should always be non-NULL if the parse operation succeeds. + + // LCOV_EXCL_START *ok = 1; v = use_float ? ((uint64_t)d) : v; goto done; + // LCOV_EXCL_STOP } cp = (char*) eat_whitespace(cp); diff --git a/src/test/test_confparse.c b/src/test/test_confparse.c index fef361ef65..dde61b1c81 100644 --- a/src/test/test_confparse.c +++ b/src/test/test_confparse.c @@ -18,6 +18,8 @@ #include "test/test.h" #include "test/log_test_helpers.h" +#include "lib/confmgt/unitparse.h" + typedef struct test_struct_t { uint32_t magic; char *s; @@ -805,6 +807,65 @@ test_confparse_extra_lines(void *arg) config_free(&etest_fmt, tst); } +static void +test_confparse_unitparse(void *args) +{ + (void)args; + /* spot-check a few memunit values. */ + int ok = 3; + tt_u64_op(config_parse_memunit("100 MB", &ok), OP_EQ, 100<<20); + tt_assert(ok); + tt_u64_op(config_parse_memunit("100 TB", &ok), OP_EQ, UINT64_C(100)<<40); + tt_assert(ok); + // This is a floating-point value, but note that 1.5 can be represented + // precisely. + tt_u64_op(config_parse_memunit("1.5 MB", &ok), OP_EQ, 3<<19); + tt_assert(ok); + + /* Try some good intervals and msec intervals */ + tt_int_op(config_parse_interval("2 days", &ok), OP_EQ, 48*3600); + tt_assert(ok); + tt_int_op(config_parse_interval("1.5 hour", &ok), OP_EQ, 5400); + tt_assert(ok); + tt_u64_op(config_parse_interval("1 minute", &ok), OP_EQ, 60); + tt_assert(ok); + tt_int_op(config_parse_msec_interval("2 days", &ok), OP_EQ, 48*3600*1000); + tt_assert(ok); + tt_int_op(config_parse_msec_interval("10 msec", &ok), OP_EQ, 10); + tt_assert(ok); + + /* Try a couple of unitless values. */ + tt_int_op(config_parse_interval("10", &ok), OP_EQ, 10); + tt_assert(ok); + tt_u64_op(config_parse_interval("15.0", &ok), OP_EQ, 15); + tt_assert(ok); + + /* u64 overflow */ + /* XXXX our implementation does not currently detect this. See bug 30920. */ + /* + tt_u64_op(config_parse_memunit("20000000 TB", &ok), OP_EQ, 0); + tt_assert(!ok); + */ + + /* i32 overflow */ + tt_int_op(config_parse_interval("1000 months", &ok), OP_EQ, -1); + tt_assert(!ok); + tt_int_op(config_parse_msec_interval("4 weeks", &ok), OP_EQ, -1); + tt_assert(!ok); + + /* bad units */ + tt_u64_op(config_parse_memunit("7 nybbles", &ok), OP_EQ, 0); + tt_assert(!ok); + // XXXX these next two should return -1 according to the documentation. + tt_int_op(config_parse_interval("7 cowznofski", &ok), OP_EQ, 0); + tt_assert(!ok); + tt_int_op(config_parse_msec_interval("1 kalpa", &ok), OP_EQ, 0); + tt_assert(!ok); + + done: + ; +} + #define CONFPARSE_TEST(name, flags) \ { #name, test_confparse_ ## name, flags, NULL, NULL } @@ -838,5 +899,6 @@ struct testcase_t confparse_tests[] = { CONFPARSE_TEST(reassign_extend, 0), CONFPARSE_TEST(get_assigned, 0), CONFPARSE_TEST(extra_lines, 0), + CONFPARSE_TEST(unitparse, 0), END_OF_TESTCASES }; From e16b90b88a76fb82702ae26c54834aca6c591d64 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 18 Jun 2019 19:16:57 -0400 Subject: [PATCH 1160/2557] Partially port routerset to being a full-fledged config type again. --- src/app/config/confparse.c | 27 +++----------- src/feature/nodelist/routerset.c | 62 ++++++++++++++++++++++++++++++++ src/feature/nodelist/routerset.h | 3 ++ 3 files changed, 70 insertions(+), 22 deletions(-) diff --git a/src/app/config/confparse.c b/src/app/config/confparse.c index bc2ab24e4f..a02aa26e82 100644 --- a/src/app/config/confparse.c +++ b/src/app/config/confparse.c @@ -175,18 +175,8 @@ config_assign_value(const config_format_t *fmt, void *options, if (var->type == CONFIG_TYPE_ROUTERSET) { // XXXX make the backend extensible so that we don't have to - // XXXX handle ROUTERSET specially. - - if (*(routerset_t**)lvalue) { - routerset_free(*(routerset_t**)lvalue); - } - *(routerset_t**)lvalue = routerset_new(); - if (routerset_parse(*(routerset_t**)lvalue, c->value, c->key)<0) { - tor_asprintf(msg, "Invalid exit list '%s' for option '%s'", - c->value, c->key); - return -1; - } - return 0; + // XXXX special-case this type. + return typed_var_kvassign_ex(lvalue, c, msg, &routerset_type_defn); } return typed_var_kvassign(lvalue, c, msg, var->type); @@ -377,10 +367,8 @@ config_get_assigned_option(const config_format_t *fmt, const void *options, if (var->type == CONFIG_TYPE_ROUTERSET) { // XXXX make the backend extensible so that we don't have to - // XXXX handle ROUTERSET specially. - result = tor_malloc_zero(sizeof(config_line_t)); - result->key = tor_strdup(var->name); - result->value = routerset_to_string(*(routerset_t**)value); + // XXXX special-case this type. + result = typed_var_kvencode_ex(var->name, value, &routerset_type_defn); } else { result = typed_var_kvencode(var->name, value, var->type); } @@ -512,12 +500,7 @@ config_clear(const config_format_t *fmt, void *options, void *lvalue = STRUCT_VAR_P(options, var->var_offset); (void)fmt; /* unused */ if (var->type == CONFIG_TYPE_ROUTERSET) { - // XXXX make the backend extensible so that we don't have to - // XXXX handle ROUTERSET specially. - if (*(routerset_t**)lvalue) { - routerset_free(*(routerset_t**)lvalue); - *(routerset_t**)lvalue = NULL; - } + typed_var_free_ex(lvalue, &routerset_type_defn); return; } diff --git a/src/feature/nodelist/routerset.c b/src/feature/nodelist/routerset.c index e801fd81b1..ad42e8e101 100644 --- a/src/feature/nodelist/routerset.c +++ b/src/feature/nodelist/routerset.c @@ -34,6 +34,9 @@ n * Copyright (c) 2001-2004, Roger Dingledine. #include "feature/nodelist/nickname.h" #include "feature/nodelist/nodelist.h" #include "feature/nodelist/routerset.h" +#include "lib/conf/conftypes.h" +#include "lib/confmgt/typedvar.h" +#include "lib/encoding/confline.h" #include "lib/geoip/geoip.h" #include "core/or/addr_policy_st.h" @@ -41,6 +44,7 @@ n * Copyright (c) 2001-2004, Roger Dingledine. #include "feature/nodelist/node_st.h" #include "feature/nodelist/routerinfo_st.h" #include "feature/nodelist/routerstatus_st.h" +#include "lib/confmgt/var_type_def_st.h" /** Return a new empty routerset. */ routerset_t * @@ -461,3 +465,61 @@ routerset_free_(routerset_t *routerset) bitarray_free(routerset->countries); tor_free(routerset); } + +static int +routerset_kv_parse(void *target, const config_line_t *line, char **errmsg, + const void *params) +{ + (void)params; + routerset_t **p = (routerset_t**)target; + routerset_free(*p); // clear the old value, if any. + routerset_t *rs = routerset_new(); + if (routerset_parse(rs, line->value, line->key) < 0) { + routerset_free(rs); + *errmsg = tor_strdup("Invalid router list."); + return -1; + } else { + *p = rs; + return 0; + } +} + +static char * +routerset_encode(const void *value, const void *params) +{ + (void)params; + const routerset_t **p = (const routerset_t**)value; + return routerset_to_string(*p); +} + +static void +routerset_clear(void *value, const void *params) +{ + (void)params; + routerset_t **p = (routerset_t**)value; + routerset_free(*p); // sets *p to NULL. +} + +static int +routerset_copy(void *dest, const void *src, const void *params) +{ + (void)params; + routerset_t **output = (routerset_t**)dest; + const routerset_t *input = *(routerset_t**)src; + routerset_free(*output); // sets *output to NULL + *output = routerset_new(); + routerset_union(*output, input); + return 0; +} + +static const var_type_fns_t routerset_type_fns = { + .kv_parse = routerset_kv_parse, + .encode = routerset_encode, + .clear = routerset_clear, + .copy = routerset_copy +}; + +const var_type_def_t routerset_type_defn = { + .name = "RouterList", + .fns = &routerset_type_fns +}; diff --git a/src/feature/nodelist/routerset.h b/src/feature/nodelist/routerset.h index ca8b6fed93..9d184c9852 100644 --- a/src/feature/nodelist/routerset.h +++ b/src/feature/nodelist/routerset.h @@ -44,6 +44,9 @@ void routerset_free_(routerset_t *routerset); #define routerset_free(rs) FREE_AND_NULL(routerset_t, routerset_free_, (rs)) int routerset_len(const routerset_t *set); +struct var_type_def_t; +extern const struct var_type_def_t routerset_type_defn; + #ifdef ROUTERSET_PRIVATE #include "lib/container/bitarray.h" From c131b0763e994ea850f457319ec6d9c487760a85 Mon Sep 17 00:00:00 2001 From: teor Date: Mon, 24 Jun 2019 21:20:34 +1000 Subject: [PATCH 1161/2557] stats: add comments about the required chunk structure in extra info files These comments should prevent future instances of 30958. And allow a larger file in practracker. Follow up after 30958. --- scripts/maint/practracker/exceptions.txt | 2 +- src/feature/relay/router.c | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/scripts/maint/practracker/exceptions.txt b/scripts/maint/practracker/exceptions.txt index 4db452b897..ffe238a679 100644 --- a/scripts/maint/practracker/exceptions.txt +++ b/scripts/maint/practracker/exceptions.txt @@ -225,7 +225,7 @@ problem function-size /src/feature/nodelist/routerlist.c:update_extrainfo_downlo problem function-size /src/feature/relay/dns.c:dns_resolve_impl() 134 problem function-size /src/feature/relay/dns.c:configure_nameservers() 161 problem function-size /src/feature/relay/dns.c:evdns_callback() 109 -problem file-size /src/feature/relay/router.c 3510 +problem file-size /src/feature/relay/router.c 3522 problem include-count /src/feature/relay/router.c 56 problem function-size /src/feature/relay/router.c:init_keys() 252 problem function-size /src/feature/relay/router.c:get_my_declared_family() 114 diff --git a/src/feature/relay/router.c b/src/feature/relay/router.c index 782084caa3..51ced6289d 100644 --- a/src/feature/relay/router.c +++ b/src/feature/relay/router.c @@ -3158,6 +3158,8 @@ extrainfo_dump_to_string_header_helper( ed_cert_line = tor_strdup(""); } + /* This is the first chunk in the file. If the file is too big, other chunks + * are removed. So we must only add one chunk here. */ tor_asprintf(&pre, "extra-info %s %s\n%spublished %s\n", extrainfo->nickname, identity, ed_cert_line, @@ -3187,6 +3189,10 @@ extrainfo_dump_to_string_stats_helper(smartlist_t *chunks, char *contents = NULL; time_t now = time(NULL); + /* If the file is too big, these chunks are removed, starting with the last + * chunk. So each chunk must be a complete line, and the file must be valid + * after each chunk. */ + /* Add information about the pluggable transports we support, even if we * are not publishing statistics. This information is needed by BridgeDB * to distribute bridges. */ @@ -3269,6 +3275,8 @@ extrainfo_dump_to_string_ed_sig_helper( char buf[ED25519_SIG_BASE64_LEN+1]; int rv = -1; + /* These are two of the three final chunks in the file. If the file is too + * big, other chunks are removed. So we must only add two chunks here. */ smartlist_add_strdup(chunks, "router-sig-ed25519 "); crypto_digest_smartlist_prefix(sha256_digest, DIGEST256_LEN, ED_DESC_SIGNATURE_PREFIX, @@ -3362,6 +3370,8 @@ extrainfo_dump_to_string(char **s_out, extrainfo_t *extrainfo, goto err; } + /* This is one of the three final chunks in the file. If the file is too big, + * other chunks are removed. So we must only add one chunk here. */ smartlist_add_strdup(chunks, "router-signature\n"); s = smartlist_join_strings(chunks, "", 0, NULL); From e4f66bf7ff62a8669341d239ef9d705f320b28a5 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 25 Jun 2019 11:55:53 -0400 Subject: [PATCH 1162/2557] bump to 0.4.1.3-alpha-dev --- configure.ac | 4 ++-- contrib/win32build/tor-mingw.nsi.in | 2 +- src/win32/orconfig.h | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/configure.ac b/configure.ac index 107f676bdb..cbad429ce3 100644 --- a/configure.ac +++ b/configure.ac @@ -4,7 +4,7 @@ dnl Copyright (c) 2007-2019, The Tor Project, Inc. dnl See LICENSE for licensing information AC_PREREQ([2.63]) -AC_INIT([tor],[0.4.1.3-alpha]) +AC_INIT([tor],[0.4.1.3-alpha-dev]) AC_CONFIG_SRCDIR([src/app/main/tor_main.c]) AC_CONFIG_MACRO_DIR([m4]) @@ -14,7 +14,7 @@ AC_CONFIG_MACRO_DIR([m4]) # version number changes. Tor uses it to make sure that it # only shuts down for missing "required protocols" when those protocols # are listed as required by a consensus after this date. -AC_DEFINE(APPROX_RELEASE_DATE, ["2019-06-24"], # for 0.4.1.3-alpha +AC_DEFINE(APPROX_RELEASE_DATE, ["2019-06-25"], # for 0.4.1.3-alpha-dev [Approximate date when this software was released. (Updated when the version changes.)]) # "foreign" means we don't follow GNU package layout standards diff --git a/contrib/win32build/tor-mingw.nsi.in b/contrib/win32build/tor-mingw.nsi.in index da4aea718e..0a56429112 100644 --- a/contrib/win32build/tor-mingw.nsi.in +++ b/contrib/win32build/tor-mingw.nsi.in @@ -8,7 +8,7 @@ !include "LogicLib.nsh" !include "FileFunc.nsh" !insertmacro GetParameters -!define VERSION "0.4.1.3-alpha" +!define VERSION "0.4.1.3-alpha-dev" !define INSTALLER "tor-${VERSION}-win32.exe" !define WEBSITE "https://www.torproject.org/" !define LICENSE "LICENSE" diff --git a/src/win32/orconfig.h b/src/win32/orconfig.h index a90a212b42..8b0855987a 100644 --- a/src/win32/orconfig.h +++ b/src/win32/orconfig.h @@ -218,7 +218,7 @@ #define USING_TWOS_COMPLEMENT /* Version number of package */ -#define VERSION "0.4.1.3-alpha" +#define VERSION "0.4.1.3-alpha-dev" From fe415043097c1d384c82d4b4bbf624cb7564d59e Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 25 Jun 2019 11:56:08 -0400 Subject: [PATCH 1163/2557] forward-port the 0.4.1.3-alpha changelog --- ChangeLog | 49 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/ChangeLog b/ChangeLog index d44ce316c7..5bfce1ac51 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,52 @@ +Changes in version 0.4.1.3-alpha - 2019-06-25 + Tor 0.4.1.3-alpha resolves numerous bugs left over from the previous + alpha, most of them from earlier release series. + + o Major bugfixes (Onion service reachability): + - Properly clean up the introduction point map when circuits change + purpose from onion service circuits to pathbias, measurement, or + other circuit types. This should fix some service-side instances + of introduction point failure. Fixes bug 29034; bugfix + on 0.3.2.1-alpha. + + o Minor features (geoip): + - Update geoip and geoip6 to the June 10 2019 Maxmind GeoLite2 + Country database. Closes ticket 30852. + + o Minor features (logging): + - Give a more useful assertion failure message if we think we have + minherit() but we fail to make a region non-inheritable. Give a + compile-time warning if our support for minherit() is incomplete. + Closes ticket 30686. + + o Minor bugfixes (circuit isolation): + - Fix a logic error that prevented the SessionGroup sub-option from + being accepted. Fixes bug 22619; bugfix on 0.2.7.2-alpha. + + o Minor bugfixes (continuous integration): + - Allow the test-stem job to fail in Travis, because it sometimes + hangs. Fixes bug 30744; bugfix on 0.3.5.4-alpha. + - Skip test_rebind on macOS in Travis, because it is unreliable on + macOS on Travis. Fixes bug 30713; bugfix on 0.3.5.1-alpha. + - Skip test_rebind when the TOR_SKIP_TEST_REBIND environment + variable is set. Fixes bug 30713; bugfix on 0.3.5.1-alpha. + + o Minor bugfixes (directory authorities): + - Stop crashing after parsing an unknown descriptor purpose + annotation. We think this bug can only be triggered by modifying a + local file. Fixes bug 30781; bugfix on 0.2.0.8-alpha. + + o Minor bugfixes (pluggable transports): + - When running as a bridge with pluggable transports, always publish + pluggable transport information in our extrainfo descriptor, even + if ExtraInfoStatistics is 0. This information is needed by + BridgeDB. Fixes bug 30956; bugfix on 0.4.1.1-alpha. + + o Documentation: + - Mention URLs for Travis/Appveyor/Jenkins in ReleasingTor.md. + Closes ticket 30630. + + Changes in version 0.4.1.2-alpha - 2019-06-06 Tor 0.4.1.2-alpha resolves numerous bugs--some of them from the previous alpha, and some much older. It also contains minor testing From 3644f4ab5f4c10e95f6ea6db26764054c73ce594 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 25 Jun 2019 11:57:08 -0400 Subject: [PATCH 1164/2557] Remove changes files that appeared in 0.4.1.3-alpha or earlier. --- changes/bug22619 | 3 --- changes/bug29034 | 5 ----- changes/bug30713 | 5 ----- changes/bug30744 | 3 --- changes/bug30781 | 4 ---- changes/bug30956 | 4 ---- changes/doc30630 | 3 --- changes/geoip-2019-06-10 | 4 ---- changes/ticket30686 | 5 ----- 9 files changed, 36 deletions(-) delete mode 100644 changes/bug22619 delete mode 100644 changes/bug29034 delete mode 100644 changes/bug30713 delete mode 100644 changes/bug30744 delete mode 100644 changes/bug30781 delete mode 100644 changes/bug30956 delete mode 100644 changes/doc30630 delete mode 100644 changes/geoip-2019-06-10 delete mode 100644 changes/ticket30686 diff --git a/changes/bug22619 b/changes/bug22619 deleted file mode 100644 index 9c71996f5b..0000000000 --- a/changes/bug22619 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor bugfixes (circuit isolation): - - Fix a logic error that prevented the SessionGroup sub-option from - being accepted. Fixes bug 22619; bugfix on 0.2.7.2-alpha. diff --git a/changes/bug29034 b/changes/bug29034 deleted file mode 100644 index e7aa9af00b..0000000000 --- a/changes/bug29034 +++ /dev/null @@ -1,5 +0,0 @@ - o Major bugfixes (Onion service reachability): - - Properly clean up the introduction point map when circuits change purpose - from onion service circuits to pathbias, measurement, or other circuit types. - This should fix some service-side instances of introduction point failure. - Fixes bug 29034; bugfix on 0.3.2.1-alpha. diff --git a/changes/bug30713 b/changes/bug30713 deleted file mode 100644 index e00b98da65..0000000000 --- a/changes/bug30713 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor bugfixes (testing): - - Skip test_rebind when the TOR_SKIP_TEST_REBIND environmental variable is - set. Fixes bug 30713; bugfix on 0.3.5.1-alpha. - - Skip test_rebind on macOS in Travis, because it is unreliable on - macOS on Travis. Fixes bug 30713; bugfix on 0.3.5.1-alpha. diff --git a/changes/bug30744 b/changes/bug30744 deleted file mode 100644 index 9f07d4855f..0000000000 --- a/changes/bug30744 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor bugfixes (continuous integration): - - Allow the test-stem job to fail in Travis, because it sometimes hangs. - Fixes bug 30744; bugfix on 0.3.5.4-alpha. diff --git a/changes/bug30781 b/changes/bug30781 deleted file mode 100644 index 7c7adf470e..0000000000 --- a/changes/bug30781 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (directory authorities): - - Stop crashing after parsing an unknown descriptor purpose annotation. - We think this bug can only be triggered by modifying a local file. - Fixes bug 30781; bugfix on 0.2.0.8-alpha. diff --git a/changes/bug30956 b/changes/bug30956 deleted file mode 100644 index 8f52a81de3..0000000000 --- a/changes/bug30956 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (pluggable transports): - - Always publish bridge pluggable transport information in the extra info - descriptor, even if ExtraInfoStatistics is 0. This information is - needed by BridgeDB. Fixes bug 30956; bugfix on 0.4.1.1-alpha. diff --git a/changes/doc30630 b/changes/doc30630 deleted file mode 100644 index 0fbd8d4dd4..0000000000 --- a/changes/doc30630 +++ /dev/null @@ -1,3 +0,0 @@ - o Documentation: - - Mention URLs for Travis/Appveyor/Jenkins in ReleasingTor.md. Closes - ticket 30630. diff --git a/changes/geoip-2019-06-10 b/changes/geoip-2019-06-10 deleted file mode 100644 index 2d1e065649..0000000000 --- a/changes/geoip-2019-06-10 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor features (geoip): - - Update geoip and geoip6 to the June 10 2019 Maxmind GeoLite2 - Country database. Closes ticket 30852. - diff --git a/changes/ticket30686 b/changes/ticket30686 deleted file mode 100644 index 36473c1a02..0000000000 --- a/changes/ticket30686 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor features (logging): - - Give a more useful assertion failure message if we think we have - minherit() but we fail to make a region non-inheritable. Give a - compile-time warning if our support for minherit() is - incomplete. Closes ticket 30686. From 0dd59fdb56c9a47a018856b689eaa3c6775ed3d4 Mon Sep 17 00:00:00 2001 From: Taylor Yu Date: Mon, 17 Jun 2019 15:08:11 -0500 Subject: [PATCH 1165/2557] Clean up some uses of low-level control replies Part of ticket 30889. --- src/feature/control/control_auth.c | 8 ++------ src/feature/control/control_cmd.c | 11 +++++------ 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/src/feature/control/control_auth.c b/src/feature/control/control_auth.c index 49d4d415c6..a574d07b33 100644 --- a/src/feature/control/control_auth.c +++ b/src/feature/control/control_auth.c @@ -151,12 +151,8 @@ handle_control_authchallenge(control_connection_t *conn, goto fail; } if (args->kwargs == NULL || args->kwargs->next != NULL) { - /* connection_write_str_to_buf("512 AUTHCHALLENGE requires exactly " - "2 arguments.\r\n", conn); - */ - control_printf_endreply(conn, 512, - "AUTHCHALLENGE dislikes argument list %s", - escaped(args->raw_body)); + control_write_endreply(conn, 512, + "Wrong number of arguments for AUTHCHALLENGE"); goto fail; } if (strcmp(args->kwargs->key, "")) { diff --git a/src/feature/control/control_cmd.c b/src/feature/control/control_cmd.c index abb579bd43..051dd12bac 100644 --- a/src/feature/control/control_cmd.c +++ b/src/feature/control/control_cmd.c @@ -703,9 +703,8 @@ handle_control_mapaddress(control_connection_t *conn, connection_buf_add(r, sz, TO_CONN(conn)); tor_free(r); } else { - const char *response = - "512 syntax error: not enough arguments to mapaddress.\r\n"; - connection_buf_add(response, strlen(response), TO_CONN(conn)); + control_write_endreply(conn, 512, "syntax error: " + "not enough arguments to mapaddress."); } SMARTLIST_FOREACH(reply, char *, cp, tor_free(cp)); @@ -845,7 +844,7 @@ handle_control_extendcircuit(control_connection_t *conn, "addresses that are allowed by the firewall configuration; " "circuit marked for closing."); circuit_mark_for_close(TO_CIRCUIT(circ), -END_CIRC_REASON_CONNECTFAILED); - connection_write_str_to_buf("551 Couldn't start circuit\r\n", conn); + control_write_endreply(conn, 551, "Couldn't start circuit"); goto done; } circuit_append_new_exit(circ, info); @@ -1876,8 +1875,8 @@ handle_control_add_onion(control_connection_t *conn, char *encoded = rend_auth_encode_cookie(ac->descriptor_cookie, auth_type); tor_assert(encoded); - connection_printf_to_buf(conn, "250-ClientAuth=%s:%s\r\n", - ac->client_name, encoded); + control_printf_midreply(conn, 250, "ClientAuth=%s:%s", + ac->client_name, encoded); memwipe(encoded, 0, strlen(encoded)); tor_free(encoded); }); From e5e6953be74ab5d09608dcf6af48caae4e57464c Mon Sep 17 00:00:00 2001 From: Taylor Yu Date: Tue, 25 Jun 2019 09:54:50 -0500 Subject: [PATCH 1166/2557] Make control_write_reply() mockable Part of ticket 30889. --- src/feature/control/control_proto.c | 5 +++-- src/feature/control/control_proto.h | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/feature/control/control_proto.c b/src/feature/control/control_proto.c index 1dd62da2be..d2541e7308 100644 --- a/src/feature/control/control_proto.c +++ b/src/feature/control/control_proto.c @@ -176,8 +176,9 @@ send_control_done(control_connection_t *conn) * @param c separator character, usually ' ', '-', or '+' * @param s string */ -void -control_write_reply(control_connection_t *conn, int code, int c, const char *s) +MOCK_IMPL(void, +control_write_reply, (control_connection_t *conn, int code, int c, + const char *s)) { connection_printf_to_buf(conn, "%03d%c%s\r\n", code, c, s); } diff --git a/src/feature/control/control_proto.h b/src/feature/control/control_proto.h index 101b808d88..3182f3d415 100644 --- a/src/feature/control/control_proto.h +++ b/src/feature/control/control_proto.h @@ -21,8 +21,8 @@ size_t write_escaped_data(const char *data, size_t len, char **out); size_t read_escaped_data(const char *data, size_t len, char **out); void send_control_done(control_connection_t *conn); -void control_write_reply(control_connection_t *conn, int code, int c, - const char *s); +MOCK_DECL(void, control_write_reply, (control_connection_t *conn, int code, + int c, const char *s)); void control_vprintf_reply(control_connection_t *conn, int code, int c, const char *fmt, va_list ap) CHECK_PRINTF(4, 0); From 5faf54970de4bc3c80a268b5b139c713413562fd Mon Sep 17 00:00:00 2001 From: Taylor Yu Date: Tue, 25 Jun 2019 10:34:53 -0500 Subject: [PATCH 1167/2557] Fix some onion helpers Fix add_onion_helper_clientauth() and add_onion_helper_keyarg() to explicitly call the appropriate control reply abstractions instead of allocating a string to pass to their callers. Part of ticket 30889. --- src/feature/control/control_cmd.c | 75 +++++++++------------ src/feature/control/control_cmd.h | 5 +- src/test/test_controller.c | 106 ++++++++++++++++++------------ 3 files changed, 100 insertions(+), 86 deletions(-) diff --git a/src/feature/control/control_cmd.c b/src/feature/control/control_cmd.c index 051dd12bac..ad4a4ef0af 100644 --- a/src/feature/control/control_cmd.c +++ b/src/feature/control/control_cmd.c @@ -1743,16 +1743,10 @@ handle_control_add_onion(control_connection_t *conn, goto out; } else if (!strcasecmp(arg->key, "ClientAuth")) { - char *err_msg = NULL; int created = 0; rend_authorized_client_t *client = - add_onion_helper_clientauth(arg->value, - &created, &err_msg); + add_onion_helper_clientauth(arg->value, &created, conn); if (!client) { - if (err_msg) { - connection_write_str_to_buf(err_msg, conn); - tor_free(err_msg); - } goto out; } @@ -1817,19 +1811,13 @@ handle_control_add_onion(control_connection_t *conn, add_onion_secret_key_t pk = { NULL }; const char *key_new_alg = NULL; char *key_new_blob = NULL; - char *err_msg = NULL; const char *onionkey = smartlist_get(args->args, 0); if (add_onion_helper_keyarg(onionkey, discard_pk, &key_new_alg, &key_new_blob, &pk, &hs_version, - &err_msg) < 0) { - if (err_msg) { - connection_write_str_to_buf(err_msg, conn); - tor_free(err_msg); - } + conn) < 0) { goto out; } - tor_assert(!err_msg); /* Hidden service version 3 don't have client authentication support so if * ClientAuth was given, send back an error. */ @@ -1929,27 +1917,30 @@ handle_control_add_onion(control_connection_t *conn, * ADD_ONION command. Return a new crypto_pk_t and if a new key was generated * and the private key not discarded, the algorithm and serialized private key, * or NULL and an optional control protocol error message on failure. The - * caller is responsible for freeing the returned key_new_blob and err_msg. + * caller is responsible for freeing the returned key_new_blob. * * Note: The error messages returned are deliberately vague to avoid echoing * key material. + * + * Note: conn is only used for writing control replies. For testing + * purposes, it can be NULL if control_write_reply() is appropriately + * mocked. */ STATIC int add_onion_helper_keyarg(const char *arg, int discard_pk, const char **key_new_alg_out, char **key_new_blob_out, add_onion_secret_key_t *decoded_key, int *hs_version, - char **err_msg_out) + control_connection_t *conn) { smartlist_t *key_args = smartlist_new(); crypto_pk_t *pk = NULL; const char *key_new_alg = NULL; char *key_new_blob = NULL; - char *err_msg = NULL; int ret = -1; smartlist_split_string(key_args, arg, ":", SPLIT_IGNORE_BLANK, 0); if (smartlist_len(key_args) != 2) { - err_msg = tor_strdup("512 Invalid key type/blob\r\n"); + control_write_endreply(conn, 512, "Invalid key type/blob"); goto err; } @@ -1966,12 +1957,12 @@ add_onion_helper_keyarg(const char *arg, int discard_pk, /* "RSA:" - Loading a pre-existing RSA1024 key. */ pk = crypto_pk_base64_decode_private(key_blob, strlen(key_blob)); if (!pk) { - err_msg = tor_strdup("512 Failed to decode RSA key\r\n"); + control_write_endreply(conn, 512, "Failed to decode RSA key"); goto err; } if (crypto_pk_num_bits(pk) != PK_BYTES*8) { crypto_pk_free(pk); - err_msg = tor_strdup("512 Invalid RSA key size\r\n"); + control_write_endreply(conn, 512, "Invalid RSA key size"); goto err; } decoded_key->v2 = pk; @@ -1982,7 +1973,7 @@ add_onion_helper_keyarg(const char *arg, int discard_pk, if (base64_decode((char *) sk->seckey, sizeof(sk->seckey), key_blob, strlen(key_blob)) != sizeof(sk->seckey)) { tor_free(sk); - err_msg = tor_strdup("512 Failed to decode ED25519-V3 key\r\n"); + control_write_endreply(conn, 512, "Failed to decode ED25519-V3 key"); goto err; } decoded_key->v3 = sk; @@ -1994,15 +1985,15 @@ add_onion_helper_keyarg(const char *arg, int discard_pk, /* "RSA1024", RSA 1024 bit, also currently "BEST" by default. */ pk = crypto_pk_new(); if (crypto_pk_generate_key(pk)) { - tor_asprintf(&err_msg, "551 Failed to generate %s key\r\n", - key_type_rsa1024); + control_printf_endreply(conn, 551, "Failed to generate %s key", + key_type_rsa1024); goto err; } if (!discard_pk) { if (crypto_pk_base64_encode_private(pk, &key_new_blob)) { crypto_pk_free(pk); - tor_asprintf(&err_msg, "551 Failed to encode %s key\r\n", - key_type_rsa1024); + control_printf_endreply(conn, 551, "Failed to encode %s key", + key_type_rsa1024); goto err; } key_new_alg = key_type_rsa1024; @@ -2013,8 +2004,8 @@ add_onion_helper_keyarg(const char *arg, int discard_pk, ed25519_secret_key_t *sk = tor_malloc_zero(sizeof(*sk)); if (ed25519_secret_key_generate(sk, 1) < 0) { tor_free(sk); - tor_asprintf(&err_msg, "551 Failed to generate %s key\r\n", - key_type_ed25519_v3); + control_printf_endreply(conn, 551, "Failed to generate %s key", + key_type_ed25519_v3); goto err; } if (!discard_pk) { @@ -2024,8 +2015,8 @@ add_onion_helper_keyarg(const char *arg, int discard_pk, sizeof(sk->seckey), 0) != (len - 1)) { tor_free(sk); tor_free(key_new_blob); - tor_asprintf(&err_msg, "551 Failed to encode %s key\r\n", - key_type_ed25519_v3); + control_printf_endreply(conn, 551, "Failed to encode %s key", + key_type_ed25519_v3); goto err; } key_new_alg = key_type_ed25519_v3; @@ -2033,11 +2024,11 @@ add_onion_helper_keyarg(const char *arg, int discard_pk, decoded_key->v3 = sk; *hs_version = HS_VERSION_THREE; } else { - err_msg = tor_strdup("513 Invalid key type\r\n"); + control_write_endreply(conn, 513, "Invalid key type"); goto err; } } else { - err_msg = tor_strdup("513 Invalid key type\r\n"); + control_write_endreply(conn, 513, "Invalid key type"); goto err; } @@ -2051,11 +2042,6 @@ add_onion_helper_keyarg(const char *arg, int discard_pk, }); smartlist_free(key_args); - if (err_msg_out) { - *err_msg_out = err_msg; - } else { - tor_free(err_msg); - } *key_new_alg_out = key_new_alg; *key_new_blob_out = key_new_blob; @@ -2065,27 +2051,30 @@ add_onion_helper_keyarg(const char *arg, int discard_pk, /** Helper function to handle parsing a ClientAuth argument to the * ADD_ONION command. Return a new rend_authorized_client_t, or NULL * and an optional control protocol error message on failure. The - * caller is responsible for freeing the returned auth_client and err_msg. + * caller is responsible for freeing the returned auth_client. * * If 'created' is specified, it will be set to 1 when a new cookie has * been generated. + * + * Note: conn is only used for writing control replies. For testing + * purposes, it can be NULL if control_write_reply() is appropriately + * mocked. */ STATIC rend_authorized_client_t * -add_onion_helper_clientauth(const char *arg, int *created, char **err_msg) +add_onion_helper_clientauth(const char *arg, int *created, + control_connection_t *conn) { int ok = 0; tor_assert(arg); tor_assert(created); - tor_assert(err_msg); - *err_msg = NULL; smartlist_t *auth_args = smartlist_new(); rend_authorized_client_t *client = tor_malloc_zero(sizeof(rend_authorized_client_t)); smartlist_split_string(auth_args, arg, ":", 0, 0); if (smartlist_len(auth_args) < 1 || smartlist_len(auth_args) > 2) { - *err_msg = tor_strdup("512 Invalid ClientAuth syntax\r\n"); + control_write_endreply(conn, 512, "Invalid ClientAuth syntax"); goto err; } client->client_name = tor_strdup(smartlist_get(auth_args, 0)); @@ -2095,7 +2084,7 @@ add_onion_helper_clientauth(const char *arg, int *created, char **err_msg) client->descriptor_cookie, NULL, &decode_err_msg) < 0) { tor_assert(decode_err_msg); - tor_asprintf(err_msg, "512 %s\r\n", decode_err_msg); + control_write_endreply(conn, 512, decode_err_msg); tor_free(decode_err_msg); goto err; } @@ -2106,7 +2095,7 @@ add_onion_helper_clientauth(const char *arg, int *created, char **err_msg) } if (!rend_valid_client_name(client->client_name)) { - *err_msg = tor_strdup("512 Invalid name in ClientAuth\r\n"); + control_write_endreply(conn, 512, "Invalid name in ClientAuth"); goto err; } diff --git a/src/feature/control/control_cmd.h b/src/feature/control/control_cmd.h index 5c3d1a1cec..4b6d54abe7 100644 --- a/src/feature/control/control_cmd.h +++ b/src/feature/control/control_cmd.h @@ -91,10 +91,11 @@ STATIC int add_onion_helper_keyarg(const char *arg, int discard_pk, const char **key_new_alg_out, char **key_new_blob_out, add_onion_secret_key_t *decoded_key, - int *hs_version, char **err_msg_out); + int *hs_version, + control_connection_t *conn); STATIC rend_authorized_client_t *add_onion_helper_clientauth(const char *arg, - int *created, char **err_msg_out); + int *created, control_connection_t *conn); STATIC control_cmd_args_t *control_cmd_parse_args( const char *command, diff --git a/src/test/test_controller.c b/src/test/test_controller.c index ee48d656bd..b9cbe0a14d 100644 --- a/src/test/test_controller.c +++ b/src/test/test_controller.c @@ -9,6 +9,7 @@ #include "feature/control/control.h" #include "feature/control/control_cmd.h" #include "feature/control/control_getinfo.h" +#include "feature/control/control_proto.h" #include "feature/client/entrynodes.h" #include "feature/hs/hs_common.h" #include "feature/nodelist/networkstatus.h" @@ -201,42 +202,58 @@ static const control_cmd_syntax_t one_arg_kwargs_syntax = { static const parse_test_params_t parse_one_arg_kwargs_params = TESTPARAMS( one_arg_kwargs_syntax, one_arg_kwargs_tests ); +static char *reply_str = NULL; +/* Mock for control_write_reply that copies the string for inspection + * by tests */ +static void +mock_control_write_reply(control_connection_t *conn, int code, int c, + const char *s) +{ + (void)conn; + (void)code; + (void)c; + tor_free(reply_str); + reply_str = tor_strdup(s); +} + static void test_add_onion_helper_keyarg_v3(void *arg) { int ret, hs_version; add_onion_secret_key_t pk; char *key_new_blob = NULL; - char *err_msg = NULL; const char *key_new_alg = NULL; (void) arg; + MOCK(control_write_reply, mock_control_write_reply); memset(&pk, 0, sizeof(pk)); /* Test explicit ED25519-V3 key generation. */ + tor_free(reply_str); ret = add_onion_helper_keyarg("NEW:ED25519-V3", 0, &key_new_alg, &key_new_blob, &pk, &hs_version, - &err_msg); + NULL); tt_int_op(ret, OP_EQ, 0); tt_int_op(hs_version, OP_EQ, HS_VERSION_THREE); tt_assert(pk.v3); tt_str_op(key_new_alg, OP_EQ, "ED25519-V3"); tt_assert(key_new_blob); - tt_ptr_op(err_msg, OP_EQ, NULL); + tt_ptr_op(reply_str, OP_EQ, NULL); tor_free(pk.v3); pk.v3 = NULL; tor_free(key_new_blob); /* Test discarding the private key. */ + tor_free(reply_str); ret = add_onion_helper_keyarg("NEW:ED25519-V3", 1, &key_new_alg, &key_new_blob, &pk, &hs_version, - &err_msg); + NULL); tt_int_op(ret, OP_EQ, 0); tt_int_op(hs_version, OP_EQ, HS_VERSION_THREE); tt_assert(pk.v3); tt_ptr_op(key_new_alg, OP_EQ, NULL); tt_ptr_op(key_new_blob, OP_EQ, NULL); - tt_ptr_op(err_msg, OP_EQ, NULL); + tt_ptr_op(reply_str, OP_EQ, NULL); tor_free(pk.v3); pk.v3 = NULL; tor_free(key_new_blob); @@ -256,9 +273,10 @@ test_add_onion_helper_keyarg_v3(void *arg) tor_asprintf(&key_blob, "ED25519-V3:%s", base64_sk); tt_assert(key_blob); + tor_free(reply_str); ret = add_onion_helper_keyarg(key_blob, 1, &key_new_alg, &key_new_blob, &pk, &hs_version, - &err_msg); + NULL); tor_free(key_blob); tt_int_op(ret, OP_EQ, 0); tt_int_op(hs_version, OP_EQ, HS_VERSION_THREE); @@ -266,7 +284,7 @@ test_add_onion_helper_keyarg_v3(void *arg) tt_mem_op(pk.v3, OP_EQ, hex_sk, 64); tt_ptr_op(key_new_alg, OP_EQ, NULL); tt_ptr_op(key_new_blob, OP_EQ, NULL); - tt_ptr_op(err_msg, OP_EQ, NULL); + tt_ptr_op(reply_str, OP_EQ, NULL); tor_free(pk.v3); pk.v3 = NULL; tor_free(key_new_blob); } @@ -274,7 +292,8 @@ test_add_onion_helper_keyarg_v3(void *arg) done: tor_free(pk.v3); tor_free(key_new_blob); - tor_free(err_msg); + tor_free(reply_str); + UNMOCK(control_write_reply); } static void @@ -285,72 +304,73 @@ test_add_onion_helper_keyarg_v2(void *arg) crypto_pk_t *pk1 = NULL; const char *key_new_alg = NULL; char *key_new_blob = NULL; - char *err_msg = NULL; char *encoded = NULL; char *arg_str = NULL; (void) arg; + MOCK(control_write_reply, mock_control_write_reply); memset(&pk, 0, sizeof(pk)); /* Test explicit RSA1024 key generation. */ + tor_free(reply_str); ret = add_onion_helper_keyarg("NEW:RSA1024", 0, &key_new_alg, &key_new_blob, - &pk, &hs_version, &err_msg); + &pk, &hs_version, NULL); tt_int_op(ret, OP_EQ, 0); tt_int_op(hs_version, OP_EQ, HS_VERSION_TWO); tt_assert(pk.v2); tt_str_op(key_new_alg, OP_EQ, "RSA1024"); tt_assert(key_new_blob); - tt_ptr_op(err_msg, OP_EQ, NULL); + tt_ptr_op(reply_str, OP_EQ, NULL); /* Test "BEST" key generation (Assumes BEST = RSA1024). */ crypto_pk_free(pk.v2); pk.v2 = NULL; tor_free(key_new_blob); ret = add_onion_helper_keyarg("NEW:BEST", 0, &key_new_alg, &key_new_blob, - &pk, &hs_version, &err_msg); + &pk, &hs_version, NULL); tt_int_op(ret, OP_EQ, 0); tt_int_op(hs_version, OP_EQ, HS_VERSION_TWO); tt_assert(pk.v2); tt_str_op(key_new_alg, OP_EQ, "RSA1024"); tt_assert(key_new_blob); - tt_ptr_op(err_msg, OP_EQ, NULL); + tt_ptr_op(reply_str, OP_EQ, NULL); /* Test discarding the private key. */ crypto_pk_free(pk.v2); pk.v2 = NULL; tor_free(key_new_blob); ret = add_onion_helper_keyarg("NEW:BEST", 1, &key_new_alg, &key_new_blob, - &pk, &hs_version, &err_msg); + &pk, &hs_version, NULL); tt_int_op(ret, OP_EQ, 0); tt_int_op(hs_version, OP_EQ, HS_VERSION_TWO); tt_assert(pk.v2); tt_ptr_op(key_new_alg, OP_EQ, NULL); tt_ptr_op(key_new_blob, OP_EQ, NULL); - tt_ptr_op(err_msg, OP_EQ, NULL); + tt_ptr_op(reply_str, OP_EQ, NULL); /* Test generating a invalid key type. */ crypto_pk_free(pk.v2); pk.v2 = NULL; ret = add_onion_helper_keyarg("NEW:RSA512", 0, &key_new_alg, &key_new_blob, - &pk, &hs_version, &err_msg); + &pk, &hs_version, NULL); tt_int_op(ret, OP_EQ, -1); tt_int_op(hs_version, OP_EQ, HS_VERSION_TWO); tt_assert(!pk.v2); tt_ptr_op(key_new_alg, OP_EQ, NULL); tt_ptr_op(key_new_blob, OP_EQ, NULL); - tt_assert(err_msg); + tt_assert(reply_str); /* Test loading a RSA1024 key. */ - tor_free(err_msg); + tor_free(reply_str); pk1 = pk_generate(0); tt_int_op(0, OP_EQ, crypto_pk_base64_encode_private(pk1, &encoded)); tor_asprintf(&arg_str, "RSA1024:%s", encoded); ret = add_onion_helper_keyarg(arg_str, 0, &key_new_alg, &key_new_blob, - &pk, &hs_version, &err_msg); + &pk, &hs_version, NULL); tt_int_op(ret, OP_EQ, 0); tt_int_op(hs_version, OP_EQ, HS_VERSION_TWO); tt_assert(pk.v2); tt_ptr_op(key_new_alg, OP_EQ, NULL); tt_ptr_op(key_new_blob, OP_EQ, NULL); - tt_ptr_op(err_msg, OP_EQ, NULL); + tt_ptr_op(reply_str, OP_EQ, NULL); tt_int_op(crypto_pk_cmp_keys(pk1, pk.v2), OP_EQ, 0); /* Test loading a invalid key type. */ @@ -359,36 +379,37 @@ test_add_onion_helper_keyarg_v2(void *arg) crypto_pk_free(pk.v2); pk.v2 = NULL; tor_asprintf(&arg_str, "RSA512:%s", encoded); ret = add_onion_helper_keyarg(arg_str, 0, &key_new_alg, &key_new_blob, - &pk, &hs_version, &err_msg); + &pk, &hs_version, NULL); tt_int_op(ret, OP_EQ, -1); tt_int_op(hs_version, OP_EQ, HS_VERSION_TWO); tt_assert(!pk.v2); tt_ptr_op(key_new_alg, OP_EQ, NULL); tt_ptr_op(key_new_blob, OP_EQ, NULL); - tt_assert(err_msg); + tt_assert(reply_str); /* Test loading a invalid key. */ tor_free(arg_str); crypto_pk_free(pk.v2); pk.v2 = NULL; - tor_free(err_msg); + tor_free(reply_str); encoded[strlen(encoded)/2] = '\0'; tor_asprintf(&arg_str, "RSA1024:%s", encoded); ret = add_onion_helper_keyarg(arg_str, 0, &key_new_alg, &key_new_blob, - &pk, &hs_version, &err_msg); + &pk, &hs_version, NULL); tt_int_op(ret, OP_EQ, -1); tt_int_op(hs_version, OP_EQ, HS_VERSION_TWO); tt_assert(!pk.v2); tt_ptr_op(key_new_alg, OP_EQ, NULL); tt_ptr_op(key_new_blob, OP_EQ, NULL); - tt_assert(err_msg); + tt_assert(reply_str); done: crypto_pk_free(pk1); crypto_pk_free(pk.v2); tor_free(key_new_blob); - tor_free(err_msg); + tor_free(reply_str); tor_free(encoded); tor_free(arg_str); + UNMOCK(control_write_reply); } static void @@ -542,49 +563,52 @@ static void test_add_onion_helper_clientauth(void *arg) { rend_authorized_client_t *client = NULL; - char *err_msg = NULL; int created = 0; (void)arg; + MOCK(control_write_reply, mock_control_write_reply); /* Test "ClientName" only. */ - client = add_onion_helper_clientauth("alice", &created, &err_msg); + tor_free(reply_str); + client = add_onion_helper_clientauth("alice", &created, NULL); tt_assert(client); tt_assert(created); - tt_ptr_op(err_msg, OP_EQ, NULL); + tt_ptr_op(reply_str, OP_EQ, NULL); rend_authorized_client_free(client); /* Test "ClientName:Blob" */ + tor_free(reply_str); client = add_onion_helper_clientauth("alice:475hGBHPlq7Mc0cRZitK/B", - &created, &err_msg); + &created, NULL); tt_assert(client); tt_assert(!created); - tt_ptr_op(err_msg, OP_EQ, NULL); + tt_ptr_op(reply_str, OP_EQ, NULL); rend_authorized_client_free(client); /* Test invalid client names */ + tor_free(reply_str); client = add_onion_helper_clientauth("no*asterisks*allowed", &created, - &err_msg); + NULL); tt_ptr_op(client, OP_EQ, NULL); - tt_assert(err_msg); - tor_free(err_msg); + tt_assert(reply_str); /* Test invalid auth cookie */ - client = add_onion_helper_clientauth("alice:12345", &created, &err_msg); + tor_free(reply_str); + client = add_onion_helper_clientauth("alice:12345", &created, NULL); tt_ptr_op(client, OP_EQ, NULL); - tt_assert(err_msg); - tor_free(err_msg); + tt_assert(reply_str); /* Test invalid syntax */ + tor_free(reply_str); client = add_onion_helper_clientauth(":475hGBHPlq7Mc0cRZitK/B", &created, - &err_msg); + NULL); tt_ptr_op(client, OP_EQ, NULL); - tt_assert(err_msg); - tor_free(err_msg); + tt_assert(reply_str); done: rend_authorized_client_free(client); - tor_free(err_msg); + tor_free(reply_str); + UNMOCK(control_write_reply); } /* Mocks and data/variables used for GETINFO download status tests */ From 5612eccef877455769f495f088f86f6e268dfd2d Mon Sep 17 00:00:00 2001 From: Taylor Yu Date: Tue, 25 Jun 2019 11:38:36 -0500 Subject: [PATCH 1168/2557] Changes file for ticket 30889 --- changes/ticket30889 | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 changes/ticket30889 diff --git a/changes/ticket30889 b/changes/ticket30889 new file mode 100644 index 0000000000..8582e2bcac --- /dev/null +++ b/changes/ticket30889 @@ -0,0 +1,3 @@ + o Code simplification and refactoring: + - Eliminate some uses of lower-level control reply abstractions, + primarily in the onion_helper functions. Closes ticket 30889. From 2da188667d37757ae999c8ab24ed35b64e08700c Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 18 Jun 2019 20:40:11 -0400 Subject: [PATCH 1169/2557] Add new "struct_var_" functions to manipulate struct fields. These functions exist one level higher than typed_var_t. They describe a type, a name, and an offset within a structure. --- src/lib/conf/conftypes.h | 40 ++++++++ src/lib/confmgt/include.am | 2 + src/lib/confmgt/structvar.c | 200 ++++++++++++++++++++++++++++++++++++ src/lib/confmgt/structvar.h | 51 +++++++++ 4 files changed, 293 insertions(+) create mode 100644 src/lib/confmgt/structvar.c create mode 100644 src/lib/confmgt/structvar.h diff --git a/src/lib/conf/conftypes.h b/src/lib/conf/conftypes.h index e66ab3d5aa..cddfeff2fd 100644 --- a/src/lib/conf/conftypes.h +++ b/src/lib/conf/conftypes.h @@ -63,8 +63,48 @@ typedef enum config_type_t { CONFIG_TYPE_ROUTERSET, /**< A list of router names, addrs, and fps, * parsed into a routerset_t. */ CONFIG_TYPE_OBSOLETE, /**< Obsolete (ignored) option. */ + CONFIG_TYPE_EXTENDED, /**< Extended type; definition will appear in + * pointer. */ } config_type_t; +/* Forward delcaration for var_type_def_t, for extended types. */ +struct var_type_def_t; + +/** Structure to specify a named, typed member within a structure. */ +typedef struct struct_member_t { + /** Name of the field. */ + const char *name; + /** Type of the field, according to the config_type_t enumeration. + * + * This value is CONFIG_TYPE_EXTENDED for any type not listed in + * config_type_t. + **/ + config_type_t type; + /** + * Pointer to a type definition for the type of this field. Overrides + * type if not NULL. + **/ + const struct var_type_def_t *type_def; + /** + * Offset of this field within the structure. Compute this with + * offsetof(structure, fieldname). + **/ + int offset; +} struct_member_t; + +/** + * Structure to describe the location and preferred value of a "magic number" + * field within a structure. + * + * These 'magic numbers' are 32-bit values used to tag objects to make sure + * that they have the correct type. + */ +typedef struct struct_magic_decl_t { + const char *typename; + uint32_t magic_val; + int magic_offset; +} struct_magic_decl_t; + #ifdef TOR_UNIT_TESTS /** * Union used when building in test mode typechecking the members of a type diff --git a/src/lib/confmgt/include.am b/src/lib/confmgt/include.am index a2c7649957..aa5b37fdb5 100644 --- a/src/lib/confmgt/include.am +++ b/src/lib/confmgt/include.am @@ -6,6 +6,7 @@ endif # ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_confmgt_a_SOURCES = \ + src/lib/confmgt/structvar.c \ src/lib/confmgt/type_defs.c \ src/lib/confmgt/typedvar.c \ src/lib/confmgt/unitparse.c @@ -17,6 +18,7 @@ src_lib_libtor_confmgt_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) # ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ + src/lib/confmgt/structvar.h \ src/lib/confmgt/type_defs.h \ src/lib/confmgt/typedvar.h \ src/lib/confmgt/unitparse.h \ diff --git a/src/lib/confmgt/structvar.c b/src/lib/confmgt/structvar.c new file mode 100644 index 0000000000..7ea00fbde9 --- /dev/null +++ b/src/lib/confmgt/structvar.c @@ -0,0 +1,200 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file structvar.c + * @brief Functions to manipulate named and typed elements of + * a structure. + * + * These functions represent a low-level API for accessing a member of a + * structure. They use typedvar.c to work, and they are used in turn by the + * configuration system to examine and set fields in configuration objects + * used by individual modules. + * + * Almost no code should call these directly. + **/ + +#include "orconfig.h" +#include "lib/confmgt/structvar.h" +#include "lib/cc/compat_compiler.h" +#include "lib/conf/conftypes.h" +#include "lib/confmgt/type_defs.h" +#include "lib/confmgt/typedvar.h" +#include "lib/log/util_bug.h" + +#include + +/** + * Set the 'magic number' on object to correspond to decl. + **/ +void +struct_set_magic(void *object, const struct_magic_decl_t *decl) +{ + tor_assert(object); + tor_assert(decl); + uint32_t *ptr = STRUCT_VAR_P(object, decl->magic_offset); + *ptr = decl->magic_val; +} + +/** + * Assert that the 'magic number' on object to corresponds to decl. + **/ +void +struct_check_magic(const void *object, const struct_magic_decl_t *decl) +{ + tor_assert(object); + tor_assert(decl); + + const uint32_t *ptr = STRUCT_VAR_P(object, decl->magic_offset); + tor_assertf(*ptr == decl->magic_val, + "Bad magic number on purported %s object. " + "Expected %"PRIu32"x but got "PRIu32"x.", + decl->magic_val, *ptr); +} + +/** + * Return a mutable pointer to the member of object described + * by member. + **/ +void * +struct_get_mptr(void *object, const struct_member_t *member) +{ + tor_assert(object); + return STRUCT_VAR_P(object, member->offset); +} + +/** + * Return a const pointer to the member of object described + * by member. + **/ +const void * +struct_get_ptr(const void *object, const struct_member_t *member) +{ + tor_assert(object); + return STRUCT_VAR_P(object, member->offset); +} + +/** + * Helper: given a struct_member_t, look up the type definition for its + * variable. + */ +static const var_type_def_t * +get_type_def(const struct_member_t *member) +{ + if (member->type_def) + return member->type_def; + + return lookup_type_def(member->type); +} + +/** + * (As typed_var_assign, but assign a value to the member of object + * defined by member.) + **/ +int +struct_var_assign(void *object, const char *value, char **errmsg, + const struct_member_t *member) +{ + void *p = struct_get_mptr(object, member); + const var_type_def_t *def = get_type_def(member); + + return typed_var_assign_ex(p, value, errmsg, def); +} + +/** + * (As typed_var_free, but free and clear the member of object defined + * by member.) + **/ +void +struct_var_free(void *object, const struct_member_t *member) +{ + void *p = struct_get_mptr(object, member); + const var_type_def_t *def = get_type_def(member); + + typed_var_free_ex(p, def); +} + +/** + * (As typed_var_encode, but encode the member of object defined + * by member.) + **/ +char * +struct_var_encode(const void *object, const struct_member_t *member) +{ + const void *p = struct_get_ptr(object, member); + const var_type_def_t *def = get_type_def(member); + + return typed_var_encode_ex(p, def); +} + +/** + * (As typed_var_copy, but copy from src to dest the member + * defined by member.) + **/ +int +struct_var_copy(void *dest, const void *src, const struct_member_t *member) +{ + void *p_dest = struct_get_mptr(dest, member); + const void *p_src = struct_get_ptr(src, member); + const var_type_def_t *def = get_type_def(member); + + return typed_var_copy_ex(p_dest, p_src, def); +} + +/** + * (As typed_var_eq, but compare the members of a and b + * defined by member.) + **/ +bool +struct_var_eq(const void *a, const void *b, const struct_member_t *member) +{ + const void *p_a = struct_get_ptr(a, member); + const void *p_b = struct_get_ptr(b, member); + const var_type_def_t *def = get_type_def(member); + + return typed_var_eq_ex(p_a, p_b, def); +} + +/** + * (As typed_var_ok, but validate the member of object defined by + * member.) + **/ +bool +struct_var_ok(const void *object, const struct_member_t *member) +{ + const void *p = struct_get_ptr(object, member); + const var_type_def_t *def = get_type_def(member); + + return typed_var_ok_ex(p, def); +} + +/** + * (As typed_var_kvassign, but assign a value to the member of object + * defined by member.) + **/ +int +struct_var_kvassign(void *object, const struct config_line_t *line, + char **errmsg, + const struct_member_t *member) +{ + void *p = struct_get_mptr(object, member); + const var_type_def_t *def = get_type_def(member); + + return typed_var_kvassign_ex(p, line, errmsg, def); +} + +/** + * (As typed_var_kvencode, but encode the value of the member of object + * defined by member.) + **/ +struct config_line_t * +struct_var_kvencode(const void *object, const struct_member_t *member) +{ + const void *p = struct_get_ptr(object, member); + const var_type_def_t *def = get_type_def(member); + + return typed_var_kvencode_ex(member->name, p, def); +} diff --git a/src/lib/confmgt/structvar.h b/src/lib/confmgt/structvar.h new file mode 100644 index 0000000000..894098e509 --- /dev/null +++ b/src/lib/confmgt/structvar.h @@ -0,0 +1,51 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file structvar.h + * @brief Header for lib/confmgt/structvar.c + **/ + +#ifndef TOR_LIB_CONFMGT_STRUCTVAR_H +#define TOR_LIB_CONFMGT_STRUCTVAR_H + +struct struct_magic_decl_t; +struct struct_member_t; +struct config_line_t; + +#include + +void struct_set_magic(void *object, + const struct struct_magic_decl_t *decl); +void struct_check_magic(const void *object, + const struct struct_magic_decl_t *decl); + +void *struct_get_mptr(void *object, + const struct struct_member_t *member); +const void *struct_get_ptr(const void *object, + const struct struct_member_t *member); + +int struct_var_assign(void *object, const char *value, char **errmsg, + const struct struct_member_t *member); +void struct_var_free(void *object, + const struct struct_member_t *member); +char *struct_var_encode(const void *object, + const struct struct_member_t *member); +int struct_var_copy(void *dest, const void *src, + const struct struct_member_t *member); +bool struct_var_eq(const void *a, const void *b, + const struct struct_member_t *member); +bool struct_var_ok(const void *object, + const struct struct_member_t *member); + +int struct_var_kvassign(void *object, const struct config_line_t *line, + char **errmsg, + const struct struct_member_t *member); +struct config_line_t *struct_var_kvencode( + const void *object, + const struct struct_member_t *member); + +#endif /* !defined(TOR_LIB_CONFMGT_STRUCTVAR_H) */ From 3a4d67cf45dc1c94a33479c852a3dd7cbd4ebc95 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 18 Jun 2019 20:40:59 -0400 Subject: [PATCH 1170/2557] Port confparse to use struct_var in place of typed_var. This requires changes to config_var_t, causing corresponding changes throughout its users. --- src/app/config/config.c | 105 +++++++++++++-------- src/app/config/confparse.c | 110 +++++++++------------- src/app/config/confparse.h | 14 ++- src/app/config/statefile.c | 13 ++- src/feature/dirauth/shared_random_state.c | 16 ++-- src/feature/nodelist/routerset.c | 2 +- src/feature/nodelist/routerset.h | 2 +- src/test/test_confparse.c | 25 +++-- 8 files changed, 153 insertions(+), 134 deletions(-) diff --git a/src/app/config/config.c b/src/app/config/config.c index 7908007051..074df07057 100644 --- a/src/app/config/config.c +++ b/src/app/config/config.c @@ -253,23 +253,45 @@ static config_abbrev_t option_abbrevs_[] = { * members with CONF_CHECK_VAR_TYPE. */ DUMMY_TYPECHECK_INSTANCE(or_options_t); -/** An entry for config_vars: "The option name has type +/** An entry for config_vars: "The option varname has type * CONFIG_TYPE_conftype, and corresponds to * or_options_t.member" */ -#define VAR(name,conftype,member,initvalue) \ - { name, CONFIG_TYPE_ ## conftype, offsetof(or_options_t, member), \ +#define VAR(varname,conftype,member,initvalue) \ + { { .name = varname, \ + .type = CONFIG_TYPE_ ## conftype, \ + .offset = offsetof(or_options_t, member), \ + }, \ initvalue CONF_TEST_MEMBERS(or_options_t, conftype, member) } -/** As VAR, but the option name and member name are the same. */ -#define V(member,conftype,initvalue) \ - VAR(#member, conftype, member, initvalue) -/** An entry for config_vars: "The option name is obsolete." */ + #ifdef TOR_UNIT_TESTS -#define OBSOLETE(name) { name, CONFIG_TYPE_OBSOLETE, 0, NULL, {.INT=NULL} } +#define DUMMY_TEST_MEMBERS , {.INT=NULL} #else -#define OBSOLETE(name) { name, CONFIG_TYPE_OBSOLETE, 0, NULL } +#define DUMMY_TEST_MEMBERS #endif +/* As VAR, but uses a type definition in addition to a type enum. */ +#define VAR_D(varname,conftype,member,initvalue) \ + { { .name = varname, \ + .type = CONFIG_TYPE_ ## conftype, \ + .type_def = &conftype ## _type_defn, \ + .offset = offsetof(or_options_t, member), \ + }, \ + initvalue DUMMY_TEST_MEMBERS } + +/** As VAR, but the option name and member name are the same. */ +#define V(member,conftype,initvalue) \ + VAR(#member, conftype, member, initvalue) + +/** As V, but uses a type definition instead of a type enum */ +#define V_D(member,type,initvalue) \ + VAR_D(#member, type, member, initvalue) + +/** An entry for config_vars: "The option varname is obsolete." */ +#define OBSOLETE(varname) \ + { { .name = varname, .type = CONFIG_TYPE_OBSOLETE, }, NULL \ + DUMMY_TEST_MEMBERS } + /** * Macro to declare *Port options. Each one comes in three entries. * For example, most users should use "SocksPort" to configure the @@ -418,17 +440,17 @@ static config_var_t option_vars_[] = { V(TestingEnableCellStatsEvent, BOOL, "0"), OBSOLETE("TestingEnableTbEmptyEvent"), V(EnforceDistinctSubnets, BOOL, "1"), - V(EntryNodes, ROUTERSET, NULL), + V_D(EntryNodes, ROUTERSET, NULL), V(EntryStatistics, BOOL, "0"), V(TestingEstimatedDescriptorPropagationTime, INTERVAL, "10 minutes"), - V(ExcludeNodes, ROUTERSET, NULL), - V(ExcludeExitNodes, ROUTERSET, NULL), + V_D(ExcludeNodes, ROUTERSET, NULL), + V_D(ExcludeExitNodes, ROUTERSET, NULL), OBSOLETE("ExcludeSingleHopRelays"), - V(ExitNodes, ROUTERSET, NULL), + V_D(ExitNodes, ROUTERSET, NULL), /* Researchers need a way to tell their clients to use specific * middles that they also control, to allow safe live-network * experimentation with new padding machines. */ - V(MiddleNodes, ROUTERSET, NULL), + V_D(MiddleNodes, ROUTERSET, NULL), V(ExitPolicy, LINELIST, NULL), V(ExitPolicyRejectPrivate, BOOL, "1"), V(ExitPolicyRejectLocalInterfaces, BOOL, "0"), @@ -507,8 +529,8 @@ static config_var_t option_vars_[] = { V(Socks5ProxyPassword, STRING, NULL), VAR("KeyDirectory", FILENAME, KeyDirectory_option, NULL), V(KeyDirectoryGroupReadable, BOOL, "0"), - VAR("HSLayer2Nodes", ROUTERSET, HSLayer2Nodes, NULL), - VAR("HSLayer3Nodes", ROUTERSET, HSLayer3Nodes, NULL), + VAR_D("HSLayer2Nodes", ROUTERSET, HSLayer2Nodes, NULL), + VAR_D("HSLayer3Nodes", ROUTERSET, HSLayer3Nodes, NULL), V(KeepalivePeriod, INTERVAL, "5 minutes"), V(KeepBindCapabilities, AUTOBOOL, "auto"), VAR("Log", LINELIST, Logs, NULL), @@ -732,11 +754,11 @@ static config_var_t option_vars_[] = { OBSOLETE("TestingDescriptorMaxDownloadTries"), OBSOLETE("TestingMicrodescMaxDownloadTries"), OBSOLETE("TestingCertMaxDownloadTries"), - V(TestingDirAuthVoteExit, ROUTERSET, NULL), + V_D(TestingDirAuthVoteExit, ROUTERSET, NULL), V(TestingDirAuthVoteExitIsStrict, BOOL, "0"), - V(TestingDirAuthVoteGuard, ROUTERSET, NULL), + V_D(TestingDirAuthVoteGuard, ROUTERSET, NULL), V(TestingDirAuthVoteGuardIsStrict, BOOL, "0"), - V(TestingDirAuthVoteHSDir, ROUTERSET, NULL), + V_D(TestingDirAuthVoteHSDir, ROUTERSET, NULL), V(TestingDirAuthVoteHSDirIsStrict, BOOL, "0"), VAR("___UsingTestNetworkDefaults", BOOL, UsingTestNetworkDefaults_, "0"), @@ -949,11 +971,11 @@ set_options(or_options_t *new_val, char **msg) * just starting up then the old_options will be undefined. */ if (old_options && old_options != global_options) { elements = smartlist_new(); - for (i=0; options_format.vars[i].name; ++i) { + for (i=0; options_format.vars[i].member.name; ++i) { const config_var_t *var = &options_format.vars[i]; - const char *var_name = var->name; - if (var->type == CONFIG_TYPE_LINELIST_S || - var->type == CONFIG_TYPE_OBSOLETE) { + const char *var_name = var->member.name; + if (var->member.type == CONFIG_TYPE_LINELIST_S || + var->member.type == CONFIG_TYPE_OBSOLETE) { continue; } if (!config_is_same(&options_format, new_val, old_options, var_name)) { @@ -969,7 +991,7 @@ set_options(or_options_t *new_val, char **msg) tor_free(line); } } else { - smartlist_add_strdup(elements, options_format.vars[i].name); + smartlist_add_strdup(elements, options_format.vars[i].member.name); smartlist_add(elements, NULL); } } @@ -2571,7 +2593,7 @@ const char * option_get_canonical_name(const char *key) { const config_var_t *var = config_find_option(&options_format, key); - return var ? var->name : NULL; + return var ? var->member.name : NULL; } /** Return a canonical list of the options assigned for key. @@ -2653,12 +2675,12 @@ static void list_torrc_options(void) { int i; - for (i = 0; option_vars_[i].name; ++i) { + for (i = 0; option_vars_[i].member.name; ++i) { const config_var_t *var = &option_vars_[i]; - if (var->type == CONFIG_TYPE_OBSOLETE || - var->type == CONFIG_TYPE_LINELIST_V) + if (var->member.type == CONFIG_TYPE_OBSOLETE || + var->member.type == CONFIG_TYPE_LINELIST_V) continue; - printf("%s\n", var->name); + printf("%s\n", var->member.name); } } @@ -5445,18 +5467,18 @@ options_init_from_string(const char *cf_defaults, const char *cf, * let's clean it up. -NM */ /* Change defaults. */ - for (int i = 0; testing_tor_network_defaults[i].name; ++i) { + for (int i = 0; testing_tor_network_defaults[i].member.name; ++i) { const config_var_t *new_var = &testing_tor_network_defaults[i]; config_var_t *old_var = - config_find_option_mutable(&options_format, new_var->name); + config_find_option_mutable(&options_format, new_var->member.name); tor_assert(new_var); tor_assert(old_var); old_var->initvalue = new_var->initvalue; - if ((config_find_deprecation(&options_format, new_var->name))) { + if ((config_find_deprecation(&options_format, new_var->member.name))) { log_warn(LD_GENERAL, "Testing options override the deprecated " "option %s. Is that intentional?", - new_var->name); + new_var->member.name); } } @@ -8168,13 +8190,13 @@ getinfo_helper_config(control_connection_t *conn, if (!strcmp(question, "config/names")) { smartlist_t *sl = smartlist_new(); int i; - for (i = 0; option_vars_[i].name; ++i) { + for (i = 0; option_vars_[i].member.name; ++i) { const config_var_t *var = &option_vars_[i]; const char *type; /* don't tell controller about triple-underscore options */ - if (!strncmp(option_vars_[i].name, "___", 3)) + if (!strncmp(option_vars_[i].member.name, "___", 3)) continue; - switch (var->type) { + switch (var->member.type) { case CONFIG_TYPE_STRING: type = "String"; break; case CONFIG_TYPE_FILENAME: type = "Filename"; break; case CONFIG_TYPE_POSINT: type = "Integer"; break; @@ -8198,11 +8220,12 @@ getinfo_helper_config(control_connection_t *conn, case CONFIG_TYPE_LINELIST_V: type = "Virtual"; break; default: case CONFIG_TYPE_OBSOLETE: + case CONFIG_TYPE_EXTENDED: type = NULL; break; } if (!type) continue; - smartlist_add_asprintf(sl, "%s %s\n",var->name,type); + smartlist_add_asprintf(sl, "%s %s\n",var->member.name,type); } *answer = smartlist_join_strings(sl, "", 0, NULL); SMARTLIST_FOREACH(sl, char *, c, tor_free(c)); @@ -8210,17 +8233,17 @@ getinfo_helper_config(control_connection_t *conn, } else if (!strcmp(question, "config/defaults")) { smartlist_t *sl = smartlist_new(); int dirauth_lines_seen = 0, fallback_lines_seen = 0; - for (int i = 0; option_vars_[i].name; ++i) { + for (int i = 0; option_vars_[i].member.name; ++i) { const config_var_t *var = &option_vars_[i]; if (var->initvalue != NULL) { - if (strcmp(option_vars_[i].name, "DirAuthority") == 0) { + if (strcmp(option_vars_[i].member.name, "DirAuthority") == 0) { /* * Count dirauth lines we have a default for; we'll use the * count later to decide whether to add the defaults manually */ ++dirauth_lines_seen; } - if (strcmp(option_vars_[i].name, "FallbackDir") == 0) { + if (strcmp(option_vars_[i].member.name, "FallbackDir") == 0) { /* * Similarly count fallback lines, so that we can decided later * to add the defaults manually. @@ -8228,7 +8251,7 @@ getinfo_helper_config(control_connection_t *conn, ++fallback_lines_seen; } char *val = esc_for_log(var->initvalue); - smartlist_add_asprintf(sl, "%s %s\n",var->name,val); + smartlist_add_asprintf(sl, "%s %s\n",var->member.name,val); tor_free(val); } } diff --git a/src/app/config/confparse.c b/src/app/config/confparse.c index a02aa26e82..be4341e3f6 100644 --- a/src/app/config/confparse.c +++ b/src/app/config/confparse.c @@ -29,8 +29,7 @@ #include "lib/confmgt/unitparse.h" #include "lib/container/bitarray.h" #include "lib/encoding/confline.h" - -#include "lib/confmgt/typedvar.h" +#include "lib/confmgt/structvar.h" static void config_reset(const config_format_t *fmt, void *options, const config_var_t *var, int use_defaults); @@ -110,17 +109,17 @@ config_find_option_mutable(config_format_t *fmt, const char *key) if (!keylen) return NULL; /* if they say "--" on the command line, it's not an option */ /* First, check for an exact (case-insensitive) match */ - for (i=0; fmt->vars[i].name; ++i) { - if (!strcasecmp(key, fmt->vars[i].name)) { + for (i=0; fmt->vars[i].member.name; ++i) { + if (!strcasecmp(key, fmt->vars[i].member.name)) { return &fmt->vars[i]; } } /* If none, check for an abbreviated match */ - for (i=0; fmt->vars[i].name; ++i) { - if (!strncasecmp(key, fmt->vars[i].name, keylen)) { + for (i=0; fmt->vars[i].member.name; ++i) { + if (!strncasecmp(key, fmt->vars[i].member.name, keylen)) { log_warn(LD_CONFIG, "The abbreviation '%s' is deprecated. " "Please use '%s' instead", - key, fmt->vars[i].name); + key, fmt->vars[i].member.name); return &fmt->vars[i]; } } @@ -144,7 +143,7 @@ static int config_count_options(const config_format_t *fmt) { int i; - for (i=0; fmt->vars[i].name; ++i) + for (i=0; fmt->vars[i].member.name; ++i) ; return i; } @@ -163,23 +162,14 @@ config_assign_value(const config_format_t *fmt, void *options, config_line_t *c, char **msg) { const config_var_t *var; - void *lvalue; CONFIG_CHECK(fmt, options); var = config_find_option(fmt, c->key); tor_assert(var); - tor_assert(!strcmp(c->key, var->name)); + tor_assert(!strcmp(c->key, var->member.name)); - lvalue = STRUCT_VAR_P(options, var->var_offset); - - if (var->type == CONFIG_TYPE_ROUTERSET) { - // XXXX make the backend extensible so that we don't have to - // XXXX special-case this type. - return typed_var_kvassign_ex(lvalue, c, msg, &routerset_type_defn); - } - - return typed_var_kvassign(lvalue, c, msg, var->type); + return struct_var_kvassign(options, c, msg, &var->member); } /** Mark every linelist in options "fragile", so that fresh assignments @@ -191,14 +181,14 @@ config_mark_lists_fragile(const config_format_t *fmt, void *options) tor_assert(fmt); tor_assert(options); - for (i = 0; fmt->vars[i].name; ++i) { + for (i = 0; fmt->vars[i].member.name; ++i) { const config_var_t *var = &fmt->vars[i]; config_line_t *list; - if (var->type != CONFIG_TYPE_LINELIST && - var->type != CONFIG_TYPE_LINELIST_V) + if (var->member.type != CONFIG_TYPE_LINELIST && + var->member.type != CONFIG_TYPE_LINELIST_V) continue; - list = *(config_line_t **)STRUCT_VAR_P(options, var->var_offset); + list = *(config_line_t **)STRUCT_VAR_P(options, var->member.offset); if (list) list->fragile = 1; } @@ -238,7 +228,7 @@ config_assign_line(const config_format_t *fmt, void *options, var = config_find_option(fmt, c->key); if (!var) { if (fmt->extra) { - void *lvalue = STRUCT_VAR_P(options, fmt->extra->var_offset); + void *lvalue = STRUCT_VAR_P(options, fmt->extra->offset); log_info(LD_CONFIG, "Found unrecognized option '%s'; saving it.", c->key); config_line_append((config_line_t**)lvalue, c->key, c->value); @@ -251,22 +241,22 @@ config_assign_line(const config_format_t *fmt, void *options, } /* Put keyword into canonical case. */ - if (strcmp(var->name, c->key)) { + if (strcmp(var->member.name, c->key)) { tor_free(c->key); - c->key = tor_strdup(var->name); + c->key = tor_strdup(var->member.name); } const char *deprecation_msg; if (warn_deprecations && - (deprecation_msg = config_find_deprecation(fmt, var->name))) { - warn_deprecated_option(var->name, deprecation_msg); + (deprecation_msg = config_find_deprecation(fmt, var->member.name))) { + warn_deprecated_option(var->member.name, deprecation_msg); } if (!strlen(c->value)) { /* reset or clear it, then return */ if (!clear_first) { - if ((var->type == CONFIG_TYPE_LINELIST || - var->type == CONFIG_TYPE_LINELIST_S) && + if ((var->member.type == CONFIG_TYPE_LINELIST || + var->member.type == CONFIG_TYPE_LINELIST_S) && c->command != CONFIG_LINE_CLEAR) { /* We got an empty linelist from the torrc or command line. As a special case, call this an error. Warn and ignore. */ @@ -283,14 +273,14 @@ config_assign_line(const config_format_t *fmt, void *options, config_reset(fmt, options, var, use_defaults); // LCOV_EXCL_LINE } - if (options_seen && (var->type != CONFIG_TYPE_LINELIST && - var->type != CONFIG_TYPE_LINELIST_S)) { + if (options_seen && (var->member.type != CONFIG_TYPE_LINELIST && + var->member.type != CONFIG_TYPE_LINELIST_S)) { /* We're tracking which options we've seen, and this option is not * supposed to occur more than once. */ int var_index = (int)(var - fmt->vars); if (bitarray_is_set(options_seen, var_index)) { log_warn(LD_CONFIG, "Option '%s' used more than once; all but the last " - "value will be ignored.", var->name); + "value will be ignored.", var->member.name); } bitarray_set(options_seen, var_index); } @@ -352,7 +342,6 @@ config_get_assigned_option(const config_format_t *fmt, const void *options, const char *key, int escape_val) { const config_var_t *var; - const void *value; config_line_t *result; tor_assert(options && key); @@ -363,15 +352,8 @@ config_get_assigned_option(const config_format_t *fmt, const void *options, log_warn(LD_CONFIG, "Unknown option '%s'. Failing.", key); return NULL; } - value = STRUCT_VAR_P(options, var->var_offset); - if (var->type == CONFIG_TYPE_ROUTERSET) { - // XXXX make the backend extensible so that we don't have to - // XXXX special-case this type. - result = typed_var_kvencode_ex(var->name, value, &routerset_type_defn); - } else { - result = typed_var_kvencode(var->name, value, var->type); - } + result = struct_var_kvencode(options, &var->member); if (escape_val) { config_line_t *line; @@ -497,14 +479,10 @@ static void config_clear(const config_format_t *fmt, void *options, const config_var_t *var) { - void *lvalue = STRUCT_VAR_P(options, var->var_offset); - (void)fmt; /* unused */ - if (var->type == CONFIG_TYPE_ROUTERSET) { - typed_var_free_ex(lvalue, &routerset_type_defn); - return; - } - typed_var_free(lvalue, var->type); + (void)fmt; /* unused */ + + struct_var_free(options, &var->member); } /** Clear the option indexed by var in options. Then if @@ -522,7 +500,7 @@ config_reset(const config_format_t *fmt, void *options, return; /* all done */ if (var->initvalue) { c = tor_malloc_zero(sizeof(config_line_t)); - c->key = tor_strdup(var->name); + c->key = tor_strdup(var->member.name); c->value = tor_strdup(var->initvalue); if (config_assign_value(fmt, options, c, &msg) < 0) { // LCOV_EXCL_START @@ -545,10 +523,11 @@ config_free_(const config_format_t *fmt, void *options) tor_assert(fmt); - for (i=0; fmt->vars[i].name; ++i) + for (i=0; fmt->vars[i].member.name; ++i) config_clear(fmt, options, &(fmt->vars[i])); + if (fmt->extra) { - config_line_t **linep = STRUCT_VAR_P(options, fmt->extra->var_offset); + config_line_t **linep = STRUCT_VAR_P(options, fmt->extra->offset); config_free_lines(*linep); *linep = NULL; } @@ -585,12 +564,12 @@ config_dup(const config_format_t *fmt, const void *old) config_line_t *line; newopts = config_new(fmt); - for (i=0; fmt->vars[i].name; ++i) { - if (fmt->vars[i].type == CONFIG_TYPE_LINELIST_S) + for (i=0; fmt->vars[i].member.name; ++i) { + if (fmt->vars[i].member.type == CONFIG_TYPE_LINELIST_S) continue; - if (fmt->vars[i].type == CONFIG_TYPE_OBSOLETE) + if (fmt->vars[i].member.type == CONFIG_TYPE_OBSOLETE) continue; - line = config_get_assigned_option(fmt, old, fmt->vars[i].name, 0); + line = config_get_assigned_option(fmt, old, fmt->vars[i].member.name, 0); if (line) { char *msg = NULL; if (config_assign(fmt, newopts, line, 0, &msg) < 0) { @@ -615,7 +594,7 @@ config_init(const config_format_t *fmt, void *options) const config_var_t *var; CONFIG_CHECK(fmt, options); - for (i=0; fmt->vars[i].name; ++i) { + for (i=0; fmt->vars[i].member.name; ++i) { var = &fmt->vars[i]; if (!var->initvalue) continue; /* defaults to NULL or 0 */ @@ -657,22 +636,23 @@ config_dump(const config_format_t *fmt, const void *default_options, } elements = smartlist_new(); - for (i=0; fmt->vars[i].name; ++i) { + for (i=0; fmt->vars[i].member.name; ++i) { int comment_option = 0; - if (fmt->vars[i].type == CONFIG_TYPE_OBSOLETE || - fmt->vars[i].type == CONFIG_TYPE_LINELIST_S) + if (fmt->vars[i].member.type == CONFIG_TYPE_OBSOLETE || + fmt->vars[i].member.type == CONFIG_TYPE_LINELIST_S) continue; /* Don't save 'hidden' control variables. */ - if (!strcmpstart(fmt->vars[i].name, "__")) + if (!strcmpstart(fmt->vars[i].member.name, "__")) continue; - if (minimal && config_is_same(fmt, options, defaults, fmt->vars[i].name)) + if (minimal && config_is_same(fmt, options, defaults, + fmt->vars[i].member.name)) continue; else if (comment_defaults && - config_is_same(fmt, options, defaults, fmt->vars[i].name)) + config_is_same(fmt, options, defaults, fmt->vars[i].member.name)) comment_option = 1; line = assigned = - config_get_assigned_option(fmt, options, fmt->vars[i].name, 1); + config_get_assigned_option(fmt, options, fmt->vars[i].member.name, 1); for (; line; line = line->next) { if (!strcmpstart(line->key, "__")) { @@ -688,7 +668,7 @@ config_dump(const config_format_t *fmt, const void *default_options, } if (fmt->extra) { - line = *(config_line_t**)STRUCT_VAR_P(options, fmt->extra->var_offset); + line = *(config_line_t**)STRUCT_VAR_P(options, fmt->extra->offset); for (; line; line = line->next) { smartlist_add_asprintf(elements, "%s %s\n", line->key, line->value); } diff --git a/src/app/config/confparse.h b/src/app/config/confparse.h index bd06a4a0d0..5897085e63 100644 --- a/src/app/config/confparse.h +++ b/src/app/config/confparse.h @@ -34,10 +34,8 @@ typedef struct config_deprecation_t { /** A variable allowed in the configuration file or on the command line. */ typedef struct config_var_t { - const char *name; /**< The full keyword (case insensitive). */ - config_type_t type; /**< How to interpret the type and turn it into a - * value. */ - off_t var_offset; /**< Offset of the corresponding member of or_options_t. */ + struct_member_t member; /** A struct member corresponding to this + * variable. */ const char *initvalue; /**< String (or null) describing initial value. */ #ifdef TOR_UNIT_TESTS @@ -74,12 +72,12 @@ typedef struct config_var_t { #define CONF_TEST_MEMBERS(tp, conftype, member) \ , CONF_CHECK_VAR_TYPE(tp, conftype, member) #define END_OF_CONFIG_VARS \ - { NULL, CONFIG_TYPE_OBSOLETE, 0, NULL, { .INT=NULL } } + { { .name = NULL }, NULL, { .INT=NULL } } #define DUMMY_TYPECHECK_INSTANCE(tp) \ static tp tp ## _dummy #else /* !(defined(TOR_UNIT_TESTS)) */ #define CONF_TEST_MEMBERS(tp, conftype, member) -#define END_OF_CONFIG_VARS { NULL, CONFIG_TYPE_OBSOLETE, 0, NULL } +#define END_OF_CONFIG_VARS { { .name = NULL, }, NULL } /* Repeatedly declarable incomplete struct to absorb redundant semicolons */ #define DUMMY_TYPECHECK_INSTANCE(tp) \ struct tor_semicolon_eater @@ -108,9 +106,9 @@ typedef struct config_format_t { * values, and where we stick them in the structure. */ validate_fn_t validate_fn; /**< Function to validate config. */ free_cfg_fn_t free_fn; /**< Function to free the configuration. */ - /** If present, extra is a LINELIST variable for unrecognized + /** If present, extra denotes a LINELIST variable for unrecognized * lines. Otherwise, unrecognized lines are an error. */ - config_var_t *extra; + struct_member_t *extra; } config_format_t; /** Macro: assert that cfg has the right magic field for format diff --git a/src/app/config/statefile.c b/src/app/config/statefile.c index c6c5ec14f5..358b02f602 100644 --- a/src/app/config/statefile.c +++ b/src/app/config/statefile.c @@ -71,8 +71,10 @@ static config_abbrev_t state_abbrevs_[] = { DUMMY_TYPECHECK_INSTANCE(or_state_t); /*XXXX these next two are duplicates or near-duplicates from config.c */ -#define VAR(name,conftype,member,initvalue) \ - { name, CONFIG_TYPE_ ## conftype, offsetof(or_state_t, member), \ +#define VAR(varname,conftype,member,initvalue) \ + { { .name = varname, \ + .type = CONFIG_TYPE_ ## conftype, \ + .offset = offsetof(or_state_t, member), }, \ initvalue CONF_TEST_MEMBERS(or_state_t, conftype, member) } /** As VAR, but the option name and member name are the same. */ #define V(member,conftype,initvalue) \ @@ -155,9 +157,10 @@ static void or_state_free_cb(void *state); /** "Extra" variable in the state that receives lines we can't parse. This * lets us preserve options from versions of Tor newer than us. */ -static config_var_t state_extra_var = { - "__extra", CONFIG_TYPE_LINELIST, offsetof(or_state_t, ExtraLines), NULL - CONF_TEST_MEMBERS(or_state_t, LINELIST, ExtraLines) +static struct_member_t state_extra_var = { + .name = "__extra", + .type = CONFIG_TYPE_LINELIST, + .offset = offsetof(or_state_t, ExtraLines), }; /** Configuration format for or_state_t. */ diff --git a/src/feature/dirauth/shared_random_state.c b/src/feature/dirauth/shared_random_state.c index b2c7acba1a..cf4a654325 100644 --- a/src/feature/dirauth/shared_random_state.c +++ b/src/feature/dirauth/shared_random_state.c @@ -52,9 +52,11 @@ static const char dstate_cur_srv_key[] = "SharedRandCurrentValue"; DUMMY_TYPECHECK_INSTANCE(sr_disk_state_t); /* These next two are duplicates or near-duplicates from config.c */ -#define VAR(name, conftype, member, initvalue) \ - { name, CONFIG_TYPE_ ## conftype, offsetof(sr_disk_state_t, member), \ - initvalue CONF_TEST_MEMBERS(sr_disk_state_t, conftype, member) } +#define VAR(varname, conftype, member, initvalue) \ + { { .name = varname, \ + .type = CONFIG_TYPE_ ## conftype, \ + .offset = offsetof(sr_disk_state_t, member), }, \ + initvalue CONF_TEST_MEMBERS(sr_disk_state_t, conftype, member) } /* As VAR, but the option name and member name are the same. */ #define V(member, conftype, initvalue) \ VAR(#member, conftype, member, initvalue) @@ -83,10 +85,10 @@ static config_var_t state_vars[] = { /* "Extra" variable in the state that receives lines we can't parse. This * lets us preserve options from versions of Tor newer than us. */ -static config_var_t state_extra_var = { - "__extra", CONFIG_TYPE_LINELIST, - offsetof(sr_disk_state_t, ExtraLines), NULL - CONF_TEST_MEMBERS(sr_disk_state_t, LINELIST, ExtraLines) +static struct_member_t state_extra_var = { + .name = "__extra", + .type = CONFIG_TYPE_LINELIST, + .offset = offsetof(sr_disk_state_t, ExtraLines), }; /* Configuration format of sr_disk_state_t. */ diff --git a/src/feature/nodelist/routerset.c b/src/feature/nodelist/routerset.c index ad42e8e101..76777847ef 100644 --- a/src/feature/nodelist/routerset.c +++ b/src/feature/nodelist/routerset.c @@ -519,7 +519,7 @@ static const var_type_fns_t routerset_type_fns = { .copy = routerset_copy }; -const var_type_def_t routerset_type_defn = { +const var_type_def_t ROUTERSET_type_defn = { .name = "RouterList", .fns = &routerset_type_fns }; diff --git a/src/feature/nodelist/routerset.h b/src/feature/nodelist/routerset.h index 9d184c9852..f3bf4a1f7c 100644 --- a/src/feature/nodelist/routerset.h +++ b/src/feature/nodelist/routerset.h @@ -45,7 +45,7 @@ void routerset_free_(routerset_t *routerset); int routerset_len(const routerset_t *set); struct var_type_def_t; -extern const struct var_type_def_t routerset_type_defn; +extern const struct var_type_def_t ROUTERSET_type_defn; #ifdef ROUTERSET_PRIVATE #include "lib/container/bitarray.h" diff --git a/src/test/test_confparse.c b/src/test/test_confparse.c index dde61b1c81..27696a537a 100644 --- a/src/test/test_confparse.c +++ b/src/test/test_confparse.c @@ -48,15 +48,17 @@ typedef struct test_struct_t { static test_struct_t test_struct_t_dummy; -#define VAR(name,conftype,member,initvalue) \ - { name, CONFIG_TYPE_##conftype, offsetof(test_struct_t, member), \ +#define VAR(varname,conftype,member,initvalue) \ + { { .name = varname, \ + .type = CONFIG_TYPE_##conftype, \ + .offset = offsetof(test_struct_t, member), }, \ initvalue CONF_TEST_MEMBERS(test_struct_t, conftype, member) } #define V(name,conftype,initvalue) \ VAR( #name, conftype, name, initvalue ) -#define OBSOLETE(name) \ - { name, CONFIG_TYPE_OBSOLETE, 0, NULL, {.INT=NULL} } +#define OBSOLETE(varname) \ + { { .name=varname, .type=CONFIG_TYPE_OBSOLETE }, NULL, {.INT=NULL} } static config_var_t test_vars[] = { V(s, STRING, "hello"), @@ -79,7 +81,14 @@ static config_var_t test_vars[] = { VAR("LineTypeA", LINELIST_S, mixed_lines, NULL), VAR("LineTypeB", LINELIST_S, mixed_lines, NULL), OBSOLETE("obsolete"), - V(routerset, ROUTERSET, NULL), + { + { .name = "routerset", + .type = CONFIG_TYPE_ROUTERSET, + .type_def = &ROUTERSET_type_defn, + .offset = offsetof(test_struct_t, routerset), + }, + NULL, {.INT=NULL} + }, VAR("__HiddenInt", POSINT, hidden_int, "0"), VAR("MixedHiddenLines", LINELIST_V, mixed_hidden_lines, NULL), VAR("__HiddenLineA", LINELIST_S, mixed_hidden_lines, NULL), @@ -757,7 +766,11 @@ test_confparse_get_assigned(void *arg) /* Another variant, which accepts and stores unrecognized lines.*/ #define ETEST_MAGIC 13371337 -static config_var_t extra = VAR("__extra", LINELIST, extra_lines, NULL); +static struct_member_t extra = { + .name = "__extra", + .type = CONFIG_TYPE_LINELIST, + .offset = offsetof(test_struct_t, extra_lines), +}; static config_format_t etest_fmt = { sizeof(test_struct_t), From 4d101b39d74fb467d7fb4ad8ddb27e07c3074a69 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 20 Jun 2019 15:21:21 -0400 Subject: [PATCH 1171/2557] Move config_var_t info conftypes.h --- src/app/config/confparse.h | 13 ------------- src/lib/conf/conftypes.h | 13 +++++++++++++ 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/app/config/confparse.h b/src/app/config/confparse.h index b91ea1c13d..f89ff3c214 100644 --- a/src/app/config/confparse.h +++ b/src/app/config/confparse.h @@ -32,19 +32,6 @@ typedef struct config_deprecation_t { * you can abbreviate toks as tok". */ #define PLURAL(tok) { #tok, #tok "s", 0, 0 } -/** A variable allowed in the configuration file or on the command line. */ -typedef struct config_var_t { - struct_member_t member; /** A struct member corresponding to this - * variable. */ - const char *initvalue; /**< String (or null) describing initial value. */ - -#ifdef TOR_UNIT_TESTS - /** Used for compiler-magic to typecheck the corresponding field in the - * corresponding struct. Only used in unit test mode, at compile-time. */ - confparse_dummy_values_t var_ptr_dummy; -#endif -} config_var_t; - /* Macros to define extra members inside config_var_t fields, and at the * end of a list of them. */ diff --git a/src/lib/conf/conftypes.h b/src/lib/conf/conftypes.h index cddfeff2fd..5f13ec3de2 100644 --- a/src/lib/conf/conftypes.h +++ b/src/lib/conf/conftypes.h @@ -136,4 +136,17 @@ typedef union { } confparse_dummy_values_t; #endif /* defined(TOR_UNIT_TESTS) */ +/** A variable allowed in the configuration file or on the command line. */ +typedef struct config_var_t { + struct_member_t member; /** A struct member corresponding to this + * variable. */ + const char *initvalue; /**< String (or null) describing initial value. */ + +#ifdef TOR_UNIT_TESTS + /** Used for compiler-magic to typecheck the corresponding field in the + * corresponding struct. Only used in unit test mode, at compile-time. */ + confparse_dummy_values_t var_ptr_dummy; +#endif +} config_var_t; + #endif /* !defined(TOR_SRC_LIB_CONF_CONFTYPES_H) */ From 59317c8a238f49ad298d4f51f42a0f7b16c9cc3c Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 19 Jun 2019 08:34:20 -0400 Subject: [PATCH 1172/2557] Use struct_magic_decl to verify magic numbers in config objects --- src/app/config/config.c | 7 +++++-- src/app/config/confparse.c | 2 +- src/app/config/confparse.h | 9 +++------ src/app/config/statefile.c | 7 +++++-- src/feature/dirauth/shared_random_state.c | 7 +++++-- src/test/test_confparse.c | 14 ++++++++++---- 6 files changed, 29 insertions(+), 17 deletions(-) diff --git a/src/app/config/config.c b/src/app/config/config.c index 074df07057..37cbe6b2ef 100644 --- a/src/app/config/config.c +++ b/src/app/config/config.c @@ -877,8 +877,11 @@ static void set_protocol_warning_severity_level(int warning_severity); /** Configuration format for or_options_t. */ STATIC config_format_t options_format = { sizeof(or_options_t), - OR_OPTIONS_MAGIC, - offsetof(or_options_t, magic_), + { + "or_options_t", + OR_OPTIONS_MAGIC, + offsetof(or_options_t, magic_), + }, option_abbrevs_, option_deprecation_notes_, option_vars_, diff --git a/src/app/config/confparse.c b/src/app/config/confparse.c index be4341e3f6..752d16c844 100644 --- a/src/app/config/confparse.c +++ b/src/app/config/confparse.c @@ -39,7 +39,7 @@ void * config_new(const config_format_t *fmt) { void *opts = tor_malloc_zero(fmt->size); - *(uint32_t*)STRUCT_VAR_P(opts, fmt->magic_offset) = fmt->magic; + struct_set_magic(opts, &fmt->magic); CONFIG_CHECK(fmt, opts); return opts; } diff --git a/src/app/config/confparse.h b/src/app/config/confparse.h index 5897085e63..4ef4e708f3 100644 --- a/src/app/config/confparse.h +++ b/src/app/config/confparse.h @@ -96,9 +96,7 @@ typedef void (*free_cfg_fn_t)(void*); * configuration or storage format. */ typedef struct config_format_t { size_t size; /**< Size of the struct that everything gets parsed into. */ - uint32_t magic; /**< Required 'magic value' to make sure we have a struct - * of the right type. */ - off_t magic_offset; /**< Offset of the magic value within the struct. */ + struct_magic_decl_t magic; /**< Magic number info for this struct. */ config_abbrev_t *abbrevs; /**< List of abbreviations that we expand when * parsing this format. */ const config_deprecation_t *deprecations; /** List of deprecated options */ @@ -114,9 +112,8 @@ typedef struct config_format_t { /** Macro: assert that cfg has the right magic field for format * fmt. */ #define CONFIG_CHECK(fmt, cfg) STMT_BEGIN \ - tor_assert(fmt && cfg); \ - tor_assert((fmt)->magic == \ - *(uint32_t*)STRUCT_VAR_P(cfg,fmt->magic_offset)); \ + tor_assert(fmt); \ + struct_check_magic((cfg), &fmt->magic); \ STMT_END #define CAL_USE_DEFAULTS (1u<<0) diff --git a/src/app/config/statefile.c b/src/app/config/statefile.c index 358b02f602..331592c3a6 100644 --- a/src/app/config/statefile.c +++ b/src/app/config/statefile.c @@ -166,8 +166,11 @@ static struct_member_t state_extra_var = { /** Configuration format for or_state_t. */ static const config_format_t state_format = { sizeof(or_state_t), - OR_STATE_MAGIC, - offsetof(or_state_t, magic_), + { + "or_state_t", + OR_STATE_MAGIC, + offsetof(or_state_t, magic_), + }, state_abbrevs_, NULL, state_vars_, diff --git a/src/feature/dirauth/shared_random_state.c b/src/feature/dirauth/shared_random_state.c index cf4a654325..da4187b38a 100644 --- a/src/feature/dirauth/shared_random_state.c +++ b/src/feature/dirauth/shared_random_state.c @@ -94,8 +94,11 @@ static struct_member_t state_extra_var = { /* Configuration format of sr_disk_state_t. */ static const config_format_t state_format = { sizeof(sr_disk_state_t), - SR_DISK_STATE_MAGIC, - offsetof(sr_disk_state_t, magic_), + { + "sr_disk_state_t", + SR_DISK_STATE_MAGIC, + offsetof(sr_disk_state_t, magic_), + }, NULL, NULL, state_vars, diff --git a/src/test/test_confparse.c b/src/test/test_confparse.c index 27696a537a..9e626356d3 100644 --- a/src/test/test_confparse.c +++ b/src/test/test_confparse.c @@ -131,8 +131,11 @@ static void test_free_cb(void *options); static config_format_t test_fmt = { sizeof(test_struct_t), - TEST_MAGIC, - offsetof(test_struct_t, magic), + { + "test_struct_t", + TEST_MAGIC, + offsetof(test_struct_t, magic), + }, test_abbrevs, test_deprecation_notes, test_vars, @@ -774,8 +777,11 @@ static struct_member_t extra = { static config_format_t etest_fmt = { sizeof(test_struct_t), - ETEST_MAGIC, - offsetof(test_struct_t, magic), + { + "test_struct_t (with extra lines)", + ETEST_MAGIC, + offsetof(test_struct_t, magic), + }, test_abbrevs, test_deprecation_notes, test_vars, From c553750e32d1bf669a3e8308fa44319954a627ca Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 20 Jun 2019 15:55:59 -0400 Subject: [PATCH 1173/2557] Move responsibility for config var macros The testing-only parts now live in a conftesting.h; the shared parts of the macros live in confmacros.h --- src/app/config/config.c | 24 +------ src/app/config/confparse.h | 39 +--------- src/app/config/statefile.c | 9 +-- src/feature/dirauth/shared_random_state.c | 12 ++-- src/lib/conf/.may_include | 1 + src/lib/conf/confmacros.h | 61 ++++++++++++++++ src/lib/conf/conftesting.h | 86 +++++++++++++++++++++++ src/lib/conf/conftypes.h | 37 +--------- src/lib/conf/include.am | 4 +- src/test/test_confparse.c | 12 ++-- 10 files changed, 168 insertions(+), 117 deletions(-) create mode 100644 src/lib/conf/confmacros.h create mode 100644 src/lib/conf/conftesting.h diff --git a/src/app/config/config.c b/src/app/config/config.c index 8da1e2acdc..c15236b0e9 100644 --- a/src/app/config/config.c +++ b/src/app/config/config.c @@ -259,28 +259,12 @@ DUMMY_TYPECHECK_INSTANCE(or_options_t); * or_options_t.member" */ #define VAR(varname,conftype,member,initvalue) \ - { { .name = varname, \ - .type = CONFIG_TYPE_ ## conftype, \ - .offset = offsetof(or_options_t, member), \ - }, \ - initvalue CONF_TEST_MEMBERS(or_options_t, conftype, member) } - -#ifdef TOR_UNIT_TESTS -#define DUMMY_TEST_MEMBERS , {.INT=NULL} -#else -#define DUMMY_TEST_MEMBERS -#endif + CONFIG_VAR_ETYPE(or_options_t, varname, conftype, member, initvalue) /* As VAR, but uses a type definition in addition to a type enum. */ #define VAR_D(varname,conftype,member,initvalue) \ - { { .name = varname, \ - .type = CONFIG_TYPE_ ## conftype, \ - .type_def = &conftype ## _type_defn, \ - .offset = offsetof(or_options_t, member), \ - }, \ - initvalue DUMMY_TEST_MEMBERS } + CONFIG_VAR_DEFN(or_options_t, varname, conftype, member, initvalue) -/** As VAR, but the option name and member name are the same. */ #define V(member,conftype,initvalue) \ VAR(#member, conftype, member, initvalue) @@ -289,9 +273,7 @@ DUMMY_TYPECHECK_INSTANCE(or_options_t); VAR_D(#member, type, member, initvalue) /** An entry for config_vars: "The option varname is obsolete." */ -#define OBSOLETE(varname) \ - { { .name = varname, .type = CONFIG_TYPE_OBSOLETE, }, NULL \ - DUMMY_TEST_MEMBERS } +#define OBSOLETE(varname) CONFIG_VAR_OBSOLETE(varname) /** * Macro to declare *Port options. Each one comes in three entries. diff --git a/src/app/config/confparse.h b/src/app/config/confparse.h index f89ff3c214..6d63ba3e0e 100644 --- a/src/app/config/confparse.h +++ b/src/app/config/confparse.h @@ -14,6 +14,7 @@ #define TOR_CONFPARSE_H #include "lib/conf/conftypes.h" +#include "lib/conf/confmacros.h" /** An abbreviation for a configuration option allowed on the command line. */ typedef struct config_abbrev_t { @@ -32,44 +33,6 @@ typedef struct config_deprecation_t { * you can abbreviate toks as tok". */ #define PLURAL(tok) { #tok, #tok "s", 0, 0 } -/* Macros to define extra members inside config_var_t fields, and at the - * end of a list of them. - */ -#ifdef TOR_UNIT_TESTS -/* This is a somewhat magic type-checking macro for users of confparse.c. - * It initializes a union member "confparse_dummy_values_t.conftype" with - * the address of a static member "tp_dummy.member". This - * will give a compiler warning unless the member field is of the correct - * type. - * - * (This warning is mandatory, because a type mismatch here violates the type - * compatibility constraint for simple assignment, and requires a diagnostic, - * according to the C spec.) - * - * For example, suppose you say: - * "CONF_CHECK_VAR_TYPE(or_options_t, STRING, Address)". - * Then this macro will evaluate to: - * { .STRING = &or_options_t_dummy.Address } - * And since confparse_dummy_values_t.STRING has type "char **", that - * expression will create a warning unless or_options_t.Address also - * has type "char *". - */ -#define CONF_CHECK_VAR_TYPE(tp, conftype, member) \ - { . conftype = &tp ## _dummy . member } -#define CONF_TEST_MEMBERS(tp, conftype, member) \ - , CONF_CHECK_VAR_TYPE(tp, conftype, member) -#define END_OF_CONFIG_VARS \ - { { .name = NULL }, NULL, { .INT=NULL } } -#define DUMMY_TYPECHECK_INSTANCE(tp) \ - static tp tp ## _dummy -#else /* !(defined(TOR_UNIT_TESTS)) */ -#define CONF_TEST_MEMBERS(tp, conftype, member) -#define END_OF_CONFIG_VARS { { .name = NULL, }, NULL } -/* Repeatedly declarable incomplete struct to absorb redundant semicolons */ -#define DUMMY_TYPECHECK_INSTANCE(tp) \ - struct tor_semicolon_eater -#endif /* defined(TOR_UNIT_TESTS) */ - /** Type of a callback to validate whether a given configuration is * well-formed and consistent. See options_trial_assign() for documentation * of arguments. */ diff --git a/src/app/config/statefile.c b/src/app/config/statefile.c index 331592c3a6..e0584c62af 100644 --- a/src/app/config/statefile.c +++ b/src/app/config/statefile.c @@ -70,14 +70,9 @@ static config_abbrev_t state_abbrevs_[] = { * members with CONF_CHECK_VAR_TYPE. */ DUMMY_TYPECHECK_INSTANCE(or_state_t); -/*XXXX these next two are duplicates or near-duplicates from config.c */ #define VAR(varname,conftype,member,initvalue) \ - { { .name = varname, \ - .type = CONFIG_TYPE_ ## conftype, \ - .offset = offsetof(or_state_t, member), }, \ - initvalue CONF_TEST_MEMBERS(or_state_t, conftype, member) } -/** As VAR, but the option name and member name are the same. */ -#define V(member,conftype,initvalue) \ + CONFIG_VAR_ETYPE(or_state_t, varname, conftype, member, initvalue) +#define V(member,conftype,initvalue) \ VAR(#member, conftype, member, initvalue) /** Array of "state" variables saved to the ~/.tor/state file. */ diff --git a/src/feature/dirauth/shared_random_state.c b/src/feature/dirauth/shared_random_state.c index da4187b38a..3cdb223d25 100644 --- a/src/feature/dirauth/shared_random_state.c +++ b/src/feature/dirauth/shared_random_state.c @@ -51,15 +51,11 @@ static const char dstate_cur_srv_key[] = "SharedRandCurrentValue"; * members with CONF_CHECK_VAR_TYPE. */ DUMMY_TYPECHECK_INSTANCE(sr_disk_state_t); -/* These next two are duplicates or near-duplicates from config.c */ -#define VAR(varname, conftype, member, initvalue) \ - { { .name = varname, \ - .type = CONFIG_TYPE_ ## conftype, \ - .offset = offsetof(sr_disk_state_t, member), }, \ - initvalue CONF_TEST_MEMBERS(sr_disk_state_t, conftype, member) } -/* As VAR, but the option name and member name are the same. */ -#define V(member, conftype, initvalue) \ +#define VAR(varname,conftype,member,initvalue) \ + CONFIG_VAR_ETYPE(sr_disk_state_t, varname, conftype, member, initvalue) +#define V(member,conftype,initvalue) \ VAR(#member, conftype, member, initvalue) + /* Our persistent state magic number. */ #define SR_DISK_STATE_MAGIC 0x98AB1254 diff --git a/src/lib/conf/.may_include b/src/lib/conf/.may_include index 4285c3dcb8..629e2f897d 100644 --- a/src/lib/conf/.may_include +++ b/src/lib/conf/.may_include @@ -1,2 +1,3 @@ orconfig.h lib/cc/*.h +lib/conf/*.h diff --git a/src/lib/conf/confmacros.h b/src/lib/conf/confmacros.h new file mode 100644 index 0000000000..29040e1f55 --- /dev/null +++ b/src/lib/conf/confmacros.h @@ -0,0 +1,61 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file confmacros.h + * @brief Macro definitions for declaring configuration variables + **/ + +#ifndef TOR_LIB_CONF_CONFMACROS_H +#define TOR_LIB_CONF_CONFMACROS_H + +#include "orconfig.h" +#include "lib/conf/conftesting.h" + +/** + * Used to indicate the end of an array of configuration variables. + **/ +#define END_OF_CONFIG_VARS \ + { { .name = NULL }, NULL DUMMY_CONF_TEST_MEMBERS } + +/** + * Declare a config_var_t as a member named membername of the structure + * structtype, whose user-visible name is varname, whose + * type corresponds to the config_type_t member CONFIG_TYPE_vartype, + * and whose initial value is intval. + * + * Most modules that use this macro should wrap it in a local macro that + * sets structtype to the local configuration type. + **/ +#define CONFIG_VAR_ETYPE(structtype, varname, vartype, membername, initval) \ + { .member = \ + { .name = varname, \ + .type = CONFIG_TYPE_ ## vartype, \ + .offset = offsetof(structtype, membername), \ + }, \ + .initvalue = initval \ + CONF_TEST_MEMBERS(structtype, vartype, membername) \ + } + +/** + * As CONFIG_VAR_XTYPE, but declares a value using an extension type whose + * type definition is vartype_type_defn. + **/ +#define CONFIG_VAR_DEFN(structtype, varname, vartype, membername, initval) \ + { .member = \ + { .name = varname, \ + .type = CONFIG_TYPE_EXTENDED, \ + .type_def = &vartype ## _type_defn, \ + .offset = offsetof(structtype, membername), \ + }, \ + .initvalue = initval \ + CONF_TEST_MEMBERS(structtype, vartype, membername) \ + } + +#define CONFIG_VAR_OBSOLETE(varname) \ + { .member = { .name = varname, .type = CONFIG_TYPE_OBSOLETE } } + +#endif /* !defined(TOR_LIB_CONF_CONFMACROS_H) */ diff --git a/src/lib/conf/conftesting.h b/src/lib/conf/conftesting.h new file mode 100644 index 0000000000..f4aca442a2 --- /dev/null +++ b/src/lib/conf/conftesting.h @@ -0,0 +1,86 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file conftesting.h + * @brief Macro and type declarations for testing + **/ + +#ifndef TOR_LIB_CONF_CONFTESTING_H +#define TOR_LIB_CONF_CONFTESTING_H + +#ifdef TOR_UNIT_TESTS +/** + * Union used when building in test mode typechecking the members of a type + * used with confparse.c. See CONF_CHECK_VAR_TYPE for a description of how + * it is used. */ +typedef union { + char **STRING; + char **FILENAME; + int *POSINT; /* yes, this is really an int, and not an unsigned int. For + * historical reasons, many configuration values are restricted + * to the range [0,INT_MAX], and stored in signed ints. + */ + uint64_t *UINT64; + int *INT; + int *INTERVAL; + int *MSEC_INTERVAL; + uint64_t *MEMUNIT; + double *DOUBLE; + int *BOOL; + int *AUTOBOOL; + time_t *ISOTIME; + struct smartlist_t **CSV; + int *CSV_INTERVAL; + struct config_line_t **LINELIST; + struct config_line_t **LINELIST_S; + struct config_line_t **LINELIST_V; + // XXXX this doesn't belong at this level of abstraction. + struct routerset_t **ROUTERSET; +} confparse_dummy_values_t; +#endif /* defined(TOR_UNIT_TESTS) */ + +/* Macros to define extra members inside config_var_t fields, and at the + * end of a list of them. + */ +#ifdef TOR_UNIT_TESTS +/* This is a somewhat magic type-checking macro for users of confparse.c. + * It initializes a union member "confparse_dummy_values_t.conftype" with + * the address of a static member "tp_dummy.member". This + * will give a compiler warning unless the member field is of the correct + * type. + * + * (This warning is mandatory, because a type mismatch here violates the type + * compatibility constraint for simple assignment, and requires a diagnostic, + * according to the C spec.) + * + * For example, suppose you say: + * "CONF_CHECK_VAR_TYPE(or_options_t, STRING, Address)". + * Then this macro will evaluate to: + * { .STRING = &or_options_t_dummy.Address } + * And since confparse_dummy_values_t.STRING has type "char **", that + * expression will create a warning unless or_options_t.Address also + * has type "char *". + */ +#define CONF_CHECK_VAR_TYPE(tp, conftype, member) \ + { . conftype = &tp ## _dummy . member } +#define CONF_TEST_MEMBERS(tp, conftype, member) \ + , CONF_CHECK_VAR_TYPE(tp, conftype, member) +#define DUMMY_CONF_TEST_MEMBERS , { .INT=NULL } +#define DUMMY_TYPECHECK_INSTANCE(tp) \ + static tp tp ## _dummy + +#else /* !(defined(TOR_UNIT_TESTS)) */ + +#define CONF_TEST_MEMBERS(tp, conftype, member) +/* Repeatedly declarable incomplete struct to absorb redundant semicolons */ +#define DUMMY_TYPECHECK_INSTANCE(tp) \ + struct tor_semicolon_eater +#define DUMMY_CONF_TEST_MEMBERS + +#endif /* defined(TOR_UNIT_TESTS) */ + +#endif /* !defined(TOR_LIB_CONF_CONFTESTING_H) */ diff --git a/src/lib/conf/conftypes.h b/src/lib/conf/conftypes.h index 5f13ec3de2..72697e8ee4 100644 --- a/src/lib/conf/conftypes.h +++ b/src/lib/conf/conftypes.h @@ -29,6 +29,9 @@ #define TOR_SRC_LIB_CONF_CONFTYPES_H #include "lib/cc/torint.h" +#ifdef TOR_UNIT_TESTS +#include "lib/conf/conftesting.h" +#endif /** Enumeration of types which option values can take */ typedef enum config_type_t { @@ -59,9 +62,6 @@ typedef enum config_type_t { CONFIG_TYPE_LINELIST_V, /**< Catch-all "virtual" option to summarize * context-sensitive config lines when fetching. */ - // XXXX this doesn't belong at this level of abstraction. - CONFIG_TYPE_ROUTERSET, /**< A list of router names, addrs, and fps, - * parsed into a routerset_t. */ CONFIG_TYPE_OBSOLETE, /**< Obsolete (ignored) option. */ CONFIG_TYPE_EXTENDED, /**< Extended type; definition will appear in * pointer. */ @@ -105,37 +105,6 @@ typedef struct struct_magic_decl_t { int magic_offset; } struct_magic_decl_t; -#ifdef TOR_UNIT_TESTS -/** - * Union used when building in test mode typechecking the members of a type - * used with confparse.c. See CONF_CHECK_VAR_TYPE for a description of how - * it is used. */ -typedef union { - char **STRING; - char **FILENAME; - int *POSINT; /* yes, this is really an int, and not an unsigned int. For - * historical reasons, many configuration values are restricted - * to the range [0,INT_MAX], and stored in signed ints. - */ - uint64_t *UINT64; - int *INT; - int *INTERVAL; - int *MSEC_INTERVAL; - uint64_t *MEMUNIT; - double *DOUBLE; - int *BOOL; - int *AUTOBOOL; - time_t *ISOTIME; - struct smartlist_t **CSV; - int *CSV_INTERVAL; - struct config_line_t **LINELIST; - struct config_line_t **LINELIST_S; - struct config_line_t **LINELIST_V; - // XXXX this doesn't belong at this level of abstraction. - struct routerset_t **ROUTERSET; -} confparse_dummy_values_t; -#endif /* defined(TOR_UNIT_TESTS) */ - /** A variable allowed in the configuration file or on the command line. */ typedef struct config_var_t { struct_member_t member; /** A struct member corresponding to this diff --git a/src/lib/conf/include.am b/src/lib/conf/include.am index 25355697d2..cb7126184d 100644 --- a/src/lib/conf/include.am +++ b/src/lib/conf/include.am @@ -1,4 +1,6 @@ # ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ - src/lib/conf/conftypes.h + src/lib/conf/conftesting.h \ + src/lib/conf/conftypes.h \ + src/lib/conf/confmacros.h diff --git a/src/test/test_confparse.c b/src/test/test_confparse.c index d0c33841fe..2f408b5b6e 100644 --- a/src/test/test_confparse.c +++ b/src/test/test_confparse.c @@ -49,13 +49,9 @@ typedef struct test_struct_t { static test_struct_t test_struct_t_dummy; #define VAR(varname,conftype,member,initvalue) \ - { { .name = varname, \ - .type = CONFIG_TYPE_##conftype, \ - .offset = offsetof(test_struct_t, member), }, \ - initvalue CONF_TEST_MEMBERS(test_struct_t, conftype, member) } - -#define V(name,conftype,initvalue) \ - VAR( #name, conftype, name, initvalue ) + CONFIG_VAR_ETYPE(test_struct_t, varname, conftype, member, initvalue) +#define V(member,conftype,initvalue) \ + VAR(#member, conftype, member, initvalue) #define OBSOLETE(varname) \ { { .name=varname, .type=CONFIG_TYPE_OBSOLETE }, NULL, {.INT=NULL} } @@ -83,7 +79,7 @@ static config_var_t test_vars[] = { OBSOLETE("obsolete"), { { .name = "routerset", - .type = CONFIG_TYPE_ROUTERSET, + .type = CONFIG_TYPE_EXTENDED, .type_def = &ROUTERSET_type_defn, .offset = offsetof(test_struct_t, routerset), }, From 53e969c137cb39bed432cd165d3d7e3825b1a2a9 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 19 Jun 2019 08:41:52 -0400 Subject: [PATCH 1174/2557] Use struct_var_{copy,eq} in confparse.c. --- src/app/config/confparse.c | 33 ++++++++++++--------------------- 1 file changed, 12 insertions(+), 21 deletions(-) diff --git a/src/app/config/confparse.c b/src/app/config/confparse.c index 752d16c844..b9b5fddb96 100644 --- a/src/app/config/confparse.c +++ b/src/app/config/confparse.c @@ -542,17 +542,15 @@ config_is_same(const config_format_t *fmt, const void *o1, const void *o2, const char *name) { - config_line_t *c1, *c2; - int r = 1; CONFIG_CHECK(fmt, o1); CONFIG_CHECK(fmt, o2); - c1 = config_get_assigned_option(fmt, o1, name, 0); - c2 = config_get_assigned_option(fmt, o2, name, 0); - r = config_lines_eq(c1, c2); - config_free_lines(c1); - config_free_lines(c2); - return r; + const config_var_t *var = config_find_option(fmt, name); + if (!var) { + return true; + } + + return struct_var_eq(o1, o2, &var->member); } /** Copy storage held by old into a new or_options_t and return it. */ @@ -561,7 +559,6 @@ config_dup(const config_format_t *fmt, const void *old) { void *newopts; int i; - config_line_t *line; newopts = config_new(fmt); for (i=0; fmt->vars[i].member.name; ++i) { @@ -569,19 +566,13 @@ config_dup(const config_format_t *fmt, const void *old) continue; if (fmt->vars[i].member.type == CONFIG_TYPE_OBSOLETE) continue; - line = config_get_assigned_option(fmt, old, fmt->vars[i].member.name, 0); - if (line) { - char *msg = NULL; - if (config_assign(fmt, newopts, line, 0, &msg) < 0) { - // LCOV_EXCL_START - log_err(LD_BUG, "config_get_assigned_option() generated " - "something we couldn't config_assign(): %s", msg); - tor_free(msg); - tor_assert(0); - // LCOV_EXCL_STOP - } + if (struct_var_copy(newopts, old, &fmt->vars[i].member) < 0) { + // LCOV_EXCL_START + log_err(LD_BUG, "Unable to copy value for %s.", + fmt->vars[i].member.name); + tor_assert_unreached(); + // LCOV_EXCL_STOP } - config_free_lines(line); } return newopts; } From 5b252d31ede521bea53c690693a2ab3d74c8fef4 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 20 Jun 2019 16:07:00 -0400 Subject: [PATCH 1175/2557] Add a "flags" member to config_var_t Additionally, adjust the macros so that we can add new members like this more easily. --- src/lib/conf/confmacros.h | 2 +- src/lib/conf/conftesting.h | 4 ++-- src/lib/conf/conftypes.h | 3 ++- src/test/test_confparse.c | 16 +++++++--------- 4 files changed, 12 insertions(+), 13 deletions(-) diff --git a/src/lib/conf/confmacros.h b/src/lib/conf/confmacros.h index 29040e1f55..4242137c5a 100644 --- a/src/lib/conf/confmacros.h +++ b/src/lib/conf/confmacros.h @@ -19,7 +19,7 @@ * Used to indicate the end of an array of configuration variables. **/ #define END_OF_CONFIG_VARS \ - { { .name = NULL }, NULL DUMMY_CONF_TEST_MEMBERS } + { .member = { .name = NULL } DUMMY_CONF_TEST_MEMBERS } /** * Declare a config_var_t as a member named membername of the structure diff --git a/src/lib/conf/conftesting.h b/src/lib/conf/conftesting.h index f4aca442a2..a40c9bc97c 100644 --- a/src/lib/conf/conftesting.h +++ b/src/lib/conf/conftesting.h @@ -68,8 +68,8 @@ typedef union { #define CONF_CHECK_VAR_TYPE(tp, conftype, member) \ { . conftype = &tp ## _dummy . member } #define CONF_TEST_MEMBERS(tp, conftype, member) \ - , CONF_CHECK_VAR_TYPE(tp, conftype, member) -#define DUMMY_CONF_TEST_MEMBERS , { .INT=NULL } + , .var_ptr_dummy=CONF_CHECK_VAR_TYPE(tp, conftype, member) +#define DUMMY_CONF_TEST_MEMBERS , .var_ptr_dummy={ .INT=NULL } #define DUMMY_TYPECHECK_INSTANCE(tp) \ static tp tp ## _dummy diff --git a/src/lib/conf/conftypes.h b/src/lib/conf/conftypes.h index 72697e8ee4..028a889917 100644 --- a/src/lib/conf/conftypes.h +++ b/src/lib/conf/conftypes.h @@ -110,7 +110,8 @@ typedef struct config_var_t { struct_member_t member; /** A struct member corresponding to this * variable. */ const char *initvalue; /**< String (or null) describing initial value. */ - + uint32_t flags; /**< One or more flags describing special handling for this + * variable */ #ifdef TOR_UNIT_TESTS /** Used for compiler-magic to typecheck the corresponding field in the * corresponding struct. Only used in unit test mode, at compile-time. */ diff --git a/src/test/test_confparse.c b/src/test/test_confparse.c index 2f408b5b6e..4bf66f3230 100644 --- a/src/test/test_confparse.c +++ b/src/test/test_confparse.c @@ -52,9 +52,8 @@ static test_struct_t test_struct_t_dummy; CONFIG_VAR_ETYPE(test_struct_t, varname, conftype, member, initvalue) #define V(member,conftype,initvalue) \ VAR(#member, conftype, member, initvalue) - -#define OBSOLETE(varname) \ - { { .name=varname, .type=CONFIG_TYPE_OBSOLETE }, NULL, {.INT=NULL} } +#define OBSOLETE(varname) \ + CONFIG_VAR_OBSOLETE(varname) static config_var_t test_vars[] = { V(s, STRING, "hello"), @@ -78,12 +77,11 @@ static config_var_t test_vars[] = { VAR("LineTypeB", LINELIST_S, mixed_lines, NULL), OBSOLETE("obsolete"), { - { .name = "routerset", - .type = CONFIG_TYPE_EXTENDED, - .type_def = &ROUTERSET_type_defn, - .offset = offsetof(test_struct_t, routerset), - }, - NULL, {.INT=NULL} + .member = { .name = "routerset", + .type = CONFIG_TYPE_EXTENDED, + .type_def = &ROUTERSET_type_defn, + .offset = offsetof(test_struct_t, routerset), + }, }, VAR("__HiddenInt", POSINT, hidden_int, "0"), VAR("MixedHiddenLines", LINELIST_V, mixed_hidden_lines, NULL), From a91ed23403ae28974639a9bdb67530c5c07a0ce6 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 19 Jun 2019 10:46:07 -0400 Subject: [PATCH 1176/2557] Use structvar to find the types for config vars. --- scripts/maint/practracker/exceptions.txt | 2 +- src/app/config/config.c | 30 ++---------------------- src/lib/confmgt/structvar.c | 26 ++++++++++++++++++++ src/lib/confmgt/structvar.h | 3 +++ 4 files changed, 32 insertions(+), 29 deletions(-) diff --git a/scripts/maint/practracker/exceptions.txt b/scripts/maint/practracker/exceptions.txt index 4db452b897..2190fb1caa 100644 --- a/scripts/maint/practracker/exceptions.txt +++ b/scripts/maint/practracker/exceptions.txt @@ -30,7 +30,7 @@ # Remember: It is better to fix the problem than to add a new exception! problem file-size /src/app/config/config.c 8518 -problem include-count /src/app/config/config.c 87 +problem include-count /src/app/config/config.c 88 problem function-size /src/app/config/config.c:options_act_reversible() 296 problem function-size /src/app/config/config.c:options_act() 589 problem function-size /src/app/config/config.c:resolve_my_address() 192 diff --git a/src/app/config/config.c b/src/app/config/config.c index 37cbe6b2ef..8da1e2acdc 100644 --- a/src/app/config/config.c +++ b/src/app/config/config.c @@ -111,6 +111,7 @@ #include "feature/stats/predict_ports.h" #include "feature/stats/rephist.h" #include "lib/compress/compress.h" +#include "lib/confmgt/structvar.h" #include "lib/crypt_ops/crypto_init.h" #include "lib/crypt_ops/crypto_rand.h" #include "lib/crypt_ops/crypto_util.h" @@ -8195,37 +8196,10 @@ getinfo_helper_config(control_connection_t *conn, int i; for (i = 0; option_vars_[i].member.name; ++i) { const config_var_t *var = &option_vars_[i]; - const char *type; /* don't tell controller about triple-underscore options */ if (!strncmp(option_vars_[i].member.name, "___", 3)) continue; - switch (var->member.type) { - case CONFIG_TYPE_STRING: type = "String"; break; - case CONFIG_TYPE_FILENAME: type = "Filename"; break; - case CONFIG_TYPE_POSINT: type = "Integer"; break; - case CONFIG_TYPE_UINT64: type = "Integer"; break; - case CONFIG_TYPE_INT: type = "SignedInteger"; break; - case CONFIG_TYPE_INTERVAL: type = "TimeInterval"; break; - case CONFIG_TYPE_MSEC_INTERVAL: type = "TimeMsecInterval"; break; - case CONFIG_TYPE_MEMUNIT: type = "DataSize"; break; - case CONFIG_TYPE_DOUBLE: type = "Float"; break; - case CONFIG_TYPE_BOOL: type = "Boolean"; break; - case CONFIG_TYPE_AUTOBOOL: type = "Boolean+Auto"; break; - case CONFIG_TYPE_ISOTIME: type = "Time"; break; - case CONFIG_TYPE_ROUTERSET: type = "RouterList"; break; - case CONFIG_TYPE_CSV: type = "CommaList"; break; - /* This type accepts more inputs than TimeInterval, but it ignores - * everything after the first entry, so we may as well pretend - * it's a TimeInterval. */ - case CONFIG_TYPE_CSV_INTERVAL: type = "TimeInterval"; break; - case CONFIG_TYPE_LINELIST: type = "LineList"; break; - case CONFIG_TYPE_LINELIST_S: type = "Dependent"; break; - case CONFIG_TYPE_LINELIST_V: type = "Virtual"; break; - default: - case CONFIG_TYPE_OBSOLETE: - case CONFIG_TYPE_EXTENDED: - type = NULL; break; - } + const char *type = struct_var_get_typename(&var->member); if (!type) continue; smartlist_add_asprintf(sl, "%s %s\n",var->member.name,type); diff --git a/src/lib/confmgt/structvar.c b/src/lib/confmgt/structvar.c index 7ea00fbde9..38f8e5dd7a 100644 --- a/src/lib/confmgt/structvar.c +++ b/src/lib/confmgt/structvar.c @@ -25,6 +25,8 @@ #include "lib/confmgt/typedvar.h" #include "lib/log/util_bug.h" +#include "lib/confmgt/var_type_def_st.h" + #include /** @@ -198,3 +200,27 @@ struct_var_kvencode(const void *object, const struct_member_t *member) return typed_var_kvencode_ex(member->name, p, def); } + +/** + * Return the official name of this struct member. + **/ +const char * +struct_var_get_name(const struct_member_t *member) +{ + return member->name; +} + +/** + * Return the type name for this struct member. + * + * Do not use the output of this function to inspect a type within Tor. It is + * suitable for debugging, informing the controller or user of a variable's + * type, etc. + **/ +const char * +struct_var_get_typename(const struct_member_t *member) +{ + const var_type_def_t *def = get_type_def(member); + + return def ? def->name : NULL; +} diff --git a/src/lib/confmgt/structvar.h b/src/lib/confmgt/structvar.h index 894098e509..92b9b6fc71 100644 --- a/src/lib/confmgt/structvar.h +++ b/src/lib/confmgt/structvar.h @@ -41,6 +41,9 @@ bool struct_var_eq(const void *a, const void *b, bool struct_var_ok(const void *object, const struct struct_member_t *member); +const char *struct_var_get_name(const struct struct_member_t *member); +const char *struct_var_get_typename(const struct struct_member_t *member); + int struct_var_kvassign(void *object, const struct config_line_t *line, char **errmsg, const struct struct_member_t *member); From b6457d4c08f601c4e42e64aad47ac9c30c36306e Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 20 Jun 2019 16:26:05 -0400 Subject: [PATCH 1177/2557] Extend macros to allow flag arguments. --- src/app/config/config.c | 4 ++-- src/app/config/statefile.c | 2 +- src/feature/dirauth/shared_random_state.c | 2 +- src/lib/conf/confmacros.h | 10 +++++++--- src/test/test_confparse.c | 2 +- 5 files changed, 12 insertions(+), 8 deletions(-) diff --git a/src/app/config/config.c b/src/app/config/config.c index c15236b0e9..5f7a82d15a 100644 --- a/src/app/config/config.c +++ b/src/app/config/config.c @@ -259,11 +259,11 @@ DUMMY_TYPECHECK_INSTANCE(or_options_t); * or_options_t.member" */ #define VAR(varname,conftype,member,initvalue) \ - CONFIG_VAR_ETYPE(or_options_t, varname, conftype, member, initvalue) + CONFIG_VAR_ETYPE(or_options_t, varname, conftype, member, 0, initvalue) /* As VAR, but uses a type definition in addition to a type enum. */ #define VAR_D(varname,conftype,member,initvalue) \ - CONFIG_VAR_DEFN(or_options_t, varname, conftype, member, initvalue) + CONFIG_VAR_DEFN(or_options_t, varname, conftype, member, 0, initvalue) #define V(member,conftype,initvalue) \ VAR(#member, conftype, member, initvalue) diff --git a/src/app/config/statefile.c b/src/app/config/statefile.c index e0584c62af..4fe415b8cf 100644 --- a/src/app/config/statefile.c +++ b/src/app/config/statefile.c @@ -71,7 +71,7 @@ static config_abbrev_t state_abbrevs_[] = { DUMMY_TYPECHECK_INSTANCE(or_state_t); #define VAR(varname,conftype,member,initvalue) \ - CONFIG_VAR_ETYPE(or_state_t, varname, conftype, member, initvalue) + CONFIG_VAR_ETYPE(or_state_t, varname, conftype, member, 0, initvalue) #define V(member,conftype,initvalue) \ VAR(#member, conftype, member, initvalue) diff --git a/src/feature/dirauth/shared_random_state.c b/src/feature/dirauth/shared_random_state.c index 3cdb223d25..d89f249a72 100644 --- a/src/feature/dirauth/shared_random_state.c +++ b/src/feature/dirauth/shared_random_state.c @@ -52,7 +52,7 @@ static const char dstate_cur_srv_key[] = "SharedRandCurrentValue"; DUMMY_TYPECHECK_INSTANCE(sr_disk_state_t); #define VAR(varname,conftype,member,initvalue) \ - CONFIG_VAR_ETYPE(sr_disk_state_t, varname, conftype, member, initvalue) + CONFIG_VAR_ETYPE(sr_disk_state_t, varname, conftype, member, 0, initvalue) #define V(member,conftype,initvalue) \ VAR(#member, conftype, member, initvalue) diff --git a/src/lib/conf/confmacros.h b/src/lib/conf/confmacros.h index 4242137c5a..ff284e681d 100644 --- a/src/lib/conf/confmacros.h +++ b/src/lib/conf/confmacros.h @@ -30,12 +30,14 @@ * Most modules that use this macro should wrap it in a local macro that * sets structtype to the local configuration type. **/ -#define CONFIG_VAR_ETYPE(structtype, varname, vartype, membername, initval) \ +#define CONFIG_VAR_ETYPE(structtype, varname, vartype, membername, \ + varflags, initval) \ { .member = \ { .name = varname, \ .type = CONFIG_TYPE_ ## vartype, \ .offset = offsetof(structtype, membername), \ }, \ + .flags = varflags, \ .initvalue = initval \ CONF_TEST_MEMBERS(structtype, vartype, membername) \ } @@ -44,13 +46,15 @@ * As CONFIG_VAR_XTYPE, but declares a value using an extension type whose * type definition is vartype_type_defn. **/ -#define CONFIG_VAR_DEFN(structtype, varname, vartype, membername, initval) \ +#define CONFIG_VAR_DEFN(structtype, varname, vartype, membername, \ + varflags, initval) \ { .member = \ - { .name = varname, \ + { .name = varname, \ .type = CONFIG_TYPE_EXTENDED, \ .type_def = &vartype ## _type_defn, \ .offset = offsetof(structtype, membername), \ }, \ + .flags = varflags, \ .initvalue = initval \ CONF_TEST_MEMBERS(structtype, vartype, membername) \ } diff --git a/src/test/test_confparse.c b/src/test/test_confparse.c index 4bf66f3230..4612419dff 100644 --- a/src/test/test_confparse.c +++ b/src/test/test_confparse.c @@ -49,7 +49,7 @@ typedef struct test_struct_t { static test_struct_t test_struct_t_dummy; #define VAR(varname,conftype,member,initvalue) \ - CONFIG_VAR_ETYPE(test_struct_t, varname, conftype, member, initvalue) + CONFIG_VAR_ETYPE(test_struct_t, varname, conftype, member, 0, initvalue) #define V(member,conftype,initvalue) \ VAR(#member, conftype, member, initvalue) #define OBSOLETE(varname) \ From a114df9a040dbdedfc89f7d2ff777476e204a2cf Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 19 Jun 2019 16:06:15 -0400 Subject: [PATCH 1178/2557] Add a function to make sure all values in a config object are ok --- src/app/config/confparse.c | 18 ++++++++++++++++++ src/app/config/confparse.h | 2 ++ 2 files changed, 20 insertions(+) diff --git a/src/app/config/confparse.c b/src/app/config/confparse.c index b9b5fddb96..2890d8c81b 100644 --- a/src/app/config/confparse.c +++ b/src/app/config/confparse.c @@ -673,3 +673,21 @@ config_dump(const config_format_t *fmt, const void *default_options, } return result; } + +/** + * Return true if every member of options is in-range and well-formed. + * Return false otherwise. Log errors at level severity. + */ +bool +config_check_ok(const config_format_t *fmt, const void *options, int severity) +{ + bool all_ok = true; + for (int i=0; fmt->vars[i].member.name; ++i) { + if (!struct_var_ok(options, &fmt->vars[i].member)) { + log_fn(severity, LD_BUG, "Invalid value for %s", + fmt->vars[i].member.name); + all_ok = false; + } + } + return all_ok; +} diff --git a/src/app/config/confparse.h b/src/app/config/confparse.h index 4ef4e708f3..b91ea1c13d 100644 --- a/src/app/config/confparse.h +++ b/src/app/config/confparse.h @@ -138,6 +138,8 @@ void *config_dup(const config_format_t *fmt, const void *old); char *config_dump(const config_format_t *fmt, const void *default_options, const void *options, int minimal, int comment_defaults); +bool config_check_ok(const config_format_t *fmt, const void *options, + int severity); int config_assign(const config_format_t *fmt, void *options, struct config_line_t *list, unsigned flags, char **msg); From a7835202cf871f68854494df904058a6e644c0b0 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 21 Jun 2019 09:58:40 -0400 Subject: [PATCH 1179/2557] Turn several properties of types or variables into flags. "unsettable" is a property of types. LINELIST_V and OBSOLETE are unsettable, meaning that they cannot be set by name. "contained" is a property of types. I'm hoping to find a better name here. LINELIST_S is "contained" because it always appears within a LINELIST_V, and as such doesn't need to be dumped ore copied independently. "cumulative" is a property of types. Cumulative types can appear more than once in a torrc without causing a warning, because they add to each other rather than replacing each other. "obsolete" is a property of variables. "marking fragile" is now a command that struct members can accept. With these changes, confparse and config no longer ever need to mention CONFIG_TYPE_XYZ values by name. --- src/app/config/config.c | 9 +++-- src/app/config/confparse.c | 45 ++++++++++++--------- src/app/config/confparse.h | 4 ++ src/lib/conf/confmacros.h | 4 +- src/lib/conf/conftypes.h | 6 +++ src/lib/confmgt/structvar.c | 37 ++++++++++++++++++ src/lib/confmgt/structvar.h | 5 +++ src/lib/confmgt/type_defs.c | 65 +++++++++++++++++++++---------- src/lib/confmgt/typedvar.c | 45 +++++++++++++++++++++ src/lib/confmgt/typedvar.h | 6 +++ src/lib/confmgt/var_type_def_st.h | 20 ++++++++++ 11 files changed, 203 insertions(+), 43 deletions(-) diff --git a/src/app/config/config.c b/src/app/config/config.c index 5f7a82d15a..d240a73fe6 100644 --- a/src/app/config/config.c +++ b/src/app/config/config.c @@ -960,8 +960,8 @@ set_options(or_options_t *new_val, char **msg) for (i=0; options_format.vars[i].member.name; ++i) { const config_var_t *var = &options_format.vars[i]; const char *var_name = var->member.name; - if (var->member.type == CONFIG_TYPE_LINELIST_S || - var->member.type == CONFIG_TYPE_OBSOLETE) { + if (config_var_is_contained(var)) { + /* something else will check this var, or it doesn't need checking */ continue; } if (!config_is_same(&options_format, new_val, old_options, var_name)) { @@ -2663,9 +2663,10 @@ list_torrc_options(void) int i; for (i = 0; option_vars_[i].member.name; ++i) { const config_var_t *var = &option_vars_[i]; - if (var->member.type == CONFIG_TYPE_OBSOLETE || - var->member.type == CONFIG_TYPE_LINELIST_V) + if (! config_var_is_settable(var)) { + /* This variable cannot be set, or cannot be set by this name. */ continue; + } printf("%s\n", var->member.name); } } diff --git a/src/app/config/confparse.c b/src/app/config/confparse.c index 2890d8c81b..0d19974d70 100644 --- a/src/app/config/confparse.c +++ b/src/app/config/confparse.c @@ -148,6 +148,24 @@ config_count_options(const config_format_t *fmt) return i; } +bool +config_var_is_cumulative(const config_var_t *var) +{ + return struct_var_is_cumulative(&var->member); +} +bool +config_var_is_settable(const config_var_t *var) +{ + if (var->flags & CVFLAG_OBSOLETE) + return false; + return struct_var_is_settable(&var->member); +} +bool +config_var_is_contained(const config_var_t *var) +{ + return struct_var_is_contained(&var->member); +} + /* * Functions to assign config options. */ @@ -183,14 +201,7 @@ config_mark_lists_fragile(const config_format_t *fmt, void *options) for (i = 0; fmt->vars[i].member.name; ++i) { const config_var_t *var = &fmt->vars[i]; - config_line_t *list; - if (var->member.type != CONFIG_TYPE_LINELIST && - var->member.type != CONFIG_TYPE_LINELIST_V) - continue; - - list = *(config_line_t **)STRUCT_VAR_P(options, var->member.offset); - if (list) - list->fragile = 1; + struct_var_mark_fragile(options, &var->member); } } @@ -255,9 +266,7 @@ config_assign_line(const config_format_t *fmt, void *options, if (!strlen(c->value)) { /* reset or clear it, then return */ if (!clear_first) { - if ((var->member.type == CONFIG_TYPE_LINELIST || - var->member.type == CONFIG_TYPE_LINELIST_S) && - c->command != CONFIG_LINE_CLEAR) { + if (config_var_is_cumulative(var) && c->command != CONFIG_LINE_CLEAR) { /* We got an empty linelist from the torrc or command line. As a special case, call this an error. Warn and ignore. */ log_warn(LD_CONFIG, @@ -273,8 +282,7 @@ config_assign_line(const config_format_t *fmt, void *options, config_reset(fmt, options, var, use_defaults); // LCOV_EXCL_LINE } - if (options_seen && (var->member.type != CONFIG_TYPE_LINELIST && - var->member.type != CONFIG_TYPE_LINELIST_S)) { + if (options_seen && ! config_var_is_cumulative(var)) { /* We're tracking which options we've seen, and this option is not * supposed to occur more than once. */ int var_index = (int)(var - fmt->vars); @@ -562,10 +570,10 @@ config_dup(const config_format_t *fmt, const void *old) newopts = config_new(fmt); for (i=0; fmt->vars[i].member.name; ++i) { - if (fmt->vars[i].member.type == CONFIG_TYPE_LINELIST_S) - continue; - if (fmt->vars[i].member.type == CONFIG_TYPE_OBSOLETE) + if (config_var_is_contained(&fmt->vars[i])) { + // Something else will copy this option, or it doesn't need copying. continue; + } if (struct_var_copy(newopts, old, &fmt->vars[i].member) < 0) { // LCOV_EXCL_START log_err(LD_BUG, "Unable to copy value for %s.", @@ -629,9 +637,10 @@ config_dump(const config_format_t *fmt, const void *default_options, elements = smartlist_new(); for (i=0; fmt->vars[i].member.name; ++i) { int comment_option = 0; - if (fmt->vars[i].member.type == CONFIG_TYPE_OBSOLETE || - fmt->vars[i].member.type == CONFIG_TYPE_LINELIST_S) + if (config_var_is_contained(&fmt->vars[i])) { + // Something else will dump this option, or it doesn't need dumping. continue; + } /* Don't save 'hidden' control variables. */ if (!strcmpstart(fmt->vars[i].member.name, "__")) continue; diff --git a/src/app/config/confparse.h b/src/app/config/confparse.h index 6d63ba3e0e..c53e3224db 100644 --- a/src/app/config/confparse.h +++ b/src/app/config/confparse.h @@ -104,6 +104,10 @@ const char *config_expand_abbrev(const config_format_t *fmt, int command_line, int warn_obsolete); void warn_deprecated_option(const char *what, const char *why); +bool config_var_is_cumulative(const config_var_t *var); +bool config_var_is_settable(const config_var_t *var); +bool config_var_is_contained(const config_var_t *var); + /* Helper macros to compare an option across two configuration objects */ #define CFG_EQ_BOOL(a,b,opt) ((a)->opt == (b)->opt) #define CFG_EQ_INT(a,b,opt) ((a)->opt == (b)->opt) diff --git a/src/lib/conf/confmacros.h b/src/lib/conf/confmacros.h index ff284e681d..aa89965e69 100644 --- a/src/lib/conf/confmacros.h +++ b/src/lib/conf/confmacros.h @@ -60,6 +60,8 @@ } #define CONFIG_VAR_OBSOLETE(varname) \ - { .member = { .name = varname, .type = CONFIG_TYPE_OBSOLETE } } + { .member = { .name = varname, .type = CONFIG_TYPE_OBSOLETE }, \ + .flags = CVFLAG_OBSOLETE \ + } #endif /* !defined(TOR_LIB_CONF_CONFMACROS_H) */ diff --git a/src/lib/conf/conftypes.h b/src/lib/conf/conftypes.h index 028a889917..6a44fb92ed 100644 --- a/src/lib/conf/conftypes.h +++ b/src/lib/conf/conftypes.h @@ -105,6 +105,12 @@ typedef struct struct_magic_decl_t { int magic_offset; } struct_magic_decl_t; +/** + * Flag to indicate that an option is obsolete. Any attempt to set or + * fetch this option should produce a warning. + **/ +#define CVFLAG_OBSOLETE (1u<<0) + /** A variable allowed in the configuration file or on the command line. */ typedef struct config_var_t { struct_member_t member; /** A struct member corresponding to this diff --git a/src/lib/confmgt/structvar.c b/src/lib/confmgt/structvar.c index 38f8e5dd7a..97a8fb3633 100644 --- a/src/lib/confmgt/structvar.c +++ b/src/lib/confmgt/structvar.c @@ -201,6 +201,19 @@ struct_var_kvencode(const void *object, const struct_member_t *member) return typed_var_kvencode_ex(member->name, p, def); } +/** + * Mark the field in object determined by member -- a variable + * that ordinarily would be extended by assignment -- as "fragile", so that it + * will get replaced by the next assignment instead. + */ +void +struct_var_mark_fragile(void *object, const struct_member_t *member) +{ + void *p = struct_get_mptr(object, member); + const var_type_def_t *def = get_type_def(member); + return typed_var_mark_fragile_ex(p, def); +} + /** * Return the official name of this struct member. **/ @@ -224,3 +237,27 @@ struct_var_get_typename(const struct_member_t *member) return def ? def->name : NULL; } + +bool +struct_var_is_cumulative(const struct_member_t *member) +{ + const var_type_def_t *def = get_type_def(member); + + return def ? def->is_cumulative : false; +} + +bool +struct_var_is_settable(const struct_member_t *member) +{ + const var_type_def_t *def = get_type_def(member); + + return def ? !def->is_unsettable : true; +} + +bool +struct_var_is_contained(const struct_member_t *member) +{ + const var_type_def_t *def = get_type_def(member); + + return def ? def->is_contained : false; +} diff --git a/src/lib/confmgt/structvar.h b/src/lib/confmgt/structvar.h index 92b9b6fc71..e6dbc6d6ec 100644 --- a/src/lib/confmgt/structvar.h +++ b/src/lib/confmgt/structvar.h @@ -40,9 +40,14 @@ bool struct_var_eq(const void *a, const void *b, const struct struct_member_t *member); bool struct_var_ok(const void *object, const struct struct_member_t *member); +void struct_var_mark_fragile(void *object, + const struct struct_member_t *member); const char *struct_var_get_name(const struct struct_member_t *member); const char *struct_var_get_typename(const struct struct_member_t *member); +bool struct_var_is_cumulative(const struct struct_member_t *member); +bool struct_var_is_settable(const struct struct_member_t *member); +bool struct_var_is_contained(const struct struct_member_t *member); int struct_var_kvassign(void *object, const struct config_line_t *line, char **errmsg, diff --git a/src/lib/confmgt/type_defs.c b/src/lib/confmgt/type_defs.c index 62b4c1019d..f8b2681aa0 100644 --- a/src/lib/confmgt/type_defs.c +++ b/src/lib/confmgt/type_defs.c @@ -620,12 +620,22 @@ linelist_copy(void *target, const void *value, const void *params) return 0; } +static void +linelist_mark_fragile(void *target, const void *params) +{ + (void)params; + config_line_t **ptr = (config_line_t **)target; + if (*ptr) + (*ptr)->fragile = 1; +} + static const var_type_fns_t linelist_fns = { .kv_parse = linelist_kv_parse, .kv_encode = linelist_kv_encode, .clear = linelist_clear, .eq = linelist_eq, .copy = linelist_copy, + .mark_fragile = linelist_mark_fragile, }; static const var_type_fns_t linelist_v_fns = { @@ -634,6 +644,7 @@ static const var_type_fns_t linelist_v_fns = { .clear = linelist_clear, .eq = linelist_eq, .copy = linelist_copy, + .mark_fragile = linelist_mark_fragile, }; static const var_type_fns_t linelist_s_fns = { @@ -690,26 +701,40 @@ static const var_type_fns_t ignore_fns = { * Table mapping conf_type_t values to var_type_def_t objects. **/ static const var_type_def_t type_definitions_table[] = { - [CONFIG_TYPE_STRING] = { "String", &string_fns, NULL }, - [CONFIG_TYPE_FILENAME] = { "Filename", &string_fns, NULL }, - [CONFIG_TYPE_INT] = { "SignedInteger", &int_fns, &INT_PARSE_UNRESTRICTED }, - [CONFIG_TYPE_POSINT] = { "Integer", &int_fns, &INT_PARSE_POSINT }, - [CONFIG_TYPE_UINT64] = { "Integer", &uint64_fns, NULL, }, - [CONFIG_TYPE_MEMUNIT] = { "DataSize", &memunit_fns, &memory_units }, - [CONFIG_TYPE_INTERVAL] = { "TimeInterval", &interval_fns, &time_units }, - [CONFIG_TYPE_MSEC_INTERVAL] = { "TimeMsecInterval", &interval_fns, - &time_msec_units }, - [CONFIG_TYPE_DOUBLE] = { "Float", &double_fns, NULL }, - [CONFIG_TYPE_BOOL] = { "Boolean", &enum_fns, &enum_table_bool }, - [CONFIG_TYPE_AUTOBOOL] = { "Boolean+Auto", &enum_fns, &enum_table_autobool }, - [CONFIG_TYPE_ISOTIME] = { "Time", &time_fns, NULL }, - [CONFIG_TYPE_CSV] = { "CommaList", &csv_fns, NULL }, - [CONFIG_TYPE_CSV_INTERVAL] = { "TimeInterval", &legacy_csv_interval_fns, - NULL }, - [CONFIG_TYPE_LINELIST] = { "LineList", &linelist_fns, NULL }, - [CONFIG_TYPE_LINELIST_S] = { "Dependent", &linelist_s_fns, NULL }, - [CONFIG_TYPE_LINELIST_V] = { "Virtual", &linelist_v_fns, NULL }, - [CONFIG_TYPE_OBSOLETE] = { "Obsolete", &ignore_fns, NULL } + [CONFIG_TYPE_STRING] = { .name="String", .fns=&string_fns }, + [CONFIG_TYPE_FILENAME] = { .name="Filename", .fns=&string_fns }, + [CONFIG_TYPE_INT] = { .name="SignedInteger", .fns=&int_fns, + .params=&INT_PARSE_UNRESTRICTED }, + [CONFIG_TYPE_POSINT] = { .name="Integer", .fns=&int_fns, + .params=&INT_PARSE_POSINT }, + [CONFIG_TYPE_UINT64] = { .name="Integer", .fns=&uint64_fns, }, + [CONFIG_TYPE_MEMUNIT] = { .name="DataSize", .fns=&memunit_fns, + .params=&memory_units }, + [CONFIG_TYPE_INTERVAL] = { .name="TimeInterval", .fns=&interval_fns, + .params=&time_units }, + [CONFIG_TYPE_MSEC_INTERVAL] = { .name="TimeMsecInterval", + .fns=&interval_fns, + .params=&time_msec_units }, + [CONFIG_TYPE_DOUBLE] = { .name="Float", .fns=&double_fns, }, + [CONFIG_TYPE_BOOL] = { .name="Boolean", .fns=&enum_fns, + .params=&enum_table_bool }, + [CONFIG_TYPE_AUTOBOOL] = { .name="Boolean+Auto", .fns=&enum_fns, + .params=&enum_table_autobool }, + [CONFIG_TYPE_ISOTIME] = { .name="Time", .fns=&time_fns, }, + [CONFIG_TYPE_CSV] = { .name="CommaList", .fns=&csv_fns, }, + [CONFIG_TYPE_CSV_INTERVAL] = { .name="TimeInterval", + .fns=&legacy_csv_interval_fns, }, + [CONFIG_TYPE_LINELIST] = { .name="LineList", .fns=&linelist_fns, + .is_cumulative=true}, + [CONFIG_TYPE_LINELIST_S] = { .name="Dependent", .fns=&linelist_s_fns, + .is_cumulative=true, + .is_contained=true, }, + [CONFIG_TYPE_LINELIST_V] = { .name="Virtual", .fns=&linelist_v_fns, + .is_cumulative=true, + .is_unsettable=true }, + [CONFIG_TYPE_OBSOLETE] = { .name="Obsolete", .fns=&ignore_fns, + .is_unsettable=true, + .is_contained=true, } }; /** diff --git a/src/lib/confmgt/typedvar.c b/src/lib/confmgt/typedvar.c index c2b9b45725..3cba075390 100644 --- a/src/lib/confmgt/typedvar.c +++ b/src/lib/confmgt/typedvar.c @@ -210,6 +210,51 @@ typed_var_ok_ex(const void *value, const var_type_def_t *def) return true; } +/** + * Mark value -- a variable that ordinarily would be extended by + * assignment -- as "fragile", so that it will get replaced by the next + * assignment instead. + **/ +void +typed_var_mark_fragile_ex(void *value, const var_type_def_t *def) +{ + if (BUG(!def)) { + return; // LCOV_EXCL_LINE + } + + if (def->fns->mark_fragile) + def->fns->mark_fragile(value, def->params); +} + +/** + * Return true iff multiple assignments to a variable will extend its + * value, rather than replacing it. + **/ +bool +var_type_is_cumulative(const var_type_def_t *def) +{ + return def->is_cumulative; +} + +/** + * Return true iff this variable type is always contained in another variable, + * and as such doesn't need to be dumped or copied independently. + **/ +bool +var_type_is_contained(const var_type_def_t *def) +{ + return def->is_contained; +} + +/** + * Return true iff this type can not be assigned directly by the user. + **/ +bool +var_type_is_settable(const var_type_def_t *def) +{ + return ! def->is_unsettable; +} + /* ===== * The functions below take a config_type_t instead of a var_type_def_t. * I'd like to deprecate them eventually and use var_type_def_t everywhere, diff --git a/src/lib/confmgt/typedvar.h b/src/lib/confmgt/typedvar.h index 720ad54fc6..2e36f9d673 100644 --- a/src/lib/confmgt/typedvar.h +++ b/src/lib/confmgt/typedvar.h @@ -46,4 +46,10 @@ int typed_var_kvassign_ex(void *target, const struct config_line_t *line, struct config_line_t *typed_var_kvencode_ex(const char *key, const void *value, const var_type_def_t *def); +void typed_var_mark_fragile_ex(void *value, const var_type_def_t *def); + +bool var_type_is_cumulative(const var_type_def_t *def); +bool var_type_is_contained(const var_type_def_t *def); +bool var_type_is_settable(const var_type_def_t *def); + #endif /* !defined(TOR_LIB_CONFMGT_TYPEDVAR_H) */ diff --git a/src/lib/confmgt/var_type_def_st.h b/src/lib/confmgt/var_type_def_st.h index c63ff66eff..c4f7acec69 100644 --- a/src/lib/confmgt/var_type_def_st.h +++ b/src/lib/confmgt/var_type_def_st.h @@ -122,6 +122,15 @@ struct var_type_fns_t { * values are valid. **/ bool (*ok)(const void *value, const void *params); + /** + * Mark a value of this variable as "fragile", so that future attempts to + * assign to this variable will replace rather than extending it. + * + * The default implementation for this function does nothing. + * + * Only meaningful for types with is_cumulative set. + **/ + void (*mark_fragile)(void *value, const void *params); }; /** @@ -142,6 +151,17 @@ struct var_type_def_t { * calling the functions in this type's function table. */ const void *params; + + /** True iff a variable of this type can never be set directly by name. */ + bool is_unsettable; + /** True iff a variable of this type is always contained in another + * variable, and as such doesn't need to be dumped or copied + * independently. */ + bool is_contained; + /** True iff a variable of this type can be set more than once without + * destroying older values. Such variables should implement "mark_fragile". + */ + bool is_cumulative; }; #endif /* !defined(TOR_LIB_CONFMGT_VAR_TYPE_DEF_ST_H) */ From c390efe84fa7a4c47e39ce0cc7e34550f515d5be Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 19 Jun 2019 16:21:21 -0400 Subject: [PATCH 1180/2557] A few more test cases and unreachable lines --- src/lib/confmgt/typedvar.c | 18 ++++++++++-------- src/test/test_confparse.c | 20 ++++++++++++++++++++ 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/src/lib/confmgt/typedvar.c b/src/lib/confmgt/typedvar.c index fc45c44481..c2b9b45725 100644 --- a/src/lib/confmgt/typedvar.c +++ b/src/lib/confmgt/typedvar.c @@ -44,7 +44,7 @@ typed_var_assign_ex(void *target, const char *value, char **errmsg, const var_type_def_t *def) { if (BUG(!def)) - return -1; + return -1; // LCOV_EXCL_LINE // clear old value if needed. typed_var_free_ex(target, def); @@ -67,7 +67,7 @@ typed_var_kvassign_ex(void *target, const config_line_t *line, char **errmsg, const var_type_def_t *def) { if (BUG(!def)) - return -1; + return -1; // LCOV_EXCL_LINE if (def->fns->kv_parse) { // We do _not_ free the old value here, since linelist options @@ -86,7 +86,7 @@ void typed_var_free_ex(void *target, const var_type_def_t *def) { if (BUG(!def)) - return; + return; // LCOV_EXCL_LINE if (def->fns->clear) { def->fns->clear(target, def->params); } @@ -102,7 +102,7 @@ char * typed_var_encode_ex(const void *value, const var_type_def_t *def) { if (BUG(!def)) - return NULL; + return NULL; // LCOV_EXCL_LINE tor_assert(def->fns->encode); return def->fns->encode(value, def->params); } @@ -121,7 +121,7 @@ typed_var_kvencode_ex(const char *key, const void *value, const var_type_def_t *def) { if (BUG(!def)) - return NULL; + return NULL; // LCOV_EXCL_LINE if (def->fns->kv_encode) { return def->fns->kv_encode(key, value, def->params); } @@ -145,7 +145,7 @@ int typed_var_copy_ex(void *dest, const void *src, const var_type_def_t *def) { if (BUG(!def)) - return -1; + return -1; // LCOV_EXCL_LINE if (def->fns->copy) { // If we have been provided a copy fuction, use it. return def->fns->copy(dest, src, def); @@ -160,8 +160,10 @@ typed_var_copy_ex(void *dest, const void *src, const var_type_def_t *def) char *err = NULL; int rv = typed_var_assign_ex(dest, enc, &err, def); if (BUG(rv < 0)) { + // LCOV_EXCL_START log_warn(LD_BUG, "Encoded value %s was not parseable as a %s: %s", escaped(enc), def->name, err?err:""); + // LCOV_EXCL_STOP } tor_free(err); tor_free(enc); @@ -176,7 +178,7 @@ bool typed_var_eq_ex(const void *a, const void *b, const var_type_def_t *def) { if (BUG(!def)) - return false; + return false; // LCOV_EXCL_LINE if (def->fns->eq) { // Use a provided eq function if we got one. @@ -200,7 +202,7 @@ bool typed_var_ok_ex(const void *value, const var_type_def_t *def) { if (BUG(!def)) - return false; + return false; // LCOV_EXCL_LINE if (def->fns->ok) return def->fns->ok(value, def->params); diff --git a/src/test/test_confparse.c b/src/test/test_confparse.c index 9e626356d3..d0c33841fe 100644 --- a/src/test/test_confparse.c +++ b/src/test/test_confparse.c @@ -290,6 +290,8 @@ test_confparse_assign_simple(void *arg) tt_str_op(tst->mixed_hidden_lines->next->value, OP_EQ, "ABC"); tt_assert(!tst->mixed_hidden_lines->next->next); + tt_assert(config_check_ok(&test_fmt, tst, LOG_ERR)); + done: config_free(&test_fmt, tst); } @@ -344,6 +346,8 @@ test_confparse_assign_deprecated(void *arg) tt_int_op(tst->deprecated_int, OP_EQ, 7); + tt_assert(config_check_ok(&test_fmt, tst, LOG_ERR)); + done: teardown_capture_of_logs(); config_free(&test_fmt, tst); @@ -489,6 +493,8 @@ static const badval_test_t bv_badabool = static const badval_test_t bv_badtime = { "time lunchtime\n", "Invalid time" }; static const badval_test_t bv_virt = { "MixedLines 7\n", "virtual option" }; static const badval_test_t bv_rs = { "Routerset 2.2.2.2.2\n", "Invalid" }; +static const badval_test_t bv_big_interval = + { "interval 1000 months", "too large" }; /* Try config_dump(), and make sure it behaves correctly */ static void @@ -885,6 +891,18 @@ test_confparse_unitparse(void *args) ; } +static void +test_confparse_check_ok_fail(void *arg) +{ + (void)arg; + test_struct_t *tst = config_new(&test_fmt); + tst->pos = -10; + tt_assert(! config_check_ok(&test_fmt, tst, LOG_INFO)); + + done: + config_free(&test_fmt, tst); +} + #define CONFPARSE_TEST(name, flags) \ { #name, test_confparse_ ## name, flags, NULL, NULL } @@ -912,6 +930,7 @@ struct testcase_t confparse_tests[] = { BADVAL_TEST(badtime), BADVAL_TEST(virt), BADVAL_TEST(rs), + BADVAL_TEST(big_interval), CONFPARSE_TEST(dump, 0), CONFPARSE_TEST(reset, 0), CONFPARSE_TEST(reassign, 0), @@ -919,5 +938,6 @@ struct testcase_t confparse_tests[] = { CONFPARSE_TEST(get_assigned, 0), CONFPARSE_TEST(extra_lines, 0), CONFPARSE_TEST(unitparse, 0), + CONFPARSE_TEST(check_ok_fail, 0), END_OF_TESTCASES }; From f3330d2be3de287d4b15bd4d9b0e86fab3ffb9fd Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 21 Jun 2019 10:17:59 -0400 Subject: [PATCH 1181/2557] Make "invisibility" and "undumpability" properties of variables. Previously, these were magical things that we detected by checking whether a variable's name was prefixed with two or three underscores. --- src/app/config/config.c | 35 +++++++++++++++++++++++------------ src/app/config/confparse.c | 2 +- src/lib/conf/conftypes.h | 10 ++++++++++ 3 files changed, 34 insertions(+), 13 deletions(-) diff --git a/src/app/config/config.c b/src/app/config/config.c index d240a73fe6..5667702a60 100644 --- a/src/app/config/config.c +++ b/src/app/config/config.c @@ -265,6 +265,13 @@ DUMMY_TYPECHECK_INSTANCE(or_options_t); #define VAR_D(varname,conftype,member,initvalue) \ CONFIG_VAR_DEFN(or_options_t, varname, conftype, member, 0, initvalue) +#define VAR_NODUMP(varname,conftype,member,initvalue) \ + CONFIG_VAR_ETYPE(or_options_t, varname, conftype, member, \ + CVFLAG_NODUMP, initvalue) +#define VAR_INVIS(varname,conftype,member,initvalue) \ + CONFIG_VAR_ETYPE(or_options_t, varname, conftype, member, \ + CVFLAG_NODUMP|CVFLAG_INVISIBLE, initvalue) + #define V(member,conftype,initvalue) \ VAR(#member, conftype, member, initvalue) @@ -285,7 +292,7 @@ DUMMY_TYPECHECK_INSTANCE(or_options_t); #define VPORT(member) \ VAR(#member "Lines", LINELIST_V, member ## _lines, NULL), \ VAR(#member, LINELIST_S, member ## _lines, NULL), \ - VAR("__" #member, LINELIST_S, member ## _lines, NULL) + VAR_NODUMP("__" #member, LINELIST_S, member ## _lines, NULL) /** UINT64_MAX as a decimal string */ #define UINT64_MAX_STRING "18446744073709551615" @@ -682,15 +689,17 @@ static config_var_t option_vars_[] = { V(WarnPlaintextPorts, CSV, "23,109,110,143"), OBSOLETE("UseFilteringSSLBufferevents"), OBSOLETE("__UseFilteringSSLBufferevents"), - VAR("__ReloadTorrcOnSIGHUP", BOOL, ReloadTorrcOnSIGHUP, "1"), - VAR("__AllDirActionsPrivate", BOOL, AllDirActionsPrivate, "0"), - VAR("__DisablePredictedCircuits",BOOL,DisablePredictedCircuits, "0"), - VAR("__DisableSignalHandlers", BOOL, DisableSignalHandlers, "0"), - VAR("__LeaveStreamsUnattached",BOOL, LeaveStreamsUnattached, "0"), - VAR("__HashedControlSessionPassword", LINELIST, HashedControlSessionPassword, + VAR_NODUMP("__ReloadTorrcOnSIGHUP", BOOL, ReloadTorrcOnSIGHUP, "1"), + VAR_NODUMP("__AllDirActionsPrivate", BOOL, AllDirActionsPrivate, "0"), + VAR_NODUMP("__DisablePredictedCircuits",BOOL,DisablePredictedCircuits, "0"), + VAR_NODUMP("__DisableSignalHandlers", BOOL, DisableSignalHandlers, "0"), + VAR_NODUMP("__LeaveStreamsUnattached",BOOL, LeaveStreamsUnattached, "0"), + VAR_NODUMP("__HashedControlSessionPassword", LINELIST, + HashedControlSessionPassword, NULL), - VAR("__OwningControllerProcess",STRING,OwningControllerProcess, NULL), - VAR("__OwningControllerFD", UINT64, OwningControllerFD, UINT64_MAX_STRING), + VAR_NODUMP("__OwningControllerProcess",STRING,OwningControllerProcess, NULL), + VAR_NODUMP("__OwningControllerFD", UINT64, OwningControllerFD, + UINT64_MAX_STRING), V(MinUptimeHidServDirectoryV2, INTERVAL, "96 hours"), V(TestingServerDownloadInitialDelay, CSV_INTERVAL, "0"), V(TestingClientDownloadInitialDelay, CSV_INTERVAL, "0"), @@ -743,7 +752,8 @@ static config_var_t option_vars_[] = { V(TestingDirAuthVoteGuardIsStrict, BOOL, "0"), V_D(TestingDirAuthVoteHSDir, ROUTERSET, NULL), V(TestingDirAuthVoteHSDirIsStrict, BOOL, "0"), - VAR("___UsingTestNetworkDefaults", BOOL, UsingTestNetworkDefaults_, "0"), + VAR_INVIS("___UsingTestNetworkDefaults", BOOL, UsingTestNetworkDefaults_, + "0"), END_OF_CONFIG_VARS }; @@ -783,7 +793,8 @@ static const config_var_t testing_tor_network_defaults[] = { V(TestingDirConnectionMaxStall, INTERVAL, "30 seconds"), V(TestingEnableConnBwEvent, BOOL, "1"), V(TestingEnableCellStatsEvent, BOOL, "1"), - VAR("___UsingTestNetworkDefaults", BOOL, UsingTestNetworkDefaults_, "1"), + VAR_INVIS("___UsingTestNetworkDefaults", BOOL, UsingTestNetworkDefaults_, + "1"), V(RendPostPeriod, INTERVAL, "2 minutes"), END_OF_CONFIG_VARS @@ -8180,7 +8191,7 @@ getinfo_helper_config(control_connection_t *conn, for (i = 0; option_vars_[i].member.name; ++i) { const config_var_t *var = &option_vars_[i]; /* don't tell controller about triple-underscore options */ - if (!strncmp(option_vars_[i].member.name, "___", 3)) + if (option_vars_[i].flags & CVFLAG_INVISIBLE) continue; const char *type = struct_var_get_typename(&var->member); if (!type) diff --git a/src/app/config/confparse.c b/src/app/config/confparse.c index 0d19974d70..450ff5e083 100644 --- a/src/app/config/confparse.c +++ b/src/app/config/confparse.c @@ -642,7 +642,7 @@ config_dump(const config_format_t *fmt, const void *default_options, continue; } /* Don't save 'hidden' control variables. */ - if (!strcmpstart(fmt->vars[i].member.name, "__")) + if (fmt->vars[i].flags & CVFLAG_NODUMP) continue; if (minimal && config_is_same(fmt, options, defaults, fmt->vars[i].member.name)) diff --git a/src/lib/conf/conftypes.h b/src/lib/conf/conftypes.h index 6a44fb92ed..3b754e07be 100644 --- a/src/lib/conf/conftypes.h +++ b/src/lib/conf/conftypes.h @@ -110,6 +110,16 @@ typedef struct struct_magic_decl_t { * fetch this option should produce a warning. **/ #define CVFLAG_OBSOLETE (1u<<0) +/** + * Flag to indicate that an option is undumpable. An undumpable option is + * never saved to disk, and is prefixed with __. + **/ +#define CVFLAG_NODUMP (1u<<1) +/** + * Flag to indicate that an option is "invisible". An invisible option + * is always undumpable, and we don't tell the controller about it. + **/ +#define CVFLAG_INVISIBLE (1u<<2) /** A variable allowed in the configuration file or on the command line. */ typedef struct config_var_t { From e8790971f6120cc3e4fd2acd41f5dd0512f52068 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 19 Jun 2019 16:24:07 -0400 Subject: [PATCH 1182/2557] changes file for ticket 30914 --- changes/ticket30914 | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 changes/ticket30914 diff --git a/changes/ticket30914 b/changes/ticket30914 new file mode 100644 index 0000000000..c8c008b3d1 --- /dev/null +++ b/changes/ticket30914 @@ -0,0 +1,4 @@ + o Code simplification and refactoring: + - Lower another layer of object management from confparse.c to + a more general tool. Now typed structure members are accessible + via an abstract type. Implements ticket 30914. From cd1de99468aa4d7b48c9f3bcb7c7d4fb5b3bfe9a Mon Sep 17 00:00:00 2001 From: teor Date: Sun, 2 Jun 2019 17:46:58 +1000 Subject: [PATCH 1183/2557] resolve: consistently parse IP addresses in square brackets When parsing addreses via Tor's internal DNS lookup API: * reject IPv4 addresses in square brackets (with or without a port), * accept IPv6 addresses in square brackets (with or without a port), and * accept IPv6 addresses without square brackets, as long as they have no port. This change completes the work started in 23082, making address parsing consistent between tor's internal DNS lookup and address parsing APIs. Fixes bug 30721; bugfix on 0.2.1.5-alpha. --- changes/bug30721 | 6 ++++ src/lib/net/address.c | 31 +++++++++++++++--- src/lib/net/resolve.c | 75 ++++++++++++++++++++++--------------------- 3 files changed, 70 insertions(+), 42 deletions(-) create mode 100644 changes/bug30721 diff --git a/changes/bug30721 b/changes/bug30721 new file mode 100644 index 0000000000..5e03653005 --- /dev/null +++ b/changes/bug30721 @@ -0,0 +1,6 @@ + o Minor bugfixes (networking): + - When parsing addreses via Tor's internal DNS lookup API, reject IPv4 + addresses in square brackets, and accept IPv6 addresses in square + brackets. This change completes the work started in 23082, making + address parsing consistent between tor's internal DNS lookup and address + parsing APIs. Fixes bug 30721; bugfix on 0.2.1.5-alpha. diff --git a/src/lib/net/address.c b/src/lib/net/address.c index 546af800a9..4989e4ab2b 100644 --- a/src/lib/net/address.c +++ b/src/lib/net/address.c @@ -373,7 +373,8 @@ tor_addr_to_str(char *dest, const tor_addr_t *addr, size_t len, int decorate) * * If accept_regular is set and the address is in neither recognized * reverse lookup hostname format, try parsing the address as a regular - * IPv4 or IPv6 address too. + * IPv4 or IPv6 address too. This mode will accept IPv6 addresses with or + * without square brackets. */ int tor_addr_parse_PTR_name(tor_addr_t *result, const char *address, @@ -1204,6 +1205,10 @@ tor_addr_parse(tor_addr_t *addr, const char *src) tor_assert(addr && src); + /* Clear the address before starting, to avoid returning uninitialised data. + */ + memset(addr, 0, sizeof(tor_addr_t)); + size_t len = strlen(src); if (len && src[0] == '[' && src[len - 1] == ']') { @@ -1718,6 +1723,11 @@ get_interface_address6_list,(int severity, * form "ip" or "ip:0". Otherwise, accept those forms, and set * *port_out to default_port. * + * This function accepts: + * - IPv6 address and port, when the IPv6 address is in square brackets, + * - IPv6 address with square brackets, + * - IPv6 address without square brackets. + * * Return 0 on success, -1 on failure. */ int tor_addr_port_parse(int severity, const char *addrport, @@ -1755,9 +1765,17 @@ tor_addr_port_parse(int severity, const char *addrport, } /** Given an address of the form "host[:port]", try to divide it into its host - * and port portions, setting *address_out to a newly allocated string - * holding the address portion and *port_out to the port (or 0 if no - * port is given). Return 0 on success, -1 on failure. */ + * and port portions. + * + * Like tor_addr_port_parse(), this function accepts: + * - IPv6 address and port, when the IPv6 address is in square brackets, + * - IPv6 address with square brackets, + * - IPv6 address without square brackets. + * + * Sets *address_out to a newly allocated string holding the address + * portion, and *port_out to the port (or 0 if no port is given). + * + * Return 0 on success, -1 on failure. */ int tor_addr_port_split(int severity, const char *addrport, char **address_out, uint16_t *port_out) @@ -1766,8 +1784,11 @@ tor_addr_port_split(int severity, const char *addrport, tor_assert(addrport); tor_assert(address_out); tor_assert(port_out); + /* We need to check for IPv6 manually because the logic below doesn't - * do a good job on IPv6 addresses that lack a port. */ + * do a good job on IPv6 addresses that lack a port. + * If an IPv6 address without square brackets is ambiguous, it gets parsed + * here as an address, rather than address:port. */ if (tor_addr_parse(&a_tmp, addrport) == AF_INET6) { *port_out = 0; *address_out = tor_strdup(addrport); diff --git a/src/lib/net/resolve.c b/src/lib/net/resolve.c index 49c263faa2..71c033fe7a 100644 --- a/src/lib/net/resolve.c +++ b/src/lib/net/resolve.c @@ -35,6 +35,8 @@ * *addr to the proper IP address, in host byte order. Returns 0 * on success, -1 on failure; 1 on transient failure. * + * This function only accepts IPv4 addresses. + * * (This function exists because standard windows gethostbyname * doesn't treat raw IP addresses properly.) */ @@ -62,6 +64,9 @@ tor_lookup_hostname,(const char *name, uint32_t *addr)) * preferred family, though another one may be returned if only one * family is implemented for this address. * + * Like tor_addr_parse(), this function accepts IPv6 addresses with or without + * square brackets. + * * Return 0 on success, -1 on failure; 1 on transient failure. */ MOCK_IMPL(int, @@ -70,26 +75,39 @@ tor_addr_lookup,(const char *name, uint16_t family, tor_addr_t *addr)) /* Perhaps eventually this should be replaced by a tor_getaddrinfo or * something. */ - struct in_addr iaddr; - struct in6_addr iaddr6; + int parsed_family = 0; + tor_assert(name); tor_assert(addr); tor_assert(family == AF_INET || family == AF_INET6 || family == AF_UNSPEC); + + /* Clear address before starting, to avoid returning uninitialised data */ + memset(addr, 0, sizeof(tor_addr_t)); + if (!*name) { /* Empty address is an error. */ return -1; - } else if (tor_inet_pton(AF_INET, name, &iaddr)) { + } + + /* Is it an IP address? */ + parsed_family = tor_addr_parse(addr, name); + + if (parsed_family == AF_INET) { /* It's an IPv4 IP. */ - if (family == AF_INET6) + if (family == AF_INET6) { + memset(addr, 0, sizeof(tor_addr_t)); return -1; - tor_addr_from_in(addr, &iaddr); + } return 0; - } else if (tor_inet_pton(AF_INET6, name, &iaddr6)) { - if (family == AF_INET) + } else if (parsed_family == AF_INET6) { + if (family == AF_INET) { + memset(addr, 0, sizeof(tor_addr_t)); return -1; - tor_addr_from_in6(addr, &iaddr6); + } return 0; } else { + /* Clear the address after a failed tor_addr_parse(). */ + memset(addr, 0, sizeof(tor_addr_t)); #ifdef HAVE_GETADDRINFO int err; struct addrinfo *res=NULL, *res_p; @@ -179,52 +197,35 @@ tor_addr_lookup,(const char *name, uint16_t family, tor_addr_t *addr)) /** Parse an address or address-port combination from s, resolve the * address as needed, and put the result in addr_out and (optionally) - * port_out. Return 0 on success, negative on failure. */ + * port_out. + * + * Like tor_addr_port_parse(), this function accepts: + * - IPv6 address and port, when the IPv6 address is in square brackets, + * - IPv6 address with square brackets, + * - IPv6 address without square brackets. + * + * Return 0 on success, negative on failure. */ int tor_addr_port_lookup(const char *s, tor_addr_t *addr_out, uint16_t *port_out) { - const char *port; tor_addr_t addr; uint16_t portval; char *tmp = NULL; + int rv = 0; tor_assert(s); tor_assert(addr_out); s = eat_whitespace(s); - if (*s == '[') { - port = strstr(s, "]"); - if (!port) - goto err; - tmp = tor_strndup(s+1, port-(s+1)); - port = port+1; - if (*port == ':') - port++; - else - port = NULL; - } else { - port = strchr(s, ':'); - if (port) - tmp = tor_strndup(s, port-s); - else - tmp = tor_strdup(s); - if (port) - ++port; - } + rv = tor_addr_port_split(LOG_WARN, s, &tmp, &portval); + if (rv < 0) + goto err; if (tor_addr_lookup(tmp, AF_UNSPEC, &addr) != 0) goto err; tor_free(tmp); - if (port) { - portval = (int) tor_parse_long(port, 10, 1, 65535, NULL, NULL); - if (!portval) - goto err; - } else { - portval = 0; - } - if (port_out) *port_out = portval; tor_addr_copy(addr_out, &addr); From 29cf64c8389fd4de1e4419cb8ac30b2ca582ba5e Mon Sep 17 00:00:00 2001 From: teor Date: Sun, 2 Jun 2019 18:20:25 +1000 Subject: [PATCH 1184/2557] resolve: refactor address family logic in tor_addr_lookup() Cleanup after 30721. --- src/lib/net/resolve.c | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/src/lib/net/resolve.c b/src/lib/net/resolve.c index 71c033fe7a..1b68a0b33a 100644 --- a/src/lib/net/resolve.c +++ b/src/lib/net/resolve.c @@ -81,30 +81,24 @@ tor_addr_lookup,(const char *name, uint16_t family, tor_addr_t *addr)) tor_assert(addr); tor_assert(family == AF_INET || family == AF_INET6 || family == AF_UNSPEC); - /* Clear address before starting, to avoid returning uninitialised data */ - memset(addr, 0, sizeof(tor_addr_t)); - if (!*name) { /* Empty address is an error. */ + memset(addr, 0, sizeof(tor_addr_t)); return -1; } /* Is it an IP address? */ parsed_family = tor_addr_parse(addr, name); - if (parsed_family == AF_INET) { - /* It's an IPv4 IP. */ - if (family == AF_INET6) { + if (parsed_family >= 0) { + /* If the IP address family matches, or was unspecified */ + if (parsed_family == family || family == AF_UNSPEC) { + return 0; + } else { + /* Clear the address before returning an error. */ memset(addr, 0, sizeof(tor_addr_t)); return -1; } - return 0; - } else if (parsed_family == AF_INET6) { - if (family == AF_INET) { - memset(addr, 0, sizeof(tor_addr_t)); - return -1; - } - return 0; } else { /* Clear the address after a failed tor_addr_parse(). */ memset(addr, 0, sizeof(tor_addr_t)); From fb93646c1c2e97a978045019403d9cfb43f55cbc Mon Sep 17 00:00:00 2001 From: teor Date: Sun, 2 Jun 2019 18:42:01 +1000 Subject: [PATCH 1185/2557] resolve: split sub-functions out of tor_addr_lookup() And remove the practracker exception for tor_addr_lookup(). Cleanup after 30721. --- scripts/maint/practracker/exceptions.txt | 1 - src/lib/net/resolve.c | 221 ++++++++++++++--------- 2 files changed, 134 insertions(+), 88 deletions(-) diff --git a/scripts/maint/practracker/exceptions.txt b/scripts/maint/practracker/exceptions.txt index 3a32e97beb..4c1ae1ba51 100644 --- a/scripts/maint/practracker/exceptions.txt +++ b/scripts/maint/practracker/exceptions.txt @@ -272,7 +272,6 @@ problem function-size /src/lib/math/prob_distr.c:sample_uniform_interval() 145 problem function-size /src/lib/net/address.c:tor_addr_parse_mask_ports() 198 problem function-size /src/lib/net/address.c:tor_addr_compare_masked() 111 problem function-size /src/lib/net/inaddr.c:tor_inet_pton() 107 -problem function-size /src/lib/net/resolve.c:tor_addr_lookup() 110 problem function-size /src/lib/net/socketpair.c:tor_ersatz_socketpair() 102 problem function-size /src/lib/osinfo/uname.c:get_uname() 116 problem function-size /src/lib/process/process_unix.c:process_unix_exec() 220 diff --git a/src/lib/net/resolve.c b/src/lib/net/resolve.c index 1b68a0b33a..a70fb0a823 100644 --- a/src/lib/net/resolve.c +++ b/src/lib/net/resolve.c @@ -58,6 +58,116 @@ tor_lookup_hostname,(const char *name, uint32_t *addr)) return -1; } +#ifdef HAVE_GETADDRINFO + +/* Host lookup helper for tor_addr_lookup(), when getaddrinfo() is + * available on this system. + * + * See tor_addr_lookup() for details. + */ +static int +tor_addr_lookup_host_getaddrinfo(const char *name, + uint16_t family, + tor_addr_t *addr) +{ + int err; + struct addrinfo *res=NULL, *res_p; + struct addrinfo *best=NULL; + struct addrinfo hints; + int result = -1; + memset(&hints, 0, sizeof(hints)); + hints.ai_family = family; + hints.ai_socktype = SOCK_STREAM; + err = tor_getaddrinfo(name, NULL, &hints, &res); + /* The check for 'res' here shouldn't be necessary, but it makes static + * analysis tools happy. */ + if (!err && res) { + best = NULL; + for (res_p = res; res_p; res_p = res_p->ai_next) { + if (family == AF_UNSPEC) { + if (res_p->ai_family == AF_INET) { + best = res_p; + break; + } else if (res_p->ai_family == AF_INET6 && !best) { + best = res_p; + } + } else if (family == res_p->ai_family) { + best = res_p; + break; + } + } + if (!best) + best = res; + if (best->ai_family == AF_INET) { + tor_addr_from_in(addr, + &((struct sockaddr_in*)best->ai_addr)->sin_addr); + result = 0; + } else if (best->ai_family == AF_INET6) { + tor_addr_from_in6(addr, + &((struct sockaddr_in6*)best->ai_addr)->sin6_addr); + result = 0; + } + tor_freeaddrinfo(res); + return result; + } + return (err == EAI_AGAIN) ? 1 : -1; +} + +#else /* !(defined(HAVE_GETADDRINFO)) */ + +/* Host lookup helper for tor_addr_lookup(), which calls getaddrinfo(). + * Used when gethostbyname() is not available on this system. + * + * See tor_addr_lookup() for details. + */ +static int +tor_addr_lookup_host_gethostbyname(const char *name, + tor_addr_t *addr) +{ + struct hostent *ent; + int err; +#ifdef HAVE_GETHOSTBYNAME_R_6_ARG + char buf[2048]; + struct hostent hostent; + int r; + r = gethostbyname_r(name, &hostent, buf, sizeof(buf), &ent, &err); +#elif defined(HAVE_GETHOSTBYNAME_R_5_ARG) + char buf[2048]; + struct hostent hostent; + ent = gethostbyname_r(name, &hostent, buf, sizeof(buf), &err); +#elif defined(HAVE_GETHOSTBYNAME_R_3_ARG) + struct hostent_data data; + struct hostent hent; + memset(&data, 0, sizeof(data)); + err = gethostbyname_r(name, &hent, &data); + ent = err ? NULL : &hent; +#else + ent = gethostbyname(name); +#ifdef _WIN32 + err = WSAGetLastError(); +#else + err = h_errno; +#endif /* defined(_WIN32) */ +#endif /* defined(HAVE_GETHOSTBYNAME_R_6_ARG) || ... */ + if (ent) { + if (ent->h_addrtype == AF_INET) { + tor_addr_from_in(addr, (struct in_addr*) ent->h_addr); + } else if (ent->h_addrtype == AF_INET6) { + tor_addr_from_in6(addr, (struct in6_addr*) ent->h_addr); + } else { + tor_assert(0); // LCOV_EXCL_LINE: gethostbyname() returned bizarre type + } + return 0; + } +#ifdef _WIN32 + return (err == WSATRY_AGAIN) ? 1 : -1; +#else + return (err == TRY_AGAIN) ? 1 : -1; +#endif +} + +#endif /* defined(HAVE_GETADDRINFO) */ + /** Similar behavior to Unix gethostbyname: resolve name, and set * *addr to the proper IP address and family. The family * argument (which must be AF_INET, AF_INET6, or AF_UNSPEC) declares a @@ -76,6 +186,7 @@ tor_addr_lookup,(const char *name, uint16_t family, tor_addr_t *addr)) * something. */ int parsed_family = 0; + int result = -1; tor_assert(name); tor_assert(addr); @@ -83,8 +194,7 @@ tor_addr_lookup,(const char *name, uint16_t family, tor_addr_t *addr)) if (!*name) { /* Empty address is an error. */ - memset(addr, 0, sizeof(tor_addr_t)); - return -1; + goto permfail; } /* Is it an IP address? */ @@ -93,100 +203,37 @@ tor_addr_lookup,(const char *name, uint16_t family, tor_addr_t *addr)) if (parsed_family >= 0) { /* If the IP address family matches, or was unspecified */ if (parsed_family == family || family == AF_UNSPEC) { - return 0; + goto success; } else { - /* Clear the address before returning an error. */ - memset(addr, 0, sizeof(tor_addr_t)); - return -1; + goto permfail; } } else { /* Clear the address after a failed tor_addr_parse(). */ memset(addr, 0, sizeof(tor_addr_t)); #ifdef HAVE_GETADDRINFO - int err; - struct addrinfo *res=NULL, *res_p; - struct addrinfo *best=NULL; - struct addrinfo hints; - int result = -1; - memset(&hints, 0, sizeof(hints)); - hints.ai_family = family; - hints.ai_socktype = SOCK_STREAM; - err = tor_getaddrinfo(name, NULL, &hints, &res); - /* The check for 'res' here shouldn't be necessary, but it makes static - * analysis tools happy. */ - if (!err && res) { - best = NULL; - for (res_p = res; res_p; res_p = res_p->ai_next) { - if (family == AF_UNSPEC) { - if (res_p->ai_family == AF_INET) { - best = res_p; - break; - } else if (res_p->ai_family == AF_INET6 && !best) { - best = res_p; - } - } else if (family == res_p->ai_family) { - best = res_p; - break; - } - } - if (!best) - best = res; - if (best->ai_family == AF_INET) { - tor_addr_from_in(addr, - &((struct sockaddr_in*)best->ai_addr)->sin_addr); - result = 0; - } else if (best->ai_family == AF_INET6) { - tor_addr_from_in6(addr, - &((struct sockaddr_in6*)best->ai_addr)->sin6_addr); - result = 0; - } - tor_freeaddrinfo(res); - return result; - } - return (err == EAI_AGAIN) ? 1 : -1; + result = tor_addr_lookup_host_getaddrinfo(name, family, addr); + goto done; #else /* !(defined(HAVE_GETADDRINFO)) */ - struct hostent *ent; - int err; -#ifdef HAVE_GETHOSTBYNAME_R_6_ARG - char buf[2048]; - struct hostent hostent; - int r; - r = gethostbyname_r(name, &hostent, buf, sizeof(buf), &ent, &err); -#elif defined(HAVE_GETHOSTBYNAME_R_5_ARG) - char buf[2048]; - struct hostent hostent; - ent = gethostbyname_r(name, &hostent, buf, sizeof(buf), &err); -#elif defined(HAVE_GETHOSTBYNAME_R_3_ARG) - struct hostent_data data; - struct hostent hent; - memset(&data, 0, sizeof(data)); - err = gethostbyname_r(name, &hent, &data); - ent = err ? NULL : &hent; -#else - ent = gethostbyname(name); -#ifdef _WIN32 - err = WSAGetLastError(); -#else - err = h_errno; -#endif -#endif /* defined(HAVE_GETHOSTBYNAME_R_6_ARG) || ... */ - if (ent) { - if (ent->h_addrtype == AF_INET) { - tor_addr_from_in(addr, (struct in_addr*) ent->h_addr); - } else if (ent->h_addrtype == AF_INET6) { - tor_addr_from_in6(addr, (struct in6_addr*) ent->h_addr); - } else { - tor_assert(0); // LCOV_EXCL_LINE: gethostbyname() returned bizarre type - } - return 0; - } -#ifdef _WIN32 - return (err == WSATRY_AGAIN) ? 1 : -1; -#else - return (err == TRY_AGAIN) ? 1 : -1; -#endif + result = tor_addr_lookup_host_gethostbyname(name, addr); + goto done; #endif /* defined(HAVE_GETADDRINFO) */ } + + /* If we weren't successful, and haven't already set the result, + * assume it's a permanent failure */ + permfail: + result = -1; + goto done; + success: + result = 0; + + /* We have set the result, now it's time to clean up */ + done: + if (result) { + /* Clear the address on error */ + memset(addr, 0, sizeof(tor_addr_t)); + } + return result; } /** Parse an address or address-port combination from s, resolve the From 308d3002130cfb5b9ad2100dab25b0c4c83162f6 Mon Sep 17 00:00:00 2001 From: teor Date: Thu, 13 Jun 2019 21:14:36 +1000 Subject: [PATCH 1186/2557] address/resolve: try harder to avoid returning uninitialised data Cleanup after 30721. --- src/lib/net/address.c | 10 ++++++++-- src/lib/net/resolve.c | 11 ++++++++++- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/src/lib/net/address.c b/src/lib/net/address.c index 4989e4ab2b..75322ad886 100644 --- a/src/lib/net/address.c +++ b/src/lib/net/address.c @@ -1760,6 +1760,13 @@ tor_addr_port_parse(int severity, const char *addrport, retval = 0; done: + /* Clear the address and port on error, to avoid returning uninitialised or + * partly parsed data. + */ + if (retval == -1) { + memset(address_out, 0, sizeof(tor_addr_t)); + *port_out = 0; + } tor_free(addr_tmp); return retval; } @@ -1828,8 +1835,7 @@ tor_addr_port_split(int severity, const char *addrport, tor_free(address_); } - if (port_out) - *port_out = ok ? ((uint16_t) port_) : 0; + *port_out = ok ? ((uint16_t) port_) : 0; return ok ? 0 : -1; } diff --git a/src/lib/net/resolve.c b/src/lib/net/resolve.c index a70fb0a823..9555720883 100644 --- a/src/lib/net/resolve.c +++ b/src/lib/net/resolve.c @@ -47,6 +47,11 @@ tor_lookup_hostname,(const char *name, uint32_t *addr)) tor_addr_t myaddr; int ret; + if (BUG(!addr)) + return -1; + + *addr = 0; + if ((ret = tor_addr_lookup(name, AF_INET, &myaddr))) return ret; @@ -250,7 +255,7 @@ int tor_addr_port_lookup(const char *s, tor_addr_t *addr_out, uint16_t *port_out) { tor_addr_t addr; - uint16_t portval; + uint16_t portval = 0; char *tmp = NULL; int rv = 0; @@ -273,6 +278,10 @@ tor_addr_port_lookup(const char *s, tor_addr_t *addr_out, uint16_t *port_out) return 0; err: + /* Clear the address and port on error */ + memset(addr_out, 0, sizeof(tor_addr_t)); + if (port_out) + *port_out = 0; tor_free(tmp); return -1; } From adb8538e7b6a7a2531233797990f6d11be86a718 Mon Sep 17 00:00:00 2001 From: teor Date: Thu, 13 Jun 2019 21:21:19 +1000 Subject: [PATCH 1187/2557] address/resolve: Require square brackets on IPv6 address:ports When parsing addreses via Tor's internal address:port parsing and DNS lookup APIs, require IPv6 addresses with ports to have square brackets. But allow IPv6 addresses without ports, whether or not they have square brackets. Fixes bug 30721; bugfix on 0.2.1.5-alpha. --- changes/bug30721 | 6 +++- src/lib/net/address.c | 77 ++++++++++++++++++++++++++++++------------- src/lib/net/resolve.c | 40 ++++++++++++++++++++-- 3 files changed, 96 insertions(+), 27 deletions(-) diff --git a/changes/bug30721 b/changes/bug30721 index 5e03653005..5ea4a14625 100644 --- a/changes/bug30721 +++ b/changes/bug30721 @@ -1,6 +1,10 @@ - o Minor bugfixes (networking): + o Minor bugfixes (networking, IP addresses): - When parsing addreses via Tor's internal DNS lookup API, reject IPv4 addresses in square brackets, and accept IPv6 addresses in square brackets. This change completes the work started in 23082, making address parsing consistent between tor's internal DNS lookup and address parsing APIs. Fixes bug 30721; bugfix on 0.2.1.5-alpha. + - When parsing addreses via Tor's internal address:port parsing and + DNS lookup APIs, require IPv6 addresses with ports to have square + brackets. But allow IPv6 addresses without ports, whether or not they + have square brackets. Fixes bug 30721; bugfix on 0.2.1.5-alpha. diff --git a/src/lib/net/address.c b/src/lib/net/address.c index 75322ad886..0a2c84caf2 100644 --- a/src/lib/net/address.c +++ b/src/lib/net/address.c @@ -1188,27 +1188,28 @@ fmt_addr32(uint32_t addr) } /** Convert the string in src to a tor_addr_t addr. The string - * may be an IPv4 address, an IPv6 address, or an IPv6 address surrounded by - * square brackets. + * may be an IPv4 address, or an IPv6 address surrounded by square brackets. * - * Return an address family on success, or -1 if an invalid address string is - * provided. */ -int -tor_addr_parse(tor_addr_t *addr, const char *src) + * If allow_ipv6_without_brackets is true, also allow IPv6 addresses + * without brackets. + * + * Always rejects IPv4 addresses with brackets. + * + * Returns an address family on success, or -1 if an invalid address string is + * provided. */ +static int +tor_addr_parse_impl(tor_addr_t *addr, const char *src, + bool allow_ipv6_without_brackets) { /* Holds substring of IPv6 address after removing square brackets */ char *tmp = NULL; - int result; + int result = -1; struct in_addr in_tmp; struct in6_addr in6_tmp; int brackets_detected = 0; tor_assert(addr && src); - /* Clear the address before starting, to avoid returning uninitialised data. - */ - memset(addr, 0, sizeof(tor_addr_t)); - size_t len = strlen(src); if (len && src[0] == '[' && src[len - 1] == ']') { @@ -1216,21 +1217,46 @@ tor_addr_parse(tor_addr_t *addr, const char *src) src = tmp = tor_strndup(src+1, strlen(src)-2); } - if (tor_inet_pton(AF_INET6, src, &in6_tmp) > 0) { - result = AF_INET6; - tor_addr_from_in6(addr, &in6_tmp); - } else if (!brackets_detected && - tor_inet_pton(AF_INET, src, &in_tmp) > 0) { - result = AF_INET; - tor_addr_from_in(addr, &in_tmp); - } else { - result = -1; + /* Try to parse an IPv6 address if it has brackets, or if IPv6 addresses + * without brackets are allowed */ + if (brackets_detected || allow_ipv6_without_brackets) { + if (tor_inet_pton(AF_INET6, src, &in6_tmp) > 0) { + result = AF_INET6; + tor_addr_from_in6(addr, &in6_tmp); + } + } + + /* Try to parse an IPv4 address without brackets */ + if (!brackets_detected) { + if (tor_inet_pton(AF_INET, src, &in_tmp) > 0) { + result = AF_INET; + tor_addr_from_in(addr, &in_tmp); + } + } + + /* Clear the address on error, to avoid returning uninitialised or partly + * parsed data. + */ + if (result == -1) { + memset(addr, 0, sizeof(tor_addr_t)); } tor_free(tmp); return result; } +/** Convert the string in src to a tor_addr_t addr. The string + * may be an IPv4 address, an IPv6 address, or an IPv6 address surrounded by + * square brackets. + * + * Returns an address family on success, or -1 if an invalid address string is + * provided. */ +int +tor_addr_parse(tor_addr_t *addr, const char *src) +{ + return tor_addr_parse_impl(addr, src, 1); +} + #ifdef HAVE_IFADDRS_TO_SMARTLIST /* * Convert a linked list consisting of ifaddrs structures @@ -1737,6 +1763,7 @@ tor_addr_port_parse(int severity, const char *addrport, int retval = -1; int r; char *addr_tmp = NULL; + bool has_port; tor_assert(addrport); tor_assert(address_out); @@ -1746,15 +1773,19 @@ tor_addr_port_parse(int severity, const char *addrport, if (r < 0) goto done; - if (!*port_out) { + has_port = !! *port_out; + /* If there's no port, use the default port, or fail if there is no default + */ + if (!has_port) { if (default_port >= 0) *port_out = default_port; else goto done; } - /* make sure that address_out is an IP address */ - if (tor_addr_parse(address_out, addr_tmp) < 0) + /* Make sure that address_out is an IP address. + * If there is no port in addrport, allow IPv6 addresses without brackets. */ + if (tor_addr_parse_impl(address_out, addr_tmp, !has_port) < 0) goto done; retval = 0; diff --git a/src/lib/net/resolve.c b/src/lib/net/resolve.c index 9555720883..1097e36bd8 100644 --- a/src/lib/net/resolve.c +++ b/src/lib/net/resolve.c @@ -258,32 +258,66 @@ tor_addr_port_lookup(const char *s, tor_addr_t *addr_out, uint16_t *port_out) uint16_t portval = 0; char *tmp = NULL; int rv = 0; + int result; tor_assert(s); tor_assert(addr_out); s = eat_whitespace(s); + /* Try parsing s as an address:port first, so we don't have to duplicate + * the logic that rejects IPv6:Port with no square brackets. */ + rv = tor_addr_port_parse(LOG_WARN, s, &addr, &portval, 0); + /* That was easy, no DNS required. */ + if (rv == 0) + goto success; + + /* Now let's check for malformed IPv6 addresses and ports: + * tor_addr_port_parse() requires squared brackes if there is a port, + * and we want tor_addr_port_lookup() to have the same requirement. + * But we strip the port using tor_addr_port_split(), so tor_addr_lookup() + * only sees the address, and will accept it without square brackets. */ + int family = tor_addr_parse(&addr, s); + /* If tor_addr_parse() succeeds where tor_addr_port_parse() failed, we need + * to reject this address as malformed. */ + if (family >= 0) { + /* Double-check it's an IPv6 address. If not, we have a parsing bug. + */ + tor_assertf_nonfatal(family == AF_INET6, + "Wrong family: %d (should be IPv6: %d) which " + "failed IP:port parsing, but passed IP parsing. " + "input string: '%s'; parsed address: '%s'.", + family, AF_INET6, s, fmt_addr(&addr)); + goto err; + } + + /* Now we have a hostname. Let's split off the port, if any. */ rv = tor_addr_port_split(LOG_WARN, s, &tmp, &portval); if (rv < 0) goto err; + /* And feed the hostname to the lookup function. */ if (tor_addr_lookup(tmp, AF_UNSPEC, &addr) != 0) goto err; - tor_free(tmp); + success: if (port_out) *port_out = portval; tor_addr_copy(addr_out, &addr); + result = 0; + goto done; - return 0; err: /* Clear the address and port on error */ memset(addr_out, 0, sizeof(tor_addr_t)); if (port_out) *port_out = 0; + result = -1; + + /* We have set the result, now it's time to clean up */ + done: tor_free(tmp); - return -1; + return result; } #ifdef USE_SANDBOX_GETADDRINFO From 1d3d6bf6b18ff7d3462d48d33d3e982f063f35d6 Mon Sep 17 00:00:00 2001 From: teor Date: Thu, 13 Jun 2019 21:22:27 +1000 Subject: [PATCH 1188/2557] test/addr: Add unit tests for the fixes in 30721 --- src/test/test_addr.c | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/src/test/test_addr.c b/src/test/test_addr.c index 05d8bf6c7b..a961a46854 100644 --- a/src/test/test_addr.c +++ b/src/test/test_addr.c @@ -788,6 +788,10 @@ test_addr_parse(void *arg) r= tor_addr_parse(&addr, "11:22:33:44:55:66:77:88:1.2.3.4"); tt_int_op(r,OP_EQ, -1); + /* IPv6 address with port and no brackets */ + r= tor_addr_parse(&addr, "11:22::33:44:12345"); + tt_int_op(r, OP_EQ, -1); + /* Correct call. */ r= tor_addr_port_parse(LOG_DEBUG, "192.0.2.1:1234", @@ -834,6 +838,18 @@ test_addr_parse(void *arg) tt_int_op(r, OP_EQ, 0); tt_int_op(port,OP_EQ,400); + /* Allow IPv6 without square brackets, when there is no port */ + r= tor_addr_port_parse(LOG_DEBUG, + "::1", + &addr, &port, -1); + tt_int_op(r, OP_EQ, -1); + + r= tor_addr_port_parse(LOG_DEBUG, + "::1", + &addr, &port, 600); + tt_int_op(r, OP_EQ, 0); + tt_int_op(port,OP_EQ,600); + /* Bad port. */ r= tor_addr_port_parse(LOG_DEBUG, "192.0.2.2:66666", @@ -854,12 +870,29 @@ test_addr_parse(void *arg) &addr, &port, 200); tt_int_op(r, OP_EQ, -1); - /* Bad IP address */ + /* Bad IPv4 address */ r= tor_addr_port_parse(LOG_DEBUG, "192.0.2:1234", &addr, &port, -1); tt_int_op(r, OP_EQ, -1); + /* Bad IPv4 address and port: brackets */ + r= tor_addr_port_parse(LOG_DEBUG, + "[192.0.2.3]:12345", + &addr, &port, -1); + tt_int_op(r, OP_EQ, -1); + + /* Bad IPv6 addresses and ports: no brackets */ + r= tor_addr_port_parse(LOG_DEBUG, + "::1:12345", + &addr, &port, -1); + tt_int_op(r, OP_EQ, -1); + + r= tor_addr_port_parse(LOG_DEBUG, + "11:22::33:44:12345", + &addr, &port, -1); + tt_int_op(r, OP_EQ, -1); + /* Make sure that the default port has lower priority than the real one */ r= tor_addr_port_parse(LOG_DEBUG, From 5a3770dc6cf5490b0e1c3951e8e12c815ec7891e Mon Sep 17 00:00:00 2001 From: teor Date: Thu, 13 Jun 2019 22:30:03 +1000 Subject: [PATCH 1189/2557] test/addr: Stop repeating so much code in test_addr_parse() Cleanup after 30721. --- src/test/test_addr.c | 286 +++++++++++++++++++++++-------------------- 1 file changed, 151 insertions(+), 135 deletions(-) diff --git a/src/test/test_addr.c b/src/test/test_addr.c index a961a46854..f8fa7b9503 100644 --- a/src/test/test_addr.c +++ b/src/test/test_addr.c @@ -724,188 +724,204 @@ test_addr_ip6_helpers(void *arg) ; } +/* Test that addr_str successfully parses, and: + * - the address has family expect_family, + * - the fmt_decorated result of tor_addr_to_str() is expect_str. + */ +#define TEST_ADDR_PARSE_FMT(addr_str, expect_family, fmt_decorated, \ + expect_str) \ + STMT_BEGIN \ + int r; \ + tor_addr_t addr; \ + char buf[TOR_ADDR_BUF_LEN]; \ + const char *sv; \ + r = tor_addr_parse(&addr, addr_str); \ + tt_int_op(r, OP_EQ, expect_family); \ + sv = tor_addr_to_str(buf, &addr, sizeof(buf), fmt_decorated); \ + tt_str_op(sv, OP_EQ, buf); \ + tt_str_op(buf, OP_EQ, expect_str); \ + STMT_END + +/* Test that addr_str fails to parse, and: + * - the returned address is null. + */ +#define TEST_ADDR_PARSE_XFAIL(addr_str) \ + STMT_BEGIN \ + int r; \ + tor_addr_t addr; \ + r = tor_addr_parse(&addr, addr_str); \ + tt_int_op(r, OP_EQ, -1); \ + tt_assert(tor_addr_is_null(&addr)); \ + STMT_END + +/* Test that addr_port_str and default_port successfully parse, and: + * - the address has family expect_family, + * - the fmt_decorated result of tor_addr_to_str() is expect_str, + * - the port is expect_port. + */ +#define TEST_ADDR_PORT_PARSE_FMT(addr_port_str, default_port, expect_family, \ + fmt_decorated, expect_str, expect_port) \ + STMT_BEGIN \ + int r; \ + tor_addr_t addr; \ + uint16_t port; \ + char buf[TOR_ADDR_BUF_LEN]; \ + const char *sv; \ + r = tor_addr_port_parse(LOG_DEBUG, addr_port_str, &addr, &port, \ + default_port); \ + tt_int_op(r, OP_EQ, 0); \ + tt_int_op(tor_addr_family(&addr), OP_EQ, expect_family); \ + sv = tor_addr_to_str(buf, &addr, sizeof(buf), fmt_decorated); \ + tt_str_op(sv, OP_EQ, buf); \ + tt_str_op(buf, OP_EQ, expect_str); \ + tt_int_op(port, OP_EQ, expect_port); \ + STMT_END + +/* Test that addr_port_str and default_port fail to parse, and: + * - the returned address is null, + * - the returned port is 0. + */ +#define TEST_ADDR_PORT_PARSE_XFAIL(addr_port_str, default_port) \ + STMT_BEGIN \ + int r; \ + tor_addr_t addr; \ + uint16_t port; \ + r = tor_addr_port_parse(LOG_DEBUG, addr_port_str, &addr, &port, \ + default_port); \ + tt_int_op(r, OP_EQ, -1); \ + tt_assert(tor_addr_is_null(&addr)); \ + tt_int_op(port, OP_EQ, 0); \ + STMT_END + +/* Test that addr_str successfully parses as a canonical IPv4 address. */ +#define TEST_ADDR_V4_PARSE_CANONICAL(addr_str) \ + TEST_ADDR_PARSE_FMT(addr_str, AF_INET, 0, addr_str) + +/* Test that addr_str successfully parses as a canonical fmt_decorated + * IPv6 address. */ +#define TEST_ADDR_V6_PARSE_CANONICAL(addr_str, fmt_decorated) \ + TEST_ADDR_PARSE_FMT(addr_str, AF_INET6, fmt_decorated, addr_str) + +/* Test that addr_str successfully parses, and the fmt_decorated canonical + * IPv6 string is expect_str. */ +#define TEST_ADDR_V6_PARSE(addr_str, fmt_decorated, expect_str) \ + TEST_ADDR_PARSE_FMT(addr_str, AF_INET6, fmt_decorated, expect_str) + +/* Test that addr_port_str successfully parses to the canonical IPv4 address + * string expect_str, and port expect_port. */ +#define TEST_ADDR_V4_PORT_PARSE(addr_port_str, expect_str, expect_port) \ + TEST_ADDR_PORT_PARSE_FMT(addr_port_str, -1, AF_INET, 0, expect_str, \ + expect_port) + +/* Test that addr_port_str successfully parses to the canonical undecorated + * IPv6 address string expect_str, and port expect_port. */ +#define TEST_ADDR_V6_PORT_PARSE(addr_port_str, expect_str, expect_port) \ + TEST_ADDR_PORT_PARSE_FMT(addr_port_str, -1, AF_INET6, 0, expect_str, \ + expect_port) + +/* Test that addr_port_str and default_port successfully parse to the canonical + * IPv4 address string expect_str, and port expect_port. */ +#define TEST_ADDR_V4_PORT_DEF_PARSE(addr_port_str, default_port, expect_str, \ + expect_port) \ + TEST_ADDR_PORT_PARSE_FMT(addr_port_str, default_port, AF_INET, 0, \ + expect_str, expect_port) + +/* Test that addr_port_str successfully parses to the canonical undecorated + * IPv6 address string expect_str, and port expect_port. */ +#define TEST_ADDR_V6_PORT_DEF_PARSE(addr_port_str, default_port, expect_str, \ + expect_port) \ + TEST_ADDR_PORT_PARSE_FMT(addr_port_str, default_port, AF_INET6, 0, \ + expect_str, expect_port) + /** Test tor_addr_parse() and tor_addr_port_parse(). */ static void test_addr_parse(void *arg) { - int r; - tor_addr_t addr; - char buf[TOR_ADDR_BUF_LEN]; - uint16_t port = 0; - - /* Correct call. */ (void)arg; - r= tor_addr_parse(&addr, "192.0.2.1"); - tt_int_op(r,OP_EQ, AF_INET); - tor_addr_to_str(buf, &addr, sizeof(buf), 0); - tt_str_op(buf,OP_EQ, "192.0.2.1"); - r= tor_addr_parse(&addr, "11:22::33:44"); - tt_int_op(r,OP_EQ, AF_INET6); - tor_addr_to_str(buf, &addr, sizeof(buf), 0); - tt_str_op(buf,OP_EQ, "11:22::33:44"); + /* Correct calls. */ + TEST_ADDR_V4_PARSE_CANONICAL("192.0.2.1"); - r= tor_addr_parse(&addr, "[11:22::33:44]"); - tt_int_op(r,OP_EQ, AF_INET6); - tor_addr_to_str(buf, &addr, sizeof(buf), 0); - tt_str_op(buf,OP_EQ, "11:22::33:44"); + TEST_ADDR_V6_PARSE_CANONICAL("11:22::33:44", 0); + TEST_ADDR_V6_PARSE_CANONICAL("[11:22::33:44]", 1); - r= tor_addr_parse(&addr, "11:22:33:44:55:66:1.2.3.4"); - tt_int_op(r,OP_EQ, AF_INET6); - tor_addr_to_str(buf, &addr, sizeof(buf), 0); - tt_str_op(buf,OP_EQ, "11:22:33:44:55:66:102:304"); + TEST_ADDR_V6_PARSE("11:22:33:44:55:66:1.2.3.4", 0, + "11:22:33:44:55:66:102:304"); - r= tor_addr_parse(&addr, "11:22::33:44:1.2.3.4"); - tt_int_op(r,OP_EQ, AF_INET6); - tor_addr_to_str(buf, &addr, sizeof(buf), 0); - tt_str_op(buf,OP_EQ, "11:22::33:44:102:304"); + TEST_ADDR_V6_PARSE("11:22::33:44:1.2.3.4", 0, + "11:22::33:44:102:304"); /* Empty string. */ - r= tor_addr_parse(&addr, ""); - tt_int_op(r,OP_EQ, -1); + TEST_ADDR_PARSE_XFAIL(""); /* Square brackets around IPv4 address. */ - r= tor_addr_parse(&addr, "[192.0.2.1]"); - tt_int_op(r,OP_EQ, -1); + TEST_ADDR_PARSE_XFAIL("[192.0.2.1]"); /* Only left square bracket. */ - r= tor_addr_parse(&addr, "[11:22::33:44"); - tt_int_op(r,OP_EQ, -1); + TEST_ADDR_PARSE_XFAIL("[11:22::33:44"); /* Only right square bracket. */ - r= tor_addr_parse(&addr, "11:22::33:44]"); - tt_int_op(r,OP_EQ, -1); + TEST_ADDR_PARSE_XFAIL("11:22::33:44]"); /* Leading colon. */ - r= tor_addr_parse(&addr, ":11:22::33:44"); - tt_int_op(r,OP_EQ, -1); + TEST_ADDR_PARSE_XFAIL(":11:22::33:44"); /* Trailing colon. */ - r= tor_addr_parse(&addr, "11:22::33:44:"); - tt_int_op(r,OP_EQ, -1); + TEST_ADDR_PARSE_XFAIL("11:22::33:44:"); /* Too many hex words in IPv4-mapped IPv6 address. */ - r= tor_addr_parse(&addr, "11:22:33:44:55:66:77:88:1.2.3.4"); - tt_int_op(r,OP_EQ, -1); + TEST_ADDR_PARSE_XFAIL("11:22:33:44:55:66:77:88:1.2.3.4"); /* IPv6 address with port and no brackets */ - r= tor_addr_parse(&addr, "11:22::33:44:12345"); - tt_int_op(r, OP_EQ, -1); + TEST_ADDR_PARSE_XFAIL("11:22::33:44:12345"); - /* Correct call. */ - r= tor_addr_port_parse(LOG_DEBUG, - "192.0.2.1:1234", - &addr, &port, -1); - tt_int_op(r, OP_EQ, 0); - tor_addr_to_str(buf, &addr, sizeof(buf), 0); - tt_str_op(buf,OP_EQ, "192.0.2.1"); - tt_int_op(port,OP_EQ, 1234); + /* Correct calls. */ + TEST_ADDR_V4_PORT_PARSE("192.0.2.1:1234", "192.0.2.1", 1234); - r= tor_addr_port_parse(LOG_DEBUG, - "[::1]:1234", - &addr, &port, -1); - tt_int_op(r, OP_EQ, 0); - tor_addr_to_str(buf, &addr, sizeof(buf), 0); - tt_str_op(buf,OP_EQ, "::1"); - tt_int_op(port,OP_EQ, 1234); + TEST_ADDR_V6_PORT_PARSE("[::1]:1234", "::1", 1234); /* Domain name. */ - r= tor_addr_port_parse(LOG_DEBUG, - "torproject.org:1234", - &addr, &port, -1); - tt_int_op(r, OP_EQ, -1); + TEST_ADDR_PORT_PARSE_XFAIL("torproject.org:1234", -1); /* Only IP. */ - r= tor_addr_port_parse(LOG_DEBUG, - "192.0.2.2", - &addr, &port, -1); - tt_int_op(r, OP_EQ, -1); + TEST_ADDR_PORT_PARSE_XFAIL("192.0.2.2", -1); + TEST_ADDR_V4_PORT_DEF_PARSE("192.0.2.2", 200, "192.0.2.2", 200); - r= tor_addr_port_parse(LOG_DEBUG, - "192.0.2.2", - &addr, &port, 200); - tt_int_op(r, OP_EQ, 0); - tt_int_op(port,OP_EQ,200); + TEST_ADDR_PORT_PARSE_XFAIL("[::1]", -1); + TEST_ADDR_V6_PORT_DEF_PARSE("[::1]", 400, "::1", 400); - r= tor_addr_port_parse(LOG_DEBUG, - "[::1]", - &addr, &port, -1); - tt_int_op(r, OP_EQ, -1); - - r= tor_addr_port_parse(LOG_DEBUG, - "[::1]", - &addr, &port, 400); - tt_int_op(r, OP_EQ, 0); - tt_int_op(port,OP_EQ,400); - - /* Allow IPv6 without square brackets, when there is no port */ - r= tor_addr_port_parse(LOG_DEBUG, - "::1", - &addr, &port, -1); - tt_int_op(r, OP_EQ, -1); - - r= tor_addr_port_parse(LOG_DEBUG, - "::1", - &addr, &port, 600); - tt_int_op(r, OP_EQ, 0); - tt_int_op(port,OP_EQ,600); + /* Allow IPv6 without square brackets, when there is no port, but only if + * there is a default port */ + TEST_ADDR_PORT_PARSE_XFAIL("::1", -1); + TEST_ADDR_V6_PORT_DEF_PARSE("::1", 600, "::1", 600); /* Bad port. */ - r= tor_addr_port_parse(LOG_DEBUG, - "192.0.2.2:66666", - &addr, &port, -1); - tt_int_op(r, OP_EQ, -1); - r= tor_addr_port_parse(LOG_DEBUG, - "192.0.2.2:66666", - &addr, &port, 200); - tt_int_op(r, OP_EQ, -1); + TEST_ADDR_PORT_PARSE_XFAIL("192.0.2.2:66666", -1); + TEST_ADDR_PORT_PARSE_XFAIL("192.0.2.2:66666", 200); /* Only domain name */ - r= tor_addr_port_parse(LOG_DEBUG, - "torproject.org", - &addr, &port, -1); - tt_int_op(r, OP_EQ, -1); - r= tor_addr_port_parse(LOG_DEBUG, - "torproject.org", - &addr, &port, 200); - tt_int_op(r, OP_EQ, -1); + TEST_ADDR_PORT_PARSE_XFAIL("torproject.org", -1); + TEST_ADDR_PORT_PARSE_XFAIL("torproject.org", 200); /* Bad IPv4 address */ - r= tor_addr_port_parse(LOG_DEBUG, - "192.0.2:1234", - &addr, &port, -1); - tt_int_op(r, OP_EQ, -1); + TEST_ADDR_PORT_PARSE_XFAIL("192.0.2:1234", -1); + TEST_ADDR_PORT_PARSE_XFAIL("192.0.2:1234", 200); /* Bad IPv4 address and port: brackets */ - r= tor_addr_port_parse(LOG_DEBUG, - "[192.0.2.3]:12345", - &addr, &port, -1); - tt_int_op(r, OP_EQ, -1); + TEST_ADDR_PORT_PARSE_XFAIL("[192.0.2.3]:12345", -1); + TEST_ADDR_PORT_PARSE_XFAIL("[192.0.2.3]:12345", 200); /* Bad IPv6 addresses and ports: no brackets */ - r= tor_addr_port_parse(LOG_DEBUG, - "::1:12345", - &addr, &port, -1); - tt_int_op(r, OP_EQ, -1); + TEST_ADDR_PORT_PARSE_XFAIL("::1:12345", -1); + TEST_ADDR_PORT_PARSE_XFAIL("::1:12345", 200); - r= tor_addr_port_parse(LOG_DEBUG, - "11:22::33:44:12345", - &addr, &port, -1); - tt_int_op(r, OP_EQ, -1); + TEST_ADDR_PORT_PARSE_XFAIL("11:22::33:44:12345", -1); + TEST_ADDR_PORT_PARSE_XFAIL("11:22::33:44:12345", 200); /* Make sure that the default port has lower priority than the real one */ - r= tor_addr_port_parse(LOG_DEBUG, - "192.0.2.2:1337", - &addr, &port, 200); - tt_int_op(r, OP_EQ, 0); - tt_int_op(port,OP_EQ,1337); - - r= tor_addr_port_parse(LOG_DEBUG, - "[::1]:1369", - &addr, &port, 200); - tt_int_op(r, OP_EQ, 0); - tt_int_op(port,OP_EQ,1369); + TEST_ADDR_V4_PORT_DEF_PARSE("192.0.2.2:1337", 200, "192.0.2.2", 1337); + TEST_ADDR_V6_PORT_DEF_PARSE("[::1]:1369", 200, "::1", 1369); done: ; From 2dbde3617f5d117e708dbd83410c8c73dc134902 Mon Sep 17 00:00:00 2001 From: teor Date: Thu, 13 Jun 2019 23:14:16 +1000 Subject: [PATCH 1190/2557] test/addr: Increase coverage in test_addr_parse() Do as many tests as possible for each input string. Then remove some redundant test cases. Cleanup after 30721. --- src/test/test_addr.c | 130 +++++++++++++++++++++++-------------------- 1 file changed, 70 insertions(+), 60 deletions(-) diff --git a/src/test/test_addr.c b/src/test/test_addr.c index f8fa7b9503..6a6796c987 100644 --- a/src/test/test_addr.c +++ b/src/test/test_addr.c @@ -793,45 +793,72 @@ test_addr_ip6_helpers(void *arg) tt_int_op(port, OP_EQ, 0); \ STMT_END -/* Test that addr_str successfully parses as a canonical IPv4 address. */ +/* Test that addr_str successfully parses as a canonical IPv4 address. + * Use tor_addr_parse(), and tor_addr_port_parse() with a default port. + * Also check that tor_addr_port_parse() fails without a default port. */ #define TEST_ADDR_V4_PARSE_CANONICAL(addr_str) \ - TEST_ADDR_PARSE_FMT(addr_str, AF_INET, 0, addr_str) + STMT_BEGIN \ + TEST_ADDR_PARSE_FMT(addr_str, AF_INET, 0, addr_str); \ + TEST_ADDR_PORT_PARSE_FMT(addr_str, 111, AF_INET, 0, \ + addr_str, 111); \ + TEST_ADDR_PORT_PARSE_XFAIL(addr_str, -1); \ + STMT_END /* Test that addr_str successfully parses as a canonical fmt_decorated - * IPv6 address. */ + * IPv6 address. + * Use tor_addr_parse(), and tor_addr_port_parse() with a default port. + * Also check that tor_addr_port_parse() fails without a default port. */ #define TEST_ADDR_V6_PARSE_CANONICAL(addr_str, fmt_decorated) \ - TEST_ADDR_PARSE_FMT(addr_str, AF_INET6, fmt_decorated, addr_str) + STMT_BEGIN \ + TEST_ADDR_PARSE_FMT(addr_str, AF_INET6, fmt_decorated, addr_str); \ + TEST_ADDR_PORT_PARSE_FMT(addr_str, 222, AF_INET6, fmt_decorated, \ + addr_str, 222); \ + TEST_ADDR_PORT_PARSE_XFAIL(addr_str, -1); \ + STMT_END /* Test that addr_str successfully parses, and the fmt_decorated canonical - * IPv6 string is expect_str. */ + * IPv6 string is expect_str. + * Use tor_addr_parse(), and tor_addr_port_parse() with a default port. + * Also check that tor_addr_port_parse() fails without a default port. */ #define TEST_ADDR_V6_PARSE(addr_str, fmt_decorated, expect_str) \ - TEST_ADDR_PARSE_FMT(addr_str, AF_INET6, fmt_decorated, expect_str) + STMT_BEGIN \ + TEST_ADDR_PARSE_FMT(addr_str, AF_INET6, fmt_decorated, expect_str); \ + TEST_ADDR_PORT_PARSE_FMT(addr_str, 333, AF_INET6, fmt_decorated, \ + expect_str, 333); \ + TEST_ADDR_PORT_PARSE_XFAIL(addr_str, -1); \ + STMT_END /* Test that addr_port_str successfully parses to the canonical IPv4 address - * string expect_str, and port expect_port. */ + * string expect_str, and port expect_port. + * Check with and without a default port. */ #define TEST_ADDR_V4_PORT_PARSE(addr_port_str, expect_str, expect_port) \ - TEST_ADDR_PORT_PARSE_FMT(addr_port_str, -1, AF_INET, 0, expect_str, \ - expect_port) + STMT_BEGIN \ + TEST_ADDR_PORT_PARSE_FMT(addr_port_str, -1, AF_INET, 0, expect_str, \ + expect_port); \ + TEST_ADDR_PORT_PARSE_FMT(addr_port_str, 444, AF_INET, 0, expect_str, \ + expect_port); \ + STMT_END /* Test that addr_port_str successfully parses to the canonical undecorated - * IPv6 address string expect_str, and port expect_port. */ + * IPv6 address string expect_str, and port expect_port. + * Check with and without a default port. */ #define TEST_ADDR_V6_PORT_PARSE(addr_port_str, expect_str, expect_port) \ - TEST_ADDR_PORT_PARSE_FMT(addr_port_str, -1, AF_INET6, 0, expect_str, \ - expect_port) + STMT_BEGIN \ + TEST_ADDR_PORT_PARSE_FMT(addr_port_str, -1, AF_INET6, 0, expect_str, \ + expect_port); \ + TEST_ADDR_PORT_PARSE_FMT(addr_port_str, 555, AF_INET6, 0, expect_str, \ + expect_port); \ + STMT_END -/* Test that addr_port_str and default_port successfully parse to the canonical - * IPv4 address string expect_str, and port expect_port. */ -#define TEST_ADDR_V4_PORT_DEF_PARSE(addr_port_str, default_port, expect_str, \ - expect_port) \ - TEST_ADDR_PORT_PARSE_FMT(addr_port_str, default_port, AF_INET, 0, \ - expect_str, expect_port) - -/* Test that addr_port_str successfully parses to the canonical undecorated - * IPv6 address string expect_str, and port expect_port. */ -#define TEST_ADDR_V6_PORT_DEF_PARSE(addr_port_str, default_port, expect_str, \ - expect_port) \ - TEST_ADDR_PORT_PARSE_FMT(addr_port_str, default_port, AF_INET6, 0, \ - expect_str, expect_port) +/* Test that addr_str fails to parse due to a bad address or port. + * Use tor_addr_parse(), and tor_addr_port_parse() with and without a + * default port. */ +#define TEST_ADDR_PARSE_XFAIL_MALFORMED(addr_str) \ + STMT_BEGIN \ + TEST_ADDR_PARSE_XFAIL(addr_str); \ + TEST_ADDR_PORT_PARSE_XFAIL(addr_str, -1); \ + TEST_ADDR_PORT_PARSE_XFAIL(addr_str, 666); \ + STMT_END /** Test tor_addr_parse() and tor_addr_port_parse(). */ static void @@ -852,76 +879,59 @@ test_addr_parse(void *arg) "11:22::33:44:102:304"); /* Empty string. */ - TEST_ADDR_PARSE_XFAIL(""); + TEST_ADDR_PARSE_XFAIL_MALFORMED(""); /* Square brackets around IPv4 address. */ - TEST_ADDR_PARSE_XFAIL("[192.0.2.1]"); + TEST_ADDR_PARSE_XFAIL_MALFORMED("[192.0.2.1]"); /* Only left square bracket. */ - TEST_ADDR_PARSE_XFAIL("[11:22::33:44"); + TEST_ADDR_PARSE_XFAIL_MALFORMED("[11:22::33:44"); /* Only right square bracket. */ - TEST_ADDR_PARSE_XFAIL("11:22::33:44]"); + TEST_ADDR_PARSE_XFAIL_MALFORMED("11:22::33:44]"); /* Leading colon. */ - TEST_ADDR_PARSE_XFAIL(":11:22::33:44"); + TEST_ADDR_PARSE_XFAIL_MALFORMED(":11:22::33:44"); /* Trailing colon. */ - TEST_ADDR_PARSE_XFAIL("11:22::33:44:"); + TEST_ADDR_PARSE_XFAIL_MALFORMED("11:22::33:44:"); /* Too many hex words in IPv4-mapped IPv6 address. */ - TEST_ADDR_PARSE_XFAIL("11:22:33:44:55:66:77:88:1.2.3.4"); + TEST_ADDR_PARSE_XFAIL_MALFORMED("11:22:33:44:55:66:77:88:1.2.3.4"); /* IPv6 address with port and no brackets */ - TEST_ADDR_PARSE_XFAIL("11:22::33:44:12345"); + TEST_ADDR_PARSE_XFAIL_MALFORMED("11:22::33:44:12345"); /* Correct calls. */ TEST_ADDR_V4_PORT_PARSE("192.0.2.1:1234", "192.0.2.1", 1234); - TEST_ADDR_V6_PORT_PARSE("[::1]:1234", "::1", 1234); /* Domain name. */ - TEST_ADDR_PORT_PARSE_XFAIL("torproject.org:1234", -1); + TEST_ADDR_PARSE_XFAIL_MALFORMED("torproject.org:1234"); /* Only IP. */ - TEST_ADDR_PORT_PARSE_XFAIL("192.0.2.2", -1); - TEST_ADDR_V4_PORT_DEF_PARSE("192.0.2.2", 200, "192.0.2.2", 200); - - TEST_ADDR_PORT_PARSE_XFAIL("[::1]", -1); - TEST_ADDR_V6_PORT_DEF_PARSE("[::1]", 400, "::1", 400); + TEST_ADDR_V4_PARSE_CANONICAL("192.0.2.2"); + TEST_ADDR_V6_PARSE_CANONICAL("[::1]", 1); /* Allow IPv6 without square brackets, when there is no port, but only if * there is a default port */ - TEST_ADDR_PORT_PARSE_XFAIL("::1", -1); - TEST_ADDR_V6_PORT_DEF_PARSE("::1", 600, "::1", 600); + TEST_ADDR_V6_PARSE_CANONICAL("::1", 0); /* Bad port. */ - TEST_ADDR_PORT_PARSE_XFAIL("192.0.2.2:66666", -1); - TEST_ADDR_PORT_PARSE_XFAIL("192.0.2.2:66666", 200); + TEST_ADDR_PARSE_XFAIL_MALFORMED("192.0.2.2:66666"); /* Only domain name */ - TEST_ADDR_PORT_PARSE_XFAIL("torproject.org", -1); - TEST_ADDR_PORT_PARSE_XFAIL("torproject.org", 200); + TEST_ADDR_PARSE_XFAIL_MALFORMED("torproject.org"); /* Bad IPv4 address */ - TEST_ADDR_PORT_PARSE_XFAIL("192.0.2:1234", -1); - TEST_ADDR_PORT_PARSE_XFAIL("192.0.2:1234", 200); + TEST_ADDR_PARSE_XFAIL_MALFORMED("192.0.2:1234"); /* Bad IPv4 address and port: brackets */ - TEST_ADDR_PORT_PARSE_XFAIL("[192.0.2.3]:12345", -1); - TEST_ADDR_PORT_PARSE_XFAIL("[192.0.2.3]:12345", 200); + TEST_ADDR_PARSE_XFAIL_MALFORMED("[192.0.2.3]:12345"); /* Bad IPv6 addresses and ports: no brackets */ - TEST_ADDR_PORT_PARSE_XFAIL("::1:12345", -1); - TEST_ADDR_PORT_PARSE_XFAIL("::1:12345", 200); - - TEST_ADDR_PORT_PARSE_XFAIL("11:22::33:44:12345", -1); - TEST_ADDR_PORT_PARSE_XFAIL("11:22::33:44:12345", 200); - - /* Make sure that the default port has lower priority than the real - one */ - TEST_ADDR_V4_PORT_DEF_PARSE("192.0.2.2:1337", 200, "192.0.2.2", 1337); - TEST_ADDR_V6_PORT_DEF_PARSE("[::1]:1369", 200, "::1", 1369); + TEST_ADDR_PARSE_XFAIL_MALFORMED("::1:12345"); + TEST_ADDR_PARSE_XFAIL_MALFORMED("11:22::33:44:12345"); done: ; From 60ce431c5440ee08e100a1f70a4d7224b7dc9c21 Mon Sep 17 00:00:00 2001 From: teor Date: Thu, 13 Jun 2019 23:15:52 +1000 Subject: [PATCH 1191/2557] test/addr: Add some ambiguous IPv6 cases to test_addr_parse() Test some strings which could be parsed as IPv6 addresses, or as IPv6:port strings. Additional tests for 30721. --- src/test/test_addr.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/test/test_addr.c b/src/test/test_addr.c index 6a6796c987..1b25c4c643 100644 --- a/src/test/test_addr.c +++ b/src/test/test_addr.c @@ -901,6 +901,23 @@ test_addr_parse(void *arg) /* IPv6 address with port and no brackets */ TEST_ADDR_PARSE_XFAIL_MALFORMED("11:22::33:44:12345"); + /* Is it a port, or are there too many hex words? + * We reject it either way. */ + TEST_ADDR_PARSE_XFAIL_MALFORMED("11:22:33:44:55:66:77:88:99"); + /* But we accept it if it has square brackets. */ + TEST_ADDR_V6_PORT_PARSE("[11:22:33:44:55:66:77:88]:99", + "11:22:33:44:55:66:77:88",99); + + /* This is an IPv6 address */ + TEST_ADDR_V6_PARSE_CANONICAL("11:22:33:44:55:66:77:88", 0); + TEST_ADDR_V6_PARSE_CANONICAL("[11:22:33:44:55:66:77:88]", 1); + + /* And this is an ambiguous case, which is interpreted as an IPv6 address. */ + TEST_ADDR_V6_PARSE_CANONICAL("11:22::88:99", 0); + /* Use square brackets to resolve the ambiguity */ + TEST_ADDR_V6_PARSE_CANONICAL("[11:22::88:99]", 1); + TEST_ADDR_V6_PORT_PARSE("[11:22::88]:99", + "11:22::88",99); /* Correct calls. */ TEST_ADDR_V4_PORT_PARSE("192.0.2.1:1234", "192.0.2.1", 1234); From 1c92d7f3882e6bd43e871ff1b17ca8c4a65b1e08 Mon Sep 17 00:00:00 2001 From: teor Date: Fri, 14 Jun 2019 00:00:05 +1000 Subject: [PATCH 1192/2557] test/addr: test that tor_lookup_hostname() handles IPv4 addresses And that it fails on IPv6 and host:port, and does something sensible with host. Tests for 30721. --- src/test/test_addr.c | 142 ++++++++++++++++++++++++++++++++++++++----- 1 file changed, 128 insertions(+), 14 deletions(-) diff --git a/src/test/test_addr.c b/src/test/test_addr.c index 1b25c4c643..53f41bcf1b 100644 --- a/src/test/test_addr.c +++ b/src/test/test_addr.c @@ -793,71 +793,187 @@ test_addr_ip6_helpers(void *arg) tt_int_op(port, OP_EQ, 0); \ STMT_END +/* Test that addr_str successfully parses as an IPv4 address using + * tor_lookup_hostname(), and: + * - the fmt_addr32() of the result is expect_str. + */ +#define TEST_ADDR_V4_LOOKUP_HOSTNAME(addr_str, expect_str) \ + STMT_BEGIN \ + int r; \ + uint32_t addr32h; \ + r = tor_lookup_hostname(addr_str, &addr32h); \ + tt_int_op(r, OP_EQ, 0); \ + tt_str_op(fmt_addr32(addr32h), OP_EQ, expect_str); \ + STMT_END + +/* Test that bad_str fails to parse using tor_lookup_hostname(), with a + * permanent failure, and: + * - the returned address is 0. + */ +#define TEST_ADDR_V4_LOOKUP_XFAIL(bad_str) \ + STMT_BEGIN \ + int r; \ + uint32_t addr32h; \ + r = tor_lookup_hostname(bad_str, &addr32h); \ + tt_int_op(r, OP_EQ, -1); \ + tt_int_op(addr32h, OP_EQ, 0); \ + STMT_END + +/* Test that looking up host_str as an IPv4 address using tor_lookup_hostname() + * does something sensible: + * - the result is -1, 0, or 1. + * - if the result is a failure, the returned address is 0. + * We can't rely on the result of this function, because it depends on the + * network. + */ +#define TEST_HOST_V4_LOOKUP(host_str) \ + STMT_BEGIN \ + int r; \ + uint32_t addr32h; \ + r = tor_lookup_hostname(host_str, &addr32h); \ + tt_int_op(r, OP_GE, -1); \ + tt_int_op(r, OP_LE, 1); \ + if (r != 0) \ + tt_int_op(addr32h, OP_EQ, 0); \ + STMT_END + /* Test that addr_str successfully parses as a canonical IPv4 address. - * Use tor_addr_parse(), and tor_addr_port_parse() with a default port. - * Also check that tor_addr_port_parse() fails without a default port. */ + * Check for successful parsing using: + * - tor_addr_parse(), + * - tor_addr_port_parse() with a default port, + * - tor_lookup_hostname(). + * Check for failures using: + * - tor_addr_port_parse() without a default port, because there is no port. + */ #define TEST_ADDR_V4_PARSE_CANONICAL(addr_str) \ STMT_BEGIN \ TEST_ADDR_PARSE_FMT(addr_str, AF_INET, 0, addr_str); \ TEST_ADDR_PORT_PARSE_FMT(addr_str, 111, AF_INET, 0, \ addr_str, 111); \ TEST_ADDR_PORT_PARSE_XFAIL(addr_str, -1); \ + TEST_ADDR_V4_LOOKUP_HOSTNAME(addr_str, addr_str); \ STMT_END /* Test that addr_str successfully parses as a canonical fmt_decorated * IPv6 address. - * Use tor_addr_parse(), and tor_addr_port_parse() with a default port. - * Also check that tor_addr_port_parse() fails without a default port. */ + * Check for successful parsing using: + * - tor_addr_parse(), + * - tor_addr_port_parse() with a default port. + * Check for failures using: + * - tor_addr_port_parse() without a default port, because there is no port, + * - tor_lookup_hostname(), because it only supports IPv4. + */ #define TEST_ADDR_V6_PARSE_CANONICAL(addr_str, fmt_decorated) \ STMT_BEGIN \ TEST_ADDR_PARSE_FMT(addr_str, AF_INET6, fmt_decorated, addr_str); \ TEST_ADDR_PORT_PARSE_FMT(addr_str, 222, AF_INET6, fmt_decorated, \ addr_str, 222); \ TEST_ADDR_PORT_PARSE_XFAIL(addr_str, -1); \ + TEST_ADDR_V4_LOOKUP_XFAIL(addr_str); \ STMT_END /* Test that addr_str successfully parses, and the fmt_decorated canonical * IPv6 string is expect_str. - * Use tor_addr_parse(), and tor_addr_port_parse() with a default port. - * Also check that tor_addr_port_parse() fails without a default port. */ + * Check for successful parsing using: + * - tor_addr_parse(), + * - tor_addr_port_parse() with a default port. + * Check for failures using: + * - tor_addr_port_parse() without a default port, because there is no port. + * - tor_lookup_hostname(), because it only supports IPv4. + */ #define TEST_ADDR_V6_PARSE(addr_str, fmt_decorated, expect_str) \ STMT_BEGIN \ TEST_ADDR_PARSE_FMT(addr_str, AF_INET6, fmt_decorated, expect_str); \ TEST_ADDR_PORT_PARSE_FMT(addr_str, 333, AF_INET6, fmt_decorated, \ expect_str, 333); \ TEST_ADDR_PORT_PARSE_XFAIL(addr_str, -1); \ + TEST_ADDR_V4_LOOKUP_XFAIL(addr_str); \ STMT_END /* Test that addr_port_str successfully parses to the canonical IPv4 address * string expect_str, and port expect_port. - * Check with and without a default port. */ + * Check for successful parsing using: + * - tor_addr_port_parse() without a default port, + * - tor_addr_port_parse() with a default port. + * Check for failures using: + * - tor_addr_parse(), because there is a port, + * - tor_lookup_hostname(), because there is a port. + */ #define TEST_ADDR_V4_PORT_PARSE(addr_port_str, expect_str, expect_port) \ STMT_BEGIN \ TEST_ADDR_PORT_PARSE_FMT(addr_port_str, -1, AF_INET, 0, expect_str, \ expect_port); \ TEST_ADDR_PORT_PARSE_FMT(addr_port_str, 444, AF_INET, 0, expect_str, \ expect_port); \ + TEST_ADDR_PARSE_XFAIL(addr_port_str); \ + TEST_ADDR_V4_LOOKUP_XFAIL(addr_port_str); \ STMT_END /* Test that addr_port_str successfully parses to the canonical undecorated * IPv6 address string expect_str, and port expect_port. - * Check with and without a default port. */ + * Check for successful parsing using: + * - tor_addr_port_parse() without a default port, + * - tor_addr_port_parse() with a default port. + * Check for failures using: + * - tor_addr_parse(), because there is a port, + * - tor_lookup_hostname(), because there is a port, and because it only + * supports IPv4. + */ #define TEST_ADDR_V6_PORT_PARSE(addr_port_str, expect_str, expect_port) \ STMT_BEGIN \ TEST_ADDR_PORT_PARSE_FMT(addr_port_str, -1, AF_INET6, 0, expect_str, \ expect_port); \ TEST_ADDR_PORT_PARSE_FMT(addr_port_str, 555, AF_INET6, 0, expect_str, \ expect_port); \ + TEST_ADDR_PARSE_XFAIL(addr_port_str); \ + TEST_ADDR_V4_LOOKUP_XFAIL(addr_port_str); \ STMT_END /* Test that addr_str fails to parse due to a bad address or port. - * Use tor_addr_parse(), and tor_addr_port_parse() with and without a - * default port. */ + * Check for failures using: + * - tor_addr_parse(), + * - tor_addr_port_parse() without a default port, + * - tor_addr_port_parse() with a default port. + * - tor_lookup_hostname(). + */ #define TEST_ADDR_PARSE_XFAIL_MALFORMED(addr_str) \ STMT_BEGIN \ TEST_ADDR_PARSE_XFAIL(addr_str); \ TEST_ADDR_PORT_PARSE_XFAIL(addr_str, -1); \ TEST_ADDR_PORT_PARSE_XFAIL(addr_str, 666); \ + TEST_ADDR_V4_LOOKUP_XFAIL(addr_str); \ + STMT_END + +/* Test that host_str is treated as a hostname, and not an address. + * Check for success or failure using: + * - tor_lookup_hostname(), because it depends on the network. + * Check for failures using: + * - tor_addr_parse(), + * - tor_addr_port_parse() without a default port, + * - tor_addr_port_parse() with a default port. + */ +#define TEST_HOSTNAME(host_str) \ + STMT_BEGIN \ + TEST_HOST_V4_LOOKUP(host_str); \ + TEST_ADDR_PARSE_XFAIL(host_str); \ + TEST_ADDR_PORT_PARSE_XFAIL(host_str, -1); \ + TEST_ADDR_PORT_PARSE_XFAIL(host_str, 777); \ + STMT_END + +/* Test that host_port_str is treated as a hostname and port, and not a + * hostname or an address. + * Check for failures using: + * - tor_lookup_hostname(), because it doesn't support ports, + * - tor_addr_parse(), + * - tor_addr_port_parse() without a default port, + * - tor_addr_port_parse() with a default port. + */ +#define TEST_HOSTNAME_PORT(host_port_str) \ + STMT_BEGIN \ + TEST_ADDR_V4_LOOKUP_XFAIL(host_port_str); \ + TEST_ADDR_PARSE_XFAIL(host_port_str); \ + TEST_ADDR_PORT_PARSE_XFAIL(host_port_str, -1); \ + TEST_ADDR_PORT_PARSE_XFAIL(host_port_str, 888); \ STMT_END /** Test tor_addr_parse() and tor_addr_port_parse(). */ @@ -924,7 +1040,8 @@ test_addr_parse(void *arg) TEST_ADDR_V6_PORT_PARSE("[::1]:1234", "::1", 1234); /* Domain name. */ - TEST_ADDR_PARSE_XFAIL_MALFORMED("torproject.org:1234"); + TEST_HOSTNAME("torproject.org"); + TEST_HOSTNAME_PORT("torproject.org:1234"); /* Only IP. */ TEST_ADDR_V4_PARSE_CANONICAL("192.0.2.2"); @@ -937,9 +1054,6 @@ test_addr_parse(void *arg) /* Bad port. */ TEST_ADDR_PARSE_XFAIL_MALFORMED("192.0.2.2:66666"); - /* Only domain name */ - TEST_ADDR_PARSE_XFAIL_MALFORMED("torproject.org"); - /* Bad IPv4 address */ TEST_ADDR_PARSE_XFAIL_MALFORMED("192.0.2:1234"); From 6079dfd1032add1c952f22b831d0f3c38147ea08 Mon Sep 17 00:00:00 2001 From: teor Date: Fri, 14 Jun 2019 00:49:45 +1000 Subject: [PATCH 1193/2557] test/addr: test that tor_addr_lookup() handles IP addresses And that it fails on IP:port and host:port, and does something sensible with host. Tests for 30721. --- src/test/test_addr.c | 138 ++++++++++++++++++++++++++++++++++++------- 1 file changed, 118 insertions(+), 20 deletions(-) diff --git a/src/test/test_addr.c b/src/test/test_addr.c index 53f41bcf1b..d5f7cda438 100644 --- a/src/test/test_addr.c +++ b/src/test/test_addr.c @@ -837,57 +837,130 @@ test_addr_ip6_helpers(void *arg) tt_int_op(addr32h, OP_EQ, 0); \ STMT_END +/* Test that addr_str successfully parses as a require_family IP address using + * tor_addr_lookup(), and: + * - the address has family expect_family, + * - the fmt_decorated result of tor_addr_to_str() is expect_str. + */ +#define TEST_ADDR_LOOKUP_FMT(addr_str, require_family, expect_family, \ + fmt_decorated, expect_str) \ + STMT_BEGIN \ + int r; \ + tor_addr_t addr; \ + char buf[TOR_ADDR_BUF_LEN]; \ + const char *sv; \ + r = tor_addr_lookup(addr_str, require_family, &addr); \ + tt_int_op(r, OP_EQ, 0); \ + tt_int_op(tor_addr_family(&addr), OP_EQ, expect_family); \ + sv = tor_addr_to_str(buf, &addr, sizeof(buf), fmt_decorated); \ + tt_str_op(sv, OP_EQ, buf); \ + tt_str_op(buf, OP_EQ, expect_str); \ + STMT_END + +/* Test that bad_str fails to parse as a require_family IP address using + * tor_addr_lookup(), with a permanent failure, and: + * - the returned address is null. + */ +#define TEST_ADDR_LOOKUP_XFAIL(bad_str, require_family) \ + STMT_BEGIN \ + int r; \ + tor_addr_t addr; \ + r = tor_addr_lookup(bad_str, require_family, &addr); \ + tt_int_op(r, OP_EQ, -1); \ + tt_assert(tor_addr_is_null(&addr)); \ + STMT_END + +/* Test that looking up host_string as a require_family IP address using + * tor_addr_lookup(), does something sensible: + * - the result is -1, 0, or 1. + * - if the result is a failure, the returned address is null. + * We can't rely on the result of this function, because it depends on the + * network. + */ +#define TEST_HOST_LOOKUP(host_str, require_family) \ + STMT_BEGIN \ + int r; \ + tor_addr_t addr; \ + r = tor_addr_lookup(host_str, require_family, &addr); \ + tt_int_op(r, OP_GE, -1); \ + tt_int_op(r, OP_LE, 1); \ + if (r != 0) \ + tt_assert(tor_addr_is_null(&addr)); \ + STMT_END + /* Test that addr_str successfully parses as a canonical IPv4 address. * Check for successful parsing using: * - tor_addr_parse(), * - tor_addr_port_parse() with a default port, - * - tor_lookup_hostname(). + * - tor_lookup_hostname(), + * - tor_addr_lookup() with AF_INET, + * - tor_addr_lookup() with AF_UNSPEC. * Check for failures using: - * - tor_addr_port_parse() without a default port, because there is no port. + * - tor_addr_port_parse() without a default port, because there is no port, + * - tor_addr_lookup() with AF_INET6. */ #define TEST_ADDR_V4_PARSE_CANONICAL(addr_str) \ STMT_BEGIN \ TEST_ADDR_PARSE_FMT(addr_str, AF_INET, 0, addr_str); \ TEST_ADDR_PORT_PARSE_FMT(addr_str, 111, AF_INET, 0, \ addr_str, 111); \ - TEST_ADDR_PORT_PARSE_XFAIL(addr_str, -1); \ TEST_ADDR_V4_LOOKUP_HOSTNAME(addr_str, addr_str); \ + TEST_ADDR_LOOKUP_FMT(addr_str, AF_INET, AF_INET, 0, addr_str); \ + TEST_ADDR_LOOKUP_FMT(addr_str, AF_UNSPEC, AF_INET, 0, addr_str); \ + TEST_ADDR_PORT_PARSE_XFAIL(addr_str, -1); \ + TEST_ADDR_LOOKUP_XFAIL(addr_str, AF_INET6); \ STMT_END /* Test that addr_str successfully parses as a canonical fmt_decorated * IPv6 address. * Check for successful parsing using: * - tor_addr_parse(), - * - tor_addr_port_parse() with a default port. + * - tor_addr_port_parse() with a default port, + * - tor_addr_lookup() with AF_INET6, + * - tor_addr_lookup() with AF_UNSPEC. * Check for failures using: * - tor_addr_port_parse() without a default port, because there is no port, - * - tor_lookup_hostname(), because it only supports IPv4. + * - tor_lookup_hostname(), because it only supports IPv4, + * - tor_addr_lookup() with AF_INET. */ #define TEST_ADDR_V6_PARSE_CANONICAL(addr_str, fmt_decorated) \ STMT_BEGIN \ TEST_ADDR_PARSE_FMT(addr_str, AF_INET6, fmt_decorated, addr_str); \ TEST_ADDR_PORT_PARSE_FMT(addr_str, 222, AF_INET6, fmt_decorated, \ addr_str, 222); \ + TEST_ADDR_LOOKUP_FMT(addr_str, AF_INET6, AF_INET6, fmt_decorated, \ + addr_str); \ + TEST_ADDR_LOOKUP_FMT(addr_str, AF_UNSPEC, AF_INET6, fmt_decorated, \ + addr_str); \ TEST_ADDR_PORT_PARSE_XFAIL(addr_str, -1); \ TEST_ADDR_V4_LOOKUP_XFAIL(addr_str); \ + TEST_ADDR_LOOKUP_XFAIL(addr_str, AF_INET); \ STMT_END /* Test that addr_str successfully parses, and the fmt_decorated canonical * IPv6 string is expect_str. * Check for successful parsing using: * - tor_addr_parse(), - * - tor_addr_port_parse() with a default port. + * - tor_addr_port_parse() with a default port, + * - tor_addr_lookup() with AF_INET6, + * - tor_addr_lookup() with AF_UNSPEC. * Check for failures using: * - tor_addr_port_parse() without a default port, because there is no port. - * - tor_lookup_hostname(), because it only supports IPv4. + * - tor_lookup_hostname(), because it only supports IPv4, + * - tor_addr_lookup() with AF_INET. */ #define TEST_ADDR_V6_PARSE(addr_str, fmt_decorated, expect_str) \ STMT_BEGIN \ TEST_ADDR_PARSE_FMT(addr_str, AF_INET6, fmt_decorated, expect_str); \ TEST_ADDR_PORT_PARSE_FMT(addr_str, 333, AF_INET6, fmt_decorated, \ expect_str, 333); \ + TEST_ADDR_LOOKUP_FMT(addr_str, AF_INET6, AF_INET6, fmt_decorated, \ + expect_str); \ + TEST_ADDR_LOOKUP_FMT(addr_str, AF_UNSPEC, AF_INET6, fmt_decorated, \ + expect_str); \ TEST_ADDR_PORT_PARSE_XFAIL(addr_str, -1); \ TEST_ADDR_V4_LOOKUP_XFAIL(addr_str); \ + TEST_ADDR_LOOKUP_XFAIL(addr_str, AF_INET); \ STMT_END /* Test that addr_port_str successfully parses to the canonical IPv4 address @@ -898,6 +971,8 @@ test_addr_ip6_helpers(void *arg) * Check for failures using: * - tor_addr_parse(), because there is a port, * - tor_lookup_hostname(), because there is a port. + * - tor_addr_lookup(), regardless of the address family, because there is a + * port. */ #define TEST_ADDR_V4_PORT_PARSE(addr_port_str, expect_str, expect_port) \ STMT_BEGIN \ @@ -907,6 +982,9 @@ test_addr_ip6_helpers(void *arg) expect_port); \ TEST_ADDR_PARSE_XFAIL(addr_port_str); \ TEST_ADDR_V4_LOOKUP_XFAIL(addr_port_str); \ + TEST_ADDR_LOOKUP_XFAIL(addr_port_str, AF_INET); \ + TEST_ADDR_LOOKUP_XFAIL(addr_port_str, AF_UNSPEC); \ + TEST_ADDR_LOOKUP_XFAIL(addr_port_str, AF_INET6); \ STMT_END /* Test that addr_port_str successfully parses to the canonical undecorated @@ -917,7 +995,9 @@ test_addr_ip6_helpers(void *arg) * Check for failures using: * - tor_addr_parse(), because there is a port, * - tor_lookup_hostname(), because there is a port, and because it only - * supports IPv4. + * supports IPv4, + * - tor_addr_lookup(), regardless of the address family, because there is a + * port. */ #define TEST_ADDR_V6_PORT_PARSE(addr_port_str, expect_str, expect_port) \ STMT_BEGIN \ @@ -927,26 +1007,34 @@ test_addr_ip6_helpers(void *arg) expect_port); \ TEST_ADDR_PARSE_XFAIL(addr_port_str); \ TEST_ADDR_V4_LOOKUP_XFAIL(addr_port_str); \ + TEST_ADDR_LOOKUP_XFAIL(addr_port_str, AF_INET6); \ + TEST_ADDR_LOOKUP_XFAIL(addr_port_str, AF_UNSPEC); \ + TEST_ADDR_LOOKUP_XFAIL(addr_port_str, AF_INET); \ STMT_END /* Test that addr_str fails to parse due to a bad address or port. * Check for failures using: * - tor_addr_parse(), * - tor_addr_port_parse() without a default port, - * - tor_addr_port_parse() with a default port. - * - tor_lookup_hostname(). + * - tor_addr_port_parse() with a default port, + * - tor_lookup_hostname(), + * - tor_addr_lookup(), regardless of the address family. */ -#define TEST_ADDR_PARSE_XFAIL_MALFORMED(addr_str) \ +#define TEST_ADDR_PARSE_XFAIL_MALFORMED(bad_str) \ STMT_BEGIN \ - TEST_ADDR_PARSE_XFAIL(addr_str); \ - TEST_ADDR_PORT_PARSE_XFAIL(addr_str, -1); \ - TEST_ADDR_PORT_PARSE_XFAIL(addr_str, 666); \ - TEST_ADDR_V4_LOOKUP_XFAIL(addr_str); \ + TEST_ADDR_PARSE_XFAIL(bad_str); \ + TEST_ADDR_PORT_PARSE_XFAIL(bad_str, -1); \ + TEST_ADDR_PORT_PARSE_XFAIL(bad_str, 666); \ + TEST_ADDR_V4_LOOKUP_XFAIL(bad_str); \ + TEST_ADDR_LOOKUP_XFAIL(bad_str, AF_UNSPEC); \ + TEST_ADDR_LOOKUP_XFAIL(bad_str, AF_INET); \ + TEST_ADDR_LOOKUP_XFAIL(bad_str, AF_INET6); \ STMT_END /* Test that host_str is treated as a hostname, and not an address. - * Check for success or failure using: - * - tor_lookup_hostname(), because it depends on the network. + * Check for success or failure using the network-dependent functions: + * - tor_lookup_hostname(), + * - tor_addr_lookup(), regardless of the address family. * Check for failures using: * - tor_addr_parse(), * - tor_addr_port_parse() without a default port, @@ -955,6 +1043,9 @@ test_addr_ip6_helpers(void *arg) #define TEST_HOSTNAME(host_str) \ STMT_BEGIN \ TEST_HOST_V4_LOOKUP(host_str); \ + TEST_HOST_LOOKUP(host_str, AF_UNSPEC); \ + TEST_HOST_LOOKUP(host_str, AF_INET); \ + TEST_HOST_LOOKUP(host_str, AF_INET6); \ TEST_ADDR_PARSE_XFAIL(host_str); \ TEST_ADDR_PORT_PARSE_XFAIL(host_str, -1); \ TEST_ADDR_PORT_PARSE_XFAIL(host_str, 777); \ @@ -963,17 +1054,22 @@ test_addr_ip6_helpers(void *arg) /* Test that host_port_str is treated as a hostname and port, and not a * hostname or an address. * Check for failures using: - * - tor_lookup_hostname(), because it doesn't support ports, * - tor_addr_parse(), * - tor_addr_port_parse() without a default port, - * - tor_addr_port_parse() with a default port. + * - tor_addr_port_parse() with a default port, + * - tor_lookup_hostname(), because it doesn't support ports, + * - tor_addr_lookup(), regardless of the address family, because it doesn't + * support ports. */ #define TEST_HOSTNAME_PORT(host_port_str) \ STMT_BEGIN \ - TEST_ADDR_V4_LOOKUP_XFAIL(host_port_str); \ TEST_ADDR_PARSE_XFAIL(host_port_str); \ TEST_ADDR_PORT_PARSE_XFAIL(host_port_str, -1); \ TEST_ADDR_PORT_PARSE_XFAIL(host_port_str, 888); \ + TEST_ADDR_V4_LOOKUP_XFAIL(host_port_str); \ + TEST_ADDR_LOOKUP_XFAIL(host_port_str, AF_UNSPEC); \ + TEST_ADDR_LOOKUP_XFAIL(host_port_str, AF_INET); \ + TEST_ADDR_LOOKUP_XFAIL(host_port_str, AF_INET6); \ STMT_END /** Test tor_addr_parse() and tor_addr_port_parse(). */ @@ -1040,6 +1136,8 @@ test_addr_parse(void *arg) TEST_ADDR_V6_PORT_PARSE("[::1]:1234", "::1", 1234); /* Domain name. */ + TEST_HOSTNAME("localhost"); + TEST_HOSTNAME_PORT("localhost:1234"); TEST_HOSTNAME("torproject.org"); TEST_HOSTNAME_PORT("torproject.org:1234"); From 6ef555bda0f7cc1248a133f197fcd0a5f92f5cf1 Mon Sep 17 00:00:00 2001 From: teor Date: Fri, 14 Jun 2019 11:06:21 +1000 Subject: [PATCH 1194/2557] test/addr: test that tor_addr_port_lookup() handles IP addresses and ports And that it does something sensible with host and host:port. Also reorder the tests into valid, invalid, and ambiguous. And add some missing cases. Note: tor_addr_port_lookup() handles ip, ip:port, host, and host:port. Tests for 30721. --- src/test/test_addr.c | 229 +++++++++++++++++++++++++++++++++---------- 1 file changed, 178 insertions(+), 51 deletions(-) diff --git a/src/test/test_addr.c b/src/test/test_addr.c index d5f7cda438..0f50a43615 100644 --- a/src/test/test_addr.c +++ b/src/test/test_addr.c @@ -888,16 +888,85 @@ test_addr_ip6_helpers(void *arg) tt_assert(tor_addr_is_null(&addr)); \ STMT_END +/* Test that addr_port_str successfully parses as an IP address and port + * using tor_addr_port_lookup(), and: + * - the address has family expect_family, + * - the fmt_decorated result of tor_addr_to_str() is expect_str, + * - the port is expect_port. + */ +#define TEST_ADDR_PORT_LOOKUP_FMT(addr_port_str, expect_family, \ + fmt_decorated, expect_str, expect_port) \ + STMT_BEGIN \ + int r; \ + tor_addr_t addr; \ + uint16_t port; \ + char buf[TOR_ADDR_BUF_LEN]; \ + const char *sv; \ + r = tor_addr_port_lookup(addr_port_str, &addr, &port); \ + tt_int_op(r, OP_EQ, 0); \ + tt_int_op(tor_addr_family(&addr), OP_EQ, expect_family); \ + sv = tor_addr_to_str(buf, &addr, sizeof(buf), fmt_decorated); \ + tt_str_op(sv, OP_EQ, buf); \ + tt_str_op(buf, OP_EQ, expect_str); \ + tt_int_op(port, OP_EQ, expect_port); \ + STMT_END + +/* Test that bad_str fails to parse as an IP address and port + * using tor_addr_port_lookup(), and: + * - the returned address is null, + * - the returned port is 0. + */ +#define TEST_ADDR_PORT_LOOKUP_XFAIL(bad_str) \ + STMT_BEGIN \ + int r; \ + tor_addr_t addr; \ + uint16_t port; \ + r = tor_addr_port_lookup(bad_str, &addr, &port); \ + tt_int_op(r, OP_EQ, -1); \ + tt_assert(tor_addr_is_null(&addr)); \ + tt_int_op(port, OP_EQ, 0); \ + STMT_END + +/* Test that looking up host_port_str as an IP address using + * tor_addr_port_lookup(), does something sensible: + * - the result is -1 or 0. + * - if the result is a failure, the returned address is null, and the + * returned port is zero, + * - if the result is a success, the returned port is expect_success_port, + * and the returned family is AF_INET or AF_INET6. + * We can't rely on the result of this function, because it depends on the + * network. + */ +#define TEST_HOST_PORT_LOOKUP(host_port_str, expect_success_port) \ + STMT_BEGIN \ + int r; \ + tor_addr_t addr; \ + uint16_t port; \ + r = tor_addr_port_lookup(host_port_str, &addr, &port); \ + tt_int_op(r, OP_GE, -1); \ + tt_int_op(r, OP_LE, 0); \ + if (r == -1) { \ + tt_assert(tor_addr_is_null(&addr)); \ + tt_int_op(port, OP_EQ, 0); \ + } else { \ + tt_assert(tor_addr_family(&addr) == AF_INET || \ + tor_addr_family(&addr) == AF_INET6); \ + tt_int_op(port, OP_EQ, expect_success_port); \ + } \ + STMT_END + /* Test that addr_str successfully parses as a canonical IPv4 address. * Check for successful parsing using: * - tor_addr_parse(), * - tor_addr_port_parse() with a default port, * - tor_lookup_hostname(), * - tor_addr_lookup() with AF_INET, - * - tor_addr_lookup() with AF_UNSPEC. + * - tor_addr_lookup() with AF_UNSPEC, + * - tor_addr_port_lookup(), with a zero port. * Check for failures using: * - tor_addr_port_parse() without a default port, because there is no port, - * - tor_addr_lookup() with AF_INET6. + * - tor_addr_lookup() with AF_INET6, + * - tor_addr_port_lookup(), because there is no port. */ #define TEST_ADDR_V4_PARSE_CANONICAL(addr_str) \ STMT_BEGIN \ @@ -905,6 +974,7 @@ test_addr_ip6_helpers(void *arg) TEST_ADDR_PORT_PARSE_FMT(addr_str, 111, AF_INET, 0, \ addr_str, 111); \ TEST_ADDR_V4_LOOKUP_HOSTNAME(addr_str, addr_str); \ + TEST_ADDR_PORT_LOOKUP_FMT(addr_str, AF_INET, 0, addr_str, 0); \ TEST_ADDR_LOOKUP_FMT(addr_str, AF_INET, AF_INET, 0, addr_str); \ TEST_ADDR_LOOKUP_FMT(addr_str, AF_UNSPEC, AF_INET, 0, addr_str); \ TEST_ADDR_PORT_PARSE_XFAIL(addr_str, -1); \ @@ -917,7 +987,8 @@ test_addr_ip6_helpers(void *arg) * - tor_addr_parse(), * - tor_addr_port_parse() with a default port, * - tor_addr_lookup() with AF_INET6, - * - tor_addr_lookup() with AF_UNSPEC. + * - tor_addr_lookup() with AF_UNSPEC, + * - tor_addr_port_lookup(), with a zero port. * Check for failures using: * - tor_addr_port_parse() without a default port, because there is no port, * - tor_lookup_hostname(), because it only supports IPv4, @@ -932,6 +1003,8 @@ test_addr_ip6_helpers(void *arg) addr_str); \ TEST_ADDR_LOOKUP_FMT(addr_str, AF_UNSPEC, AF_INET6, fmt_decorated, \ addr_str); \ + TEST_ADDR_PORT_LOOKUP_FMT(addr_str, AF_INET6, fmt_decorated, addr_str, \ + 0); \ TEST_ADDR_PORT_PARSE_XFAIL(addr_str, -1); \ TEST_ADDR_V4_LOOKUP_XFAIL(addr_str); \ TEST_ADDR_LOOKUP_XFAIL(addr_str, AF_INET); \ @@ -943,7 +1016,8 @@ test_addr_ip6_helpers(void *arg) * - tor_addr_parse(), * - tor_addr_port_parse() with a default port, * - tor_addr_lookup() with AF_INET6, - * - tor_addr_lookup() with AF_UNSPEC. + * - tor_addr_lookup() with AF_UNSPEC, + * - tor_addr_port_lookup(), with a zero port. * Check for failures using: * - tor_addr_port_parse() without a default port, because there is no port. * - tor_lookup_hostname(), because it only supports IPv4, @@ -958,6 +1032,8 @@ test_addr_ip6_helpers(void *arg) expect_str); \ TEST_ADDR_LOOKUP_FMT(addr_str, AF_UNSPEC, AF_INET6, fmt_decorated, \ expect_str); \ + TEST_ADDR_PORT_LOOKUP_FMT(addr_str, AF_INET6, fmt_decorated, expect_str, \ + 0); \ TEST_ADDR_PORT_PARSE_XFAIL(addr_str, -1); \ TEST_ADDR_V4_LOOKUP_XFAIL(addr_str); \ TEST_ADDR_LOOKUP_XFAIL(addr_str, AF_INET); \ @@ -967,7 +1043,8 @@ test_addr_ip6_helpers(void *arg) * string expect_str, and port expect_port. * Check for successful parsing using: * - tor_addr_port_parse() without a default port, - * - tor_addr_port_parse() with a default port. + * - tor_addr_port_parse() with a default port, + * - tor_addr_port_lookup(). * Check for failures using: * - tor_addr_parse(), because there is a port, * - tor_lookup_hostname(), because there is a port. @@ -980,6 +1057,8 @@ test_addr_ip6_helpers(void *arg) expect_port); \ TEST_ADDR_PORT_PARSE_FMT(addr_port_str, 444, AF_INET, 0, expect_str, \ expect_port); \ + TEST_ADDR_PORT_LOOKUP_FMT(addr_port_str, AF_INET, 0, expect_str, \ + expect_port); \ TEST_ADDR_PARSE_XFAIL(addr_port_str); \ TEST_ADDR_V4_LOOKUP_XFAIL(addr_port_str); \ TEST_ADDR_LOOKUP_XFAIL(addr_port_str, AF_INET); \ @@ -991,7 +1070,8 @@ test_addr_ip6_helpers(void *arg) * IPv6 address string expect_str, and port expect_port. * Check for successful parsing using: * - tor_addr_port_parse() without a default port, - * - tor_addr_port_parse() with a default port. + * - tor_addr_port_parse() with a default port, + * - tor_addr_port_lookup(). * Check for failures using: * - tor_addr_parse(), because there is a port, * - tor_lookup_hostname(), because there is a port, and because it only @@ -1005,6 +1085,8 @@ test_addr_ip6_helpers(void *arg) expect_port); \ TEST_ADDR_PORT_PARSE_FMT(addr_port_str, 555, AF_INET6, 0, expect_str, \ expect_port); \ + TEST_ADDR_PORT_LOOKUP_FMT(addr_port_str, AF_INET6, 0, expect_str, \ + expect_port); \ TEST_ADDR_PARSE_XFAIL(addr_port_str); \ TEST_ADDR_V4_LOOKUP_XFAIL(addr_port_str); \ TEST_ADDR_LOOKUP_XFAIL(addr_port_str, AF_INET6); \ @@ -1012,13 +1094,14 @@ test_addr_ip6_helpers(void *arg) TEST_ADDR_LOOKUP_XFAIL(addr_port_str, AF_INET); \ STMT_END -/* Test that addr_str fails to parse due to a bad address or port. +/* Test that bad_str fails to parse due to a bad address or port. * Check for failures using: * - tor_addr_parse(), * - tor_addr_port_parse() without a default port, * - tor_addr_port_parse() with a default port, * - tor_lookup_hostname(), - * - tor_addr_lookup(), regardless of the address family. + * - tor_addr_lookup(), regardless of the address family, + * - tor_addr_port_lookup(). */ #define TEST_ADDR_PARSE_XFAIL_MALFORMED(bad_str) \ STMT_BEGIN \ @@ -1029,12 +1112,14 @@ test_addr_ip6_helpers(void *arg) TEST_ADDR_LOOKUP_XFAIL(bad_str, AF_UNSPEC); \ TEST_ADDR_LOOKUP_XFAIL(bad_str, AF_INET); \ TEST_ADDR_LOOKUP_XFAIL(bad_str, AF_INET6); \ + TEST_ADDR_PORT_LOOKUP_XFAIL(bad_str); \ STMT_END /* Test that host_str is treated as a hostname, and not an address. * Check for success or failure using the network-dependent functions: * - tor_lookup_hostname(), - * - tor_addr_lookup(), regardless of the address family. + * - tor_addr_lookup(), regardless of the address family, + * - tor_addr_port_lookup(), expecting a zero port. * Check for failures using: * - tor_addr_parse(), * - tor_addr_port_parse() without a default port, @@ -1046,6 +1131,7 @@ test_addr_ip6_helpers(void *arg) TEST_HOST_LOOKUP(host_str, AF_UNSPEC); \ TEST_HOST_LOOKUP(host_str, AF_INET); \ TEST_HOST_LOOKUP(host_str, AF_INET6); \ + TEST_HOST_PORT_LOOKUP(host_str, 0); \ TEST_ADDR_PARSE_XFAIL(host_str); \ TEST_ADDR_PORT_PARSE_XFAIL(host_str, -1); \ TEST_ADDR_PORT_PARSE_XFAIL(host_str, 777); \ @@ -1053,6 +1139,9 @@ test_addr_ip6_helpers(void *arg) /* Test that host_port_str is treated as a hostname and port, and not a * hostname or an address. + * Check for success or failure using the network-dependent function: + * - tor_addr_port_lookup(), expecting expect_success_port if the lookup is + * successful. * Check for failures using: * - tor_addr_parse(), * - tor_addr_port_parse() without a default port, @@ -1061,8 +1150,9 @@ test_addr_ip6_helpers(void *arg) * - tor_addr_lookup(), regardless of the address family, because it doesn't * support ports. */ -#define TEST_HOSTNAME_PORT(host_port_str) \ +#define TEST_HOSTNAME_PORT(host_port_str, expect_success_port) \ STMT_BEGIN \ + TEST_HOST_PORT_LOOKUP(host_port_str, expect_success_port); \ TEST_ADDR_PARSE_XFAIL(host_port_str); \ TEST_ADDR_PORT_PARSE_XFAIL(host_port_str, -1); \ TEST_ADDR_PORT_PARSE_XFAIL(host_port_str, 888); \ @@ -1080,21 +1170,51 @@ test_addr_parse(void *arg) /* Correct calls. */ TEST_ADDR_V4_PARSE_CANONICAL("192.0.2.1"); + TEST_ADDR_V4_PARSE_CANONICAL("192.0.2.2"); - TEST_ADDR_V6_PARSE_CANONICAL("11:22::33:44", 0); TEST_ADDR_V6_PARSE_CANONICAL("[11:22::33:44]", 1); + TEST_ADDR_V6_PARSE_CANONICAL("[::1]", 1); + TEST_ADDR_V6_PARSE_CANONICAL("[::]", 1); + TEST_ADDR_V6_PARSE_CANONICAL("[2::]", 1); + TEST_ADDR_V6_PARSE_CANONICAL("[11:22:33:44:55:66:77:88]", 1); + /* Allow IPv6 without square brackets, when there is no port, but only if + * there is a default port */ + TEST_ADDR_V6_PARSE_CANONICAL("11:22::33:44", 0); + TEST_ADDR_V6_PARSE_CANONICAL("::1", 0); + TEST_ADDR_V6_PARSE_CANONICAL("::", 0); + TEST_ADDR_V6_PARSE_CANONICAL("2::", 0); + TEST_ADDR_V6_PARSE_CANONICAL("11:22:33:44:55:66:77:88", 0); + + /* IPv6-mapped IPv4 addresses. Tor doesn't really use these. */ TEST_ADDR_V6_PARSE("11:22:33:44:55:66:1.2.3.4", 0, "11:22:33:44:55:66:102:304"); TEST_ADDR_V6_PARSE("11:22::33:44:1.2.3.4", 0, "11:22::33:44:102:304"); + /* Ports. */ + TEST_ADDR_V4_PORT_PARSE("192.0.2.1:1234", "192.0.2.1", 1234); + TEST_ADDR_V6_PORT_PARSE("[::1]:1234", "::1", 1234); + + /* Host names. */ + TEST_HOSTNAME("localhost"); + TEST_HOSTNAME_PORT("localhost:1234", 1234); + TEST_HOSTNAME_PORT("localhost:0", 0); + + TEST_HOSTNAME("torproject.org"); + TEST_HOSTNAME_PORT("torproject.org:56", 56); + + TEST_HOSTNAME("probably-not-a-valid-dns.name-tld"); + TEST_HOSTNAME_PORT("probably-not-a-valid-dns.name-tld:789", 789); + + /* Malformed addresses. */ /* Empty string. */ TEST_ADDR_PARSE_XFAIL_MALFORMED(""); /* Square brackets around IPv4 address. */ TEST_ADDR_PARSE_XFAIL_MALFORMED("[192.0.2.1]"); + TEST_ADDR_PARSE_XFAIL_MALFORMED("[192.0.2.3]:12345"); /* Only left square bracket. */ TEST_ADDR_PARSE_XFAIL_MALFORMED("[11:22::33:44"); @@ -1107,22 +1227,60 @@ test_addr_parse(void *arg) /* Trailing colon. */ TEST_ADDR_PARSE_XFAIL_MALFORMED("11:22::33:44:"); + TEST_ADDR_PARSE_XFAIL_MALFORMED("[::1]:"); + TEST_ADDR_PARSE_XFAIL_MALFORMED("localhost:"); - /* Too many hex words in IPv4-mapped IPv6 address. */ - TEST_ADDR_PARSE_XFAIL_MALFORMED("11:22:33:44:55:66:77:88:1.2.3.4"); + /* Bad port. */ + TEST_ADDR_PARSE_XFAIL_MALFORMED("192.0.2.2:66666"); + TEST_ADDR_PARSE_XFAIL_MALFORMED("[::1]:77777"); + TEST_ADDR_PARSE_XFAIL_MALFORMED("::1:88888"); + TEST_ADDR_PARSE_XFAIL_MALFORMED("localhost:99999"); - /* IPv6 address with port and no brackets */ - TEST_ADDR_PARSE_XFAIL_MALFORMED("11:22::33:44:12345"); + TEST_ADDR_PARSE_XFAIL_MALFORMED("192.0.2.2:-1"); + TEST_ADDR_PARSE_XFAIL_MALFORMED("[::1]:-2"); + TEST_ADDR_PARSE_XFAIL_MALFORMED("::1:-3"); + TEST_ADDR_PARSE_XFAIL_MALFORMED("localhost:-4"); + + TEST_ADDR_PARSE_XFAIL_MALFORMED("192.0.2.2:1 bad"); + TEST_ADDR_PARSE_XFAIL_MALFORMED("192.0.2.2:bad-port"); + TEST_ADDR_PARSE_XFAIL_MALFORMED("[::1]:bad-port-1"); + TEST_ADDR_PARSE_XFAIL_MALFORMED("::1:1-bad-port"); + TEST_ADDR_PARSE_XFAIL_MALFORMED("localhost:1-bad-port"); + TEST_ADDR_PARSE_XFAIL_MALFORMED("localhost:1-bad-port-1"); + + /* Bad hostname */ + TEST_ADDR_PARSE_XFAIL_MALFORMED("definitely invalid"); + TEST_ADDR_PARSE_XFAIL_MALFORMED("definitely invalid:22222"); + + /* Ambiguous cases */ + /* Too many hex words in IPv4-mapped IPv6 address. + * But some OS host lookup routines accept it as a hostname, or + * as an IP address?? (I assume they discard unused characters). */ + TEST_HOSTNAME("11:22:33:44:55:66:77:88:1.2.3.4"); + + /* IPv6 address with port and no brackets + * We reject it, but some OS host lookup routines accept it as an + * IPv6 address:port ? */ + TEST_HOSTNAME_PORT("11:22::33:44:12345", 12345); /* Is it a port, or are there too many hex words? - * We reject it either way. */ - TEST_ADDR_PARSE_XFAIL_MALFORMED("11:22:33:44:55:66:77:88:99"); + * We reject it either way, but some OS host lookup routines accept it as an + * IPv6 address:port */ + TEST_HOSTNAME_PORT("11:22:33:44:55:66:77:88:99", 99); /* But we accept it if it has square brackets. */ TEST_ADDR_V6_PORT_PARSE("[11:22:33:44:55:66:77:88]:99", "11:22:33:44:55:66:77:88",99); - /* This is an IPv6 address */ - TEST_ADDR_V6_PARSE_CANONICAL("11:22:33:44:55:66:77:88", 0); - TEST_ADDR_V6_PARSE_CANONICAL("[11:22:33:44:55:66:77:88]", 1); + /* Bad IPv4 address + * We reject it, but some OS host lookup routines accept it as an + * IPv4 address[:port], with a zero last octet */ + TEST_HOSTNAME("192.0.1"); + TEST_HOSTNAME_PORT("192.0.2:1234", 1234); + + /* More bad IPv6 addresses and ports: no brackets + * We reject it, but some OS host lookup routines accept it as an + * IPv6 address[:port] */ + TEST_HOSTNAME_PORT("::1:12345", 12345); + TEST_HOSTNAME_PORT("11:22::33:44:12345", 12345); /* And this is an ambiguous case, which is interpreted as an IPv6 address. */ TEST_ADDR_V6_PARSE_CANONICAL("11:22::88:99", 0); @@ -1131,37 +1289,6 @@ test_addr_parse(void *arg) TEST_ADDR_V6_PORT_PARSE("[11:22::88]:99", "11:22::88",99); - /* Correct calls. */ - TEST_ADDR_V4_PORT_PARSE("192.0.2.1:1234", "192.0.2.1", 1234); - TEST_ADDR_V6_PORT_PARSE("[::1]:1234", "::1", 1234); - - /* Domain name. */ - TEST_HOSTNAME("localhost"); - TEST_HOSTNAME_PORT("localhost:1234"); - TEST_HOSTNAME("torproject.org"); - TEST_HOSTNAME_PORT("torproject.org:1234"); - - /* Only IP. */ - TEST_ADDR_V4_PARSE_CANONICAL("192.0.2.2"); - TEST_ADDR_V6_PARSE_CANONICAL("[::1]", 1); - - /* Allow IPv6 without square brackets, when there is no port, but only if - * there is a default port */ - TEST_ADDR_V6_PARSE_CANONICAL("::1", 0); - - /* Bad port. */ - TEST_ADDR_PARSE_XFAIL_MALFORMED("192.0.2.2:66666"); - - /* Bad IPv4 address */ - TEST_ADDR_PARSE_XFAIL_MALFORMED("192.0.2:1234"); - - /* Bad IPv4 address and port: brackets */ - TEST_ADDR_PARSE_XFAIL_MALFORMED("[192.0.2.3]:12345"); - - /* Bad IPv6 addresses and ports: no brackets */ - TEST_ADDR_PARSE_XFAIL_MALFORMED("::1:12345"); - TEST_ADDR_PARSE_XFAIL_MALFORMED("11:22::33:44:12345"); - done: ; } From 362afa8c4e9fc2afeef703950e1f9c715cd1aef4 Mon Sep 17 00:00:00 2001 From: teor Date: Mon, 24 Jun 2019 13:50:26 +1000 Subject: [PATCH 1195/2557] doc: update the man page entries for DirAuthority and FallbackDir Improve the documentation for the DirAuthority and FallbackDir torrc options. Closes ticket 30955. --- changes/ticket30955 | 3 +++ doc/tor.1.txt | 45 +++++++++++++++++++++++++-------------------- 2 files changed, 28 insertions(+), 20 deletions(-) create mode 100644 changes/ticket30955 diff --git a/changes/ticket30955 b/changes/ticket30955 new file mode 100644 index 0000000000..7715a07569 --- /dev/null +++ b/changes/ticket30955 @@ -0,0 +1,3 @@ + o Documentation (hard-coded directories): + - Improve the documentation for the DirAuthority and FallbackDir torrc + options. Closes ticket 30955. diff --git a/doc/tor.1.txt b/doc/tor.1.txt index 064259b15f..dc261cb9c9 100644 --- a/doc/tor.1.txt +++ b/doc/tor.1.txt @@ -447,13 +447,18 @@ GENERAL OPTIONS setting for DataDirectoryGroupReadable when the CacheDirectory is the same as the DataDirectory, and 0 otherwise. (Default: auto) -[[FallbackDir]] **FallbackDir** __ipv4address__:__port__ orport=__port__ id=__fingerprint__ [weight=__num__] [ipv6=**[**__ipv6address__**]**:__orport__]:: - When we're unable to connect to any directory cache for directory info - (usually because we don't know about any yet) we try a directory authority. - Clients also simultaneously try a FallbackDir, to avoid hangs on client - startup if a directory authority is down. Clients retry FallbackDirs more - often than directory authorities, to reduce the load on the directory - authorities. +[[FallbackDir]] **FallbackDir** __ipv4address__:__dirport__ orport=__orport__ id=__fingerprint__ [weight=__num__] [ipv6=**[**__ipv6address__**]**:__orport__]:: + When tor is unable to connect to any directory cache for directory info + (usually because it doesn't know about any yet) it tries a hard-coded + directory. Relays try one directory authority at a time. Clients try + multiple directory authorities and FallbackDirs, to avoid hangs on + startup if a hard-coded directory is down. Clients wait for a few seconds + between each attempt, and retry FallbackDirs more often than directory + authorities, to reduce the load on the directory authorities. + + + + FallbackDirs should be stable relays with stable IP addresses, ports, + and identity keys. They must have a DirPort. + + + By default, the directory authorities are also FallbackDirs. Specifying a FallbackDir replaces Tor's default hard-coded FallbackDirs (if any). (See the **DirAuthority** entry for an explanation of each flag.) @@ -463,30 +468,30 @@ GENERAL OPTIONS FallbackDir line is present, it replaces the hard-coded FallbackDirs, regardless of the value of UseDefaultFallbackDirs.) (Default: 1) -[[DirAuthority]] **DirAuthority** [__nickname__] [**flags**] __ipv4address__:__port__ __fingerprint__:: +[[DirAuthority]] **DirAuthority** [__nickname__] [**flags**] __ipv4address__:__dirport__ __fingerprint__:: Use a nonstandard authoritative directory server at the provided address and port, with the specified key fingerprint. This option can be repeated many times, for multiple authoritative directory servers. Flags are separated by spaces, and determine what kind of an authority this directory is. By default, an authority is not authoritative for any directory style - or version unless an appropriate flag is given. + or version unless an appropriate flag is given. + + + Tor will use this authority as a bridge authoritative directory if the - "bridge" flag is set. If a flag "orport=**port**" is given, Tor will use the - given port when opening encrypted tunnels to the dirserver. If a flag - "weight=**num**" is given, then the directory server is chosen randomly - with probability proportional to that weight (default 1.0). If a + "bridge" flag is set. If a flag "orport=**orport**" is given, Tor will + use the given port when opening encrypted tunnels to the dirserver. If a + flag "weight=**num**" is given, then the directory server is chosen + randomly with probability proportional to that weight (default 1.0). If a flag "v3ident=**fp**" is given, the dirserver is a v3 directory authority whose v3 long-term signing key has the fingerprint **fp**. Lastly, if an "ipv6=**[**__ipv6address__**]**:__orport__" flag is present, then - the directory - authority is listening for IPv6 connections on the indicated IPv6 address - and OR Port. + + the directory authority is listening for IPv6 connections on the + indicated IPv6 address and OR Port. + + Tor will contact the authority at __ipv4address__ to - download directory documents. The provided __port__ value is a dirport; - clients ignore this in favor of the specified "orport=" value. If an - IPv6 ORPort is supplied, Tor will - also download directory documents at the IPv6 ORPort. + + download directory documents. Clients always use the ORPort. Relays + usually use the DirPort, but will use the ORPort in some circumstances. + If an IPv6 ORPort is supplied, clients will also download directory + documents at the IPv6 ORPort, if they are configured to use IPv6. + + If no **DirAuthority** line is given, Tor will use the default directory authorities. NOTE: this option is intended for setting up a private Tor From f446db59e1fc3f87f1312defbb12894ad006d978 Mon Sep 17 00:00:00 2001 From: David Goulet Date: Thu, 27 Jun 2019 14:47:11 -0400 Subject: [PATCH 1196/2557] man: Fix -help typo to --help Signed-off-by: David Goulet --- changes/ticket31008 | 3 +++ doc/tor.1.txt | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 changes/ticket31008 diff --git a/changes/ticket31008 b/changes/ticket31008 new file mode 100644 index 0000000000..c7077de6c6 --- /dev/null +++ b/changes/ticket31008 @@ -0,0 +1,3 @@ + o Documentation (tor.1 man page): + - Fix typo -help to --help in tor.1 man page. Fixes bug 31008; bugfix on + 0.2.2.9-alpha. diff --git a/doc/tor.1.txt b/doc/tor.1.txt index 975a198182..3909a829b5 100644 --- a/doc/tor.1.txt +++ b/doc/tor.1.txt @@ -37,7 +37,7 @@ Project's website. COMMAND-LINE OPTIONS -------------------- -[[opt-h]] **-h**, **-help**:: +[[opt-h]] **-h**, **--help**:: Display a short help message and exit. [[opt-f]] **-f** __FILE__:: From 0fa3dc3228a32fd21ff71e24e3c2e456e342e3b8 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 28 Jun 2019 11:27:59 -0400 Subject: [PATCH 1197/2557] begin_cell_parse(): Add an assertion to please coverity. Coverity doesn't understand that if begin_cell_parse() returns 0 and sets is_begindir to 0, its address field will always be set. Fixes bug 30126; bugfix on 0.2.4.7-alpha; Fixes CID 1447296. --- changes/ticket31026 | 5 +++++ scripts/maint/practracker/exceptions.txt | 4 ++-- src/core/or/connection_edge.c | 1 + 3 files changed, 8 insertions(+), 2 deletions(-) create mode 100644 changes/ticket31026 diff --git a/changes/ticket31026 b/changes/ticket31026 new file mode 100644 index 0000000000..6f6abcffba --- /dev/null +++ b/changes/ticket31026 @@ -0,0 +1,5 @@ + o Minor bugfixes (coverity compliance): + - Add an assertion when parsing a BEGIN cell so that coverity can be sure + that we are not about to dereference a NULL address. + Fixes bug 31026; bugfix on 0.2.4.7-alpha. This is CID + 1447296. diff --git a/scripts/maint/practracker/exceptions.txt b/scripts/maint/practracker/exceptions.txt index 3ed76a2bfd..e29d3b6076 100644 --- a/scripts/maint/practracker/exceptions.txt +++ b/scripts/maint/practracker/exceptions.txt @@ -101,7 +101,7 @@ problem function-size /src/core/or/circuituse.c:circuit_get_open_circ_or_launch( problem function-size /src/core/or/circuituse.c:connection_ap_handshake_attach_circuit() 244 problem function-size /src/core/or/command.c:command_process_create_cell() 156 problem function-size /src/core/or/command.c:command_process_relay_cell() 132 -problem file-size /src/core/or/connection_edge.c 4595 +problem file-size /src/core/or/connection_edge.c 4596 problem include-count /src/core/or/connection_edge.c 65 problem function-size /src/core/or/connection_edge.c:connection_ap_expire_beginning() 117 problem function-size /src/core/or/connection_edge.c:connection_ap_handshake_rewrite() 192 @@ -109,7 +109,7 @@ problem function-size /src/core/or/connection_edge.c:connection_ap_handle_onion( problem function-size /src/core/or/connection_edge.c:connection_ap_handshake_rewrite_and_attach() 423 problem function-size /src/core/or/connection_edge.c:connection_ap_handshake_send_begin() 111 problem function-size /src/core/or/connection_edge.c:connection_ap_handshake_socks_resolved() 106 -problem function-size /src/core/or/connection_edge.c:connection_exit_begin_conn() 184 +problem function-size /src/core/or/connection_edge.c:connection_exit_begin_conn() 185 problem function-size /src/core/or/connection_edge.c:connection_exit_connect() 102 problem file-size /src/core/or/connection_or.c 3124 problem include-count /src/core/or/connection_or.c 51 diff --git a/src/core/or/connection_edge.c b/src/core/or/connection_edge.c index c08d2a9ff5..091d9c9b09 100644 --- a/src/core/or/connection_edge.c +++ b/src/core/or/connection_edge.c @@ -3833,6 +3833,7 @@ connection_exit_begin_conn(cell_t *cell, circuit_t *circ) if (! bcell.is_begindir) { /* Steal reference */ + tor_assert(bcell.address); address = bcell.address; port = bcell.port; From f55598f870b2346ed48a32befc51b9a548b8b5fa Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 28 Jun 2019 11:57:36 -0400 Subject: [PATCH 1198/2557] Coverity: different implementation for csiphash Coverity has had trouble figuring out our csiphash implementation, and has given spurious warnings about its behavior. This patch changes the csiphash implementation when coverity is in use, so that coverity can figure out that we are not about to read beyond the provided input. Closes ticket 31025. --- changes/ticket31025 | 5 +++++ src/ext/csiphash.c | 8 ++++++++ 2 files changed, 13 insertions(+) create mode 100644 changes/ticket31025 diff --git a/changes/ticket31025 b/changes/ticket31025 new file mode 100644 index 0000000000..c572288239 --- /dev/null +++ b/changes/ticket31025 @@ -0,0 +1,5 @@ + o Minor bugfixes (coverity): + - In our siphash implementation, when building for coverity, use memcpy + in place of a switch statement, so that coverity can tell we are not + accessing out-of-bounds memory. Fixes bug 31025; bugfix on + 0.2.8.1-alpha. This is tracked as CID 1447293 and 1447295. diff --git a/src/ext/csiphash.c b/src/ext/csiphash.c index af8559a476..faa52ae4e1 100644 --- a/src/ext/csiphash.c +++ b/src/ext/csiphash.c @@ -87,6 +87,13 @@ uint64_t siphash24(const void *src, unsigned long src_sz, const struct sipkey *k v0 ^= mi; } +#ifdef __COVERITY__ + { + uint64_t mi = 0; + memcpy(&mi, m+i, (src_sz-blocks)); + last7 = _le64toh(mi) | (uint64_t)(src_sz & 0xff) << 56; + } +#else switch (src_sz - blocks) { case 7: last7 |= (uint64_t)m[i + 6] << 48; /* Falls through. */ case 6: last7 |= (uint64_t)m[i + 5] << 40; /* Falls through. */ @@ -98,6 +105,7 @@ uint64_t siphash24(const void *src, unsigned long src_sz, const struct sipkey *k case 0: default:; } +#endif v3 ^= last7; DOUBLE_ROUND(v0,v1,v2,v3); v0 ^= last7; From 68792f77e51f84d0fb6758ef9491a70570ac9a53 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 28 Jun 2019 12:21:49 -0400 Subject: [PATCH 1199/2557] Fix a few coverity unitinitialzed-value warnings in the unit tests. Coverity can't see that it is not in fact going to read uninitialized memory here, so we initialize these values unconditionally. Bugfix on 0.4.0.1-alpha. --- src/test/test_btrack.c | 4 ++++ src/test/test_controller_events.c | 3 +++ 2 files changed, 7 insertions(+) diff --git a/src/test/test_btrack.c b/src/test/test_btrack.c index 9e5d0d0723..21e88a57b6 100644 --- a/src/test/test_btrack.c +++ b/src/test/test_btrack.c @@ -44,6 +44,8 @@ test_btrack_launch(void *arg) { orconn_state_msg_t conn; ocirc_chan_msg_t circ; + memset(&conn, 0, sizeof(conn)); + memset(&circ, 0, sizeof(circ)); (void)arg; conn.gid = 1; @@ -93,6 +95,8 @@ test_btrack_delete(void *arg) { orconn_state_msg_t state; orconn_status_msg_t status; + memset(&state, 0, sizeof(state)); + memset(&status, 0, sizeof(status)); (void)arg; state.gid = 1; diff --git a/src/test/test_controller_events.c b/src/test/test_controller_events.c index a8967bba50..9fb2bc7256 100644 --- a/src/test/test_controller_events.c +++ b/src/test/test_controller_events.c @@ -429,6 +429,7 @@ static void test_cntev_orconn_state(void *arg) { orconn_state_msg_t conn; + memset(&conn, 0, sizeof(conn)); (void)arg; MOCK(queue_control_event_string, mock_queue_control_event_string); @@ -468,6 +469,7 @@ static void test_cntev_orconn_state_pt(void *arg) { orconn_state_msg_t conn; + memset(&conn, 0, sizeof(conn)); (void)arg; MOCK(queue_control_event_string, mock_queue_control_event_string); @@ -503,6 +505,7 @@ static void test_cntev_orconn_state_proxy(void *arg) { orconn_state_msg_t conn; + memset(&conn, 0, sizeof(conn)); (void)arg; MOCK(queue_control_event_string, mock_queue_control_event_string); From 5fa2b322005d1860d39e420cb6d3ed25f5073389 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 28 Jun 2019 12:24:26 -0400 Subject: [PATCH 1200/2557] Coverity: fix test issues with always-present 'service' var. Coverity is worried that we check "service" at the end of these test functions, since it doesn't see any way to reach the cleanup code without having first dereferenced the variable. Removing the check would be unwise in this case: instead we add a tt_assert check before using "service" so that coverity thinks that the check is doing something useful. Bugfix on 0.3.2.1-alpha. --- src/test/test_hs_common.c | 1 + src/test/test_hs_service.c | 1 + 2 files changed, 2 insertions(+) diff --git a/src/test/test_hs_common.c b/src/test/test_hs_common.c index abded6021e..de3f7e04f7 100644 --- a/src/test/test_hs_common.c +++ b/src/test/test_hs_common.c @@ -502,6 +502,7 @@ test_desc_reupload_logic(void *arg) pubkey_hex, strlen(pubkey_hex)); hs_build_address(&pubkey, HS_VERSION_THREE, onion_addr); service = tor_malloc_zero(sizeof(hs_service_t)); + tt_assert(service); memcpy(service->onion_address, onion_addr, sizeof(service->onion_address)); ed25519_secret_key_generate(&service->keys.identity_sk, 0); ed25519_public_key_generate(&service->keys.identity_pk, diff --git a/src/test/test_hs_service.c b/src/test/test_hs_service.c index a303f10411..2e4be4e295 100644 --- a/src/test/test_hs_service.c +++ b/src/test/test_hs_service.c @@ -1265,6 +1265,7 @@ test_service_event(void *arg) /* Set a service for this circuit. */ service = helper_create_service(); + tt_assert(service); ed25519_pubkey_copy(&circ->hs_ident->identity_pk, &service->keys.identity_pk); From ea154a6108bae597cb37e6bc53036b6dd2ed6187 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 28 Jun 2019 12:27:51 -0400 Subject: [PATCH 1201/2557] Coverity: fix memory leak on error in test function. The function make_intro_from_plaintext() in test_introduce.c would leak memory if we ever hit a failure from our underlying crypto functions. This kind of failure should be impossible, but it's best to be safe here. Bugfix on 0.2.4.1-alpha. --- src/test/test_introduce.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/test/test_introduce.c b/src/test/test_introduce.c index 4a6d90d97e..104e973b1f 100644 --- a/src/test/test_introduce.c +++ b/src/test/test_introduce.c @@ -383,8 +383,10 @@ make_intro_from_plaintext( /* Output the cell */ *cell_out = cell; + cell = NULL; done: + tor_free(cell); return cell_len; } @@ -535,4 +537,3 @@ struct testcase_t introduce_tests[] = { INTRODUCE_LEGACY(late_parse_v3), END_OF_TESTCASES }; - From 75ea7514e1e87124bf250b36a1a1c243b6b0cea0 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 28 Jun 2019 12:36:12 -0400 Subject: [PATCH 1202/2557] Add a changes file for coverity test fixes of 31030. --- changes/ticket31030 | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 changes/ticket31030 diff --git a/changes/ticket31030 b/changes/ticket31030 new file mode 100644 index 0000000000..4d99323b4e --- /dev/null +++ b/changes/ticket31030 @@ -0,0 +1,3 @@ + o Minor bugfixes (coverity, tests): + - Fix several coverity warnings from our unit tests. Fixes bug 31030; + bugfix on 0.2.4.1-alpha, 0.3.2.1-alpha, and 0.4.0.1-alpha. From 0e4753e579c0288a295d1e0a585e1156dd26590a Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Sat, 29 Jun 2019 19:38:14 +0300 Subject: [PATCH 1203/2557] Remove dead code from circpad_machine_remove_token(). --- changes/bug31027 | 3 +++ src/core/or/circuitpadding.c | 9 ++++----- 2 files changed, 7 insertions(+), 5 deletions(-) create mode 100644 changes/bug31027 diff --git a/changes/bug31027 b/changes/bug31027 new file mode 100644 index 0000000000..dd3ce20b60 --- /dev/null +++ b/changes/bug31027 @@ -0,0 +1,3 @@ + o Code simplification and refactoring: + - Remove some dead code from circpad_machine_remove_token() to fix some + Coverity warnings (CID 1447298). Fixes bug 31027; bugfix on 0.4.1.1-alpha. \ No newline at end of file diff --git a/src/core/or/circuitpadding.c b/src/core/or/circuitpadding.c index 0214cc4219..626787da7b 100644 --- a/src/core/or/circuitpadding.c +++ b/src/core/or/circuitpadding.c @@ -1083,8 +1083,11 @@ circpad_machine_remove_token(circpad_machine_runtime_t *mi) state = circpad_machine_current_state(mi); + /* If we are not in a padding state (like start or end), we're done */ + if (!state) + return; /* Don't remove any tokens if we're not doing token removal */ - if (!state || state->token_removal == CIRCPAD_TOKEN_REMOVAL_NONE) + if (state->token_removal == CIRCPAD_TOKEN_REMOVAL_NONE) return; current_time = monotime_absolute_usec(); @@ -1103,10 +1106,6 @@ circpad_machine_remove_token(circpad_machine_runtime_t *mi) timer_disable(mi->padding_timer); } - /* If we are not in a padding state (like start or end), we're done */ - if (!state) - return; - /* Perform the specified token removal strategy */ switch (state->token_removal) { case CIRCPAD_TOKEN_REMOVAL_CLOSEST_USEC: From a5911c4551d001c6106ca780bf4c3c1d4078fc76 Mon Sep 17 00:00:00 2001 From: Roger Dingledine Date: Sat, 29 Jun 2019 22:26:00 -0400 Subject: [PATCH 1204/2557] get rid of accidental second space --- src/core/or/circuitpadding.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/or/circuitpadding.c b/src/core/or/circuitpadding.c index 0214cc4219..017ef7b6ba 100644 --- a/src/core/or/circuitpadding.c +++ b/src/core/or/circuitpadding.c @@ -210,7 +210,7 @@ circpad_marked_circuit_for_padding(circuit_t *circ, int reason) } log_info(LD_CIRC, "Circuit %d is not marked for close because of a " - " pending padding machine.", CIRCUIT_IS_ORIGIN(circ) ? + "pending padding machine.", CIRCUIT_IS_ORIGIN(circ) ? TO_ORIGIN_CIRCUIT(circ)->global_identifier : 0); /* If the machine has had no network events at all within the @@ -222,7 +222,7 @@ circpad_marked_circuit_for_padding(circuit_t *circ, int reason) if (circ->padding_info[i]->last_cell_time_sec + (time_t)CIRCPAD_DELAY_MAX_SECS < approx_time()) { log_notice(LD_BUG, "Circuit %d was not marked for close because of a " - " pending padding machine for over an hour. Circuit is a %s", + "pending padding machine for over an hour. Circuit is a %s", CIRCUIT_IS_ORIGIN(circ) ? TO_ORIGIN_CIRCUIT(circ)->global_identifier : 0, circuit_purpose_to_string(circ->purpose)); From 98c1262b2a374de9dc2579a71a1c7464f4435f1d Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Sat, 29 Jun 2019 19:46:43 +0300 Subject: [PATCH 1205/2557] Always check the retval of circpad_machine_current_state(). --- changes/bug31024 | 4 ++++ src/core/or/circuitpadding.c | 6 ++++++ 2 files changed, 10 insertions(+) create mode 100644 changes/bug31024 diff --git a/changes/bug31024 b/changes/bug31024 new file mode 100644 index 0000000000..888fb2a26b --- /dev/null +++ b/changes/bug31024 @@ -0,0 +1,4 @@ + o Minor bugfixes (circuitpadding): + - Add two NULL checks in unreachable places to silence Coverity (CID 144729 + and 1447291) and better future proof ourselves. Fixes bug 31024; bugfix + on 0.4.1.1-alpha. \ No newline at end of file diff --git a/src/core/or/circuitpadding.c b/src/core/or/circuitpadding.c index 626787da7b..c4670bbc2f 100644 --- a/src/core/or/circuitpadding.c +++ b/src/core/or/circuitpadding.c @@ -450,6 +450,9 @@ circpad_is_token_removal_supported(circpad_machine_runtime_t *mi) /* Machines that do want token removal are less sensitive to performance. * Let's spend some time to check that our state is consistent and sane */ const circpad_state_t *state = circpad_machine_current_state(mi); + if (BUG(!state)) { + return 1; + } tor_assert_nonfatal(state->token_removal != CIRCPAD_TOKEN_REMOVAL_NONE); tor_assert_nonfatal(state->histogram_len == mi->histogram_len); tor_assert_nonfatal(mi->histogram_len != 0); @@ -1667,6 +1670,9 @@ circpad_estimate_circ_rtt_on_received(circuit_t *circ, } } else { const circpad_state_t *state = circpad_machine_current_state(mi); + if (BUG(!state)) { + return; + } /* Since monotime is unpredictably expensive, only update this field * if rtt estimates are needed. Otherwise, stop the rtt update. */ From 59e523f058ee0953deacf1370d546cabd192584d Mon Sep 17 00:00:00 2001 From: rl1987 Date: Tue, 2 Jul 2019 20:06:23 +0300 Subject: [PATCH 1206/2557] Early exit from post-merge git hook script when not merging to master --- changes/bug31040 | 3 +++ scripts/git/post-merge.git-hook | 6 ++++++ 2 files changed, 9 insertions(+) create mode 100644 changes/bug31040 diff --git a/changes/bug31040 b/changes/bug31040 new file mode 100644 index 0000000000..81f6d7e795 --- /dev/null +++ b/changes/bug31040 @@ -0,0 +1,3 @@ + o Minor bugfixes (developer tooling): + - Only log git script changes in post-merge script when merge was to the + master branch. Fixes bug 31040; bugfix on 0.4.1.1-alpha. diff --git a/scripts/git/post-merge.git-hook b/scripts/git/post-merge.git-hook index 176b7c9bbd..eae4f999e7 100755 --- a/scripts/git/post-merge.git-hook +++ b/scripts/git/post-merge.git-hook @@ -35,6 +35,12 @@ check_for_script_update() { fi } +cur_branch=$(git rev-parse --abbrev-ref HEAD) +if [ "$cur_branch" != "master" ]; then + echo "post-merge: Not a master branch. Skipping." + exit 0 +fi + check_for_diffs "pre-push" check_for_diffs "pre-commit" check_for_diffs "post-merge" From ef2123c7c7bbf0cb6ec2a5528bae082d51ea8962 Mon Sep 17 00:00:00 2001 From: David Goulet Date: Wed, 19 Jun 2019 12:02:41 -0400 Subject: [PATCH 1207/2557] hs-v3: Disallow single hop client to post/get a descriptor Closes #24964 Signed-off-by: David Goulet --- changes/ticket24964 | 4 ++ src/feature/dircache/dircache.c | 13 +++--- src/feature/dircommon/directory.c | 69 +++++++++++++++++++++++++++++++ src/feature/dircommon/directory.h | 1 + src/test/test_hs_cache.c | 23 ++++++++++- 5 files changed, 102 insertions(+), 8 deletions(-) create mode 100644 changes/ticket24964 diff --git a/changes/ticket24964 b/changes/ticket24964 new file mode 100644 index 0000000000..171c86eb1d --- /dev/null +++ b/changes/ticket24964 @@ -0,0 +1,4 @@ + o Minor feature (onion service v3): + - Do not allow single hop client to fetch or post an HS descriptor from an + HSDir. Closes ticket 24964; + diff --git a/src/feature/dircache/dircache.c b/src/feature/dircache/dircache.c index 1b36f716f4..7c6af3582b 100644 --- a/src/feature/dircache/dircache.c +++ b/src/feature/dircache/dircache.c @@ -1390,8 +1390,9 @@ handle_get_hs_descriptor_v3(dir_connection_t *conn, const char *pubkey_str = NULL; const char *url = args->url; - /* Reject unencrypted dir connections */ - if (!connection_dir_is_encrypted(conn)) { + /* Reject non anonymous dir connections (which also tests if encrypted). We + * do not allow single hop clients to query an HSDir. */ + if (!connection_dir_is_anonymous(conn)) { write_short_http_response(conn, 404, "Not found"); goto done; } @@ -1632,10 +1633,10 @@ directory_handle_command_post,(dir_connection_t *conn, const char *headers, goto done; } - /* Handle HS descriptor publish request. */ - /* XXX: This should be disabled with a consensus param until we want to - * the prop224 be deployed and thus use. */ - if (connection_dir_is_encrypted(conn) && !strcmpstart(url, "/tor/hs/")) { + /* Handle HS descriptor publish request. We force an anonymous connection + * (which also tests for encrypted). We do not allow single-hop client to + * post a descriptor onto an HSDir. */ + if (connection_dir_is_anonymous(conn) && !strcmpstart(url, "/tor/hs/")) { const char *msg = "HS descriptor stored successfully."; /* We most probably have a publish request for an HS descriptor. */ diff --git a/src/feature/dircommon/directory.c b/src/feature/dircommon/directory.c index 9e6f72e9ac..b3db0aa108 100644 --- a/src/feature/dircommon/directory.c +++ b/src/feature/dircommon/directory.c @@ -7,6 +7,10 @@ #include "app/config/config.h" #include "core/mainloop/connection.h" +#include "core/or/circuitlist.h" +#include "core/or/connection_edge.h" +#include "core/or/connection_or.h" +#include "core/or/channeltls.h" #include "feature/dircache/dircache.h" #include "feature/dircache/dirserv.h" #include "feature/dirclient/dirclient.h" @@ -15,6 +19,10 @@ #include "feature/stats/geoip_stats.h" #include "lib/compress/compress.h" +#include "core/or/circuit_st.h" +#include "core/or/or_circuit_st.h" +#include "core/or/edge_connection_st.h" +#include "core/or/or_connection_st.h" #include "feature/dircommon/dir_connection_st.h" #include "feature/nodelist/routerinfo_st.h" @@ -167,6 +175,67 @@ connection_dir_is_encrypted(const dir_connection_t *conn) return TO_CONN(conn)->linked; } +/** Return true iff the given directory connection dir_conn is + * anonymous, that is, it is on a circuit via a public relay and not directly + * from a client or bridge. + * + * For client circuits via relays: true for 2-hop+ paths. + * For client circuits via bridges: true for 3-hop+ paths. + * + * This first test if the connection is encrypted since it is a strong + * requirement for anonymity. */ +bool +connection_dir_is_anonymous(const dir_connection_t *dir_conn) +{ + const connection_t *conn, *linked_conn; + const edge_connection_t *edge_conn; + const circuit_t *circ; + + tor_assert(dir_conn); + + if (!connection_dir_is_encrypted(dir_conn)) { + return false; + } + + /* + * Buckle up, we'll do a deep dive into the connection in order to get the + * final connection channel of that connection in order to figure out if + * this is a client or relay link. + * + * We go: dir_conn -> linked_conn -> edge_conn -> on_circuit -> p_chan. + */ + + conn = TO_CONN(dir_conn); + linked_conn = conn->linked_conn; + + /* The dir connection should be connected to an edge connection. It can not + * be closed or marked for close. */ + if (linked_conn == NULL || linked_conn->magic != EDGE_CONNECTION_MAGIC || + conn->linked_conn_is_closed || conn->linked_conn->marked_for_close) { + log_info(LD_DIR, "Rejected HSDir request: not linked to edge"); + return false; + } + + edge_conn = TO_EDGE_CONN((connection_t *) linked_conn); + circ = edge_conn->on_circuit; + + /* Can't be a circuit we initiated and without a circuit, no channel. */ + if (circ == NULL || CIRCUIT_IS_ORIGIN(circ)) { + log_info(LD_DIR, "Rejected HSDir request: not on OR circuit"); + return false; + } + + /* Get the previous channel to learn if it is a client or relay link. */ + if (BUG(CONST_TO_OR_CIRCUIT(circ)->p_chan == NULL)) { + log_info(LD_DIR, "Rejected HSDir request: no p_chan"); + return false; + } + + /* Will be true if the channel is an unauthenticated peer which is only true + * for clients and bridges. */ + return !channel_is_client(CONST_TO_OR_CIRCUIT(circ)->p_chan); +} + /** Parse an HTTP request line at the start of a headers string. On failure, * return -1. On success, set *command_out to a copy of the HTTP * command ("get", "post", etc), set *url_out to a copy of the URL, and diff --git a/src/feature/dircommon/directory.h b/src/feature/dircommon/directory.h index ba3f8c1b0e..4fc743ad3d 100644 --- a/src/feature/dircommon/directory.h +++ b/src/feature/dircommon/directory.h @@ -94,6 +94,7 @@ int parse_http_command(const char *headers, char *http_get_header(const char *headers, const char *which); int connection_dir_is_encrypted(const dir_connection_t *conn); +bool connection_dir_is_anonymous(const dir_connection_t *conn); int connection_dir_reached_eof(dir_connection_t *conn); int connection_dir_process_inbuf(dir_connection_t *conn); int connection_dir_finished_flushing(dir_connection_t *conn); diff --git a/src/test/test_hs_cache.c b/src/test/test_hs_cache.c index d71f8b6b18..86ac7e7fb1 100644 --- a/src/test/test_hs_cache.c +++ b/src/test/test_hs_cache.c @@ -10,6 +10,7 @@ #define DIRCACHE_PRIVATE #define DIRCLIENT_PRIVATE #define HS_CACHE_PRIVATE +#define TOR_CHANNEL_INTERNAL_ #include "trunnel/ed25519_cert.h" #include "feature/hs/hs_cache.h" @@ -20,7 +21,12 @@ #include "core/mainloop/connection.h" #include "core/proto/proto_http.h" #include "lib/crypt_ops/crypto_format.h" +#include "core/or/circuitlist.h" +#include "core/or/channel.h" +#include "core/or/edge_connection_st.h" +#include "core/or/or_circuit_st.h" +#include "core/or/or_connection_st.h" #include "feature/dircommon/dir_connection_st.h" #include "feature/nodelist/networkstatus_st.h" @@ -232,6 +238,8 @@ helper_fetch_desc_from_hsdir(const ed25519_public_key_t *blinded_key) /* The dir conn we are going to simulate */ dir_connection_t *conn = NULL; + edge_connection_t *edge_conn = NULL; + or_circuit_t *or_circ = NULL; /* First extract the blinded public key that we are going to use in our query, and then build the actual query string. */ @@ -245,8 +253,16 @@ helper_fetch_desc_from_hsdir(const ed25519_public_key_t *blinded_key) /* Simulate an HTTP GET request to the HSDir */ conn = dir_connection_new(AF_INET); tt_assert(conn); + TO_CONN(conn)->linked = 1; /* Signal that it is encrypted. */ tor_addr_from_ipv4h(&conn->base_.addr, 0x7f000001); - TO_CONN(conn)->linked = 1;/* Pretend the conn is encrypted :) */ + + /* Pretend this conn is anonymous. */ + edge_conn = edge_connection_new(CONN_TYPE_EXIT, AF_INET); + TO_CONN(conn)->linked_conn = TO_CONN(edge_conn); + or_circ = or_circuit_new(0, NULL); + or_circ->p_chan = tor_malloc_zero(sizeof(channel_t)); + edge_conn->on_circuit = TO_CIRCUIT(or_circ); + retval = directory_handle_command_get(conn, hsdir_query_str, NULL, 0); tt_int_op(retval, OP_EQ, 0); @@ -263,8 +279,11 @@ helper_fetch_desc_from_hsdir(const ed25519_public_key_t *blinded_key) done: tor_free(hsdir_query_str); - if (conn) + if (conn) { + tor_free(or_circ->p_chan); + connection_free_minimal(TO_CONN(conn)->linked_conn); connection_free_minimal(TO_CONN(conn)); + } return received_desc; } From e3ccf37e2545adf42e9b5975b51fbb4b46d91da4 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 3 Jul 2019 10:19:57 -0400 Subject: [PATCH 1208/2557] Fix @file directive in var_type_def_st.h --- src/lib/confmgt/var_type_def_st.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/confmgt/var_type_def_st.h b/src/lib/confmgt/var_type_def_st.h index c63ff66eff..d142ee1104 100644 --- a/src/lib/confmgt/var_type_def_st.h +++ b/src/lib/confmgt/var_type_def_st.h @@ -5,7 +5,7 @@ /* See LICENSE for licensing information */ /** - * @file typedvar.h + * @file var_type_def_st.h * @brief Structure declarations for typedvar type definitions. * * This structure is used for defining new variable types. If you are not From daed2e39ada7efcd8d61a306f8a5f4e22316350f Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 3 Jul 2019 10:21:49 -0400 Subject: [PATCH 1209/2557] Revert "Add a function to append an existing line to a config line list." This reverts commit 5a2ab886baaa125fe715acca8f7daf35031855aa. --- src/lib/encoding/confline.c | 15 +++------------ src/lib/encoding/confline.h | 3 +-- 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/src/lib/encoding/confline.c b/src/lib/encoding/confline.c index b36e83dcc6..fdb575e03f 100644 --- a/src/lib/encoding/confline.c +++ b/src/lib/encoding/confline.c @@ -33,23 +33,14 @@ config_line_append(config_line_t **lst, const char *key, const char *val) { + tor_assert(lst); + config_line_t *newline; newline = tor_malloc_zero(sizeof(config_line_t)); newline->key = tor_strdup(key); newline->value = tor_strdup(val); newline->next = NULL; - - config_line_append_line(lst, newline); -} - -/** Helper: append newline to the end of lst. */ -void -config_line_append_line(config_line_t **lst, - config_line_t *newline) -{ - tor_assert(lst); - while (*lst) lst = &((*lst)->next); @@ -265,7 +256,7 @@ config_lines_dup_and_filter(const config_line_t *inp, /** Return true iff a and b contain identical keys and values in identical * order. */ int -config_lines_eq(const config_line_t *a, const config_line_t *b) +config_lines_eq(config_line_t *a, config_line_t *b) { while (a && b) { if (strcasecmp(a->key, b->key) || strcmp(a->value, b->value)) diff --git a/src/lib/encoding/confline.h b/src/lib/encoding/confline.h index 7d0b9ce5fc..56ea36bf61 100644 --- a/src/lib/encoding/confline.h +++ b/src/lib/encoding/confline.h @@ -41,7 +41,6 @@ typedef struct config_line_t { void config_line_append(config_line_t **lst, const char *key, const char *val); -void config_line_append_line(config_line_t **lst, config_line_t *newline); void config_line_prepend(config_line_t **lst, const char *key, const char *val); config_line_t *config_lines_dup(const config_line_t *inp); @@ -51,7 +50,7 @@ const config_line_t *config_line_find(const config_line_t *lines, const char *key); const config_line_t *config_line_find_case(const config_line_t *lines, const char *key); -int config_lines_eq(const config_line_t *a, const config_line_t *b); +int config_lines_eq(config_line_t *a, config_line_t *b); int config_count_key(const config_line_t *a, const char *key); void config_free_lines_(config_line_t *front); #define config_free_lines(front) \ From 3e34840a77c45da3575b8a929ed8bbd7370aad63 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 3 Jul 2019 10:27:38 -0400 Subject: [PATCH 1210/2557] Make config_lines_eq() take const arguments. --- src/lib/encoding/confline.c | 2 +- src/lib/encoding/confline.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/encoding/confline.c b/src/lib/encoding/confline.c index fdb575e03f..0d8384db13 100644 --- a/src/lib/encoding/confline.c +++ b/src/lib/encoding/confline.c @@ -256,7 +256,7 @@ config_lines_dup_and_filter(const config_line_t *inp, /** Return true iff a and b contain identical keys and values in identical * order. */ int -config_lines_eq(config_line_t *a, config_line_t *b) +config_lines_eq(const config_line_t *a, const config_line_t *b) { while (a && b) { if (strcasecmp(a->key, b->key) || strcmp(a->value, b->value)) diff --git a/src/lib/encoding/confline.h b/src/lib/encoding/confline.h index 56ea36bf61..12c554c6e7 100644 --- a/src/lib/encoding/confline.h +++ b/src/lib/encoding/confline.h @@ -50,7 +50,7 @@ const config_line_t *config_line_find(const config_line_t *lines, const char *key); const config_line_t *config_line_find_case(const config_line_t *lines, const char *key); -int config_lines_eq(config_line_t *a, config_line_t *b); +int config_lines_eq(const config_line_t *a, const config_line_t *b); int config_count_key(const config_line_t *a, const char *key); void config_free_lines_(config_line_t *front); #define config_free_lines(front) \ From ebf1fc64fe3e7fbebb1640ddd8e59dd065cbe2ea Mon Sep 17 00:00:00 2001 From: Roger Dingledine Date: Wed, 3 Jul 2019 15:56:17 -0400 Subject: [PATCH 1211/2557] clarify a comment in the tor-exit-notice text (a relay operator in #tor-relays just now was confused and thought that dirportfrontpage could serve multiple files, like a real webserver.) --- contrib/operator-tools/tor-exit-notice.html | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/contrib/operator-tools/tor-exit-notice.html b/contrib/operator-tools/tor-exit-notice.html index 8cf5c294f2..c9f939f0d9 100644 --- a/contrib/operator-tools/tor-exit-notice.html +++ b/contrib/operator-tools/tor-exit-notice.html @@ -38,8 +38,11 @@ router IP should be generating no other traffic, unless it has been compromised.

- +

From 85473f9aaf0f54e44aae413d838821956371901a Mon Sep 17 00:00:00 2001 From: Roger Dingledine Date: Wed, 3 Jul 2019 16:01:05 -0400 Subject: [PATCH 1212/2557] typo fix --- contrib/operator-tools/tor-exit-notice.html | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/contrib/operator-tools/tor-exit-notice.html b/contrib/operator-tools/tor-exit-notice.html index c9f939f0d9..7f3d7525d0 100644 --- a/contrib/operator-tools/tor-exit-notice.html +++ b/contrib/operator-tools/tor-exit-notice.html @@ -37,8 +37,7 @@ privacy to people who need it most: average computer users. This router IP should be generating no other traffic, unless it has been compromised.

- - + +Smartlists are created empty with smartlist_new() and freed with +smartlist_free(). See the containers.h module documentation for more +information; there are many convenience functions for commonly needed +operations. + + +### Digest maps, string maps, and more. + +Tor makes frequent use of maps from 160-bit digests, 256-bit digests, +or nul-terminated strings to void \*. These types are digestmap_t, +digest256map_t, and strmap_t respectively. See the containers.h +module documentation for more information. + + +### Intrusive lists and hashtables + +For performance-sensitive cases, we sometimes want to use "intrusive" +collections: ones where the bookkeeping pointers are stuck inside the +structures that belong to the collection. If you've used the +BSD-style sys/queue.h macros, you'll be familiar with these. + +Unfortunately, the sys/queue.h macros vary significantly between the +platforms that have them, so we provide our own variants in +src/ext/tor_queue.h . + +We also provide an intrusive hashtable implementation in src/ext/ht.h +. When you're using it, you'll need to define your own hash +functions. If attacker-induced collisions are a worry here, use the +cryptographic siphash24g function to extract hashes. + diff --git a/doc/HACKING/design/01c-time.md b/doc/HACKING/design/01c-time.md new file mode 100644 index 0000000000..5cd0b354fd --- /dev/null +++ b/doc/HACKING/design/01c-time.md @@ -0,0 +1,75 @@ + +## Time in tor ## + +### What time is it? ### + +We have several notions of the current time in Tor. + +The *wallclock time* is available from time(NULL) with +second-granularity and tor_gettimeofday() with microsecond +granularity. It corresponds most closely to "the current time and date". + +The *monotonic time* is available with the set of monotime_\* +functions declared in compat_time.h. Unlike the wallclock time, it +can only move forward. It does not necessarily correspond to a real +world time, and it is not portable between systems. + +The *coarse monotonic time* is available from the set of +monotime_coarse_\* functions in compat_time.h. It is the same as +monotime_\* on some platforms. On others, it gives a monotonic timer +with less precision, but which it's more efficient to access. + +### Cached views of time. ### + +On some systems (like Linux), many time functions use a VDSO to avoid +the overhead of a system call. But on other systems, gettimeofday() +and time() can be costly enough that you wouldn't want to call them +tens of thousands of times. To get a recent, but not especially +accurate, view of the current time, see approx_time() and +tor_gettimeofday_cached(). + + +### Parsing and encoding time values ### + +Tor has functions to parse and format time in these formats: + + * RFC1123 format. ("Fri, 29 Sep 2006 15:54:20 GMT"). For this, + use format_rfc1123_time() and parse_rfc1123_time. + + * ISO8601 format. ("2006-10-29 10:57:20") For this, use + format_local_iso_time and format_iso_time. We also support the + variant format "2006-10-29T10:57:20" with format_iso_time_nospace, and + "2006-10-29T10:57:20.123456" with format_iso_time_nospace_usec. + + * HTTP format collections (preferably "Mon, 25 Jul 2016 04:01:11 + GMT" or possibly "Wed Jun 30 21:49:08 1993" or even "25-Jul-16 + 04:01:11 GMT"). For this, use parse_http_time. Don't generate anything + but the first format. + +Some of these functions use struct tm. You can use the standard +tor_localtime_r and tor_gmtime_r() to wrap these in a safe way. We +also have a tor_timegm() function. + +### Scheduling events ### + +The main way to schedule a not-too-frequent periodic event with +respect to the Tor mainloop is via the mechanism in periodic.c. +There's a big table of periodic_events in main.c, each of which gets +invoked on its own schedule. You should not expect more than about +one second of accuracy with these timers. + +You can create an independent timer using libevent directly, or using +the periodic_timer_new() function. But you should avoid doing this +for per-connection or per-circuit timers: Libevent's internal timer +implementation uses a min-heap, and those tend to start scaling poorly +once you have a few thousand entries. + +If you need to create a large number of fine-grained timers for some +purpose, you should consider the mechanism in src/common/timers.c, +which is optimized for the case where you have a large number of +timers with not-too-long duration, many of which will be deleted +before they actually expire. These timers should be reasonably +accurate within a handful of milliseconds -- possibly better on some +platforms. (The timers.c module uses William Ahern's timeout.c +implementation as its backend, which is based on a hierarchical timing +wheel algorithm. It's cool stuff; check it out.) diff --git a/doc/HACKING/design/01d-crypto.md b/doc/HACKING/design/01d-crypto.md new file mode 100644 index 0000000000..d4def947d1 --- /dev/null +++ b/doc/HACKING/design/01d-crypto.md @@ -0,0 +1,169 @@ + +## Lower-level cryptography functionality in Tor ## + +Generally speaking, Tor code shouldn't be calling OpenSSL (or any +other crypto library) directly. Instead, we should indirect through +one of the functions in src/common/crypto\*.c or src/common/tortls.c. + +Cryptography functionality that's available is described below. + +### RNG facilities ### + +The most basic RNG capability in Tor is the crypto_rand() family of +functions. These currently use OpenSSL's RAND_() backend, but may use +something faster in the future. + +In addition to crypto_rand(), which fills in a buffer with random +bytes, we also have functions to produce random integers in certain +ranges; to produce random hostnames; to produce random doubles, etc. + +When you're creating a long-term cryptographic secret, you might want +to use crypto_strongest_rand() instead of crypto_rand(). It takes the +operating system's entropy source and combines it with output from +crypto_rand(). This is a pure paranoia measure, but it might help us +someday. + +You can use smartlist_choose() to pick a random element from a smartlist +and smartlist_shuffle() to randomize the order of a smartlist. Both are +potentially a bit slow. + +### Cryptographic digests and related functions ### + +We treat digests as separate types based on the length of their +outputs. We support one 160-bit digest (SHA1), two 256-bit digests +(SHA256 and SHA3-256), and two 512-bit digests (SHA512 and SHA3-512). + +You should not use SHA1 for anything new. + +The crypto_digest\*() family of functions manipulates digests. You +can either compute a digest of a chunk of memory all at once using +crypto_digest(), crypto_digest256(), or crypto_digest512(). Or you +can create a crypto_digest_t object with +crypto_digest{,256,512}_new(), feed information to it in chunks using +crypto_digest_add_bytes(), and then extract the final digest using +crypto_digest_get_digest(). You can copy the state of one of these +objects using crypto_digest_dup() or crypto_digest_assign(). + +We support the HMAC hash-based message authentication code +instantiated using SHA256. See crypto_hmac_sha256. (You should not +add any HMAC users with SHA1, and HMAC is not necessary with SHA3.) + +We also support the SHA3 cousins, SHAKE128 and SHAKE256. Unlike +digests, these are extendable output functions (or XOFs) where you can +get any amount of output. Use the crypto_xof_\*() functions to access +these. + +We have several ways to derive keys from cryptographically strong secret +inputs (like diffie-hellman outputs). The old +crypto_expand_key_material-TAP() performs an ad-hoc KDF based on SHA1 -- you +shouldn't use it for implementing anything but old versions of the Tor +protocol. You can use HKDF-SHA256 (as defined in RFC5869) for more modern +protocols. Also consider SHAKE256. + +If your input is potentially weak, like a password or passphrase, use a salt +along with the secret_to_key() functions as defined in crypto_s2k.c. Prefer +scrypt over other hashing methods when possible. If you're using a password +to encrypt something, see the "boxed file storage" section below. + +Finally, in order to store objects in hash tables, Tor includes the +randomized SipHash 2-4 function. Call it via the siphash24g() function in +src/ext/siphash.h whenever you're creating a hashtable whose keys may be +manipulated by an attacker in order to DoS you with collisions. + + +### Stream ciphers ### + +You can create instances of a stream cipher using crypto_cipher_new(). +These are stateful objects of type crypto_cipher_t. Note that these +objects only support AES-128 right now; a future version should add +support for AES-128 and/or ChaCha20. + +You can encrypt/decrypt with crypto_cipher_encrypt or +crypto_cipher_decrypt. The crypto_cipher_crypt_inplace function performs +an encryption without a copy. + +Note that sensible people should not use raw stream ciphers; they should +probably be using some kind of AEAD. Sorry. + +### Public key functionality ### + +We support four public key algorithms: DH1024, RSA, Curve25519, and +Ed25519. + +We support DH1024 over two prime groups. You access these via the +crypto_dh_\*() family of functions. + +We support RSA in many bit sizes for signing and encryption. You access +it via the crypto_pk_*() family of functions. Note that a crypto_pk_t +may or may not include a private key. See the crypto_pk_* functions in +crypto.c for a full list of functions here. + +For Curve25519 functionality, see the functions and types in +crypto_curve25519.c. Curve25519 is generally suitable for when you need +a secure fast elliptic-curve diffie hellman implementation. When +designing new protocols, prefer it over DH in Z_p. + +For Ed25519 functionality, see the functions and types in +crypto_ed25519.c. Ed25519 is a generally suitable as a secure fast +elliptic curve signature method. For new protocols, prefer it over RSA +signatures. + +### Metaformats for storage ### + +When OpenSSL manages the storage of some object, we use whatever format +OpenSSL provides -- typically, some kind of PEM-wrapped base 64 encoding +that starts with "----- BEGIN CRYPTOGRAPHIC OBJECT ----". + +When we manage the storage of some cryptographic object, we prefix the +object with 32-byte NUL-padded prefix in order to avoid accidental +object confusion; see the crypto_read_tagged_contents_from_file() and +crypto_write_tagged_contents_to_file() functions for manipulating +these. The prefix is "== type: tag ==", where type describes the object +and its encoding, and tag indicates which one it is. + +### Boxed-file storage ### + +When managing keys, you frequently want to have some way to write a +secret object to disk, encrypted with a passphrase. The crypto_pwbox +and crypto_unpwbox functions do so in a way that's likely to be +readable by future versions of Tor. + +### Certificates ### + +We have, alas, several certificate types in Tor. + +The tor_x509_cert_t type represents an X.509 certificate. This document +won't explain X.509 to you -- possibly, no document can. (OTOH, Peter +Gutmann's "x.509 style guide", though severely dated, does a good job of +explaining how awful x.509 can be.) Do not introduce any new usages of +X.509. Right now we only use it in places where TLS forces us to do so. + +The authority_cert_t type is used only for directory authority keys. It +has a medium-term signing key (which the authorities actually keep +online) signed by a long-term identity key (which the authority operator +had really better be keeping offline). Don't use it for any new kind of +certificate. + +For new places where you need a certificate, consider tor_cert_t: it +represents a typed and dated _something_ signed by an Ed25519 key. The +format is described in tor-spec. Unlike x.509, you can write it on a +napkin. + +(Additionally, the Tor directory design uses a fairly wide variety of +documents that include keys and which are signed by keys. You can +consider these documents to be an additional kind of certificate if you +want.) + +### TLS ### + +Tor's TLS implementation is more tightly coupled to OpenSSL than we'd +prefer. You can read most of it in tortls.c. + +Unfortunately, TLS's state machine and our requirement for nonblocking +IO support means that using TLS in practice is a bit hairy, since +logical writes can block on a physical reads, and vice versa. + +If you are lucky, you will never have to look at the code here. + + + diff --git a/doc/HACKING/design/01e-os-compat.md b/doc/HACKING/design/01e-os-compat.md new file mode 100644 index 0000000000..072e95bc8a --- /dev/null +++ b/doc/HACKING/design/01e-os-compat.md @@ -0,0 +1,50 @@ + +## OS compatibility functions ## + +We've got a bunch of functions to wrap differences between various +operating systems where we run. + +### The filesystem ### + +We wrap the most important filesystem functions with load-file, +save-file, and map-file abstractions declared in util.c or compat.c. If +you're messing about with file descriptors yourself, you might be doing +things wrong. Most of the time, write_str_to_file() and +read_str_from_file() are all you need. + +Use the check_private_directory() function to create or verify the +presence of directories, and tor_listdir() to list the files in a +directory. + +Those modules also have functions for manipulating paths a bit. + +### Networking ### + +Nearly all the world is on a Berkeley sockets API, except for +windows, whose version of the Berkeley API was corrupted by late-90s +insistence on backward compatibility with the +sort-of-berkeley-sort-of-not add-on *thing* that was WinSocks. + +What's more, everybody who implemented sockets realized that select() +wasn't a very good way to do nonblocking IO... and then the various +implementations all decided to so something different. + +You can forget about most of these differences, fortunately: We use +libevent to hide most of the differences between the various networking +backends, and we add a few of our own functions to hide the differences +that Libevent doesn't. + +To create a network connection, the right level of abstraction to look +at is probably the connection_t system in connection.c. Most of the +lower level work has already been done for you. If you need to +instantiate something that doesn't fit well with connection_t, you +should see whether you can instantiate it with connection_t anyway -- or +you might need to refactor connection.c a little. + +Whenever possible, represent network addresses as tor_addr_t. + +### Process launch and monitoring ### + +Launching and/or monitoring a process is tricky business. You can use +the mechanisms in procmon.c and tor_spawn_background(), but they're both +a bit wonky. A refactoring would not be out of order. diff --git a/doc/HACKING/design/01f-threads.md b/doc/HACKING/design/01f-threads.md new file mode 100644 index 0000000000..a0dfa2d40e --- /dev/null +++ b/doc/HACKING/design/01f-threads.md @@ -0,0 +1,26 @@ + +## Threads in Tor ## + +Tor is based around a single main thread and one or more worker +threads. We aim (with middling success) to use worker threads for +CPU-intensive activities and the main thread for our networking. +Fortunately (?) we have enough cryptography that moving what we can of the +cryptographic processes to the workers should achieve good parallelism under most +loads. Unfortunately, we only have a small fraction of our +cryptography done in our worker threads right now. + +Our threads-and-workers abstraction is defined in workqueue.c, which +combines a work queue with a thread pool, and integrates the +signalling with libevent. Tor main instance of a work queue is +instantiated in cpuworker.c. It will probably need some refactoring +as more types of work are added. + +On a lower level, we provide locks with tor_mutex_t, conditions with +tor_cond_t, and thread-local storage with tor_threadlocal_t, all of +which are specified in compat_threads.h and implemented in an OS- +specific compat_\*threads.h module. + +Try to minimize sharing between threads: it is usually best to simply +make the worker "own" all the data it needs while the work is in +progress, and to give up ownership when it's complete. + diff --git a/doc/HACKING/design/01g-strings.md b/doc/HACKING/design/01g-strings.md new file mode 100644 index 0000000000..145a35cd6f --- /dev/null +++ b/doc/HACKING/design/01g-strings.md @@ -0,0 +1,95 @@ + +## String processing in Tor ## + +Since you're reading about a C program, you probably expected this +section: it's full of functions for manipulating the (notoriously +dubious) C string abstraction. I'll describe some often-missed +highlights here. + +### Comparing strings and memory chunks ### + +We provide strcmpstart() and strcmpend() to perform a strcmp with the start +or end of a string. + + tor_assert(!strcmpstart("Hello world","Hello")); + tor_assert(!strcmpend("Hello world","world")); + + tor_assert(!strcasecmpstart("HELLO WORLD","Hello")); + tor_assert(!strcasecmpend("HELLO WORLD","world")); + +To compare two string pointers, either of which might be NULL, use +strcmp_opt(). + +To search for a string or a chunk of memory within a non-null +terminated memory block, use tor_memstr or tor_memmem respectively. + +We avoid using memcmp() directly, since it tends to be used in cases +when having a constant-time operation would be better. Instead, we +recommend tor_memeq() and tor_memneq() for when you need a +constant-time operation. In cases when you need a fast comparison, +and timing leaks are not a danger, you can use fast_memeq() and +fast_memneq(). + +It's a common pattern to take a string representing one or more lines +of text, and search within it for some other string, at the start of a +line. You could search for "\\ntarget", but that would miss the first +line. Instead, use find_str_at_start_of_line. + +### Parsing text ### + +Over the years, we have accumulated lots of ways to parse text -- +probably too many. Refactoring them to be safer and saner could be a +good project! The one that seems most error-resistant is tokenizing +text with smartlist_split_strings(). This function takes a smartlist, +a string, and a separator, and splits the string along occurrences of +the separator, adding new strings for the sub-elements to the given +smartlist. + +To handle time, you can use one of the functions mentioned above in +"Parsing and encoding time values". + +For numbers in general, use the tor_parse_{long,ulong,double,uint64} +family of functions. Each of these can be called in a few ways. The +most general is as follows: + + const int BASE = 10; + const int MINVAL = 10, MAXVAL = 10000; + const char *next; + int ok; + long lng = tor_parse_long("100", BASE, MINVAL, MAXVAL, &ok, &next); + +The return value should be ignored if "ok" is set to false. The input +string needs to contain an entire number, or it's considered +invalid... unless the "next" pointer is available, in which case extra +characters at the end are allowed, and "next" is set to point to the +first such character. + +### Generating blocks of text ### + +For not-too-large blocks of text, we provide tor_asprintf(), which +behaves like other members of the sprintf() family, except that it +always allocates enough memory on the heap for its output. + +For larger blocks: Rather than using strlcat and strlcpy to build +text, or keeping pointers to the interior of a memory block, we +recommend that you use the smartlist_* functions to build a smartlist +full of substrings in order. Then you can concatenate them into a +single string with smartlist_join_strings(), which also takes optional +separator and terminator arguments. + +As a convenience, we provide smartlist_add_asprintf(), which combines +the two methods above together. Many of the cryptographic digest +functions also accept a not-yet-concatenated smartlist of strings. + +### Logging helpers ### + +Often we'd like to log a value that comes from an untrusted source. +To do this, use escaped() to escape the nonprintable characters and +other confusing elements in a string, and surround it by quotes. (Use +esc_for_log() if you need to allocate a new string.) + +It's also handy to put memory chunks into hexadecimal before logging; +you can use hex_str(memory, length) for that. + +The escaped() and hex_str() functions both provide outputs that are +only valid till they are next invoked; they are not threadsafe. diff --git a/doc/HACKING/design/02-dataflow.md b/doc/HACKING/design/02-dataflow.md new file mode 100644 index 0000000000..39f21a908c --- /dev/null +++ b/doc/HACKING/design/02-dataflow.md @@ -0,0 +1,236 @@ + +## Data flow in the Tor process ## + +We read bytes from the network, we write bytes to the network. For the +most part, the bytes we write correspond roughly to bytes we have read, +with bits of cryptography added in. + +The rest is a matter of details. + +![Diagram of main data flows in Tor](./diagrams/02/02-dataflow.png "Diagram of main data flows in Tor") + +### Connections and buffers: reading, writing, and interpreting. ### + +At a low level, Tor's networking code is based on "connections". Each +connection represents an object that can send or receive network-like +events. For the most part, each connection has a single underlying TCP +stream (I'll discuss counterexamples below). + +A connection that behaves like a TCP stream has an input buffer and an +output buffer. Incoming data is +written into the input buffer ("inbuf"); data to be written to the +network is queued on an output buffer ("outbuf"). + +Buffers are implemented in buffers.c. Each of these buffers is +implemented as a linked queue of memory extents, in the style of classic +BSD mbufs, or Linux skbufs. + +A connection's reading and writing can be enabled or disabled. Under +the hood, this functionality is implemented using libevent events: one +for reading, one for writing. These events are turned on/off in +main.c, in the functions connection_{start,stop}_{reading,writing}. + +When a read or write event is turned on, the main libevent loop polls +the kernel, asking which sockets are ready to read or write. (This +polling happens in the event_base_loop() call in run_main_loop_once() +in main.c.) When libevent finds a socket that's ready to read or write, +it invokes conn_{read,write}_callback(), also in main.c + +These callback functions delegate to connection_handle_read() and +connection_handle_write() in connection.c, which read or write on the +network as appropriate, possibly delegating to openssl. + +After data is read or written, or other event occurs, these +connection_handle_read_write() functions call logic functions whose job is +to respond to the information. Some examples included: + + * connection_flushed_some() -- called after a connection writes any + amount of data from its outbuf. + * connection_finished_flushing() -- called when a connection has + emptied its outbuf. + * connection_finished_connecting() -- called when an in-process connection + finishes making a remote connection. + * connection_reached_eof() -- called after receiving a FIN from the + remote server. + * connection_process_inbuf() -- called when more data arrives on + the inbuf. + +These functions then call into specific implementations depending on +the type of the connection. For example, if the connection is an +edge_connection_t, connection_reached_eof() will call +connection_edge_reached_eof(). + +> **Note:** "Also there are bufferevents!" We have vestigial +> code for an alternative low-level networking +> implementation, based on Libevent's evbuffer and bufferevent +> code. These two object types take on (most of) the roles of +> buffers and connections respectively. It isn't working in today's +> Tor, due to code rot and possible lingering libevent bugs. More +> work is needed; it would be good to get this working efficiently +> again, to have IOCP support on Windows. + + +#### Controlling connections #### + +A connection can have reading or writing enabled or disabled for a +wide variety of reasons, including: + + * Writing is disabled when there is no more data to write + * For some connection types, reading is disabled when the inbuf is + too full. + * Reading/writing is temporarily disabled on connections that have + recently read/written enough data up to their bandwidth + * Reading is disabled on connections when reading more data from them + would require that data to be buffered somewhere else that is + already full. + +Currently, these conditions are checked in a diffuse set of +increasingly complex conditional expressions. In the future, it could +be helpful to transition to a unified model for handling temporary +read/write suspensions. + +#### Kinds of connections #### + +Today Tor has the following connection and pseudoconnection types. +For the most part, each type of channel has an associated C module +that implements its underlying logic. + +**Edge connections** receive data from and deliver data to points +outside the onion routing network. See `connection_edge.c`. They fall into two types: + +**Entry connections** are a type of edge connection. They receive data +from the user running a Tor client, and deliver data to that user. +They are used to implement SOCKSPort, TransPort, NATDPort, and so on. +Sometimes they are called "AP" connections for historical reasons (it +used to stand for "Application Proxy"). + +**Exit connections** are a type of edge connection. They exist at an +exit node, and transmit traffic to and from the network. + +(Entry connections and exit connections are also used as placeholders +when performing a remote DNS request; they are not decoupled from the +notion of "stream" in the Tor protocol. This is implemented partially +in `connection_edge.c`, and partially in `dnsserv.c` and `dns.c`.) + +**OR connections** send and receive Tor cells over TLS, using some +version of the Tor link protocol. Their implementation is spread +across `connection_or.c`, with a bit of logic in `command.c`, +`relay.c`, and `channeltls.c`. + +**Extended OR connections** are a type of OR connection for use on +bridges using pluggable transports, so that the PT can tell the bridge +some information about the incoming connection before passing on its +data. They are implemented in `ext_orport.c`. + +**Directory connections** are server-side or client-side connections +that implement Tor's HTTP-based directory protocol. These are +instantiated using a socket when Tor is making an unencrypted HTTP +connection. When Tor is tunneling a directory request over a Tor +circuit, directory connections are implemented using a linked +connection pair (see below). Directory connections are implemented in +`directory.c`; some of the server-side logic is implemented in +`dirserver.c`. + +**Controller connections** are local connections to a controller +process implementing the controller protocol from +control-spec.txt. These are in `control.c`. + +**Listener connections** are not stream oriented! Rather, they wrap a +listening socket in order to detect new incoming connections. They +bypass most of stream logic. They don't have associated buffers. +They are implemented in `connection.c`. + +![structure hierarchy for connection types](./diagrams/02/02-connection-types.png "structure hierarchy for connection types") + +>**Note**: "History Time!" You might occasionally find reference to a couple types of connections +> which no longer exist in modern Tor. A *CPUWorker connection* +>connected the main Tor process to a thread or process used for +>computation. (Nowadays we use in-process communication.) Even more +>anciently, a *DNSWorker connection* connected the main tor process to +>a separate thread or process used for running `gethostbyname()` or +>`getaddrinfo()`. (Nowadays we use Libevent's evdns facility to +>perform DNS requests asynchronously.) + +#### Linked connections #### + +Sometimes two channels are joined together, such that data which the +Tor process sends on one should immediately be received by the same +Tor process on the other. (For example, when Tor makes a tunneled +directory connection, this is implemented on the client side as a +directory connection whose output goes, not to the network, but to a +local entry connection. And when a directory receives a tunnelled +directory connection, this is implemented as an exit connection whose +output goes, not to the network, but to a local directory connection.) + +The earliest versions of Tor to support linked connections used +socketpairs for the purpose. But using socketpairs forced us to copy +data through kernelspace, and wasted limited file descriptors. So +instead, a pair of connections can be linked in-process. Each linked +connection has a pointer to the other, such that data written on one +is immediately readable on the other, and vice versa. + +### From connections to channels ### + +There's an abstraction layer above OR connections (the ones that +handle cells) and below cells called **Channels**. A channel's +purpose is to transmit authenticated cells from one Tor instance +(relay or client) to another. + +Currently, only one implementation exists: Channel_tls, which sends +and receiveds cells over a TLS-based OR connection. + +Cells are sent on a channel using +`channel_write_{,packed_,var_}cell()`. Incoming cells arrive on a +channel from its backend using `channel_queue*_cell()`, and are +immediately processed using `channel_process_cells()`. + +Some cell types are handled below the channel layer, such as those +that affect handshaking only. And some others are passed up to the +generic cross-channel code in `command.c`: cells like `DESTROY` and +`CREATED` are all trivial to handle. But relay cells +require special handling... + +### From channels through circuits ### + +When a relay cell arrives on an existing circuit, it is handled in +`circuit_receive_relay_cell()` -- one of the innermost functions in +Tor. This function encrypts or decrypts the relay cell as +appropriate, and decides whether the cell is intended for the current +hop of the circuit. + +If the cell *is* intended for the current hop, we pass it to +`connection_edge_process_relay_cell()` in `relay.c`, which acts on it +based on its relay command, and (possibly) queues its data on an +`edge_connection_t`. + +If the cell *is not* intended for the current hop, we queue it for the +next channel in sequence with `append cell_to_circuit_queue()`. This +places the cell on a per-circuit queue for cells headed out on that +particular channel. + +### Sending cells on circuits: the complicated bit. ### + +Relay cells are queued onto circuits from one of two (main) sources: +reading data from edge connections, and receiving a cell to be relayed +on a circuit. Both of these sources place their cells on cell queue: +each circuit has one cell queue for each direction that it travels. + +A naive implementation would skip using cell queues, and instead write +each outgoing relay cell. (Tor did this in its earlier versions.) +But such an approach tends to give poor performance, because it allows +high-volume circuits to clog channels, and it forces the Tor server to +send data queued on a circuit even after that circuit has been closed. + +So by using queues on each circuit, we can add cells to each channel +on a just-in-time basis, choosing the cell at each moment based on +a performance-aware algorithm. + +This logic is implemented in two main modules: `scheduler.c` and +`circuitmux*.c`. The scheduler code is responsible for determining +globally, across all channels that could write cells, which one should +next receive queued cells. The circuitmux code determines, for all +of the circuits with queued cells for a channel, which one should +queue the next cell. + +(This logic applies to outgoing relay cells only; incoming relay cells +are processed as they arrive.) diff --git a/doc/HACKING/design/03-modules.md b/doc/HACKING/design/03-modules.md new file mode 100644 index 0000000000..93eb9d3089 --- /dev/null +++ b/doc/HACKING/design/03-modules.md @@ -0,0 +1,247 @@ + +## Tor's modules ## + +### Generic modules ### + +`buffers.c` +: Implements the `buf_t` buffered data type for connections, and several +low-level data handling functions to handle network protocols on it. + +`channel.c` +: Generic channel implementation. Channels handle sending and receiving cells +among tor nodes. + +`channeltls.c` +: Channel implementation for TLS-based OR connections. Uses `connection_or.c`. + +`circuitbuild.c` +: Code for constructing circuits and choosing their paths. (*Note*: +this module could plausibly be split into handling the client side, +the server side, and the path generation aspects of circuit building.) + +`circuitlist.c` +: Code for maintaining and navigating the global list of circuits. + +`circuitmux.c` +: Generic circuitmux implementation. A circuitmux handles deciding, for a +particular channel, which circuit should write next. + +`circuitmux_ewma.c` +: A circuitmux implementation based on the EWMA (exponentially +weighted moving average) algorithm. + +`circuituse.c` +: Code to actually send and receive data on circuits. + +`command.c` +: Handles incoming cells on channels. + +`config.c` +: Parses options from torrc, and uses them to configure the rest of Tor. + +`confparse.c` +: Generic torrc-style parser. Used to parse torrc and state files. + +`connection.c` +: Generic and common connection tools, and implementation for the simpler +connection types. + +`connection_edge.c` +: Implementation for entry and exit connections. + +`connection_or.c` +: Implementation for OR connections (the ones that send cells over TLS). + +`main.c` +: Principal entry point, main loops, scheduled events, and network +management for Tor. + +`ntmain.c` +: Implements Tor as a Windows service. (Not very well.) + +`onion.c` +: Generic code for generating and responding to CREATE and CREATED +cells, and performing the appropriate onion handshakes. Also contains +code to manage the server-side onion queue. + +`onion_fast.c` +: Implements the old SHA1-based CREATE_FAST/CREATED_FAST circuit +creation handshake. (Now deprecated.) + +`onion_ntor.c` +: Implements the Curve25519-based NTOR circuit creation handshake. + +`onion_tap.c` +: Implements the old RSA1024/DH1024-based TAP circuit creation handshake. (Now +deprecated.) + +`relay.c` +: Handles particular types of relay cells, and provides code to receive, +encrypt, route, and interpret relay cells. + +`scheduler.c` +: Decides which channel/circuit pair is ready to receive the next cell. + +`statefile.c` +: Handles loading and storing Tor's state file. + +`tor_main.c` +: Contains the actual `main()` function. (This is placed in a separate +file so that the unit tests can have their own `main()`.) + + +### Node-status modules ### + +`directory.c` +: Implements the HTTP-based directory protocol, including sending, +receiving, and handling most request types. (*Note*: The client parts +of this, and the generic-HTTP parts of this, could plausibly be split +off.) + +`microdesc.c` +: Implements the compact "microdescriptor" format for keeping track of +what we know about a router. + +`networkstatus.c` +: Code for fetching, storing, and interpreting consensus vote documents. + +`nodelist.c` +: Higher-level view of our knowledge of which Tor servers exist. Each +`node_t` corresponds to a router we know about. + +`routerlist.c` +: Code for storing and retrieving router descriptors and extrainfo +documents. + +`routerparse.c` +: Generic and specific code for parsing all Tor directory information +types. + +`routerset.c` +: Parses and interprets a specification for a set of routers (by IP +range, fingerprint, nickname (deprecated), or country). + + +### Client modules ### + +`addressmap.c` +: Handles client-side associations between one address and another. +These are used to implement client-side DNS caching (NOT RECOMMENDED), +MapAddress directives, Automapping, and more. + +`circpathbias.c` +: Path bias attack detection for circuits: tracks whether +connections made through a particular guard have an unusually high failure rate. + +`circuitstats.c` +: Code to track circuit performance statistics in order to adapt our behavior. +Notably includes an algorithm to track circuit build times. + +`dnsserv.c` +: Implements DNSPort for clients. (Note that in spite of the word +"server" in this module's name, it is used for Tor clients. It +implements a DNS server, not DNS for servers.) + +`entrynodes.c` +: Chooses, monitors, and remembers guard nodes. Also contains some +bridge-related code. + +`torcert.c` +: Code to interpret and generate Ed25519-based certificates. + +### Server modules ### + +`dns.c` +: Server-side DNS code. Handles sending and receiving DNS requests on +exit nodes, and implements the server-side DNS cache. + +`dirserv.c` +: Implements part of directory caches that handles responding to +client requests. + +`ext_orport.c` +: Implements the extended ORPort protocol for communication between +server-side pluggable transports and Tor servers. + +`hibernate.c` +: Performs bandwidth accounting, and puts Tor relays into hibernation +when their bandwidth is exhausted. + +`router.c` +: Management code for running a Tor server. In charge of RSA key +maintenance, descriptor generation and uploading. + +`routerkeys.c` +: Key handling code for a Tor server. (Currently handles only the +Ed25519 keys, but the RSA keys could be moved here too.) + + +### Onion service modules ### + +`rendcache.c` +: Stores onion service descriptors. + +`rendclient.c` +: Client-side implementation of the onion service protocol. + +`rendcommon.c` +: Parts of the onion service protocol that are shared by clients, +services, and/or Tor servers. + +`rendmid.c` +: Tor-server-side implementation of the onion service protocol. (Handles +acting as an introduction point or a rendezvous point.) + +`rendservice.c` +: Service-side implementation of the onion service protocol. + +`replaycache.c` +: Backend to check introduce2 requests for replay attempts. + + +### Authority modules ### + +`dircollate.c` +: Helper for `dirvote.c`: Given a set of votes, each containing a list +of Tor nodes, determines which entries across all the votes correspond +to the same nodes, and yields them in a useful order. + +`dirvote.c` +: Implements the directory voting algorithms that authorities use. + +`keypin.c` +: Implements a persistent key-pinning mechanism to tie RSA1024 +identities to ed25519 identities. + +### Miscellaneous modules ### + +`control.c` +: Implements the Tor controller protocol. + +`cpuworker.c` +: Implements the inner work queue function. We use this to move the +work of circuit creation (on server-side) to other CPUs. + +`fp_pair.c` +: Types for handling 2-tuples of 20-byte fingerprints. + +`geoip.c` +: Parses geoip files (which map IP addresses to country codes), and +performs lookups on the internal geoip table. Also stores some +geoip-related statistics. + +`policies.c` +: Parses and implements Tor exit policies. + +`reasons.c` +: Maps internal reason-codes to human-readable strings. + +`rephist.c` +: Tracks Tor servers' performance over time. + +`status.c` +: Writes periodic "heartbeat" status messages about the state of the Tor +process. + +`transports.c` +: Implements management for the pluggable transports subsystem. diff --git a/doc/HACKING/design/Makefile b/doc/HACKING/design/Makefile new file mode 100644 index 0000000000..e126130970 --- /dev/null +++ b/doc/HACKING/design/Makefile @@ -0,0 +1,34 @@ + + + +HTML= \ + 00-overview.html \ + 01-common-utils.html \ + 01a-memory.html \ + 01b-collections.html \ + 01c-time.html \ + 01d-crypto.html \ + 01e-os-compat.html \ + 01f-threads.html \ + 01g-strings.html \ + 02-dataflow.html \ + 03-modules.html \ + this-not-that.html + +PNG = \ + diagrams/02/02-dataflow.png \ + diagrams/02/02-connection-types.png + +all: generated + +generated: $(HTML) $(PNG) + +%.html: %.md + maruku $< -o $@ + +%.png: %.dia + dia $< --export=$@ + +clean: + rm -f $(HTML) + rm -f $(PNG) diff --git a/doc/HACKING/design/this-not-that.md b/doc/HACKING/design/this-not-that.md new file mode 100644 index 0000000000..815c7b2fbc --- /dev/null +++ b/doc/HACKING/design/this-not-that.md @@ -0,0 +1,51 @@ + +Don't use memcmp. Use {tor,fast}_{memeq,memneq,memcmp}. + +Don't use assert. Use tor_assert or tor_assert_nonfatal or BUG. Prefer +nonfatal assertions or BUG()s. + +Don't use sprintf or snprintf. Use tor_asprintf or tor_snprintf. + +Don't write hand-written binary parsers. Use trunnel. + +Don't use malloc, realloc, calloc, free, strdup, etc. Use tor_malloc, +tor_realloc, tor_calloc, tor_free, tor_strdup, etc. + +Don't use tor_realloc(x, y\*z). Use tor_reallocarray(x, y, z); + +Don't say "if (x) foo_free(x)". Just foo_free(x) and make sure that +foo_free(NULL) is a no-op. + +Don't use toupper or tolower; use TOR_TOUPPER and TOR_TOLOWER. + +Don't use isalpha, isalnum, etc. Instead use TOR_ISALPHA, TOR_ISALNUM, etc. + +Don't use strcat, strcpy, strncat, or strncpy. Use strlcat and strlcpy +instead. + +Don't use tor_asprintf then smartlist_add; use smartlist_add_asprintf. + +Don't use any of these functions: they aren't portable. Use the +version prefixed with `tor_` instead: strtok_r, memmem, memstr, +asprintf, localtime_r, gmtime_r, inet_aton, inet_ntop, inet_pton, +getpass, ntohll, htonll, strdup, (This list is incomplete.) + +Don't create or close sockets directly. Instead use the wrappers in +compat.h. + +When creating new APIs, only use 'char \*' to represent 'pointer to a +nul-terminated string'. Represent 'pointer to a chunk of memory' as +'uint8_t \*'. (Many older Tor APIs ignore this rule.) + +Don't encode/decode u32, u64, or u16 to byte arrays by casting +pointers. That can crash if the pointers aren't aligned, and can cause +endianness problems. Instead say something more like set_uint32(ptr, +htonl(foo)) to encode, and ntohl(get_uint32(ptr)) to decode. + +Don't declare a 0-argument function with "void foo()". That's C++ +syntax. In C you say "void foo(void)". + +When creating new APIs, use const everywhere you reasonably can. + +Sockets should have type tor_socket_t, not int. + From b03cb0cc269548f103c2fc3ff611ea3cf90437a5 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 24 Sep 2019 19:35:42 -0400 Subject: [PATCH 1586/2557] Add a changes file about the introduction of doc/HACKING/design. --- changes/ticket31849 | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 changes/ticket31849 diff --git a/changes/ticket31849 b/changes/ticket31849 new file mode 100644 index 0000000000..9d12d938c4 --- /dev/null +++ b/changes/ticket31849 @@ -0,0 +1,5 @@ + o Documentation: + - The Tor source code repository now includes a (somewhat dated) + description of Tor's modular architecture, in doc/HACKING/design. + This is based on the old "tor-guts.git" repository, which we are + adopting and superseding. Closes ticket 31849. From 1ad1e84b17267f04960b322effaf57ea8a9690c0 Mon Sep 17 00:00:00 2001 From: teor Date: Mon, 5 Aug 2019 20:04:08 +1000 Subject: [PATCH 1587/2557] log: Move SEVERITY_MASK_IDX() to log.h Move SEVERITY_MASK_IDX() to log.h private/unit tests section, so that we can use it in log.c, the unit tests, and the fuzzers. (The test and fuzzer code changes are in a subsequent commit.) Preparation for bug 31334. --- src/lib/log/log.c | 4 ---- src/lib/log/log.h | 6 ++++++ 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/lib/log/log.c b/src/lib/log/log.c index be6f459554..9373fd4042 100644 --- a/src/lib/log/log.c +++ b/src/lib/log/log.c @@ -55,10 +55,6 @@ #include #endif // HAVE_ANDROID_LOG_H. -/** Given a severity, yields an index into log_severity_list_t.masks to use - * for that severity. */ -#define SEVERITY_MASK_IDX(sev) ((sev) - LOG_ERR) - /** @{ */ /** The string we stick at the end of a log message when it is too long, * and its length. */ diff --git a/src/lib/log/log.h b/src/lib/log/log.h index 4291418eb6..da4bcbe608 100644 --- a/src/lib/log/log.h +++ b/src/lib/log/log.h @@ -297,4 +297,10 @@ MOCK_DECL(STATIC void, logv, (int severity, log_domain_mask_t domain, va_list ap) CHECK_PRINTF(5,0)); #endif +#if defined(LOG_PRIVATE) || defined(TOR_UNIT_TESTS) +/** Given a severity, yields an index into log_severity_list_t.masks to use + * for that severity. */ +#define SEVERITY_MASK_IDX(sev) ((sev) - LOG_ERR) +#endif + #endif /* !defined(TOR_TORLOG_H) */ From d30a042fa8348028e0bea6f3e46cba1ffbe5adcc Mon Sep 17 00:00:00 2001 From: teor Date: Wed, 25 Sep 2019 16:35:02 +1000 Subject: [PATCH 1588/2557] test: Use SEVERITY_MASK_IDX() to find the LOG_* mask indexes In the unit tests and fuzzers. Fixes bug 31334; bugfix on 0.2.5.2-alpha. --- changes/bug31334 | 4 ++++ src/test/fuzz/fuzzing_common.c | 2 +- src/test/test_logging.c | 2 +- src/test/test_options.c | 6 +++--- src/test/testing_common.c | 4 ++-- 5 files changed, 11 insertions(+), 7 deletions(-) create mode 100644 changes/bug31334 diff --git a/changes/bug31334 b/changes/bug31334 new file mode 100644 index 0000000000..dfc9cc530e --- /dev/null +++ b/changes/bug31334 @@ -0,0 +1,4 @@ + o Code simplification and refactoring: + - Use SEVERITY_MASK_IDX() to find the LOG_* mask indexes in the unit + tests and fuzzers, rather than using hard-coded values. + Closes ticket 31334. diff --git a/src/test/fuzz/fuzzing_common.c b/src/test/fuzz/fuzzing_common.c index 862acb2b35..e269c36b42 100644 --- a/src/test/fuzz/fuzzing_common.c +++ b/src/test/fuzz/fuzzing_common.c @@ -167,7 +167,7 @@ main(int argc, char **argv) memset(&s, 0, sizeof(s)); set_log_severity_config(loglevel, LOG_ERR, &s); /* ALWAYS log bug warnings. */ - s.masks[LOG_WARN-LOG_ERR] |= LD_BUG; + s.masks[SEVERITY_MASK_IDX(LOG_WARN)] |= LD_BUG; add_stream_log(&s, "", fileno(stdout)); } diff --git a/src/test/test_logging.c b/src/test/test_logging.c index bb7018fe1c..203ce64e32 100644 --- a/src/test/test_logging.c +++ b/src/test/test_logging.c @@ -35,7 +35,7 @@ test_get_sigsafe_err_fds(void *arg) set_log_severity_config(LOG_WARN, LOG_ERR, &include_bug); set_log_severity_config(LOG_WARN, LOG_ERR, &no_bug); - no_bug.masks[0] &= ~(LD_BUG|LD_GENERAL); + no_bug.masks[SEVERITY_MASK_IDX(LOG_ERR)] &= ~(LD_BUG|LD_GENERAL); set_log_severity_config(LOG_INFO, LOG_NOTICE, &no_bug2); /* Add some logs; make sure the output is as expected. */ diff --git a/src/test/test_options.c b/src/test/test_options.c index 0747a2e062..b3654ede7d 100644 --- a/src/test/test_options.c +++ b/src/test/test_options.c @@ -54,9 +54,9 @@ setup_log_callback(void) { log_severity_list_t lst; memset(&lst, 0, sizeof(lst)); - lst.masks[LOG_ERR - LOG_ERR] = ~0; - lst.masks[LOG_WARN - LOG_ERR] = ~0; - lst.masks[LOG_NOTICE - LOG_ERR] = ~0; + lst.masks[SEVERITY_MASK_IDX(LOG_ERR)] = ~0; + lst.masks[SEVERITY_MASK_IDX(LOG_WARN)] = ~0; + lst.masks[SEVERITY_MASK_IDX(LOG_NOTICE)] = ~0; add_callback_log(&lst, log_cback); mark_logs_temp(); } diff --git a/src/test/testing_common.c b/src/test/testing_common.c index ad22898ce5..9e7d83dcdc 100644 --- a/src/test/testing_common.c +++ b/src/test/testing_common.c @@ -295,7 +295,7 @@ main(int c, const char **v) memset(&s, 0, sizeof(s)); set_log_severity_config(loglevel, LOG_ERR, &s); /* ALWAYS log bug warnings. */ - s.masks[LOG_WARN-LOG_ERR] |= LD_BUG; + s.masks[SEVERITY_MASK_IDX(LOG_WARN)] |= LD_BUG; add_stream_log(&s, "", fileno(stdout)); } { @@ -303,7 +303,7 @@ main(int c, const char **v) log_severity_list_t s; memset(&s, 0, sizeof(s)); set_log_severity_config(LOG_ERR, LOG_ERR, &s); - s.masks[LOG_WARN-LOG_ERR] |= LD_BUG; + s.masks[SEVERITY_MASK_IDX(LOG_WARN)] |= LD_BUG; add_callback_log(&s, log_callback_failure); } flush_log_messages_from_startup(); From b4aeeb77ba97436a220cfce2fa42b9717a6facea Mon Sep 17 00:00:00 2001 From: rl1987 Date: Wed, 25 Sep 2019 17:15:12 +0300 Subject: [PATCH 1589/2557] Merge advice from this_not_that.md in torguts repo into our main coding standard doc --- changes/doc31853 | 3 +++ doc/HACKING/CodingStandards.md | 33 +++++++++++++++++++++++++++++---- 2 files changed, 32 insertions(+), 4 deletions(-) create mode 100644 changes/doc31853 diff --git a/changes/doc31853 b/changes/doc31853 new file mode 100644 index 0000000000..9118a4f8b1 --- /dev/null +++ b/changes/doc31853 @@ -0,0 +1,3 @@ + o Documentation: + - Unite coding advice from this_not_that.md in torguts repo into our + coding standards document. Resolves ticket 31853. diff --git a/doc/HACKING/CodingStandards.md b/doc/HACKING/CodingStandards.md index 74db2a39a3..966a2b8104 100644 --- a/doc/HACKING/CodingStandards.md +++ b/doc/HACKING/CodingStandards.md @@ -212,6 +212,8 @@ deviations from our C whitespace style. Generally, we use: - No space between a function name and an opening paren. `puts(x)`, not `puts (x)`. - Function declarations at the start of the line. + - Use `void foo(void)` to declare a function with no arguments. + - Use `const` for new APIs. If you use an editor that has plugins for editorconfig.org, the file `.editorconfig` will help you to conform this coding style. @@ -240,8 +242,21 @@ old C functions. Use `strlcat`, `strlcpy`, or `tor_snprintf/tor_asprintf` inste We don't call `memcmp()` directly. Use `fast_memeq()`, `fast_memneq()`, `tor_memeq()`, or `tor_memneq()` for most purposes. -Also see a longer list of functions to avoid in: -https://people.torproject.org/~nickm/tor-auto/internal/this-not-that.html +Don't call `assert()` directly. For hard asserts, use `tor_assert()`. For +soft asserts, use `tor_assert_nonfatal()` or `BUG()`. If you need to print +debug information in assert error message, consider using `tor_assertf()` and +`tor_assertf_nonfatal()`. + +Don't use `toupper()` and `tolower()` functions. Use `TOR_TOUPPER` and +`TOR_TOLOWER` macros instead. Similarly, use `TOR_ISALPHA`, `TOR_ISALNUM` et. +al. instead of `isalpha()`, `isalnum()`, etc. + +When allocating new string to be added to a smartlist, use +`smartlist_add_asprintf()` to do both at once. + +Avoid calling BSD socket functions directly. Use portable wrappers to work +with sockets and socket addresses. Also, sockets should be of type +`tor_socket_t`. What code can use what other code? ---------------------------------- @@ -331,8 +346,15 @@ definitions when necessary.) Assignment operators shouldn't nest inside other expressions. (You can ignore this inside macro definitions when necessary.) -Functions not to write ----------------------- +Binary data and wire formats +---------------------------- + +Use pointer to `char` when representing NUL-terminated string. To represent +arbitrary binary data, use pointer to `uint8_t`. + +Refrain from attempting to encode integers by casting their pointers to byte +arrays. Use something like `set_uint32()`/`get_uint32()` instead and don't +forget about endianness. Try to never hand-write new code to parse or generate binary formats. Instead, use trunnel if at all possible. See @@ -451,6 +473,9 @@ to use it as a function callback), define it with a name like abc_free_(obj); } +When deallocating, don't say e.g. `if (x) tor_free(x)`. The convention is to +have deallocators do nothing when NULL pointer is passed. + Doxygen comment conventions --------------------------- From 315f14c709d019c55731faedb19a97b771f9c42f Mon Sep 17 00:00:00 2001 From: teor Date: Fri, 20 Sep 2019 11:40:05 +1000 Subject: [PATCH 1590/2557] backtrace: avoid undefined behaviour on re-initialisation cb_buf_mutex is statically initialised, so we can not destroy it when we are shutting down the err subsystem. If we destroy it, and then re-initialise tor, all our backtraces will fail. Part of 31736, but committed in this branch to avoid merge conflicts. --- src/lib/err/backtrace.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/lib/err/backtrace.c b/src/lib/err/backtrace.c index bf833c1621..2a956e6115 100644 --- a/src/lib/err/backtrace.c +++ b/src/lib/err/backtrace.c @@ -251,6 +251,10 @@ remove_bt_handler(void) * It's not a fatal error, so we just ignore it. */ (void)sigaction(trap_signals[i], &sa, NULL); } + + /* cb_buf_mutex is statically initialised, so we can not destroy it. + * If we destroy it, and then re-initialise tor, all our backtraces will + * fail. */ } #endif /* defined(USE_BACKTRACE) */ From c9c046c365f10f5cbffada921931413edc690dbe Mon Sep 17 00:00:00 2001 From: teor Date: Thu, 26 Sep 2019 12:09:56 +1000 Subject: [PATCH 1591/2557] changes: file for 31614 --- changes/bug31614 | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 changes/bug31614 diff --git a/changes/bug31614 b/changes/bug31614 new file mode 100644 index 0000000000..c425a9fcd4 --- /dev/null +++ b/changes/bug31614 @@ -0,0 +1,9 @@ + o Minor bugfixes (logging): + - Disable backtrace signal handlers when shutting down tor. + Fixes bug 31614; bugfix on 0.2.5.2-alpha. + - Add a missing check for HAVE_PTHREAD_H, because the backtrace code uses + mutexes. Fixes bug 31614; bugfix on 0.2.5.2-alpha. + o Documentation: + - Explain why we can't destroy the backtrace buffer mutex. Explain why + we don't need to destroy the log mutex. + Closes ticket 31736. From 2f8a9a2db692a101b7e6241cbcdf9ed87841310b Mon Sep 17 00:00:00 2001 From: teor Date: Tue, 24 Sep 2019 13:51:38 +1000 Subject: [PATCH 1592/2557] sandbox: Allow backtrace signals to be disabled Part of 31614. --- src/lib/sandbox/sandbox.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/lib/sandbox/sandbox.c b/src/lib/sandbox/sandbox.c index e2356a1720..5c02c012e8 100644 --- a/src/lib/sandbox/sandbox.c +++ b/src/lib/sandbox/sandbox.c @@ -295,6 +295,7 @@ sb_rt_sigaction(scmp_filter_ctx ctx, sandbox_cfg_t *filter) unsigned i; int rc; int param[] = { SIGINT, SIGTERM, SIGPIPE, SIGUSR1, SIGUSR2, SIGHUP, SIGCHLD, + SIGSEGV, SIGILL, SIGFPE, SIGBUS, SIGSYS, SIGIO, #ifdef SIGXFSZ SIGXFSZ #endif From 749c2e1761c753992fb2549e7ee912e568f563d6 Mon Sep 17 00:00:00 2001 From: teor Date: Thu, 26 Sep 2019 12:18:23 +1000 Subject: [PATCH 1593/2557] log: explain why it is safe to leave the log mutex initialized The log mutex is dynamically initialized, guarded by log_mutex_initialized. We don't want to destroy it, because after it is destroyed, we won't see any more logs. If tor is re-initialized, log_mutex_initialized will still be 1. So we won't trigger any undefined behaviour by trying to re-initialize the log mutex. Part of 31736, but committed in this branch to avoid merge conflicts. --- src/lib/log/log.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/lib/log/log.c b/src/lib/log/log.c index a9ad38fb25..eacd413a53 100644 --- a/src/lib/log/log.c +++ b/src/lib/log/log.c @@ -804,7 +804,10 @@ logs_free_all(void) } /* We _could_ destroy the log mutex here, but that would screw up any logs - * that happened between here and the end of execution. */ + * that happened between here and the end of execution. + * If tor is re-initialized, log_mutex_initialized will still be 1. So we + * won't trigger any undefined behaviour by trying to re-initialize the + * log mutex. */ } /** Remove and free the log entry victim from the linked-list From d1eab05834566f998721d3a16107767885711c57 Mon Sep 17 00:00:00 2001 From: teor Date: Fri, 20 Sep 2019 11:27:05 +1000 Subject: [PATCH 1594/2557] lock: Avoid some undefined behaviour when freeing mutexes. Fixes bug 31736; bugfix on 0.0.7. --- changes/bug31736 | 3 +++ src/app/config/config.c | 6 +++++- src/feature/relay/router.c | 4 ++++ src/lib/crypt_ops/crypto_openssl_mgt.c | 4 ++++ src/lib/lock/compat_mutex.c | 10 +++++++++- src/lib/lock/compat_mutex_pthreads.c | 16 +++++++++++++++- src/lib/thread/compat_threads.c | 10 +++++++++- src/lib/thread/threads.h | 12 +++++++++++- 8 files changed, 60 insertions(+), 5 deletions(-) create mode 100644 changes/bug31736 diff --git a/changes/bug31736 b/changes/bug31736 new file mode 100644 index 0000000000..beb09e5069 --- /dev/null +++ b/changes/bug31736 @@ -0,0 +1,3 @@ + o Minor bugfixes (multithreading): + - Avoid some undefined behaviour when freeing mutexes. + Fixes bug 31736; bugfix on 0.0.7. diff --git a/src/app/config/config.c b/src/app/config/config.c index 0b1b758d96..2416da2b26 100644 --- a/src/app/config/config.c +++ b/src/app/config/config.c @@ -1187,7 +1187,11 @@ init_protocol_warning_severity_level(void) static void cleanup_protocol_warning_severity_level(void) { - atomic_counter_destroy(&protocol_warning_severity_level); + /* Destroying a locked mutex is undefined behaviour. This mutex may be + * locked, because multiple threads can access it. But we need to destroy + * it, otherwise re-initialisation will trigger undefined behaviour. + * See #31735 for details. */ + atomic_counter_destroy(&protocol_warning_severity_level); } /** List of default directory authorities */ diff --git a/src/feature/relay/router.c b/src/feature/relay/router.c index dad2c6a50f..af47e79d28 100644 --- a/src/feature/relay/router.c +++ b/src/feature/relay/router.c @@ -3072,6 +3072,10 @@ router_free_all(void) crypto_pk_free(server_identitykey); crypto_pk_free(client_identitykey); + /* Destroying a locked mutex is undefined behaviour. This mutex may be + * locked, because multiple threads can access it. But we need to destroy + * it, otherwise re-initialisation will trigger undefined behaviour. + * See #31735 for details. */ tor_mutex_free(key_lock); routerinfo_free(desc_routerinfo); extrainfo_free(desc_extrainfo); diff --git a/src/lib/crypt_ops/crypto_openssl_mgt.c b/src/lib/crypt_ops/crypto_openssl_mgt.c index c97815f9a4..a245ef3810 100644 --- a/src/lib/crypt_ops/crypto_openssl_mgt.c +++ b/src/lib/crypt_ops/crypto_openssl_mgt.c @@ -176,6 +176,10 @@ crypto_openssl_free_all(void) tor_free(crypto_openssl_version_str); tor_free(crypto_openssl_header_version_str); + /* Destroying a locked mutex is undefined behaviour. This mutex may be + * locked, because multiple threads can access it. But we need to destroy + * it, otherwise re-initialisation will trigger undefined behaviour. + * See #31735 for details. */ #ifndef NEW_THREAD_API if (n_openssl_mutexes_) { int n = n_openssl_mutexes_; diff --git a/src/lib/lock/compat_mutex.c b/src/lib/lock/compat_mutex.c index 4ad5929715..670bd0174c 100644 --- a/src/lib/lock/compat_mutex.c +++ b/src/lib/lock/compat_mutex.c @@ -29,7 +29,15 @@ tor_mutex_new_nonrecursive(void) tor_mutex_init_nonrecursive(m); return m; } -/** Release all storage and system resources held by m. */ +/** Release all storage and system resources held by m. + * + * Destroying a locked mutex is undefined behaviour. Global mutexes may be + * locked when they are passed to this function, because multiple threads can + * still access them. So we can either: + * - destroy on shutdown, and re-initialise when tor re-initialises, or + * - skip destroying and re-initialisation, using a sentinel variable. + * See #31735 for details. + */ void tor_mutex_free_(tor_mutex_t *m) { diff --git a/src/lib/lock/compat_mutex_pthreads.c b/src/lib/lock/compat_mutex_pthreads.c index ee5f520cd0..f82ad9f0e8 100644 --- a/src/lib/lock/compat_mutex_pthreads.c +++ b/src/lib/lock/compat_mutex_pthreads.c @@ -88,12 +88,26 @@ tor_mutex_release(tor_mutex_t *m) } /** Clean up the mutex m so that it no longer uses any system * resources. Does not free m. This function must only be called on - * mutexes from tor_mutex_init(). */ + * mutexes from tor_mutex_init(). + * + * Destroying a locked mutex is undefined behaviour. Global mutexes may be + * locked when they are passed to this function, because multiple threads can + * still access them. So we can either: + * - destroy on shutdown, and re-initialise when tor re-initialises, or + * - skip destroying and re-initialisation, using a sentinel variable. + * See #31735 for details. + */ void tor_mutex_uninit(tor_mutex_t *m) { int err; raw_assert(m); + /* If the mutex is already locked, wait until after it is unlocked to destroy + * it. Locking and releasing the mutex makes undefined behaviour less likely, + * but does not prevent it. Another thread can lock the mutex between release + * and destroy. */ + tor_mutex_acquire(m); + tor_mutex_release(m); err = pthread_mutex_destroy(&m->mutex); if (PREDICT_UNLIKELY(err)) { // LCOV_EXCL_START diff --git a/src/lib/thread/compat_threads.c b/src/lib/thread/compat_threads.c index 94ab021c52..16cece6125 100644 --- a/src/lib/thread/compat_threads.c +++ b/src/lib/thread/compat_threads.c @@ -65,7 +65,15 @@ atomic_counter_init(atomic_counter_t *counter) memset(counter, 0, sizeof(*counter)); tor_mutex_init_nonrecursive(&counter->mutex); } -/** Clean up all resources held by an atomic counter. */ +/** Clean up all resources held by an atomic counter. + * + * Destroying a locked mutex is undefined behaviour. Global mutexes may be + * locked when they are passed to this function, because multiple threads can + * still access them. So we can either: + * - destroy on shutdown, and re-initialise when tor re-initialises, or + * - skip destroying and re-initialisation, using a sentinel variable. + * See #31735 for details. + */ void atomic_counter_destroy(atomic_counter_t *counter) { diff --git a/src/lib/thread/threads.h b/src/lib/thread/threads.h index ecf60641b5..de3da6a585 100644 --- a/src/lib/thread/threads.h +++ b/src/lib/thread/threads.h @@ -131,7 +131,17 @@ atomic_counter_init(atomic_counter_t *counter) { atomic_init(&counter->val, 0); } -/** Clean up all resources held by an atomic counter. */ +/** Clean up all resources held by an atomic counter. + * + * This usage note applies to the compat_threads implementation of + * atomic_counter_destroy(): + * Destroying a locked mutex is undefined behaviour. Global mutexes may be + * locked when they are passed to this function, because multiple threads can + * still access them. So we can either: + * - destroy on shutdown, and re-initialise when tor re-initialises, or + * - skip destroying and re-initialisation, using a sentinel variable. + * See #31735 for details. + */ static inline void atomic_counter_destroy(atomic_counter_t *counter) { From 65e63e746120a03b54de51f1db148a7fa1aa27e2 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 18 Sep 2019 10:59:35 -0400 Subject: [PATCH 1595/2557] annotate_ifdef_directives: remove some cases of double negation This change should reduce the number of cases where we say "/* !(!defined(foo)) */" . This only does cases where we can use a regex to make sure that the simplification is guaranteed to be correct. Full boolean simplification would require this script to parse C, and nobody wants that. --- scripts/maint/annotate_ifdef_directives | 44 +++++++++++++++++++++++-- 1 file changed, 42 insertions(+), 2 deletions(-) diff --git a/scripts/maint/annotate_ifdef_directives b/scripts/maint/annotate_ifdef_directives index b784ca71ba..4463d83828 100755 --- a/scripts/maint/annotate_ifdef_directives +++ b/scripts/maint/annotate_ifdef_directives @@ -54,6 +54,46 @@ def commented_line(fmt, argument, maxwidth=LINE_WIDTH): assert len(result) <= maxwidth return result +def negate(expr): + """Return a negated version of expr; try to avoid double-negation. + + We usually wrap expressions in parentheses and add a "!". + >>> negate("A && B") + '!(A && B)' + + But if we recognize the expression as negated, we can restore it. + >>> negate(negate("A && B")) + 'A && B' + + The same applies for defined(FOO). + >>> negate("defined(FOO)") + '!defined(FOO)' + >>> negate(negate("defined(FOO)")) + 'defined(FOO)' + + Internal parentheses don't confuse us: + >>> negate("!(FOO) && !(BAR)") + '!(!(FOO) && !(BAR))' + + """ + expr = expr.strip() + # See whether we match !(...), with no intervening close-parens. + m = re.match(r'^!\s*\(([^\)]*)\)$', expr) + if m: + return m.group(1) + + + # See whether we match !?defined(...), with no intervening close-parens. + m = re.match(r'^(!?)\s*(defined\([^\)]*\))$', expr) + if m: + if m.group(1) == "!": + prefix = "" + else: + prefix = "!" + return prefix + m.group(2) + + return "!(%s)" % expr + def uncomment(s): """ Remove existing trailing comments from an #else or #endif line. @@ -108,8 +148,8 @@ def translate(f_in, f_out): raise Problem("Unexpected #%s on %d"% (command,lineno)) if (len(cur_level) == 1 and command == 'else' and lineno > cur_level[0][2] + LINE_OBVIOUSNESS_LIMIT): - f_out.write(commented_line("#else /* !(%s) */\n", - cur_level[0][1])) + f_out.write(commented_line("#else /* %s */\n", + negate(cur_level[0][1]))) else: f_out.write(line) cur_level.append((command, rest, lineno)) From 3283fd7e79913e25cd5e626d6bb3a12a05b2f3fc Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 18 Sep 2019 11:06:54 -0400 Subject: [PATCH 1596/2557] Changes file for 31759 and 31779 --- changes/ticket31759 | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 changes/ticket31759 diff --git a/changes/ticket31759 b/changes/ticket31759 new file mode 100644 index 0000000000..f7428f711c --- /dev/null +++ b/changes/ticket31759 @@ -0,0 +1,5 @@ + o Minor features (auto-formatting scripts): + - When annotating C macros, never generate a line that our check-spaces + script would reject. Closes ticket 31759. + - When annotating C macros, try to remove cases of double-negation. + Closes ticket 31779. From 194dbea24d1d05fa7b63b361b06054da4df011b9 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 18 Sep 2019 11:01:12 -0400 Subject: [PATCH 1597/2557] Run "make autostyle" with new "annotate_ifdef_directives" --- src/app/config/config.c | 6 +++--- src/lib/crypt_ops/compat_openssl.h | 2 +- src/lib/fs/dir.c | 2 +- src/lib/log/util_bug.h | 2 +- src/lib/math/fp.c | 2 +- src/lib/memarea/memarea.c | 2 +- src/lib/process/daemon.c | 2 +- src/lib/process/process.c | 2 +- src/lib/process/restrict.c | 2 +- src/lib/process/setuid.c | 2 +- src/lib/tls/tortls_openssl.c | 4 ++-- 11 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/app/config/config.c b/src/app/config/config.c index bdfa547fd7..6ee818ab0c 100644 --- a/src/app/config/config.c +++ b/src/app/config/config.c @@ -1455,7 +1455,7 @@ options_act_reversible(const or_options_t *old_options, char **msg) "on this OS/with this build."); goto rollback; } -#else /* !(!defined(HAVE_SYS_UN_H)) */ +#else /* defined(HAVE_SYS_UN_H) */ if (options->ControlSocketsGroupWritable && !options->ControlSocket) { *msg = tor_strdup("Setting ControlSocketGroupWritable without setting" "a ControlSocket makes no sense."); @@ -5101,7 +5101,7 @@ find_torrc_filename(config_line_t *cmd_arg, } else { fname = dflt ? tor_strdup(dflt) : NULL; } -#else /* !(!defined(_WIN32)) */ +#else /* defined(_WIN32) */ fname = dflt ? tor_strdup(dflt) : NULL; #endif /* !defined(_WIN32) */ } @@ -8425,7 +8425,7 @@ init_cookie_authentication(const char *fname, const char *header, log_warn(LD_FS,"Unable to make %s group-readable.", escaped(fname)); } } -#else /* !(!defined(_WIN32)) */ +#else /* defined(_WIN32) */ (void) group_readable; #endif /* !defined(_WIN32) */ diff --git a/src/lib/crypt_ops/compat_openssl.h b/src/lib/crypt_ops/compat_openssl.h index 9c10386c34..61ca51315f 100644 --- a/src/lib/crypt_ops/compat_openssl.h +++ b/src/lib/crypt_ops/compat_openssl.h @@ -45,7 +45,7 @@ ((st) == SSL3_ST_SW_SRVR_HELLO_B)) #define OSSL_HANDSHAKE_STATE int #define CONST_IF_OPENSSL_1_1_API -#else /* !(!defined(OPENSSL_1_1_API)) */ +#else /* defined(OPENSSL_1_1_API) */ #define STATE_IS_SW_SERVER_HELLO(st) \ ((st) == TLS_ST_SW_SRVR_HELLO) #define CONST_IF_OPENSSL_1_1_API const diff --git a/src/lib/fs/dir.c b/src/lib/fs/dir.c index 3c31e00d99..291f1bbf04 100644 --- a/src/lib/fs/dir.c +++ b/src/lib/fs/dir.c @@ -262,7 +262,7 @@ check_private_dir,(const char *dirname, cpd_check_t check, } } close(fd); -#else /* !(!defined(_WIN32)) */ +#else /* defined(_WIN32) */ /* Win32 case: we can't open() a directory. */ (void)effective_user; diff --git a/src/lib/log/util_bug.h b/src/lib/log/util_bug.h index 546ae1e3ef..d7f01618e8 100644 --- a/src/lib/log/util_bug.h +++ b/src/lib/log/util_bug.h @@ -96,7 +96,7 @@ (void)(a); \ (void)(fmt); \ STMT_END -#else /* !(defined(TOR_UNIT_TESTS) && ... */ +#else /* !(defined(TOR_UNIT_TESTS) && defined(DISABLE_ASSERTS_IN_UNIT_TES... */ /** Like assert(3), but send assertion failures to the log as well as to * stderr. */ #define tor_assert(expr) tor_assertf(expr, NULL) diff --git a/src/lib/math/fp.c b/src/lib/math/fp.c index 616e4f15c0..49a2a6a2ca 100644 --- a/src/lib/math/fp.c +++ b/src/lib/math/fp.c @@ -75,7 +75,7 @@ clamp_double_to_int64(double number) */ #define PROBLEMATIC_FLOAT_CONVERSION_WARNING DISABLE_GCC_WARNING(float-conversion) -#endif /* defined(MINGW_ANY) && GCC_VERSION >= 409 */ +#endif /* (defined(MINGW_ANY)||defined(__FreeBSD__)) && GCC_VERSION >= 409 */ /* With clang 4.0 we apparently run into "double promotion" warnings here, diff --git a/src/lib/memarea/memarea.c b/src/lib/memarea/memarea.c index 84c73b0b95..0a88210906 100644 --- a/src/lib/memarea/memarea.c +++ b/src/lib/memarea/memarea.c @@ -315,7 +315,7 @@ memarea_assert_ok(memarea_t *area) } } -#else /* !(!defined(DISABLE_MEMORY_SENTINELS)) */ +#else /* defined(DISABLE_MEMORY_SENTINELS) */ struct memarea_t { smartlist_t *pieces; diff --git a/src/lib/process/daemon.c b/src/lib/process/daemon.c index 3b90bef671..ae34b5bcb8 100644 --- a/src/lib/process/daemon.c +++ b/src/lib/process/daemon.c @@ -165,7 +165,7 @@ finish_daemon(const char *desired_cwd) return 0; } -#else /* !(!defined(_WIN32)) */ +#else /* defined(_WIN32) */ /* defined(_WIN32) */ int start_daemon(void) diff --git a/src/lib/process/process.c b/src/lib/process/process.c index 631c7169f1..2194a603ff 100644 --- a/src/lib/process/process.c +++ b/src/lib/process/process.c @@ -513,7 +513,7 @@ process_get_unix_process(const process_t *process) tor_assert(process->unix_process); return process->unix_process; } -#else /* !(!defined(_WIN32)) */ +#else /* defined(_WIN32) */ /** Get the internal handle for Windows backend. */ process_win32_t * process_get_win32_process(const process_t *process) diff --git a/src/lib/process/restrict.c b/src/lib/process/restrict.c index 534b39d101..93d06de9a2 100644 --- a/src/lib/process/restrict.c +++ b/src/lib/process/restrict.c @@ -214,7 +214,7 @@ set_max_file_descriptors(rlim_t limit, int *max_out) return -1; } limit = MAX_CONNECTIONS; -#else /* !(!defined(HAVE_GETRLIMIT)) */ +#else /* defined(HAVE_GETRLIMIT) */ struct rlimit rlim; if (getrlimit(RLIMIT_NOFILE, &rlim) != 0) { diff --git a/src/lib/process/setuid.c b/src/lib/process/setuid.c index 6e8258f279..e132787943 100644 --- a/src/lib/process/setuid.c +++ b/src/lib/process/setuid.c @@ -376,7 +376,7 @@ switch_id(const char *user, const unsigned flags) #endif /* defined(__linux__) && defined(HAVE_SYS_PRCTL_H) && ... */ return 0; -#else /* !(!defined(_WIN32)) */ +#else /* defined(_WIN32) */ (void)user; (void)flags; diff --git a/src/lib/tls/tortls_openssl.c b/src/lib/tls/tortls_openssl.c index 86f0ac42cc..58a7b20dec 100644 --- a/src/lib/tls/tortls_openssl.c +++ b/src/lib/tls/tortls_openssl.c @@ -657,7 +657,7 @@ tor_tls_context_new(crypto_pk_t *identity, unsigned int key_lifetime, if (r < 0) goto error; } -#else /* !(defined(SSL_CTX_set1_groups_list) || ...) */ +#else /* !(defined(SSL_CTX_set1_groups_list) || defined(HAVE_SSL_CTX_SET1... */ if (! is_client) { int nid; EC_KEY *ec_key; @@ -673,7 +673,7 @@ tor_tls_context_new(crypto_pk_t *identity, unsigned int key_lifetime, SSL_CTX_set_tmp_ecdh(result->ctx, ec_key); EC_KEY_free(ec_key); } -#endif /* defined(SSL_CTX_set1_groups_list) || ...) */ +#endif /* defined(SSL_CTX_set1_groups_list) || defined(HAVE_SSL_CTX_SET1_... */ SSL_CTX_set_verify(result->ctx, SSL_VERIFY_PEER, always_accept_verify_cb); /* let us realloc bufs that we're writing from */ From 21cc9d13f34d116b521c4415098ca1c9993bd198 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 25 Sep 2019 21:13:30 -0400 Subject: [PATCH 1598/2557] annotate_ifdef_directives: clarify situation with newlines Our line limit is 80 characters, assuming that there is a single terminating newline character that counts towards the limit. On Windows, this might go as high as 81 characters, if we count CRLF as two characters. --- scripts/maint/annotate_ifdef_directives | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/scripts/maint/annotate_ifdef_directives b/scripts/maint/annotate_ifdef_directives index 4463d83828..b6bb147ce2 100755 --- a/scripts/maint/annotate_ifdef_directives +++ b/scripts/maint/annotate_ifdef_directives @@ -26,7 +26,10 @@ import re # Any block with fewer than this many lines does not need annotations. LINE_OBVIOUSNESS_LIMIT = 4 -# Maximum line width. +# Maximum line width. This includes a terminating newline character. +# +# (This is the maximum before encoding, so that if the the operating system +# uses multiple characers to encode newline, that's still okay.) LINE_WIDTH=80 class Problem(Exception): @@ -38,7 +41,10 @@ def commented_line(fmt, argument, maxwidth=LINE_WIDTH): be longer than maxwidth, truncate argument. Requires that fmt%"..." will fit into maxwidth characters. + + Requires that fmt ends with a newline. """ + assert fmt.endswith("\n") result = fmt % argument if len(result) <= maxwidth: return result From 195aa2f5f73e0cd9462afd4f21f3f0dac36bbc82 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 25 Sep 2019 21:27:17 -0400 Subject: [PATCH 1599/2557] annotate_ifdef_directives: generate paren-balanced expressions This algorithm is not fully general, but it strikes a balance between efficiency, simplicity, and correctness. --- scripts/maint/annotate_ifdef_directives | 46 ++++++++++++++++++++----- 1 file changed, 38 insertions(+), 8 deletions(-) diff --git a/scripts/maint/annotate_ifdef_directives b/scripts/maint/annotate_ifdef_directives index b6bb147ce2..f88dd4fdfe 100755 --- a/scripts/maint/annotate_ifdef_directives +++ b/scripts/maint/annotate_ifdef_directives @@ -35,7 +35,41 @@ LINE_WIDTH=80 class Problem(Exception): pass +def close_parens_needed(expr): + """Return the number of left-parentheses needed to make 'expr' + balanced. + """ + return expr.count("(") - expr.count(")") + +def truncate_expression(expr, new_width): + """Given a parenthesized C expression in 'expr', try to return a new + expression that is similar to 'expr', but no more than 'new_width' + characters long. + + Try to return an expression with balanced parentheses. + """ + if len(expr) <= new_width: + # The expression is already short enough. + return expr + + ellipsis = "..." + + # Start this at the minimum that we might truncate. + n_to_remove = len(expr) + len(ellipsis) - new_width + + # Try removing characters, one by one, until we get something where + # re-balancing the parentheses still fits within the limit. + while n_to_remove < len(expr): + truncated = expr[:-n_to_remove] + ellipsis + truncated += ")" * close_parens_needed(truncated) + if len(truncated) <= new_width: + return truncated + n_to_remove += 1 + + return ellipsis + def commented_line(fmt, argument, maxwidth=LINE_WIDTH): + """ Return fmt%argument, for use as a commented line. If the line would be longer than maxwidth, truncate argument. @@ -49,14 +83,10 @@ def commented_line(fmt, argument, maxwidth=LINE_WIDTH): if len(result) <= maxwidth: return result else: - # figure out how much we need to truncate by to fit the argument, - # plus an ellipsis. - ellipsis = "..." - result = fmt % (argument + ellipsis) - overrun = len(result) - maxwidth - truncated_argument = argument[:-overrun] + ellipsis - - result = fmt % truncated_argument + # How long can we let the argument be? Try filling in the + # format with an empty argument to find out. + max_arg_width = maxwidth - len(fmt % "") + result = fmt % truncate_expression(argument, max_arg_width) assert len(result) <= maxwidth return result From 6f0e697e4155ca567ddfd46a7f4e7c013287c42a Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 26 Sep 2019 10:03:28 -0400 Subject: [PATCH 1600/2557] Use Doctests to test the behavior of annotate_ifdef_directives. --- scripts/maint/annotate_ifdef_directives | 115 +++++++++++++++++++----- 1 file changed, 94 insertions(+), 21 deletions(-) diff --git a/scripts/maint/annotate_ifdef_directives b/scripts/maint/annotate_ifdef_directives index f88dd4fdfe..514b5e58bb 100755 --- a/scripts/maint/annotate_ifdef_directives +++ b/scripts/maint/annotate_ifdef_directives @@ -2,24 +2,60 @@ # Copyright (c) 2017-2019, The Tor Project, Inc. # See LICENSE for licensing information -# This script iterates over a list of C files. For each file, it looks at the -# #if/#else C macros, and annotates them with comments explaining what they -# match. -# -# For example, it replaces this: -# -# #ifdef HAVE_OCELOT -# // 500 lines of ocelot code -# #endif -# -# with this: -# -# #ifdef HAVE_OCELOT -# // 500 lines of ocelot code -# #endif /* defined(HAVE_OCELOT) */ -# -# Note that only #else and #endif lines are annotated. Existing comments -# on those lines are removed. +r""" +This script iterates over a list of C files. For each file, it looks at the +#if/#else C macros, and annotates them with comments explaining what they +match. + +For example, it replaces this kind of input... + +>>> INPUT = ''' +... #ifdef HAVE_OCELOT +... C code here +... #if MIMSY == BOROGROVE +... block 1 +... block 1 +... block 1 +... block 1 +... #else +... block 2 +... block 2 +... block 2 +... block 2 +... #endif +... #endif +... ''' + +With this kind of output: +>>> EXPECTED_OUTPUT = ''' +... #ifdef HAVE_OCELOT +... C code here +... #if MIMSY == BOROGROVE +... block 1 +... block 1 +... block 1 +... block 1 +... #else /* !(MIMSY == BOROGROVE) */ +... block 2 +... block 2 +... block 2 +... block 2 +... #endif /* MIMSY == BOROGROVE */ +... #endif /* defined(HAVE_OCELOT) */ +... ''' + +Here's how to use it: +>>> import sys +>>> if sys.version_info.major < 3: from cStringIO import StringIO +>>> if sys.version_info.major >= 3: from io import StringIO + +>>> OUTPUT = StringIO() +>>> translate(StringIO(INPUT), OUTPUT) +>>> assert OUTPUT.getvalue() == EXPECTED_OUTPUT + +Note that only #else and #endif lines are annotated. Existing comments +on those lines are removed. +""" import re @@ -38,6 +74,17 @@ class Problem(Exception): def close_parens_needed(expr): """Return the number of left-parentheses needed to make 'expr' balanced. + + >>> close_parens_needed("1+2") + 0 + >>> close_parens_needed("(1 + 2)") + 0 + >>> close_parens_needed("(1 + 2") + 1 + >>> close_parens_needed("(1 + (2 *") + 2 + >>> close_parens_needed("(1 + (2 * 3) + (4") + 2 """ return expr.count("(") - expr.count(")") @@ -47,6 +94,17 @@ def truncate_expression(expr, new_width): characters long. Try to return an expression with balanced parentheses. + + >>> truncate_expression("1+2+3", 8) + '1+2+3' + >>> truncate_expression("1+2+3+4+5", 8) + '1+2+3...' + >>> truncate_expression("(1+2+3+4)", 8) + '(1+2...)' + >>> truncate_expression("(1+(2+3+4))", 8) + '(1+...)' + >>> truncate_expression("(((((((((", 8) + '((...))' """ if len(expr) <= new_width: # The expression is already short enough. @@ -69,14 +127,23 @@ def truncate_expression(expr, new_width): return ellipsis def commented_line(fmt, argument, maxwidth=LINE_WIDTH): - - """ + # (This is a raw docstring so that our doctests can use \.) + r""" Return fmt%argument, for use as a commented line. If the line would - be longer than maxwidth, truncate argument. + be longer than maxwidth, truncate argument but try to keep its + parentheses balanced. Requires that fmt%"..." will fit into maxwidth characters. Requires that fmt ends with a newline. + + >>> commented_line("/* %s */\n", "hello world", 32) + '/* hello world */\n' + >>> commented_line("/* %s */\n", "hello world", 15) + '/* hello... */\n' + >>> commented_line("#endif /* %s */\n", "((1+2) && defined(FOO))", 32) + '#endif /* ((1+2) && defi...) */\n' + """ assert fmt.endswith("\n") result = fmt % argument @@ -208,6 +275,12 @@ def translate(f_in, f_out): raise Problem("Missing #endif") import sys,os + +if sys.argv[1] == "--self-test": + import doctest + doctest.testmod() + sys.exit(0) + for fn in sys.argv[1:]: with open(fn+"_OUT", 'w') as output_file: translate(open(fn, 'r'), output_file) From d229399e77f17a8ad19a793fcc7252027c0d3758 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 26 Sep 2019 15:36:20 -0400 Subject: [PATCH 1601/2557] annotate_ifdef_directives: Allow it to be imported as a module. --- scripts/maint/annotate_ifdef_directives | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/scripts/maint/annotate_ifdef_directives b/scripts/maint/annotate_ifdef_directives index 514b5e58bb..6ff9b8ec4c 100755 --- a/scripts/maint/annotate_ifdef_directives +++ b/scripts/maint/annotate_ifdef_directives @@ -274,14 +274,16 @@ def translate(f_in, f_out): if len(stack) or cur_level != whole_file: raise Problem("Missing #endif") -import sys,os +if __name__ == '__main__': -if sys.argv[1] == "--self-test": - import doctest - doctest.testmod() - sys.exit(0) + import sys,os -for fn in sys.argv[1:]: - with open(fn+"_OUT", 'w') as output_file: - translate(open(fn, 'r'), output_file) - os.rename(fn+"_OUT", fn) + if sys.argv[1] == "--self-test": + import doctest + doctest.testmod() + sys.exit(0) + + for fn in sys.argv[1:]: + with open(fn+"_OUT", 'w') as output_file: + translate(open(fn, 'r'), output_file) + os.rename(fn+"_OUT", fn) From f1e0665c934db49cc86936a17a3a1247db9e3337 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 26 Sep 2019 15:43:40 -0400 Subject: [PATCH 1602/2557] Rename annotate_ifdef_directives to end with .py. This allows the python doctest module to process it correctly when invoked as: python -m doctest -v annotate_ifdef_directives.py --- Makefile.am | 2 +- .../{annotate_ifdef_directives => annotate_ifdef_directives.py} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename scripts/maint/{annotate_ifdef_directives => annotate_ifdef_directives.py} (100%) diff --git a/Makefile.am b/Makefile.am index 491b4c8f9f..e52b1f742a 100644 --- a/Makefile.am +++ b/Makefile.am @@ -477,7 +477,7 @@ version: .PHONY: autostyle-ifdefs autostyle-ifdefs: - $(PYTHON) scripts/maint/annotate_ifdef_directives $(OWNED_TOR_C_FILES) + $(PYTHON) scripts/maint/annotate_ifdef_directives.py $(OWNED_TOR_C_FILES) .PHONY: autostyle-ifdefs autostyle-operators: diff --git a/scripts/maint/annotate_ifdef_directives b/scripts/maint/annotate_ifdef_directives.py similarity index 100% rename from scripts/maint/annotate_ifdef_directives rename to scripts/maint/annotate_ifdef_directives.py From 21c9f7c85e90f4d3ef539d41a36a24b2f26ad3d1 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 26 Sep 2019 19:57:41 -0400 Subject: [PATCH 1603/2557] Annotate_ifdef_directives: doctest for 80-column lines. --- scripts/maint/annotate_ifdef_directives.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/scripts/maint/annotate_ifdef_directives.py b/scripts/maint/annotate_ifdef_directives.py index 6ff9b8ec4c..b4326f9822 100755 --- a/scripts/maint/annotate_ifdef_directives.py +++ b/scripts/maint/annotate_ifdef_directives.py @@ -144,6 +144,18 @@ def commented_line(fmt, argument, maxwidth=LINE_WIDTH): >>> commented_line("#endif /* %s */\n", "((1+2) && defined(FOO))", 32) '#endif /* ((1+2) && defi...) */\n' + + The default line limit is 80 characters including the newline: + + >>> long_argument = "long " * 100 + >>> long_line = commented_line("#endif /* %s */\n", long_argument) + >>> len(long_line) + 80 + + >>> long_line[:40] + '#endif /* long long long long long long ' + >>> long_line[40:] + 'long long long long long long lon... */\n' """ assert fmt.endswith("\n") result = fmt % argument From fc1134e3e59df2ca473e787f70a57ebf659d78f2 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 26 Sep 2019 20:30:41 -0400 Subject: [PATCH 1604/2557] annotate_ifdef_directives: test edge-case of 80-char line An 80-character line (79 characters if you don't count the newline) should not be truncated, and should not have a "..." insterted. --- scripts/maint/annotate_ifdef_directives.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/scripts/maint/annotate_ifdef_directives.py b/scripts/maint/annotate_ifdef_directives.py index b4326f9822..102128bfa0 100755 --- a/scripts/maint/annotate_ifdef_directives.py +++ b/scripts/maint/annotate_ifdef_directives.py @@ -156,6 +156,22 @@ def commented_line(fmt, argument, maxwidth=LINE_WIDTH): '#endif /* long long long long long long ' >>> long_line[40:] 'long long long long long long lon... */\n' + + If a line works out to being 80 characters naturally, it isn't truncated, + and no ellipsis is added. + + >>> medium_argument = "a"*66 + >>> medium_line = commented_line("#endif /* %s */\n", medium_argument) + >>> len(medium_line) + 80 + >>> "..." in medium_line + False + >>> medium_line[:40] + '#endif /* aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' + >>> medium_line[40:] + 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa */\n' + + """ assert fmt.endswith("\n") result = fmt % argument From 2420c8c9366e498dfaf3b4b6389ece9dc27ca537 Mon Sep 17 00:00:00 2001 From: teor Date: Mon, 30 Sep 2019 14:54:56 +1000 Subject: [PATCH 1605/2557] test: Avoid a map_anon_nofork test failure on SunOS This test failure happened due to a signed/unsigned integer comparison. This bug occurred on SunOS, it may also occur on other systems that use signed char as the default. (And cast 1-byte integer constants to an unsigned integer.) Fixes bug 31897; bugfix on 0.4.1.1-alpha. --- changes/bug31897 | 3 +++ src/test/test_util.c | 5 +++-- 2 files changed, 6 insertions(+), 2 deletions(-) create mode 100644 changes/bug31897 diff --git a/changes/bug31897 b/changes/bug31897 new file mode 100644 index 0000000000..81c63e704e --- /dev/null +++ b/changes/bug31897 @@ -0,0 +1,3 @@ + o Minor bugfixes (tests, SunOS): + - Avoid a map_anon_nofork test failure due to a signed/unsigned integer + comparison. Fixes bug 31897; bugfix on 0.4.1.1-alpha. diff --git a/src/test/test_util.c b/src/test/test_util.c index 2faadd4e19..6ecff6f1c3 100644 --- a/src/test/test_util.c +++ b/src/test/test_util.c @@ -6182,6 +6182,7 @@ test_util_map_anon_nofork(void *arg) * crash, or send zero. */ char *ptr = NULL; + const char TEST_VALUE = 0xd0; size_t sz = 16384; int pipefd[2] = {-1, -1}; unsigned inherit=0; @@ -6189,7 +6190,7 @@ test_util_map_anon_nofork(void *arg) tor_munmap_anonymous(ptr, sz); ptr = tor_mmap_anonymous(sz, ANONMAP_NOINHERIT, &inherit); tt_ptr_op(ptr, OP_NE, 0); - memset(ptr, 0xd0, sz); + memset(ptr, TEST_VALUE, sz); tt_int_op(0, OP_EQ, pipe(pipefd)); pid_t child = fork(); @@ -6220,7 +6221,7 @@ test_util_map_anon_nofork(void *arg) // noinherit isn't implemented. tt_int_op(inherit, OP_EQ, INHERIT_RES_KEEP); tt_int_op((int)r, OP_EQ, 1); // child should send us a byte. - tt_int_op(buf[0], OP_EQ, 0xd0); // that byte should what we set it to. + tt_int_op(buf[0], OP_EQ, TEST_VALUE); // that byte should be TEST_VALUE. } int ws; From 4c88ebcf4728084cad0a1f177eb9cdb107388581 Mon Sep 17 00:00:00 2001 From: teor Date: Thu, 26 Sep 2019 13:31:32 +1000 Subject: [PATCH 1606/2557] log: Remove duplicate code and an outdated comment Preparation for 31854. --- src/lib/log/log.c | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/lib/log/log.c b/src/lib/log/log.c index d2002f6eae..994fdbdd9e 100644 --- a/src/lib/log/log.c +++ b/src/lib/log/log.c @@ -584,8 +584,7 @@ logv,(int severity, log_domain_mask_t domain, const char *funcname, /* check that severity is sane. Overrunning the masks array leads to * interesting and hard to diagnose effects */ raw_assert(severity >= LOG_ERR && severity <= LOG_DEBUG); - /* check that we've initialised the log mutex before we try to lock it */ - raw_assert(log_mutex_initialized); + LOCK_LOGS(); if ((! (domain & LD_NOCB)) && pending_cb_messages @@ -866,9 +865,6 @@ logs_close_sigsafe(void) * logfiles (it is probably present, but it might not be due to thread * racing issues). After this function is called, the caller shouldn't * refer to victim anymore. - * - * Long-term, we need to do something about races in the log subsystem - * in general. See bug 222 for more details. */ static void delete_log(logfile_t *victim) From 25c5322dfe228178391b37fffd3333abdf57baa5 Mon Sep 17 00:00:00 2001 From: teor Date: Thu, 26 Sep 2019 13:33:17 +1000 Subject: [PATCH 1607/2557] log: Define count, boundary, and all macros for domains and flags And do static checks on those macro definitions. Part of 31854. --- src/lib/log/log.c | 4 +++- src/lib/log/log.h | 15 +++++++++++++-- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/src/lib/log/log.c b/src/lib/log/log.c index 994fdbdd9e..db4f154418 100644 --- a/src/lib/log/log.c +++ b/src/lib/log/log.c @@ -1337,7 +1337,9 @@ static const char *domain_list[] = { CTASSERT(ARRAY_LENGTH(domain_list) == N_LOGGING_DOMAINS + 1); -CTASSERT((UINT64_C(1)<<(N_LOGGING_DOMAINS-1)) < LOWEST_RESERVED_LD_FLAG_); +CTASSERT(HIGHEST_RESERVED_LD_DOMAIN_ < LD_ALL_DOMAINS); +CTASSERT(LD_ALL_DOMAINS < LOWEST_RESERVED_LD_FLAG_); +CTASSERT(LOWEST_RESERVED_LD_FLAG_ < LD_ALL_FLAGS); /** Return a bitmask for the log domain for which domain is the name, * or 0 if there is no such name. */ diff --git a/src/lib/log/log.h b/src/lib/log/log.h index da4bcbe608..b957fd1567 100644 --- a/src/lib/log/log.h +++ b/src/lib/log/log.h @@ -117,10 +117,21 @@ #define LD_BTRACK (UINT64_C(1)<<28) /** Message-passing backend. */ #define LD_MESG (UINT64_C(1)<<29) -#define N_LOGGING_DOMAINS 30 +/** The number of log domains. */ +#define N_LOGGING_DOMAINS 30 +/** The highest log domain */ +#define HIGHEST_RESERVED_LD_DOMAIN_ (UINT64_C(1)<<(N_LOGGING_DOMAINS - 1)) +/** All log domains. */ +#define LD_ALL_DOMAINS ((~(UINT64_C(0)))>>(64 - N_LOGGING_DOMAINS)) + +/** The number of log flags. */ +#define N_LOGGING_FLAGS 3 /** First bit that is reserved in log_domain_mask_t for non-domain flags. */ -#define LOWEST_RESERVED_LD_FLAG_ (UINT64_C(1)<<61) +#define LOWEST_RESERVED_LD_FLAG_ (UINT64_C(1)<<(64 - N_LOGGING_FLAGS)) +/** All log flags. */ +#define LD_ALL_FLAGS ((~(UINT64_C(0)))<<(64 - N_LOGGING_FLAGS)) + #ifdef TOR_UNIT_TESTS /** This log message should not be intercepted by mock_saving_logv */ #define LD_NO_MOCK (UINT64_C(1)<<61) From db329522ef6f0d0971111cbd07f35d54e0c7eced Mon Sep 17 00:00:00 2001 From: teor Date: Thu, 26 Sep 2019 13:37:06 +1000 Subject: [PATCH 1608/2557] log: When initialising log domain masks, only set known log domains And add a runtime test that checks for unknown domains and flags. Fixes bug 31854; bugfix on 0.2.1.1-alpha. --- changes/bug31854 | 3 +++ src/lib/log/log.c | 12 ++++++++---- src/test/test_options.c | 6 +++--- 3 files changed, 14 insertions(+), 7 deletions(-) create mode 100644 changes/bug31854 diff --git a/changes/bug31854 b/changes/bug31854 new file mode 100644 index 0000000000..692a192fd9 --- /dev/null +++ b/changes/bug31854 @@ -0,0 +1,3 @@ + o Minor bugfixes (logging): + - When initialising log domain masks, only set known log domains. + Fixes bug 31854; bugfix on 0.2.1.1-alpha. diff --git a/src/lib/log/log.c b/src/lib/log/log.c index db4f154418..7c18bea0d9 100644 --- a/src/lib/log/log.c +++ b/src/lib/log/log.c @@ -628,6 +628,10 @@ void tor_log(int severity, log_domain_mask_t domain, const char *format, ...) { va_list ap; + + /* check that domain is composed of known domains and flags */ + raw_assert((domain & (LD_ALL_DOMAINS|LD_ALL_FLAGS)) == domain); + if (severity > log_global_min_severity_) return; va_start(ap,format); @@ -927,7 +931,7 @@ set_log_severity_config(int loglevelMin, int loglevelMax, raw_assert(loglevelMax >= LOG_ERR && loglevelMax <= LOG_DEBUG); memset(severity_out, 0, sizeof(log_severity_list_t)); for (i = loglevelMin; i >= loglevelMax; --i) { - severity_out->masks[SEVERITY_MASK_IDX(i)] = ~0u; + severity_out->masks[SEVERITY_MASK_IDX(i)] = LD_ALL_DOMAINS; } } @@ -1421,7 +1425,7 @@ parse_log_severity_config(const char **cfg_ptr, const char *dash, *space; char *sev_lo, *sev_hi; int low, high, i; - log_domain_mask_t domains = ~0u; + log_domain_mask_t domains = LD_ALL_DOMAINS; if (*cfg == '[') { int err = 0; @@ -1439,7 +1443,7 @@ parse_log_severity_config(const char **cfg_ptr, tor_free(domains_str); SMARTLIST_FOREACH_BEGIN(domains_list, const char *, domain) { if (!strcmp(domain, "*")) { - domains = ~0u; + domains = LD_ALL_DOMAINS; } else { log_domain_mask_t d; int negate=0; @@ -1535,7 +1539,7 @@ switch_logs_debug(void) LOCK_LOGS(); for (lf = logfiles; lf; lf=lf->next) { for (i = LOG_DEBUG; i >= LOG_ERR; --i) - lf->severities->masks[SEVERITY_MASK_IDX(i)] = ~0u; + lf->severities->masks[SEVERITY_MASK_IDX(i)] = LD_ALL_DOMAINS; } log_global_min_severity_ = get_min_log_level(); UNLOCK_LOGS(); diff --git a/src/test/test_options.c b/src/test/test_options.c index b3654ede7d..9eb5a43924 100644 --- a/src/test/test_options.c +++ b/src/test/test_options.c @@ -54,9 +54,9 @@ setup_log_callback(void) { log_severity_list_t lst; memset(&lst, 0, sizeof(lst)); - lst.masks[SEVERITY_MASK_IDX(LOG_ERR)] = ~0; - lst.masks[SEVERITY_MASK_IDX(LOG_WARN)] = ~0; - lst.masks[SEVERITY_MASK_IDX(LOG_NOTICE)] = ~0; + lst.masks[SEVERITY_MASK_IDX(LOG_ERR)] = LD_ALL_DOMAINS; + lst.masks[SEVERITY_MASK_IDX(LOG_WARN)] = LD_ALL_DOMAINS; + lst.masks[SEVERITY_MASK_IDX(LOG_NOTICE)] = LD_ALL_DOMAINS; add_callback_log(&lst, log_cback); mark_logs_temp(); } From 53116ca0b71898b46a6d678407fb1b03c2ab8a93 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 30 Sep 2019 08:57:37 -0400 Subject: [PATCH 1609/2557] Re-run "make autostyle" with improved annotate_ifdef_directives --- src/app/config/config.c | 6 +++--- src/core/mainloop/connection.c | 4 ++-- src/core/or/channeltls.c | 2 +- src/core/or/command.c | 2 +- src/core/or/connection_edge.c | 2 +- src/core/or/scheduler.c | 2 +- src/core/or/scheduler_kist.c | 6 +++--- src/feature/api/tor_api.c | 2 +- src/feature/client/circpathbias.c | 2 +- src/feature/client/dnsserv.c | 2 +- src/feature/dirauth/authmode.h | 2 +- src/feature/dirauth/dirauth_periodic.h | 2 +- src/feature/dirauth/dirvote.h | 2 +- src/feature/dirauth/process_descs.h | 2 +- src/feature/dirauth/reachability.h | 2 +- src/feature/dirauth/shared_random.h | 2 +- src/feature/dircache/conscache.c | 2 +- src/feature/dirparse/unparseable.h | 2 +- src/feature/hs/hs_common.c | 2 +- src/lib/arch/bytes.h | 2 +- src/lib/cc/compat_compiler.h | 2 +- src/lib/cc/torint.h | 4 ++-- src/lib/compress/compress_lzma.c | 4 ++-- src/lib/compress/compress_zstd.c | 6 +++--- src/lib/conf/conftesting.h | 2 +- src/lib/crypt_ops/aes_openssl.c | 2 +- src/lib/crypt_ops/crypto_dh_openssl.c | 6 +++--- src/lib/crypt_ops/crypto_digest.c | 6 +++--- src/lib/crypt_ops/crypto_digest_openssl.c | 6 +++--- src/lib/crypt_ops/crypto_hkdf.c | 4 ++-- src/lib/crypt_ops/crypto_ope.c | 2 +- src/lib/crypt_ops/crypto_openssl_mgt.c | 2 +- src/lib/crypt_ops/crypto_rand.c | 4 ++-- src/lib/crypt_ops/crypto_rsa_openssl.c | 4 ++-- src/lib/crypt_ops/crypto_s2k.c | 4 ++-- src/lib/err/backtrace.c | 2 +- src/lib/evloop/procmon.c | 2 +- src/lib/fs/dir.c | 2 +- src/lib/fs/files.h | 4 ++-- src/lib/fs/path.c | 10 +++++----- src/lib/log/log.c | 2 +- src/lib/log/log.h | 2 +- src/lib/log/util_bug.c | 2 +- src/lib/log/util_bug.h | 4 ++-- src/lib/log/win32err.c | 2 +- src/lib/malloc/malloc.h | 2 +- src/lib/malloc/map_anon.c | 2 +- src/lib/memarea/memarea.c | 2 +- src/lib/meminfo/meminfo.c | 2 +- src/lib/net/nettypes.h | 2 +- src/lib/net/resolve.c | 4 ++-- src/lib/net/resolve.h | 2 +- src/lib/net/socket.c | 8 ++++---- src/lib/net/socket.h | 2 +- src/lib/net/socketpair.c | 2 +- src/lib/osinfo/uname.c | 2 +- src/lib/process/env.c | 2 +- src/lib/process/process_unix.c | 2 +- src/lib/process/restrict.c | 2 +- src/lib/process/setuid.c | 8 ++++---- src/lib/process/winprocess_sys.c | 2 +- src/lib/sandbox/sandbox.c | 2 +- src/lib/sandbox/sandbox.h | 2 +- src/lib/smartlist_core/smartlist_core.h | 2 +- src/lib/testsupport/testsupport.h | 4 ++-- src/lib/thread/threads.h | 4 ++-- src/lib/time/compat_time.c | 4 ++-- src/lib/time/compat_time.h | 4 ++-- src/lib/tls/tortls.h | 4 ++-- src/lib/tls/tortls_openssl.c | 6 +++--- src/lib/tls/x509_openssl.c | 2 +- src/lib/trace/events.h | 2 +- src/test/fuzz/fuzzing_common.c | 2 +- src/test/test_crypto.c | 2 +- src/test/test_options.c | 2 +- src/test/test_protover.c | 2 +- src/test/test_switch_id.c | 2 +- src/test/test_util.c | 4 ++-- src/test/test_workqueue.c | 2 +- src/test/testing_common.c | 2 +- src/test/testing_rsakeys.c | 2 +- 81 files changed, 121 insertions(+), 121 deletions(-) diff --git a/src/app/config/config.c b/src/app/config/config.c index 451593a5fa..deda2448b6 100644 --- a/src/app/config/config.c +++ b/src/app/config/config.c @@ -3518,7 +3518,7 @@ options_validate(or_options_t *old_options, or_options_t *options, REJECT("Cannot use TransProxyType without any valid TransPort."); } } -#else /* !(defined(USE_TRANSPARENT)) */ +#else /* !defined(USE_TRANSPARENT) */ if (options->TransPort_set) REJECT("TransPort is disabled in this build."); #endif /* defined(USE_TRANSPARENT) */ @@ -7795,7 +7795,7 @@ get_data_directory(const char *val) } else { return tor_strdup(get_windows_conf_root()); } -#else /* !(defined(_WIN32)) */ +#else /* !defined(_WIN32) */ const char *d = val; if (!d) d = "~/.tor"; @@ -8341,7 +8341,7 @@ config_load_geoip_file_(sa_family_t family, } r = geoip_load_file(family, fname, severity); tor_free(free_fname); -#else /* !(defined(_WIN32)) */ +#else /* !defined(_WIN32) */ (void)default_fname; r = geoip_load_file(family, fname, severity); #endif /* defined(_WIN32) */ diff --git a/src/core/mainloop/connection.c b/src/core/mainloop/connection.c index 127f08683f..348d90a63e 100644 --- a/src/core/mainloop/connection.c +++ b/src/core/mainloop/connection.c @@ -1192,7 +1192,7 @@ make_win32_socket_exclusive(tor_socket_t sock) return -1; } return 0; -#else /* !(defined(SO_EXCLUSIVEADDRUSE)) */ +#else /* !defined(SO_EXCLUSIVEADDRUSE) */ (void) sock; return 0; #endif /* defined(SO_EXCLUSIVEADDRUSE) */ @@ -3957,7 +3957,7 @@ update_send_buffer_size(tor_socket_t sock) &isb, sizeof(isb), &bytesReturned, NULL, NULL)) { setsockopt(sock, SOL_SOCKET, SO_SNDBUF, (const char*)&isb, sizeof(isb)); } -#else /* !(defined(_WIN32)) */ +#else /* !defined(_WIN32) */ (void) sock; #endif /* defined(_WIN32) */ } diff --git a/src/core/or/channeltls.c b/src/core/or/channeltls.c index f727799387..e9497651ef 100644 --- a/src/core/or/channeltls.c +++ b/src/core/or/channeltls.c @@ -1033,7 +1033,7 @@ channel_tls_time_process_cell(cell_t *cell, channel_tls_t *chan, int *time, channel_tls_time_process_cell(cl, cn, & tp ## time , \ channel_tls_process_ ## tp ## _cell); \ } STMT_END -#else /* !(defined(KEEP_TIMING_STATS)) */ +#else /* !defined(KEEP_TIMING_STATS) */ #define PROCESS_CELL(tp, cl, cn) channel_tls_process_ ## tp ## _cell(cl, cn) #endif /* defined(KEEP_TIMING_STATS) */ diff --git a/src/core/or/command.c b/src/core/or/command.c index 77e5447ce3..1c97437769 100644 --- a/src/core/or/command.c +++ b/src/core/or/command.c @@ -182,7 +182,7 @@ command_process_cell(channel_t *chan, cell_t *cell) command_time_process_cell(cl, cn, & tp ## time , \ command_process_ ## tp ## _cell); \ } STMT_END -#else /* !(defined(KEEP_TIMING_STATS)) */ +#else /* !defined(KEEP_TIMING_STATS) */ #define PROCESS_CELL(tp, cl, cn) command_process_ ## tp ## _cell(cl, cn) #endif /* defined(KEEP_TIMING_STATS) */ diff --git a/src/core/or/connection_edge.c b/src/core/or/connection_edge.c index 40d3351a82..3122a190dd 100644 --- a/src/core/or/connection_edge.c +++ b/src/core/or/connection_edge.c @@ -1224,7 +1224,7 @@ connection_ap_rescan_and_attach_pending(void) entry_conn->marked_pending_circ_line = 0; \ entry_conn->marked_pending_circ_file = 0; \ } while (0) -#else /* !(defined(DEBUGGING_17659)) */ +#else /* !defined(DEBUGGING_17659) */ #define UNMARK() do { } while (0) #endif /* defined(DEBUGGING_17659) */ diff --git a/src/core/or/scheduler.c b/src/core/or/scheduler.c index ee22a38142..063342480a 100644 --- a/src/core/or/scheduler.c +++ b/src/core/or/scheduler.c @@ -267,7 +267,7 @@ select_scheduler(void) log_notice(LD_SCHED, "Scheduler type KIST has been disabled by " "the consensus or no kernel support."); } -#else /* !(defined(HAVE_KIST_SUPPORT)) */ +#else /* !defined(HAVE_KIST_SUPPORT) */ log_info(LD_SCHED, "Scheduler type KIST not built in"); #endif /* defined(HAVE_KIST_SUPPORT) */ continue; diff --git a/src/core/or/scheduler_kist.c b/src/core/or/scheduler_kist.c index 1ec1c49c73..35b613cb8a 100644 --- a/src/core/or/scheduler_kist.c +++ b/src/core/or/scheduler_kist.c @@ -104,7 +104,7 @@ static unsigned int kist_lite_mode = 0; * changed and it doesn't recognized the values passed to the syscalls needed * by KIST. In that case, fallback to the naive approach. */ static unsigned int kist_no_kernel_support = 0; -#else /* !(defined(HAVE_KIST_SUPPORT)) */ +#else /* !defined(HAVE_KIST_SUPPORT) */ static unsigned int kist_lite_mode = 1; #endif /* defined(HAVE_KIST_SUPPORT) */ @@ -298,7 +298,7 @@ update_socket_info_impl, (socket_table_ent_t *ent)) } return; -#else /* !(defined(HAVE_KIST_SUPPORT)) */ +#else /* !defined(HAVE_KIST_SUPPORT) */ goto fallback; #endif /* defined(HAVE_KIST_SUPPORT) */ @@ -833,7 +833,7 @@ scheduler_can_use_kist(void) return run_interval > 0; } -#else /* !(defined(HAVE_KIST_SUPPORT)) */ +#else /* !defined(HAVE_KIST_SUPPORT) */ int scheduler_can_use_kist(void) diff --git a/src/feature/api/tor_api.c b/src/feature/api/tor_api.c index fd9d241353..e270c51ac9 100644 --- a/src/feature/api/tor_api.c +++ b/src/feature/api/tor_api.c @@ -40,7 +40,7 @@ #define raw_socketpair tor_ersatz_socketpair #define raw_closesocket closesocket #define snprintf _snprintf -#else /* !(defined(_WIN32)) */ +#else /* !defined(_WIN32) */ #define raw_socketpair socketpair #define raw_closesocket close #endif /* defined(_WIN32) */ diff --git a/src/feature/client/circpathbias.c b/src/feature/client/circpathbias.c index 3544dbb859..60a52664f1 100644 --- a/src/feature/client/circpathbias.c +++ b/src/feature/client/circpathbias.c @@ -303,7 +303,7 @@ pathbias_is_new_circ_attempt(origin_circuit_t *circ) return circ->cpath && circ->cpath->next != circ->cpath && circ->cpath->next->state == CPATH_STATE_AWAITING_KEYS; -#else /* !(defined(N2N_TAGGING_IS_POSSIBLE)) */ +#else /* !defined(N2N_TAGGING_IS_POSSIBLE) */ /* If tagging attacks are no longer possible, we probably want to * count bias from the first hop. However, one could argue that * timing-based tagging is still more useful than per-hop failure. diff --git a/src/feature/client/dnsserv.c b/src/feature/client/dnsserv.c index 7fb3fff6c1..f9e436e88f 100644 --- a/src/feature/client/dnsserv.c +++ b/src/feature/client/dnsserv.c @@ -238,7 +238,7 @@ dnsserv_launch_request(const char *name, int reverse, TO_CONN(conn)->port = control_conn->base_.port; TO_CONN(conn)->address = tor_addr_to_str_dup(&control_conn->base_.addr); } -#else /* !(defined(AF_UNIX)) */ +#else /* !defined(AF_UNIX) */ TO_CONN(conn)->port = control_conn->base_.port; TO_CONN(conn)->address = tor_addr_to_str_dup(&control_conn->base_.addr); #endif /* defined(AF_UNIX) */ diff --git a/src/feature/dirauth/authmode.h b/src/feature/dirauth/authmode.h index 48afc3cdb4..bfd5f4dc04 100644 --- a/src/feature/dirauth/authmode.h +++ b/src/feature/dirauth/authmode.h @@ -29,7 +29,7 @@ authdir_mode_v3(const or_options_t *options) #define have_module_dirauth() (1) -#else /* !(defined(HAVE_MODULE_DIRAUTH)) */ +#else /* !defined(HAVE_MODULE_DIRAUTH) */ #define authdir_mode(options) (((void)(options)),0) #define authdir_mode_handles_descs(options,purpose) \ diff --git a/src/feature/dirauth/dirauth_periodic.h b/src/feature/dirauth/dirauth_periodic.h index 1124fae952..866fbd35de 100644 --- a/src/feature/dirauth/dirauth_periodic.h +++ b/src/feature/dirauth/dirauth_periodic.h @@ -12,7 +12,7 @@ void dirauth_register_periodic_events(void); void reschedule_dirvote(const or_options_t *options); -#else /* !(defined(HAVE_MODULE_DIRAUTH)) */ +#else /* !defined(HAVE_MODULE_DIRAUTH) */ static inline void reschedule_dirvote(const or_options_t *options) diff --git a/src/feature/dirauth/dirvote.h b/src/feature/dirauth/dirvote.h index a0cfe0a34c..b7df33a3a9 100644 --- a/src/feature/dirauth/dirvote.h +++ b/src/feature/dirauth/dirvote.h @@ -128,7 +128,7 @@ struct config_line_t; char *format_recommended_version_list(const struct config_line_t *line, int warn); -#else /* !(defined(HAVE_MODULE_DIRAUTH)) */ +#else /* !defined(HAVE_MODULE_DIRAUTH) */ static inline time_t dirvote_act(const or_options_t *options, time_t now) diff --git a/src/feature/dirauth/process_descs.h b/src/feature/dirauth/process_descs.h index 9c9b2e354e..5e879bd6bc 100644 --- a/src/feature/dirauth/process_descs.h +++ b/src/feature/dirauth/process_descs.h @@ -38,7 +38,7 @@ uint32_t dirserv_router_get_status(const routerinfo_t *router, int severity); void dirserv_set_node_flags_from_authoritative_status(node_t *node, uint32_t authstatus); -#else /* !(defined(HAVE_MODULE_DIRAUTH)) */ +#else /* !defined(HAVE_MODULE_DIRAUTH) */ static inline int dirserv_load_fingerprint_file(void) { diff --git a/src/feature/dirauth/reachability.h b/src/feature/dirauth/reachability.h index 6624b516a1..46d0e7ee2e 100644 --- a/src/feature/dirauth/reachability.h +++ b/src/feature/dirauth/reachability.h @@ -34,7 +34,7 @@ void dirserv_orconn_tls_done(const tor_addr_t *addr, uint16_t or_port, const char *digest_rcvd, const struct ed25519_public_key_t *ed_id_rcvd); -#else /* !(defined(HAVE_MODULE_DIRAUTH)) */ +#else /* !defined(HAVE_MODULE_DIRAUTH) */ static inline int dirserv_should_launch_reachability_test(const routerinfo_t *ri, const routerinfo_t *ri_old) diff --git a/src/feature/dirauth/shared_random.h b/src/feature/dirauth/shared_random.h index 1d8fa89b0f..7ff9f15512 100644 --- a/src/feature/dirauth/shared_random.h +++ b/src/feature/dirauth/shared_random.h @@ -110,7 +110,7 @@ int sr_init(int save_to_disk); void sr_save_and_cleanup(void); void sr_act_post_consensus(const networkstatus_t *consensus); -#else /* !(defined(HAVE_MODULE_DIRAUTH)) */ +#else /* !defined(HAVE_MODULE_DIRAUTH) */ static inline int sr_init(int save_to_disk) diff --git a/src/feature/dircache/conscache.c b/src/feature/dircache/conscache.c index cf4fe8701d..2ec9981c03 100644 --- a/src/feature/dircache/conscache.c +++ b/src/feature/dircache/conscache.c @@ -92,7 +92,7 @@ consensus_cache_open(const char *subdir, int max_entries) */ #define VERY_LARGE_STORAGEDIR_LIMIT (1000*1000) storagedir_max_entries = VERY_LARGE_STORAGEDIR_LIMIT; -#else /* !(defined(MUST_UNMAP_TO_UNLINK)) */ +#else /* !defined(MUST_UNMAP_TO_UNLINK) */ /* Otherwise, we can just tell the storagedir to use the same limits * as this cache. */ storagedir_max_entries = max_entries; diff --git a/src/feature/dirparse/unparseable.h b/src/feature/dirparse/unparseable.h index 49e047961f..36c6b5a1ec 100644 --- a/src/feature/dirparse/unparseable.h +++ b/src/feature/dirparse/unparseable.h @@ -26,7 +26,7 @@ void dump_desc_init(void); log_debug(LD_MM, "Area for %s has %lu allocated; using %lu.", \ name, (unsigned long)alloc, (unsigned long)used); \ STMT_END -#else /* !(defined(DEBUG_AREA_ALLOC)) */ +#else /* !defined(DEBUG_AREA_ALLOC) */ #define DUMP_AREA(a,name) STMT_NIL #endif /* defined(DEBUG_AREA_ALLOC) */ diff --git a/src/feature/hs/hs_common.c b/src/feature/hs/hs_common.c index 8661ce046a..036d23a6b0 100644 --- a/src/feature/hs/hs_common.c +++ b/src/feature/hs/hs_common.c @@ -86,7 +86,7 @@ set_unix_port(edge_connection_t *conn, rend_service_port_config_t *p) return 0; } -#else /* !(defined(HAVE_SYS_UN_H)) */ +#else /* !defined(HAVE_SYS_UN_H) */ static int set_unix_port(edge_connection_t *conn, rend_service_port_config_t *p) diff --git a/src/lib/arch/bytes.h b/src/lib/arch/bytes.h index b8b6288139..4756ca2beb 100644 --- a/src/lib/arch/bytes.h +++ b/src/lib/arch/bytes.h @@ -129,7 +129,7 @@ tor_ntohll(uint64_t a) { return a; } -#else /* !(defined(WORDS_BIGENDIAN)) */ +#else /* !defined(WORDS_BIGENDIAN) */ static inline uint16_t tor_htons(uint16_t a) { diff --git a/src/lib/cc/compat_compiler.h b/src/lib/cc/compat_compiler.h index 92301449e8..3ef866ecce 100644 --- a/src/lib/cc/compat_compiler.h +++ b/src/lib/cc/compat_compiler.h @@ -82,7 +82,7 @@ # define ENABLE_GCC_WARNING(warningopt) \ PRAGMA_DIAGNOSTIC_(warning PRAGMA_JOIN_STRINGIFY_(-W,warningopt)) #endif /* defined(__clang__) || GCC_VERSION >= 406 */ -#else /* !(defined(__GNUC__)) */ +#else /* !defined(__GNUC__) */ /* not gcc at all */ # define DISABLE_GCC_WARNING(warning) # define ENABLE_GCC_WARNING(warning) diff --git a/src/lib/cc/torint.h b/src/lib/cc/torint.h index 523f378ed7..94b79d30a1 100644 --- a/src/lib/cc/torint.h +++ b/src/lib/cc/torint.h @@ -96,7 +96,7 @@ typedef int32_t ssize_t; # else # define TOR_PRIuSZ PRIu32 # endif -#else /* !(defined(_WIN32)) */ +#else /* !defined(_WIN32) */ # define TOR_PRIuSZ "zu" #endif /* defined(_WIN32) */ @@ -106,7 +106,7 @@ typedef int32_t ssize_t; # else # define TOR_PRIdSZ PRId32 # endif -#else /* !(defined(_WIN32)) */ +#else /* !defined(_WIN32) */ # define TOR_PRIdSZ "zd" #endif /* defined(_WIN32) */ diff --git a/src/lib/compress/compress_lzma.c b/src/lib/compress/compress_lzma.c index 2dab37e433..915f4949ae 100644 --- a/src/lib/compress/compress_lzma.c +++ b/src/lib/compress/compress_lzma.c @@ -221,7 +221,7 @@ tor_lzma_compress_new(int compress, tor_free(result); return NULL; /* LCOV_EXCL_STOP */ -#else /* !(defined(HAVE_LZMA)) */ +#else /* !defined(HAVE_LZMA) */ (void)compress; (void)method; (void)level; @@ -312,7 +312,7 @@ tor_lzma_compress_process(tor_lzma_compress_state_t *state, lzma_error_str(retval)); return TOR_COMPRESS_ERROR; } -#else /* !(defined(HAVE_LZMA)) */ +#else /* !defined(HAVE_LZMA) */ (void)state; (void)out; (void)out_len; diff --git a/src/lib/compress/compress_zstd.c b/src/lib/compress/compress_zstd.c index a99ea67e0b..9076665295 100644 --- a/src/lib/compress/compress_zstd.c +++ b/src/lib/compress/compress_zstd.c @@ -93,7 +93,7 @@ tor_zstd_get_version_str(void) ZSTD_versionNumber()); return version_str; -#else /* !(defined(HAVE_ZSTD)) */ +#else /* !defined(HAVE_ZSTD) */ return NULL; #endif /* defined(HAVE_ZSTD) */ } @@ -317,7 +317,7 @@ tor_zstd_compress_new(int compress, tor_free(result); return NULL; // LCOV_EXCL_STOP -#else /* !(defined(HAVE_ZSTD)) */ +#else /* !defined(HAVE_ZSTD) */ (void)compress; (void)method; (void)level; @@ -454,7 +454,7 @@ tor_zstd_compress_process(tor_zstd_compress_state_t *state, return TOR_COMPRESS_OK; } -#else /* !(defined(HAVE_ZSTD)) */ +#else /* !defined(HAVE_ZSTD) */ (void)state; (void)out; (void)out_len; diff --git a/src/lib/conf/conftesting.h b/src/lib/conf/conftesting.h index a40c9bc97c..f01f52d59e 100644 --- a/src/lib/conf/conftesting.h +++ b/src/lib/conf/conftesting.h @@ -73,7 +73,7 @@ typedef union { #define DUMMY_TYPECHECK_INSTANCE(tp) \ static tp tp ## _dummy -#else /* !(defined(TOR_UNIT_TESTS)) */ +#else /* !defined(TOR_UNIT_TESTS) */ #define CONF_TEST_MEMBERS(tp, conftype, member) /* Repeatedly declarable incomplete struct to absorb redundant semicolons */ diff --git a/src/lib/crypt_ops/aes_openssl.c b/src/lib/crypt_ops/aes_openssl.c index 2f985d4512..64564892ad 100644 --- a/src/lib/crypt_ops/aes_openssl.c +++ b/src/lib/crypt_ops/aes_openssl.c @@ -148,7 +148,7 @@ evaluate_ctr_for_aes(void) { return 0; } -#else /* !(defined(USE_EVP_AES_CTR)) */ +#else /* !defined(USE_EVP_AES_CTR) */ /*======================================================================*/ /* Interface to AES code, and counter implementation */ diff --git a/src/lib/crypt_ops/crypto_dh_openssl.c b/src/lib/crypt_ops/crypto_dh_openssl.c index 75cee1b596..8ae97373e8 100644 --- a/src/lib/crypt_ops/crypto_dh_openssl.c +++ b/src/lib/crypt_ops/crypto_dh_openssl.c @@ -68,7 +68,7 @@ crypto_validate_dh_params(const BIGNUM *p, const BIGNUM *g) goto out; if (!DH_set0_pqg(dh, dh_p, NULL, dh_g)) goto out; -#else /* !(defined(OPENSSL_1_1_API)) */ +#else /* !defined(OPENSSL_1_1_API) */ if (!(dh->p = BN_dup(p))) goto out; if (!(dh->g = BN_dup(g))) @@ -231,7 +231,7 @@ new_openssl_dh_from_params(BIGNUM *p, BIGNUM *g) if (!DH_set_length(res_dh, DH_PRIVATE_KEY_BITS)) goto err; -#else /* !(defined(OPENSSL_1_1_API)) */ +#else /* !defined(OPENSSL_1_1_API) */ res_dh->p = dh_p; res_dh->g = dh_g; res_dh->length = DH_PRIVATE_KEY_BITS; @@ -298,7 +298,7 @@ crypto_dh_generate_public(crypto_dh_t *dh) "the-universe chances really do happen. Treating as a failure."); return -1; } -#else /* !(defined(OPENSSL_1_1_API)) */ +#else /* !defined(OPENSSL_1_1_API) */ if (tor_check_dh_key(LOG_WARN, dh->dh->pub_key)<0) { /* LCOV_EXCL_START * If this happens, then openssl's DH implementation is busted. */ diff --git a/src/lib/crypt_ops/crypto_digest.c b/src/lib/crypt_ops/crypto_digest.c index 64a7d2d52c..ba226f8756 100644 --- a/src/lib/crypt_ops/crypto_digest.c +++ b/src/lib/crypt_ops/crypto_digest.c @@ -149,7 +149,7 @@ struct crypto_xof_t { * outside the tests yet. */ EVP_MD_CTX *ctx; -#else /* !(defined(OPENSSL_HAS_SHAKE3_EVP)) */ +#else /* !defined(OPENSSL_HAS_SHAKE3_EVP) */ keccak_state s; #endif /* defined(OPENSSL_HAS_SHAKE3_EVP) */ }; @@ -169,7 +169,7 @@ crypto_xof_new(void) tor_assert(xof->ctx); int r = EVP_DigestInit(xof->ctx, EVP_shake256()); tor_assert(r == 1); -#else /* !(defined(OPENSSL_HAS_SHAKE256)) */ +#else /* !defined(OPENSSL_HAS_SHAKE256) */ keccak_xof_init(&xof->s, 256); #endif /* defined(OPENSSL_HAS_SHAKE256) */ return xof; @@ -236,7 +236,7 @@ crypto_xof(uint8_t *output, size_t output_len, r = EVP_DigestFinalXOF(ctx, output, output_len); tor_assert(r == 1); EVP_MD_CTX_free(ctx); -#else /* !(defined(OPENSSL_HAS_SHA3)) */ +#else /* !defined(OPENSSL_HAS_SHA3) */ crypto_xof_t *xof = crypto_xof_new(); crypto_xof_add_bytes(xof, input, input_len); crypto_xof_squeeze_bytes(xof, output, output_len); diff --git a/src/lib/crypt_ops/crypto_digest_openssl.c b/src/lib/crypt_ops/crypto_digest_openssl.c index c631b0eac0..b0d8b6aee9 100644 --- a/src/lib/crypt_ops/crypto_digest_openssl.c +++ b/src/lib/crypt_ops/crypto_digest_openssl.c @@ -212,7 +212,7 @@ crypto_digest_new_internal(digest_algorithm_t algorithm) return NULL; } break; -#else /* !(defined(OPENSSL_HAS_SHA3)) */ +#else /* !defined(OPENSSL_HAS_SHA3) */ case DIGEST_SHA3_256: keccak_digest_init(&r->d.sha3, 256); break; @@ -310,7 +310,7 @@ crypto_digest_add_bytes(crypto_digest_t *digest, const char *data, tor_assert(r); } break; -#else /* !(defined(OPENSSL_HAS_SHA3)) */ +#else /* !defined(OPENSSL_HAS_SHA3) */ case DIGEST_SHA3_256: /* FALLSTHROUGH */ case DIGEST_SHA3_512: keccak_digest_update(&digest->d.sha3, (const uint8_t *)data, len); @@ -354,7 +354,7 @@ crypto_digest_get_digest(crypto_digest_t *digest, EVP_MD_CTX_free(tmp); tor_assert(res == 1); goto done; -#else /* !(defined(OPENSSL_HAS_SHA3)) */ +#else /* !defined(OPENSSL_HAS_SHA3) */ /* Tiny-Keccak handles copying into a temporary ctx, and also can handle * short output buffers by truncating appropriately. */ keccak_digest_sum(&digest->d.sha3, (uint8_t *)out, out_len); diff --git a/src/lib/crypt_ops/crypto_hkdf.c b/src/lib/crypt_ops/crypto_hkdf.c index e0f3d65ad1..e0d241d4ea 100644 --- a/src/lib/crypt_ops/crypto_hkdf.c +++ b/src/lib/crypt_ops/crypto_hkdf.c @@ -109,7 +109,7 @@ crypto_expand_key_material_rfc5869_sha256_openssl( return 0; } -#else /* !(defined(HAVE_OPENSSL_HKDF)) */ +#else /* !defined(HAVE_OPENSSL_HKDF) */ /** * Perform RFC5869 HKDF computation using our own legacy implementation. @@ -191,7 +191,7 @@ crypto_expand_key_material_rfc5869_sha256( salt_in_len, info_in, info_in_len, key_out, key_out_len); -#else /* !(defined(HAVE_OPENSSL_HKDF)) */ +#else /* !defined(HAVE_OPENSSL_HKDF) */ return crypto_expand_key_material_rfc5869_sha256_legacy(key_in, key_in_len, salt_in, salt_in_len, info_in, diff --git a/src/lib/crypt_ops/crypto_ope.c b/src/lib/crypt_ops/crypto_ope.c index 4bd4b35706..ed832d852e 100644 --- a/src/lib/crypt_ops/crypto_ope.c +++ b/src/lib/crypt_ops/crypto_ope.c @@ -57,7 +57,7 @@ ope_val_from_le(ope_val_t x) ((x) >> 8) | (((x)&0xff) << 8); } -#else /* !(defined(WORDS_BIGENDIAN)) */ +#else /* !defined(WORDS_BIGENDIAN) */ #define ope_val_from_le(x) (x) #endif /* defined(WORDS_BIGENDIAN) */ diff --git a/src/lib/crypt_ops/crypto_openssl_mgt.c b/src/lib/crypt_ops/crypto_openssl_mgt.c index 6d8364ebf8..f51309219a 100644 --- a/src/lib/crypt_ops/crypto_openssl_mgt.c +++ b/src/lib/crypt_ops/crypto_openssl_mgt.c @@ -204,7 +204,7 @@ crypto_openssl_early_init(void) OPENSSL_INIT_LOAD_CRYPTO_STRINGS | OPENSSL_INIT_ADD_ALL_CIPHERS | OPENSSL_INIT_ADD_ALL_DIGESTS, NULL); -#else /* !(defined(OPENSSL_1_1_API)) */ +#else /* !defined(OPENSSL_1_1_API) */ ERR_load_crypto_strings(); OpenSSL_add_all_algorithms(); #endif /* defined(OPENSSL_1_1_API) */ diff --git a/src/lib/crypt_ops/crypto_rand.c b/src/lib/crypt_ops/crypto_rand.c index a80a98f267..afbafbfa35 100644 --- a/src/lib/crypt_ops/crypto_rand.c +++ b/src/lib/crypt_ops/crypto_rand.c @@ -248,7 +248,7 @@ crypto_strongest_rand_fallback(uint8_t *out, size_t out_len) (void)out; (void)out_len; return -1; -#else /* !(defined(_WIN32)) */ +#else /* !defined(_WIN32) */ static const char *filenames[] = { "/dev/srandom", "/dev/urandom", "/dev/random", NULL }; @@ -520,7 +520,7 @@ crypto_rand_unmocked(char *to, size_t n) #undef BUFLEN } -#else /* !(defined(ENABLE_NSS)) */ +#else /* !defined(ENABLE_NSS) */ int r = RAND_bytes((unsigned char*)to, (int)n); /* We consider a PRNG failure non-survivable. Let's assert so that we get a * stack trace about where it happened. diff --git a/src/lib/crypt_ops/crypto_rsa_openssl.c b/src/lib/crypt_ops/crypto_rsa_openssl.c index fbdc76ccd6..05d7c26b25 100644 --- a/src/lib/crypt_ops/crypto_rsa_openssl.c +++ b/src/lib/crypt_ops/crypto_rsa_openssl.c @@ -53,7 +53,7 @@ crypto_pk_key_is_private(const crypto_pk_t *k) const BIGNUM *p, *q; RSA_get0_factors(k->key, &p, &q); return p != NULL; /* XXX/yawning: Should we check q? */ -#else /* !(defined(OPENSSL_1_1_API)) */ +#else /* !defined(OPENSSL_1_1_API) */ return k && k->key && k->key->p; #endif /* defined(OPENSSL_1_1_API) */ } @@ -287,7 +287,7 @@ crypto_pk_num_bits(crypto_pk_t *env) tor_assert(n != NULL); return RSA_bits(env->key); -#else /* !(defined(OPENSSL_1_1_API)) */ +#else /* !defined(OPENSSL_1_1_API) */ tor_assert(env->key->n); return BN_num_bits(env->key->n); #endif /* defined(OPENSSL_1_1_API) */ diff --git a/src/lib/crypt_ops/crypto_s2k.c b/src/lib/crypt_ops/crypto_s2k.c index 5cf98e3e64..361db18927 100644 --- a/src/lib/crypt_ops/crypto_s2k.c +++ b/src/lib/crypt_ops/crypto_s2k.c @@ -285,7 +285,7 @@ secret_to_key_compute_key(uint8_t *key_out, size_t key_out_len, if (rv < 0) return S2K_FAILED; return (int)key_out_len; -#else /* !(defined(ENABLE_OPENSSL)) */ +#else /* !defined(ENABLE_OPENSSL) */ SECItem passItem = { .type = siBuffer, .data = (unsigned char *) secret, .len = (int)secret_len }; @@ -348,7 +348,7 @@ secret_to_key_compute_key(uint8_t *key_out, size_t key_out_len, if (rv != 0) return S2K_FAILED; return (int)key_out_len; -#else /* !(defined(HAVE_SCRYPT)) */ +#else /* !defined(HAVE_SCRYPT) */ return S2K_NO_SCRYPT_SUPPORT; #endif /* defined(HAVE_SCRYPT) */ } diff --git a/src/lib/err/backtrace.c b/src/lib/err/backtrace.c index 8bc7e6965c..62eb9194ee 100644 --- a/src/lib/err/backtrace.c +++ b/src/lib/err/backtrace.c @@ -105,7 +105,7 @@ clean_backtrace(void **stack, size_t depth, const ucontext_t *ctx) return; stack[n] = (void*) ctx->PC_FROM_UCONTEXT; -#else /* !(defined(PC_FROM_UCONTEXT)) */ +#else /* !defined(PC_FROM_UCONTEXT) */ (void) depth; (void) ctx; (void) stack; diff --git a/src/lib/evloop/procmon.c b/src/lib/evloop/procmon.c index 52469fa5fc..b2d81fc14b 100644 --- a/src/lib/evloop/procmon.c +++ b/src/lib/evloop/procmon.c @@ -303,7 +303,7 @@ tor_process_monitor_poll_cb(periodic_timer_t *event, void *procmon_) tor_free(errmsg); } } -#else /* !(defined(_WIN32)) */ +#else /* !defined(_WIN32) */ /* Unix makes this part easy, if a bit racy. */ its_dead_jim = kill(procmon->pid, 0); its_dead_jim = its_dead_jim && (errno == ESRCH); diff --git a/src/lib/fs/dir.c b/src/lib/fs/dir.c index 291f1bbf04..390836b048 100644 --- a/src/lib/fs/dir.c +++ b/src/lib/fs/dir.c @@ -347,7 +347,7 @@ tor_listdir, (const char *dirname)) } FindClose(handle); tor_free(pattern); -#else /* !(defined(_WIN32)) */ +#else /* !defined(_WIN32) */ const char *prot_dname = sandbox_intern_string(dirname); DIR *d; struct dirent *de; diff --git a/src/lib/fs/files.h b/src/lib/fs/files.h index 81dba8c140..ed983f3b3c 100644 --- a/src/lib/fs/files.h +++ b/src/lib/fs/files.h @@ -123,7 +123,7 @@ ssize_t compat_getdelim_(char **lineptr, size_t *n, int delim, FILE *stream); */ #define tor_getdelim(lineptr, n, delim, stream) \ getdelim((lineptr), (n), (delim), (stream)) -#else /* !(defined(HAVE_GETDELIM)) */ +#else /* !defined(HAVE_GETDELIM) */ #define tor_getdelim(lineptr, n, delim, stream) \ compat_getdelim_((lineptr), (n), (delim), (stream)) #endif /* defined(HAVE_GETDELIM) */ @@ -137,7 +137,7 @@ ssize_t compat_getdelim_(char **lineptr, size_t *n, int delim, FILE *stream); */ #define tor_getline(lineptr, n, stream) \ getline((lineptr), (n), (stream)) -#else /* !(defined(HAVE_GETLINE)) */ +#else /* !defined(HAVE_GETLINE) */ #define tor_getline(lineptr, n, stream) \ tor_getdelim((lineptr), (n), '\n', (stream)) #endif /* defined(HAVE_GETLINE) */ diff --git a/src/lib/fs/path.c b/src/lib/fs/path.c index b3ef61979d..28dde62aea 100644 --- a/src/lib/fs/path.c +++ b/src/lib/fs/path.c @@ -72,7 +72,7 @@ expand_filename(const char *filename) * Chapter+3.+Input+Validation/3.7+Validating+Filenames+and+Paths/ */ return tor_strdup(filename); -#else /* !(defined(_WIN32)) */ +#else /* !defined(_WIN32) */ if (*filename == '~') { char *home, *result=NULL; const char *rest; @@ -102,7 +102,7 @@ expand_filename(const char *filename) } tor_free(username); rest = slash ? (slash+1) : ""; -#else /* !(defined(HAVE_PWD_H)) */ +#else /* !defined(HAVE_PWD_H) */ log_warn(LD_CONFIG, "Couldn't expand homedir on system without pwd.h"); return tor_strdup(filename); #endif /* defined(HAVE_PWD_H) */ @@ -153,7 +153,7 @@ clean_fname_for_stat(char *name) return; name[len-1]='\0'; } -#else /* !(defined(_WIN32)) */ +#else /* !defined(_WIN32) */ (void)name; #endif /* defined(_WIN32) */ } @@ -233,7 +233,7 @@ alloc_getcwd(void) raw_free(cwd); // alias for free to avoid tripping check-spaces. } return result; -#else /* !(defined(HAVE_GET_CURRENT_DIR_NAME)) */ +#else /* !defined(HAVE_GET_CURRENT_DIR_NAME) */ size_t size = 1024; char *buf = NULL; char *ptr = NULL; @@ -268,7 +268,7 @@ make_path_absolute(char *fname) if (absfname_malloced) raw_free(absfname_malloced); return absfname; -#else /* !(defined(_WIN32)) */ +#else /* !defined(_WIN32) */ char *absfname = NULL, *path = NULL; tor_assert(fname); diff --git a/src/lib/log/log.c b/src/lib/log/log.c index d2002f6eae..f0f4182b7d 100644 --- a/src/lib/log/log.c +++ b/src/lib/log/log.c @@ -532,7 +532,7 @@ logfile_deliver(logfile_t *lf, const char *buf, size_t msg_len, if (m != msg_after_prefix) { tor_free(m); } -#else /* !(defined(MAXLINE)) */ +#else /* !defined(MAXLINE) */ /* We have syslog but not MAXLINE. That's promising! */ syslog(severity, "%s", msg_after_prefix); #endif /* defined(MAXLINE) */ diff --git a/src/lib/log/log.h b/src/lib/log/log.h index da4bcbe608..2abe752171 100644 --- a/src/lib/log/log.h +++ b/src/lib/log/log.h @@ -26,7 +26,7 @@ #error "Your syslog.h thinks high numbers are more important. " \ "We aren't prepared to deal with that." #endif -#else /* !(defined(HAVE_SYSLOG_H)) */ +#else /* !defined(HAVE_SYSLOG_H) */ /* Note: Syslog's logging code refers to priorities, with 0 being the most * important. Thus, all our comparisons needed to be reversed when we added * syslog support. diff --git a/src/lib/log/util_bug.c b/src/lib/log/util_bug.c index 0e99be35a4..72c614a3b2 100644 --- a/src/lib/log/util_bug.c +++ b/src/lib/log/util_bug.c @@ -64,7 +64,7 @@ tor_set_failed_assertion_callback(void (*fn)(void)) { failed_assertion_cb = fn; } -#else /* !(defined(TOR_UNIT_TESTS)) */ +#else /* !defined(TOR_UNIT_TESTS) */ #define capturing_bugs() (0) #define add_captured_bug(s) do { } while (0) #endif /* defined(TOR_UNIT_TESTS) */ diff --git a/src/lib/log/util_bug.h b/src/lib/log/util_bug.h index d7f01618e8..c3141754de 100644 --- a/src/lib/log/util_bug.h +++ b/src/lib/log/util_bug.h @@ -96,7 +96,7 @@ (void)(a); \ (void)(fmt); \ STMT_END -#else /* !(defined(TOR_UNIT_TESTS) && defined(DISABLE_ASSERTS_IN_UNIT_TES... */ +#else /* !(defined(TOR_UNIT_TESTS) && defined(DISABLE_ASSERTS_IN_UNIT_T...)) */ /** Like assert(3), but send assertion failures to the log as well as to * stderr. */ #define tor_assert(expr) tor_assertf(expr, NULL) @@ -211,7 +211,7 @@ "!("#cond")", 1, NULL); \ } \ bool_result; } )) -#else /* !(defined(__GNUC__)) */ +#else /* !defined(__GNUC__) */ #define IF_BUG_ONCE__(cond,var) \ static int var = 0; \ if ((cond) ? \ diff --git a/src/lib/log/win32err.c b/src/lib/log/win32err.c index dc45cb4c3d..03d5c9fad2 100644 --- a/src/lib/log/win32err.c +++ b/src/lib/log/win32err.c @@ -47,7 +47,7 @@ format_win32_error(DWORD err) result = tor_malloc(len); wcstombs(result,str,len); result[len-1] = '\0'; -#else /* !(defined(UNICODE)) */ +#else /* !defined(UNICODE) */ result = tor_strdup(str); #endif /* defined(UNICODE) */ } else { diff --git a/src/lib/malloc/malloc.h b/src/lib/malloc/malloc.h index 8c81d30dd5..39a45901a1 100644 --- a/src/lib/malloc/malloc.h +++ b/src/lib/malloc/malloc.h @@ -48,7 +48,7 @@ void tor_free_(void *mem); raw_free(*tor_free__tmpvar); \ *tor_free__tmpvar=NULL; \ STMT_END -#else /* !(defined(__GNUC__)) */ +#else /* !defined(__GNUC__) */ #define tor_free(p) STMT_BEGIN \ raw_free(p); \ (p)=NULL; \ diff --git a/src/lib/malloc/map_anon.c b/src/lib/malloc/map_anon.c index 22190070b0..9559cbe2d4 100644 --- a/src/lib/malloc/map_anon.c +++ b/src/lib/malloc/map_anon.c @@ -122,7 +122,7 @@ nodump_mem(void *mem, size_t sz) NULL); return -1; } -#else /* !(defined(MADV_DONTDUMP)) */ +#else /* !defined(MADV_DONTDUMP) */ (void) mem; (void) sz; return 0; diff --git a/src/lib/memarea/memarea.c b/src/lib/memarea/memarea.c index 0a88210906..f3bb79a1e2 100644 --- a/src/lib/memarea/memarea.c +++ b/src/lib/memarea/memarea.c @@ -68,7 +68,7 @@ uint32_t sent_val = get_uint32(&(chunk)->U_MEM[chunk->mem_size]); \ tor_assert(sent_val == SENTINEL_VAL); \ STMT_END -#else /* !(defined(USE_SENTINELS)) */ +#else /* !defined(USE_SENTINELS) */ #define SENTINEL_LEN 0 #define SET_SENTINEL(chunk) STMT_NIL #define CHECK_SENTINEL(chunk) STMT_NIL diff --git a/src/lib/meminfo/meminfo.c b/src/lib/meminfo/meminfo.c index f4fa45167e..bff71c2f05 100644 --- a/src/lib/meminfo/meminfo.c +++ b/src/lib/meminfo/meminfo.c @@ -54,7 +54,7 @@ tor_log_mallinfo(int severity) mi.arena, mi.ordblks, mi.smblks, mi.hblks, mi.hblkhd, mi.usmblks, mi.fsmblks, mi.uordblks, mi.fordblks, mi.keepcost); -#else /* !(defined(HAVE_MALLINFO)) */ +#else /* !defined(HAVE_MALLINFO) */ (void)severity; #endif /* defined(HAVE_MALLINFO) */ } diff --git a/src/lib/net/nettypes.h b/src/lib/net/nettypes.h index 0eb352c657..60039bac09 100644 --- a/src/lib/net/nettypes.h +++ b/src/lib/net/nettypes.h @@ -31,7 +31,7 @@ typedef int socklen_t; #define TOR_SOCKET_T_FORMAT "%"PRIuPTR #define SOCKET_OK(s) ((SOCKET)(s) != INVALID_SOCKET) #define TOR_INVALID_SOCKET INVALID_SOCKET -#else /* !(defined(_WIN32)) */ +#else /* !defined(_WIN32) */ /** Type used for a network socket. */ #define tor_socket_t int #define TOR_SOCKET_T_FORMAT "%d" diff --git a/src/lib/net/resolve.c b/src/lib/net/resolve.c index e8d7d0d94d..78e72fba4f 100644 --- a/src/lib/net/resolve.c +++ b/src/lib/net/resolve.c @@ -118,7 +118,7 @@ tor_addr_lookup_host_getaddrinfo(const char *name, return (err == EAI_AGAIN) ? 1 : -1; } -#else /* !(defined(HAVE_GETADDRINFO)) */ +#else /* !defined(HAVE_GETADDRINFO) */ /* Host lookup helper for tor_addr_lookup(), which calls getaddrinfo(). * Used when gethostbyname() is not available on this system. @@ -506,7 +506,7 @@ tor_make_getaddrinfo_cache_active(void) { sandbox_getaddrinfo_is_active = 1; } -#else /* !(defined(USE_SANDBOX_GETADDRINFO)) */ +#else /* !defined(USE_SANDBOX_GETADDRINFO) */ void sandbox_disable_getaddrinfo_cache(void) { diff --git a/src/lib/net/resolve.h b/src/lib/net/resolve.h index d11c902a91..d7b60be917 100644 --- a/src/lib/net/resolve.h +++ b/src/lib/net/resolve.h @@ -42,7 +42,7 @@ int tor_getaddrinfo(const char *name, const char *servname, struct addrinfo **res); void tor_freeaddrinfo(struct addrinfo *addrinfo); void tor_free_getaddrinfo_cache(void); -#else /* !(defined(USE_SANDBOX_GETADDRINFO)) */ +#else /* !defined(USE_SANDBOX_GETADDRINFO) */ #define tor_getaddrinfo(name, servname, hints, res) \ getaddrinfo((name),(servname), (hints),(res)) #define tor_add_addrinfo(name) \ diff --git a/src/lib/net/socket.c b/src/lib/net/socket.c index e824a05045..e1b82251ed 100644 --- a/src/lib/net/socket.c +++ b/src/lib/net/socket.c @@ -84,7 +84,7 @@ check_network_configuration(bool server_mode) "so your relay makes it harder to figure out how busy it is."); } } -#else /* !(defined(__FreeBSD__)) */ +#else /* !defined(__FreeBSD__) */ (void) server_mode; #endif /* defined(__FreeBSD__) */ } @@ -206,7 +206,7 @@ mark_socket_closed(tor_socket_t s) bitarray_clear(open_sockets, s); } } -#else /* !(defined(DEBUG_SOCKET_COUNTING)) */ +#else /* !defined(DEBUG_SOCKET_COUNTING) */ #define mark_socket_open(s) ((void) (s)) #define mark_socket_closed(s) ((void) (s)) #endif /* defined(DEBUG_SOCKET_COUNTING) */ @@ -308,7 +308,7 @@ tor_open_socket_with_extensions(int domain, int type, int protocol, return TOR_INVALID_SOCKET; } } -#else /* !(defined(FD_CLOEXEC)) */ +#else /* !defined(FD_CLOEXEC) */ (void)cloexec; #endif /* defined(FD_CLOEXEC) */ @@ -418,7 +418,7 @@ tor_accept_socket_with_extensions(tor_socket_t sockfd, struct sockaddr *addr, return TOR_INVALID_SOCKET; } } -#else /* !(defined(FD_CLOEXEC)) */ +#else /* !defined(FD_CLOEXEC) */ (void)cloexec; #endif /* defined(FD_CLOEXEC) */ diff --git a/src/lib/net/socket.h b/src/lib/net/socket.h index 193ad91e4c..53a9f1bb92 100644 --- a/src/lib/net/socket.h +++ b/src/lib/net/socket.h @@ -92,7 +92,7 @@ ssize_t read_all_from_socket(tor_socket_t fd, char *buf, size_t count); #define ERRNO_IS_EINTR(e) ((e) == WSAEINTR || 0) int tor_socket_errno(tor_socket_t sock); const char *tor_socket_strerror(int e); -#else /* !(defined(_WIN32)) */ +#else /* !defined(_WIN32) */ #define SOCK_ERRNO(e) e #if EAGAIN == EWOULDBLOCK /* || 0 is for -Wparentheses-equality (-Wall?) appeasement under clang */ diff --git a/src/lib/net/socketpair.c b/src/lib/net/socketpair.c index 3be7b26f7f..f3a0c3770a 100644 --- a/src/lib/net/socketpair.c +++ b/src/lib/net/socketpair.c @@ -22,7 +22,7 @@ #include #define socket_errno() (WSAGetLastError()) #define SOCKET_EPROTONOSUPPORT WSAEPROTONOSUPPORT -#else /* !(defined(_WIN32)) */ +#else /* !defined(_WIN32) */ #define closesocket(x) close(x) #define socket_errno() (errno) #define SOCKET_EPROTONOSUPPORT EPROTONOSUPPORT diff --git a/src/lib/osinfo/uname.c b/src/lib/osinfo/uname.c index 2b37ff136c..34860c407a 100644 --- a/src/lib/osinfo/uname.c +++ b/src/lib/osinfo/uname.c @@ -137,7 +137,7 @@ get_uname,(void)) if (!is_server && !is_client) { strlcat(uname_result, " [client or server]", sizeof(uname_result)); } -#else /* !(defined(_WIN32)) */ +#else /* !defined(_WIN32) */ /* LCOV_EXCL_START -- can't provoke uname failure */ strlcpy(uname_result, "Unknown platform", sizeof(uname_result)); /* LCOV_EXCL_STOP */ diff --git a/src/lib/process/env.c b/src/lib/process/env.c index 0060200ba1..3912ade197 100644 --- a/src/lib/process/env.c +++ b/src/lib/process/env.c @@ -47,7 +47,7 @@ get_environment(void) * when we do a mostly-static build on OSX 10.7, the resulting binary won't * work on OSX 10.6. */ return *_NSGetEnviron(); -#else /* !(defined(HAVE__NSGETENVIRON)) */ +#else /* !defined(HAVE__NSGETENVIRON) */ return environ; #endif /* defined(HAVE__NSGETENVIRON) */ } diff --git a/src/lib/process/process_unix.c b/src/lib/process/process_unix.c index 17ade87463..332b432c54 100644 --- a/src/lib/process/process_unix.c +++ b/src/lib/process/process_unix.c @@ -199,7 +199,7 @@ process_unix_exec(process_t *process) "Cannot find maximum file descriptor, assuming: %d", max_fd); } } -#else /* !(defined(_SC_OPEN_MAX)) */ +#else /* !defined(_SC_OPEN_MAX) */ max_fd = DEFAULT_MAX_FD; #endif /* defined(_SC_OPEN_MAX) */ diff --git a/src/lib/process/restrict.c b/src/lib/process/restrict.c index 93d06de9a2..fda284f3d9 100644 --- a/src/lib/process/restrict.c +++ b/src/lib/process/restrict.c @@ -152,7 +152,7 @@ tor_mlockall(void) "pages: %s", strerror(errno)); return -1; } -#else /* !(defined(HAVE_UNIX_MLOCKALL)) */ +#else /* !defined(HAVE_UNIX_MLOCKALL) */ log_warn(LD_GENERAL, "Unable to lock memory pages. mlockall() unsupported?"); return -1; #endif /* defined(HAVE_UNIX_MLOCKALL) */ diff --git a/src/lib/process/setuid.c b/src/lib/process/setuid.c index e132787943..3c94ce4bac 100644 --- a/src/lib/process/setuid.c +++ b/src/lib/process/setuid.c @@ -72,7 +72,7 @@ log_credential_status(void) "UID is %u (real), %u (effective), %u (saved)", (unsigned)ruid, (unsigned)euid, (unsigned)suid); } -#else /* !(defined(HAVE_GETRESUID)) */ +#else /* !defined(HAVE_GETRESUID) */ /* getresuid is not present on MacOS X, so we can't get the saved (E)UID */ ruid = getuid(); euid = geteuid(); @@ -93,7 +93,7 @@ log_credential_status(void) "GID is %u (real), %u (effective), %u (saved)", (unsigned)rgid, (unsigned)egid, (unsigned)sgid); } -#else /* !(defined(HAVE_GETRESGID)) */ +#else /* !defined(HAVE_GETRESGID) */ /* getresgid is not present on MacOS X, so we can't get the saved (E)GID */ rgid = getgid(); egid = getegid(); @@ -154,7 +154,7 @@ have_capability_support(void) return 0; cap_free(caps); return 1; -#else /* !(defined(HAVE_LINUX_CAPABILITIES)) */ +#else /* !defined(HAVE_LINUX_CAPABILITIES) */ return 0; #endif /* defined(HAVE_LINUX_CAPABILITIES) */ } @@ -265,7 +265,7 @@ switch_id(const char *user, const unsigned flags) if (drop_capabilities(1)) return -1; } -#else /* !(defined(HAVE_LINUX_CAPABILITIES)) */ +#else /* !defined(HAVE_LINUX_CAPABILITIES) */ (void) keep_bindlow; if (warn_if_no_caps) { log_warn(LD_CONFIG, "KeepBindCapabilities set, but no capability support " diff --git a/src/lib/process/winprocess_sys.c b/src/lib/process/winprocess_sys.c index ff9bc1ba04..ad65886422 100644 --- a/src/lib/process/winprocess_sys.c +++ b/src/lib/process/winprocess_sys.c @@ -51,7 +51,7 @@ subsys_winprocess_initialize(void) return 0; } -#else /* !(defined(_WIN32)) */ +#else /* !defined(_WIN32) */ #define WINPROCESS_SYS_ENABLED false #define subsys_winprocess_initialize NULL #endif /* defined(_WIN32) */ diff --git a/src/lib/sandbox/sandbox.c b/src/lib/sandbox/sandbox.c index faaf463f29..0b316e9c6a 100644 --- a/src/lib/sandbox/sandbox.c +++ b/src/lib/sandbox/sandbox.c @@ -444,7 +444,7 @@ libc_uses_openat_for_everything(void) return 1; else return 0; -#else /* !(defined(CHECK_LIBC_VERSION)) */ +#else /* !defined(CHECK_LIBC_VERSION) */ return 0; #endif /* defined(CHECK_LIBC_VERSION) */ } diff --git a/src/lib/sandbox/sandbox.h b/src/lib/sandbox/sandbox.h index 5bec09a36a..b4ae6e5c07 100644 --- a/src/lib/sandbox/sandbox.h +++ b/src/lib/sandbox/sandbox.h @@ -108,7 +108,7 @@ typedef struct { * it matches the parameter. */ const char* sandbox_intern_string(const char *param); -#else /* !(defined(USE_LIBSECCOMP)) */ +#else /* !defined(USE_LIBSECCOMP) */ #define sandbox_intern_string(s) (s) #endif /* defined(USE_LIBSECCOMP) */ diff --git a/src/lib/smartlist_core/smartlist_core.h b/src/lib/smartlist_core/smartlist_core.h index 795741c447..36f23e2009 100644 --- a/src/lib/smartlist_core/smartlist_core.h +++ b/src/lib/smartlist_core/smartlist_core.h @@ -77,7 +77,7 @@ static inline void smartlist_set(smartlist_t *sl, int idx, void *val) { raw_assert(sl->num_used > idx); sl->list[idx] = val; } -#else /* !(defined(DEBUG_SMARTLIST)) */ +#else /* !defined(DEBUG_SMARTLIST) */ #define smartlist_len(sl) ((sl)->num_used) #define smartlist_get(sl, idx) ((sl)->list[idx]) #define smartlist_set(sl, idx, val) ((sl)->list[idx] = (val)) diff --git a/src/lib/testsupport/testsupport.h b/src/lib/testsupport/testsupport.h index 631ec0228c..90b7c43b19 100644 --- a/src/lib/testsupport/testsupport.h +++ b/src/lib/testsupport/testsupport.h @@ -21,7 +21,7 @@ * tests. */ #define STATIC #define EXTERN(type, name) extern type name; -#else /* !(defined(TOR_UNIT_TESTS)) */ +#else /* !defined(TOR_UNIT_TESTS) */ #define STATIC static #define EXTERN(type, name) #endif /* defined(TOR_UNIT_TESTS) */ @@ -90,7 +90,7 @@ do { \ func = func ##__real; \ } while (0) -#else /* !(defined(TOR_UNIT_TESTS)) */ +#else /* !defined(TOR_UNIT_TESTS) */ #define MOCK_DECL(rv, funcname, arglist) \ rv funcname arglist #define MOCK_DECL_ATTR(rv, funcname, arglist, attr) \ diff --git a/src/lib/thread/threads.h b/src/lib/thread/threads.h index de3da6a585..4b42b9abd9 100644 --- a/src/lib/thread/threads.h +++ b/src/lib/thread/threads.h @@ -107,7 +107,7 @@ typedef struct atomic_counter_t { atomic_size_t val; } atomic_counter_t; #define ATOMIC_LINKAGE static -#else /* !(defined(HAVE_WORKING_STDATOMIC)) */ +#else /* !defined(HAVE_WORKING_STDATOMIC) */ typedef struct atomic_counter_t { tor_mutex_t mutex; size_t val; @@ -172,7 +172,7 @@ atomic_counter_exchange(atomic_counter_t *counter, size_t newval) return atomic_exchange(&counter->val, newval); } -#else /* !(defined(HAVE_WORKING_STDATOMIC)) */ +#else /* !defined(HAVE_WORKING_STDATOMIC) */ #endif /* defined(HAVE_WORKING_STDATOMIC) */ #endif /* !defined(TOR_COMPAT_THREADS_H) */ diff --git a/src/lib/time/compat_time.c b/src/lib/time/compat_time.c index 3f41500f3a..ab45224a7f 100644 --- a/src/lib/time/compat_time.c +++ b/src/lib/time/compat_time.c @@ -833,7 +833,7 @@ monotime_coarse_absolute_msec(void) { return monotime_coarse_absolute_nsec() / ONE_MILLION; } -#else /* !(defined(MONOTIME_COARSE_FN_IS_DIFFERENT)) */ +#else /* !defined(MONOTIME_COARSE_FN_IS_DIFFERENT) */ #define initialized_at_coarse initialized_at #endif /* defined(MONOTIME_COARSE_FN_IS_DIFFERENT) */ @@ -865,7 +865,7 @@ monotime_msec_to_approx_coarse_stamp_units(uint64_t msec) mach_time_info.numer; return abstime_val >> monotime_shift; } -#else /* !(defined(__APPLE__)) */ +#else /* !defined(__APPLE__) */ uint64_t monotime_coarse_stamp_units_to_approx_msec(uint64_t units) { diff --git a/src/lib/time/compat_time.h b/src/lib/time/compat_time.h index 8c7661d7cb..4d16effd29 100644 --- a/src/lib/time/compat_time.h +++ b/src/lib/time/compat_time.h @@ -259,7 +259,7 @@ void monotime_coarse_get(monotime_coarse_t *out); uint64_t monotime_coarse_absolute_nsec(void); uint64_t monotime_coarse_absolute_usec(void); uint64_t monotime_coarse_absolute_msec(void); -#else /* !(defined(MONOTIME_COARSE_FN_IS_DIFFERENT)) */ +#else /* !defined(MONOTIME_COARSE_FN_IS_DIFFERENT) */ #define monotime_coarse_get monotime_get #define monotime_coarse_absolute_nsec monotime_absolute_nsec #define monotime_coarse_absolute_usec monotime_absolute_usec @@ -304,7 +304,7 @@ void monotime_coarse_zero(monotime_coarse_t *out); int monotime_coarse_is_zero(const monotime_coarse_t *val); void monotime_coarse_add_msec(monotime_coarse_t *out, const monotime_coarse_t *val, uint32_t msec); -#else /* !(defined(MONOTIME_COARSE_TYPE_IS_DIFFERENT)) */ +#else /* !defined(MONOTIME_COARSE_TYPE_IS_DIFFERENT) */ #define monotime_coarse_diff_nsec monotime_diff_nsec #define monotime_coarse_diff_usec monotime_diff_usec #define monotime_coarse_diff_msec monotime_diff_msec diff --git a/src/lib/tls/tortls.h b/src/lib/tls/tortls.h index 9e195c6af2..799bd6aaeb 100644 --- a/src/lib/tls/tortls.h +++ b/src/lib/tls/tortls.h @@ -25,7 +25,7 @@ struct ssl_ctx_st; struct ssl_session_st; typedef struct ssl_ctx_st tor_tls_context_impl_t; typedef struct ssl_st tor_tls_impl_t; -#else /* !(defined(ENABLE_OPENSSL)) */ +#else /* !defined(ENABLE_OPENSSL) */ struct PRFileDesc; typedef struct PRFileDesc tor_tls_context_impl_t; typedef struct PRFileDesc tor_tls_impl_t; @@ -144,7 +144,7 @@ void check_no_tls_errors_(const char *fname, int line); void tor_tls_log_one_error(tor_tls_t *tls, unsigned long err, int severity, int domain, const char *doing); -#else /* !(defined(ENABLE_OPENSSL)) */ +#else /* !defined(ENABLE_OPENSSL) */ #define check_no_tls_errors() STMT_NIL #endif /* defined(ENABLE_OPENSSL) */ diff --git a/src/lib/tls/tortls_openssl.c b/src/lib/tls/tortls_openssl.c index 58a7b20dec..5bafcf676d 100644 --- a/src/lib/tls/tortls_openssl.c +++ b/src/lib/tls/tortls_openssl.c @@ -657,7 +657,7 @@ tor_tls_context_new(crypto_pk_t *identity, unsigned int key_lifetime, if (r < 0) goto error; } -#else /* !(defined(SSL_CTX_set1_groups_list) || defined(HAVE_SSL_CTX_SET1... */ +#else /* !(defined(SSL_CTX_set1_groups_list) || defined(HAVE_SSL_CTX_SE...)) */ if (! is_client) { int nid; EC_KEY *ec_key; @@ -673,7 +673,7 @@ tor_tls_context_new(crypto_pk_t *identity, unsigned int key_lifetime, SSL_CTX_set_tmp_ecdh(result->ctx, ec_key); EC_KEY_free(ec_key); } -#endif /* defined(SSL_CTX_set1_groups_list) || defined(HAVE_SSL_CTX_SET1_... */ +#endif /* defined(SSL_CTX_set1_groups_list) || defined(HAVE_SSL_CTX_SET1...) */ SSL_CTX_set_verify(result->ctx, SSL_VERIFY_PEER, always_accept_verify_cb); /* let us realloc bufs that we're writing from */ @@ -764,7 +764,7 @@ find_cipher_by_id(const SSL *ssl, const SSL_METHOD *m, uint16_t cipher) tor_assert((SSL_CIPHER_get_id(c) & 0xffff) == cipher); return c != NULL; } -#else /* !(defined(HAVE_SSL_CIPHER_FIND)) */ +#else /* !defined(HAVE_SSL_CIPHER_FIND) */ # if defined(HAVE_STRUCT_SSL_METHOD_ST_GET_CIPHER_BY_CHAR) if (m && m->get_cipher_by_char) { diff --git a/src/lib/tls/x509_openssl.c b/src/lib/tls/x509_openssl.c index 03f65049cf..7724288279 100644 --- a/src/lib/tls/x509_openssl.c +++ b/src/lib/tls/x509_openssl.c @@ -59,7 +59,7 @@ ENABLE_GCC_WARNING(redundant-decls) #define X509_get_notAfter(cert) \ X509_getm_notAfter(cert) #endif -#else /* !(defined(OPENSSL_1_1_API)) */ +#else /* !defined(OPENSSL_1_1_API) */ #define X509_get_notBefore_const(cert) \ ((const ASN1_TIME*) X509_get_notBefore((X509 *)cert)) #define X509_get_notAfter_const(cert) \ diff --git a/src/lib/trace/events.h b/src/lib/trace/events.h index 0674f7d501..9de86d63f2 100644 --- a/src/lib/trace/events.h +++ b/src/lib/trace/events.h @@ -34,7 +34,7 @@ #include "lib/trace/debug.h" #endif -#else /* !(defined(TOR_EVENT_TRACING_ENABLED)) */ +#else /* !defined(TOR_EVENT_TRACING_ENABLED) */ /* Reaching this point, we NOP every event declaration because event tracing * is not been enabled at compile time. */ diff --git a/src/test/fuzz/fuzzing_common.c b/src/test/fuzz/fuzzing_common.c index e269c36b42..604aba7a7f 100644 --- a/src/test/fuzz/fuzzing_common.c +++ b/src/test/fuzz/fuzzing_common.c @@ -138,7 +138,7 @@ LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) return fuzz_main(Data, Size); } -#else /* !(defined(LLVM_FUZZ)) */ +#else /* !defined(LLVM_FUZZ) */ int main(int argc, char **argv) diff --git a/src/test/test_crypto.c b/src/test/test_crypto.c index 178a9a5097..67940aeac5 100644 --- a/src/test/test_crypto.c +++ b/src/test/test_crypto.c @@ -1923,7 +1923,7 @@ test_crypto_curve25519_impl(void *arg) "e0544770bc7de853b38f9100489e3e79"; const char e1e2k_expected[] = "cd6e8269104eb5aaee886bd2071fba88" "bd13861475516bc2cd2b6e005e805064"; -#else /* !(defined(SLOW_CURVE25519_TEST)) */ +#else /* !defined(SLOW_CURVE25519_TEST) */ const int loop_max=200; const char e1_expected[] = "bc7112cde03f97ef7008cad1bdc56be3" "c6a1037d74cceb3712e9206871dcf654"; diff --git a/src/test/test_options.c b/src/test/test_options.c index b3654ede7d..2d45ecd189 100644 --- a/src/test/test_options.c +++ b/src/test/test_options.c @@ -1172,7 +1172,7 @@ test_options_validate__transproxy(void *ignored) // Assert that a test has run for some TransProxyType tt_assert(tdata); -#else /* !(defined(USE_TRANSPARENT)) */ +#else /* !defined(USE_TRANSPARENT) */ tdata = get_options_test_data("TransPort 127.0.0.1:555\n"); ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg); diff --git a/src/test/test_protover.c b/src/test/test_protover.c index 1759aef97d..0254501165 100644 --- a/src/test/test_protover.c +++ b/src/test/test_protover.c @@ -22,7 +22,7 @@ test_protover_parse(void *arg) tt_skip(); done: ; -#else /* !(defined(HAVE_RUST)) */ +#else /* !defined(HAVE_RUST) */ char *re_encoded = NULL; const char *orig = "Foo=1,3 Bar=3 Baz= Quux=9-12,14,15-16,900"; diff --git a/src/test/test_switch_id.c b/src/test/test_switch_id.c index baddf8d66e..19483713f7 100644 --- a/src/test/test_switch_id.c +++ b/src/test/test_switch_id.c @@ -87,7 +87,7 @@ main(int argc, char **argv) fprintf(stderr, "This test is not supported on your OS.\n"); return 77; -#else /* !(defined(_WIN32)) */ +#else /* !defined(_WIN32) */ const char *username; const char *testname; if (argc != 3) { diff --git a/src/test/test_util.c b/src/test/test_util.c index 84834f4d6c..705ae862a8 100644 --- a/src/test/test_util.c +++ b/src/test/test_util.c @@ -6123,7 +6123,7 @@ test_util_log_mallinfo(void *arg) } else { tt_u64_op(mem1, OP_LT, mem2); } -#else /* !(defined(HAVE_MALLINFO)) */ +#else /* !defined(HAVE_MALLINFO) */ tt_skip(); #endif /* defined(HAVE_MALLINFO) */ done: @@ -6182,7 +6182,7 @@ test_util_map_anon_nofork(void *arg) tt_skip(); done: ; -#else /* !(defined(_WIN32)) */ +#else /* !defined(_WIN32) */ /* We have the right OS support. We're going to try marking the buffer as * either zero-on-fork or as drop-on-fork, whichever is supported. Then we * will fork and send a byte back to the parent process. This will either diff --git a/src/test/test_workqueue.c b/src/test/test_workqueue.c index c58634da5c..ba478a45a4 100644 --- a/src/test/test_workqueue.c +++ b/src/test/test_workqueue.c @@ -63,7 +63,7 @@ mark_handled(int serial) tor_assert(! bitarray_is_set(handled, serial)); bitarray_set(handled, serial); tor_mutex_release(&bitmap_mutex); -#else /* !(defined(TRACK_RESPONSES)) */ +#else /* !defined(TRACK_RESPONSES) */ (void)serial; #endif /* defined(TRACK_RESPONSES) */ } diff --git a/src/test/testing_common.c b/src/test/testing_common.c index 9e7d83dcdc..ff6028ddb4 100644 --- a/src/test/testing_common.c +++ b/src/test/testing_common.c @@ -89,7 +89,7 @@ setup_directory(void) (int)getpid(), rnd32); r = mkdir(temp_dir); } -#else /* !(defined(_WIN32)) */ +#else /* !defined(_WIN32) */ tor_snprintf(temp_dir, sizeof(temp_dir), "/tmp/tor_test_%d_%s", (int) getpid(), rnd32); r = mkdir(temp_dir, 0700); diff --git a/src/test/testing_rsakeys.c b/src/test/testing_rsakeys.c index 8ba6bf9fe4..727449e6eb 100644 --- a/src/test/testing_rsakeys.c +++ b/src/test/testing_rsakeys.c @@ -468,7 +468,7 @@ pk_generate_internal(int bits) *idxp += crypto_rand_int_range(1,3); *idxp %= n_pregen; return crypto_pk_dup_key(pregen_array[*idxp]); -#else /* !(defined(USE_PREGENERATED_RSA_KEYS)) */ +#else /* !defined(USE_PREGENERATED_RSA_KEYS) */ crypto_pk_t *result; int res; result = crypto_pk_new(); From c23986246b970bd01d887fa151a5312a6dc7db04 Mon Sep 17 00:00:00 2001 From: teor Date: Mon, 30 Sep 2019 23:12:54 +1000 Subject: [PATCH 1610/2557] err: Always lock the backtrace buffer before it is used Fixes bug 31734; bugfix on 0.2.5.3-alpha. --- changes/bug31734 | 3 +++ src/lib/err/backtrace.c | 47 ++++++++++++++++++++++++++++++++++++----- 2 files changed, 45 insertions(+), 5 deletions(-) create mode 100644 changes/bug31734 diff --git a/changes/bug31734 b/changes/bug31734 new file mode 100644 index 0000000000..ce989ea5db --- /dev/null +++ b/changes/bug31734 @@ -0,0 +1,3 @@ + o Minor bugfixes (error handling): + - Always lock the backtrace buffer before it is used. + Fixes bug 31734; bugfix on 0.2.5.3-alpha. diff --git a/src/lib/err/backtrace.c b/src/lib/err/backtrace.c index 8bc7e6965c..4e881b979e 100644 --- a/src/lib/err/backtrace.c +++ b/src/lib/err/backtrace.c @@ -52,6 +52,8 @@ #include #endif +#include "lib/cc/ctassert.h" + #define EXPOSE_CLEAN_BACKTRACE #include "lib/err/backtrace.h" #include "lib/err/torerr.h" @@ -73,15 +75,40 @@ static char bt_version[128] = ""; #ifdef USE_BACKTRACE + /** Largest stack depth to try to dump. */ #define MAX_DEPTH 256 -/** Static allocation of stack to dump. This is static so we avoid stack - * pressure. */ -static void *cb_buf[MAX_DEPTH]; +/** The size of the callback buffer, so we can clear it in unlock_cb_buf(). */ +#define SIZEOF_CB_BUF (MAX_DEPTH * sizeof(void *)) /** Protects cb_buf from concurrent access. Pthreads, since this code * is Unix-only, and since this code needs to be lowest-level. */ static pthread_mutex_t cb_buf_mutex = PTHREAD_MUTEX_INITIALIZER; +/** Lock and return a static stack pointer buffer that can hold up to + * MAX_DEPTH function pointers. */ +static void * +lock_cb_buf(void) +{ + /* Lock the mutex first, before even declaring the buffer. */ + pthread_mutex_lock(&cb_buf_mutex); + + /** Static allocation of stack to dump. This is static so we avoid stack + * pressure. */ + static void *cb_buf[MAX_DEPTH]; + CTASSERT(SIZEOF_CB_BUF == sizeof(cb_buf)); + memset(cb_buf, 0, SIZEOF_CB_BUF); + + return cb_buf; +} + +/** Unlock the static stack pointer buffer. */ +static void +unlock_cb_buf(void *cb_buf) +{ + memset(cb_buf, 0, SIZEOF_CB_BUF); + pthread_mutex_unlock(&cb_buf_mutex); +} + /** Change a stacktrace in stack of depth depth so that it will * log the correct function from which a signal was received with context * ctx. (When we get a signal, the current function will not have @@ -123,7 +150,7 @@ log_backtrace_impl(int severity, log_domain_mask_t domain, const char *msg, char **symbols; size_t i; - pthread_mutex_lock(&cb_buf_mutex); + void *cb_buf = lock_cb_buf(); depth = backtrace(cb_buf, MAX_DEPTH); symbols = backtrace_symbols(cb_buf, (int)depth); @@ -141,7 +168,7 @@ log_backtrace_impl(int severity, log_domain_mask_t domain, const char *msg, raw_free(symbols); done: - pthread_mutex_unlock(&cb_buf_mutex); + unlock_cb_buf(cb_buf); } static void crash_handler(int sig, siginfo_t *si, void *ctx_) @@ -157,6 +184,8 @@ crash_handler(int sig, siginfo_t *si, void *ctx_) int n_fds, i; const int *fds = NULL; + void *cb_buf = lock_cb_buf(); + (void) si; depth = backtrace(cb_buf, MAX_DEPTH); @@ -173,6 +202,8 @@ crash_handler(int sig, siginfo_t *si, void *ctx_) for (i=0; i < n_fds; ++i) backtrace_symbols_fd(cb_buf, (int)depth, fds[i]); + unlock_cb_buf(cb_buf); + tor_raw_abort_(); } @@ -184,11 +215,15 @@ dump_stack_symbols_to_error_fds(void) const int *fds = NULL; size_t depth; + void *cb_buf = lock_cb_buf(); + depth = backtrace(cb_buf, MAX_DEPTH); n_fds = tor_log_get_sigsafe_err_fds(&fds); for (i=0; i < n_fds; ++i) backtrace_symbols_fd(cb_buf, (int)depth, fds[i]); + + unlock_cb_buf(cb_buf); } /* The signals that we want our backtrace handler to trap */ @@ -222,10 +257,12 @@ install_bt_handler(void) * libc has pre-loaded the symbols we need to dump things, so that later * reads won't be denied by the sandbox code */ char **symbols; + void *cb_buf = lock_cb_buf(); size_t depth = backtrace(cb_buf, MAX_DEPTH); symbols = backtrace_symbols(cb_buf, (int) depth); if (symbols) raw_free(symbols); + unlock_cb_buf(cb_buf); } return rv; From 56d0655ed85b98592201ddba0f46905b29388a6f Mon Sep 17 00:00:00 2001 From: teor Date: Mon, 30 Sep 2019 23:13:32 +1000 Subject: [PATCH 1611/2557] err: Remove a duplicate header in backtrace.c --- src/lib/err/backtrace.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/lib/err/backtrace.c b/src/lib/err/backtrace.c index 4e881b979e..8f26f8f4ca 100644 --- a/src/lib/err/backtrace.c +++ b/src/lib/err/backtrace.c @@ -56,7 +56,6 @@ #define EXPOSE_CLEAN_BACKTRACE #include "lib/err/backtrace.h" -#include "lib/err/torerr.h" #if defined(HAVE_EXECINFO_H) && defined(HAVE_BACKTRACE) && \ defined(HAVE_BACKTRACE_SYMBOLS_FD) && defined(HAVE_SIGACTION) && \ From 6581f3e2faf412357212b2ee2cec0db5410d3761 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 30 Sep 2019 14:50:57 -0400 Subject: [PATCH 1612/2557] Fix the documentation for GuardLifetime. --- changes/ticket31189 | 3 +++ doc/tor.1.txt | 7 +++---- 2 files changed, 6 insertions(+), 4 deletions(-) create mode 100644 changes/ticket31189 diff --git a/changes/ticket31189 b/changes/ticket31189 new file mode 100644 index 0000000000..318941c794 --- /dev/null +++ b/changes/ticket31189 @@ -0,0 +1,3 @@ + o Documentation: + - Correct the description of "GuardLifetime". Fixes bug 31189; bugfix on + 0.3.0.1-alpha. diff --git a/doc/tor.1.txt b/doc/tor.1.txt index 21b482802e..98ef7c7334 100644 --- a/doc/tor.1.txt +++ b/doc/tor.1.txt @@ -1395,10 +1395,9 @@ The following options are useful only for clients (that is, if default to 3 if the consensus parameter isn't set. (Default: 0) [[GuardLifetime]] **GuardLifetime** __N__ **days**|**weeks**|**months**:: - If nonzero, and UseEntryGuards is set, minimum time to keep a guard before - picking a new one. If zero, we use the GuardLifetime parameter from the - consensus directory. No value here may be less than 1 month or greater - than 5 years; out-of-range values are clamped. (Default: 0) + If UseEntryGuards is set, minimum time to keep a guard on our guard list + before picking a new one. If less than one day, we use defaults from the + consensus directory. (Default: 0) [[SafeSocks]] **SafeSocks** **0**|**1**:: When this option is enabled, Tor will reject application connections that From 723288a32fb5773899790f22ea028db8e3c3d05b Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Tue, 1 Oct 2019 13:39:34 +0300 Subject: [PATCH 1613/2557] Improve v3 client auth documentation in the man page. --- doc/tor.1.txt | 53 +++++++++++++++++++++++++++++++-------------------- 1 file changed, 32 insertions(+), 21 deletions(-) diff --git a/doc/tor.1.txt b/doc/tor.1.txt index 6ba23ac62a..e8d0fd2cbd 100644 --- a/doc/tor.1.txt +++ b/doc/tor.1.txt @@ -1142,7 +1142,7 @@ The following options are useful only for clients (that is, if information) to port 80. [[HidServAuth]] **HidServAuth** __onion-address__ __auth-cookie__ [__service-name__]:: - Client authorization for a hidden service. Valid onion addresses contain 16 + Client authorization for a v2 hidden service. Valid onion addresses contain 16 characters in a-z2-7 plus ".onion", and valid auth cookies contain 22 characters in A-Za-z0-9+/. The service name is only used for internal purposes, e.g., for Tor controllers. This option may be used multiple times @@ -2961,7 +2961,7 @@ The next section describes the per service options that can only be set service. Currently, versions 2 and 3 are supported. (Default: 3) [[HiddenServiceAuthorizeClient]] **HiddenServiceAuthorizeClient** __auth-type__ __client-name__,__client-name__,__...__:: - If configured, the hidden service is accessible for authorized clients + If configured, the v2 hidden service is accessible for authorized clients only. The auth-type can either be \'basic' for a general-purpose authorization protocol or \'stealth' for a less scalable protocol that also hides service activity from unauthorized clients. Only clients that are @@ -3105,31 +3105,42 @@ Client Authorization (Version 3 only) -To configure client authorization on the service side, the -"/authorized_clients/" directory needs to exist. Each file -in that directory should be suffixed with ".auth" (i.e. "alice.auth"; the -file name is irrelevant) and its content format MUST be: +Service side: - :: + To configure client authorization on the service side, the + "/authorized_clients/" directory needs to exist. Each file + in that directory should be suffixed with ".auth" (i.e. "alice.auth"; the + file name is irrelevant) and its content format MUST be: -The supported are: "descriptor". The supported are: -"x25519". The is the base32 representation of -the raw key bytes only (32 bytes for x25519). + :: -Each file MUST contain one line only. Any malformed file will be -ignored. Client authorization will only be enabled for the service if tor -successfully loads at least one authorization file. + The supported are: "descriptor". The supported are: + "x25519". The is the base32 representation of + the raw key bytes only (32 bytes for x25519). -Note that once you've configured client authorization, anyone else with the -address won't be able to access it from this point on. If no authorization is -configured, the service will be accessible to anyone with the onion address. + Each file MUST contain one line only. Any malformed file will be + ignored. Client authorization will only be enabled for the service if tor + successfully loads at least one authorization file. -Revoking a client can be done by removing their ".auth" file, however the -revocation will be in effect only after the tor process gets restarted even if -a SIGHUP takes place. + Note that once you've configured client authorization, anyone else with the + address won't be able to access it from this point on. If no authorization is + configured, the service will be accessible to anyone with the onion address. -See the Appendix G in the rend-spec-v3.txt file of -https://spec.torproject.org/[torspec] for more information. + Revoking a client can be done by removing their ".auth" file, however the + revocation will be in effect only after the tor process gets restarted even if + a SIGHUP takes place. + +Client side: + + To access a v3 onion service with client authorization as a client, make sure + you have ClientOnionAuthDir set in your torrc. Then, in the + directory, create an .auth_private file for the onion + service corresponding to this key (i.e. 'bob_onion.auth_private'). The + contents of the /.auth_private file should look like: + + <56-char-onion-addr-without-.onion-part>:descriptor:x25519: + +For more information, please see https://2019.www.torproject.org/docs/tor-onion-service.html.en#ClientAuthorization . TESTING NETWORK OPTIONS ----------------------- From e8e42f4af97190d5df0b5fa1895f74194650df0b Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Tue, 1 Oct 2019 13:40:04 +0300 Subject: [PATCH 1614/2557] Help users who try to use v2 client auth in v3 onions. --- changes/bug28966 | 4 ++++ src/feature/hs/hs_config.c | 10 ++++++++++ 2 files changed, 14 insertions(+) create mode 100644 changes/bug28966 diff --git a/changes/bug28966 b/changes/bug28966 new file mode 100644 index 0000000000..61123a21eb --- /dev/null +++ b/changes/bug28966 @@ -0,0 +1,4 @@ + o Minor features (onion services v3): + - Assist users who try to setup v2 client authorization in v3 onion + services by pointing them to the right documentation. Closes ticket + 28966. diff --git a/src/feature/hs/hs_config.c b/src/feature/hs/hs_config.c index 7424d7d3ce..3b6caaec6a 100644 --- a/src/feature/hs/hs_config.c +++ b/src/feature/hs/hs_config.c @@ -253,6 +253,16 @@ config_has_invalid_options(const config_line_t *line_, "version %" PRIu32 " of service in %s", opt, service->config.version, service->config.directory_path); + + if (!strcasecmp(line->key, "HiddenServiceAuthorizeClient")) { + /* Special case this v2 option so that we can offer alternatives. + * If more such special cases appear, it would be good to + * generalize the exception mechanism here. */ + log_warn(LD_CONFIG, "For v3 onion service client authorization, " + "please read the 'CLIENT AUTHORIZATION' section in the " + "manual."); + } + ret = 1; /* Continue the loop so we can find all possible options. */ continue; From 39640728c332980daf7ca639827735a1c359669a Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 1 Oct 2019 09:42:10 -0400 Subject: [PATCH 1615/2557] Add comments to try to prevent recurrence of #31495. There is a bad design choice in two of our configuration types, where the empty string encodes a value that is not the same as the default value. This design choice, plus an implementation mistake, meant that config_dup() did not preserve the value of routerset_t, and thereby caused bug #31495. This comment-only patch documents the two types with the problem, and suggests that implementors try to avoid it in the future. Closes ticket 31907. --- src/feature/nodelist/routerset.c | 7 +++++++ src/lib/confmgt/type_defs.c | 4 ++++ src/lib/confmgt/var_type_def_st.h | 6 ++++++ 3 files changed, 17 insertions(+) diff --git a/src/feature/nodelist/routerset.c b/src/feature/nodelist/routerset.c index 73c2b1b1de..9a205d39b7 100644 --- a/src/feature/nodelist/routerset.c +++ b/src/feature/nodelist/routerset.c @@ -473,6 +473,13 @@ routerset_free_(routerset_t *routerset) * routerset_t** passed as target. On success return 0; on failure * return -1 and store an error message into *errmsg. **/ +/* + * Warning: For this type, the default value (NULL) and "" are sometimes + * considered different values. That is generally risky, and best avoided for + * other types in the future. For cases where we want the default to be "all + * routers" (like EntryNodes) we should add a new routerset value indicating + * "all routers" (see #31908) + */ static int routerset_kv_parse(void *target, const config_line_t *line, char **errmsg, const void *params) diff --git a/src/lib/confmgt/type_defs.c b/src/lib/confmgt/type_defs.c index 62c12fcddd..ed930fb02a 100644 --- a/src/lib/confmgt/type_defs.c +++ b/src/lib/confmgt/type_defs.c @@ -44,6 +44,10 @@ // CONFIG_TYPE_FILENAME // // These two types are the same for now, but they have different names. +// +// Warning: For this type, the default value (NULL) and "" are considered +// different values. That is generally risky, and best avoided for other +// types in the future. ////// static int diff --git a/src/lib/confmgt/var_type_def_st.h b/src/lib/confmgt/var_type_def_st.h index f1131ff116..2bf3d37cae 100644 --- a/src/lib/confmgt/var_type_def_st.h +++ b/src/lib/confmgt/var_type_def_st.h @@ -39,6 +39,12 @@ struct config_line_t; * All functions here take a params argument, whose value * is determined by the type definition. Two types may have the * same functions, but differ only in parameters. + * + * Implementation considerations: If "" encodes a valid value for a type, try + * to make sure that it encodes the same thing as the default value for the + * type (that is, the value that is set by config_clear() or memset(0)). If + * this is not the case, you need to make extra certain that your parse/encode + * implementations preserve the NULL/"" distinction. **/ struct var_type_fns_t { /** From f17591b8e504af9e088bd9df900639c83f5300fe Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 1 Oct 2019 12:42:26 -0400 Subject: [PATCH 1616/2557] Rename max_in_sl to max_in_u16_sl, and expose it as STATIC. Since we want to make this function slightly more visible for testing purposes, it needs a better name. --- src/lib/dispatch/.may_include | 1 + src/lib/dispatch/dispatch_cfg.h | 6 ++++++ src/lib/dispatch/dispatch_new.c | 10 ++++++---- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/src/lib/dispatch/.may_include b/src/lib/dispatch/.may_include index 7f2df5859f..884f4c0dbc 100644 --- a/src/lib/dispatch/.may_include +++ b/src/lib/dispatch/.may_include @@ -8,3 +8,4 @@ lib/dispatch/*.h lib/intmath/*.h lib/log/*.h lib/malloc/*.h +lib/testsupport/*.h \ No newline at end of file diff --git a/src/lib/dispatch/dispatch_cfg.h b/src/lib/dispatch/dispatch_cfg.h index 61fade7240..348dce8d40 100644 --- a/src/lib/dispatch/dispatch_cfg.h +++ b/src/lib/dispatch/dispatch_cfg.h @@ -8,6 +8,7 @@ #define TOR_DISPATCH_CFG_H #include "lib/dispatch/msgtypes.h" +#include "lib/testsupport/testsupport.h" /** * A "dispatch_cfg" is the configuration used to set up a dispatcher. @@ -36,4 +37,9 @@ int dcfg_add_recv(dispatch_cfg_t *cfg, message_id_t msg, void dcfg_free_(dispatch_cfg_t *cfg); +#ifdef DISPATCH_NEW_PRIVATE +struct smartlist_t; +STATIC int max_in_u16_sl(const struct smartlist_t *sl, int dflt); +#endif + #endif /* !defined(TOR_DISPATCH_CFG_H) */ diff --git a/src/lib/dispatch/dispatch_new.c b/src/lib/dispatch/dispatch_new.c index b89ef43ea7..4467813064 100644 --- a/src/lib/dispatch/dispatch_new.c +++ b/src/lib/dispatch/dispatch_new.c @@ -9,6 +9,7 @@ * \brief Code to construct a dispatch_t from a dispatch_cfg_t. **/ +#define DISPATCH_NEW_PRIVATE #define DISPATCH_PRIVATE #include "orconfig.h" @@ -26,8 +27,8 @@ /** Given a smartlist full of (possibly NULL) pointers to uint16_t values, * return the largest value, or dflt if the list is empty. */ -static int -max_in_sl(const smartlist_t *sl, int dflt) +STATIC int +max_in_u16_sl(const smartlist_t *sl, int dflt) { uint16_t *maxptr = NULL; SMARTLIST_FOREACH_BEGIN(sl, uint16_t *, u) { @@ -118,11 +119,12 @@ dispatch_new(const dispatch_cfg_t *cfg) smartlist_len(cfg->recv_by_msg)) + 1; /* Any channel that any message has counts towards the number of channels. */ - const size_t n_chans = (size_t) MAX(1, max_in_sl(cfg->chan_by_msg,0)) + 1; + const size_t n_chans = (size_t) + MAX(1, max_in_u16_sl(cfg->chan_by_msg,0)) + 1; /* Any type that a message has, or that has functions, counts towards * the number of types. */ - const size_t n_types = (size_t) MAX(max_in_sl(cfg->type_by_msg,0), + const size_t n_types = (size_t) MAX(max_in_u16_sl(cfg->type_by_msg,0), smartlist_len(cfg->fns_by_type)) + 1; d->n_msgs = n_msgs; From 34bbdaf5d4673491216b64f5ab983518b3f8c7d1 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 1 Oct 2019 12:52:13 -0400 Subject: [PATCH 1617/2557] Add a test for max_u16_in_sl(). This test does not currently pass, because of bug 31898. --- src/test/test_dispatch.c | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/src/test/test_dispatch.c b/src/test/test_dispatch.c index d6fe7e781a..a62c18e0c9 100644 --- a/src/test/test_dispatch.c +++ b/src/test/test_dispatch.c @@ -1,6 +1,7 @@ /* Copyright (c) 2018, The Tor Project, Inc. */ /* See LICENSE for licensing information */ +#define DISPATCH_NEW_PRIVATE #define DISPATCH_PRIVATE #include "test/test.h" @@ -19,6 +20,33 @@ static dispatch_t *dispatcher_in_use=NULL; +static void +test_dispatch_max_in_u16_sl(void *arg) +{ + (void)arg; + smartlist_t *sl = smartlist_new(); + uint16_t nums[] = { 10, 20, 30 }; + tt_int_op(-1, OP_EQ, max_in_u16_sl(sl, -1)); + + smartlist_add(sl, NULL); + tt_int_op(-1, OP_EQ, max_in_u16_sl(sl, -1)); + + smartlist_add(sl, &nums[1]); + tt_int_op(20, OP_EQ, max_in_u16_sl(sl, -1)); + + smartlist_add(sl, &nums[0]); + tt_int_op(20, OP_EQ, max_in_u16_sl(sl, -1)); + + smartlist_add(sl, NULL); + tt_int_op(20, OP_EQ, max_in_u16_sl(sl, -1)); + + smartlist_add(sl, &nums[2]); + tt_int_op(30, OP_EQ, max_in_u16_sl(sl, -1)); + + done: + smartlist_free(sl); +} + /* Construct an empty dispatch_t. */ static void test_dispatch_empty(void *arg) @@ -240,6 +268,7 @@ test_dispatch_bad_type_setup(void *arg) { #name, test_dispatch_ ## name, TT_FORK, NULL, NULL } struct testcase_t dispatch_tests[] = { + T(max_in_u16_sl), T(empty), T(simple), T(no_recipient), From 2b825a1a2e6e79fa71b0e038241d2107aaf30d4b Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 1 Oct 2019 12:55:42 -0400 Subject: [PATCH 1618/2557] Fix a crash bug in max_u16_in_sl() The documentation for this function says that the smartlist can contain NULLs, but the code only handled NULLs if they were at the start of the list. We didn't notice this for a long time, because when Tor is run normally, the sequence of msg_id_t is densely packed, and so this list (mapping msg_id_t to channel_id_t) contains no NULL elements. We could only run into this bug: * when Tor was running in embedded mode, and starting more than once. * when Tor ran first with more pubsub messages enabled, and then later with fewer. * When the second run (the one with fewer enabled pubsub messages) had at least some messages enabled, and those messages were not the ones with numerically highest msg_id_t values. Fixes bug 31898; bugfix on 47de9c7b0a828de7fb8129413db70bc4e4ecac6d in 0.4.1.1-alpha. --- changes/bug31898 | 4 ++++ src/lib/dispatch/dispatch_new.c | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 changes/bug31898 diff --git a/changes/bug31898 b/changes/bug31898 new file mode 100644 index 0000000000..6f3e0a5465 --- /dev/null +++ b/changes/bug31898 @@ -0,0 +1,4 @@ + o Major bugfixes (embedded Tor): + - Avoid a possible crash when restarting Tor in embedded mode and + enabling a different set of publish/subscribe messages. Fixes bug + 31898; bugfix on 0.4.1.1-alpha. diff --git a/src/lib/dispatch/dispatch_new.c b/src/lib/dispatch/dispatch_new.c index 4467813064..d8e59d610a 100644 --- a/src/lib/dispatch/dispatch_new.c +++ b/src/lib/dispatch/dispatch_new.c @@ -34,7 +34,7 @@ max_in_u16_sl(const smartlist_t *sl, int dflt) SMARTLIST_FOREACH_BEGIN(sl, uint16_t *, u) { if (!maxptr) maxptr = u; - else if (*u > *maxptr) + else if (u && *u > *maxptr) maxptr = u; } SMARTLIST_FOREACH_END(u); From b0bf7e7b606e45cff5988106febe157009c2f561 Mon Sep 17 00:00:00 2001 From: teor Date: Wed, 2 Oct 2019 10:09:02 +1000 Subject: [PATCH 1619/2557] Travis: Allow the build to finish before the macOS Rust job When we merged TOR_RUST_VERSION from master, the allow_failures rule did not match any more. Update it to make it match. Closes 31859 for master. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index ec3827d316..00c967e341 100644 --- a/.travis.yml +++ b/.travis.yml @@ -74,7 +74,7 @@ matrix: ## macOS rust and chutney are very slow, so we let the build finish before ## they are done. We'd like to fast finish, but still eventually show ## any failures in the build status. But Travis doesn't have that ability. - - env: RUST_OPTIONS="--enable-rust --enable-cargo-online-mode" + - env: RUST_VERSION="nightly" RUST_OPTIONS="--enable-rust --enable-cargo-online-mode" compiler: clang os: osx - env: CHUTNEY="yes" CHUTNEY_ALLOW_FAILURES="2" SKIP_MAKE_CHECK="yes" From 3f94441bfeaba23854869a7ae8ce3d5ff26a80d3 Mon Sep 17 00:00:00 2001 From: teor Date: Wed, 2 Oct 2019 12:12:26 +1000 Subject: [PATCH 1620/2557] log: fix a typo in the function comment for log_fn_() Closes 31923. --- src/lib/log/log.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/log/log.c b/src/lib/log/log.c index f0f4182b7d..5f8de619a4 100644 --- a/src/lib/log/log.c +++ b/src/lib/log/log.c @@ -755,7 +755,7 @@ tor_log_get_logfile_names(smartlist_t *out) /** Implementation of the log_fn backend, used when we have * variadic macros. All arguments are as for log_fn, except for - * fn, which is the name of the calling functions. */ + * fn, which is the name of the calling function. */ void log_fn_(int severity, log_domain_mask_t domain, const char *fn, const char *format, ...) From 3d17fafa04baf23d124edf56560abf3b19ad4425 Mon Sep 17 00:00:00 2001 From: teor Date: Tue, 6 Aug 2019 00:18:41 +1000 Subject: [PATCH 1621/2557] control/control: Refactor some error handling code Split some protocol error handling out of connection_control_process_inbuf(). This refactor reduces the size of a practracker exception. Closes 31840. --- changes/ticket31840 | 3 + scripts/maint/practracker/exceptions.txt | 2 +- src/feature/control/control.c | 79 +++++++++++++++++------- 3 files changed, 59 insertions(+), 25 deletions(-) create mode 100644 changes/ticket31840 diff --git a/changes/ticket31840 b/changes/ticket31840 new file mode 100644 index 0000000000..c75c5629f9 --- /dev/null +++ b/changes/ticket31840 @@ -0,0 +1,3 @@ + o Code simplification and refactoring: + - Refactor connection_control_process_inbuf() to reduce the size of a + practracker exception. Closes ticket 31840. diff --git a/scripts/maint/practracker/exceptions.txt b/scripts/maint/practracker/exceptions.txt index 2e676d9045..88b34b21f6 100644 --- a/scripts/maint/practracker/exceptions.txt +++ b/scripts/maint/practracker/exceptions.txt @@ -186,7 +186,7 @@ problem file-size /src/feature/client/entrynodes.h 639 problem function-size /src/feature/client/transports.c:handle_proxy_line() 108 problem function-size /src/feature/client/transports.c:parse_method_line_helper() 110 problem function-size /src/feature/client/transports.c:create_managed_proxy_environment() 109 -problem function-size /src/feature/control/control.c:connection_control_process_inbuf() 136 +problem function-size /src/feature/control/control.c:connection_control_process_inbuf() 113 problem function-size /src/feature/control/control_auth.c:handle_control_authenticate() 186 problem function-size /src/feature/control/control_cmd.c:handle_control_extendcircuit() 150 problem function-size /src/feature/control/control_cmd.c:handle_control_add_onion() 256 diff --git a/src/feature/control/control.c b/src/feature/control/control.c index 436bf423cf..d6581808c0 100644 --- a/src/feature/control/control.c +++ b/src/feature/control/control.c @@ -339,6 +339,60 @@ static const char CONTROLPORT_IS_NOT_AN_HTTP_PROXY_MSG[] = "\n" "\n"; +/** Return an error on a control connection that tried to use the v0 protocol. + */ +static void +control_send_v0_reject(control_connection_t *conn) +{ + size_t body_len; + char buf[128]; + set_uint16(buf+2, htons(0x0000)); /* type == error */ + set_uint16(buf+4, htons(0x0001)); /* code == internal error */ + strlcpy(buf+6, "The v0 control protocol is not supported by Tor 0.1.2.17 " + "and later; upgrade your controller.", + sizeof(buf)-6); + body_len = 2+strlen(buf+6)+2; /* code, msg, nul. */ + set_uint16(buf+0, htons(body_len)); + connection_buf_add(buf, 4+body_len, TO_CONN(conn)); + + connection_mark_and_flush(TO_CONN(conn)); +} + +/** Return an error on a control connection that tried to use HTTP. + */ +static void +control_send_http_reject(control_connection_t *conn) +{ + connection_write_str_to_buf(CONTROLPORT_IS_NOT_AN_HTTP_PROXY_MSG, conn); + log_notice(LD_CONTROL, "Received HTTP request on ControlPort"); + connection_mark_and_flush(TO_CONN(conn)); +} + +/** Check if a control connection has tried to use a known invalid protocol. + * If it has, then: + * - send a reject response, + * - log a notice-level message, and + * - return false. */ +static bool +control_protocol_is_valid(control_connection_t *conn) +{ + /* Detect v0 commands and send a "no more v0" message. */ + if (conn->base_.state == CONTROL_CONN_STATE_NEEDAUTH && + peek_connection_has_control0_command(TO_CONN(conn))) { + control_send_v0_reject(conn); + return 0; + } + + /* If the user has the HTTP proxy port and the control port confused. */ + if (conn->base_.state == CONTROL_CONN_STATE_NEEDAUTH && + peek_connection_has_http_command(TO_CONN(conn))) { + control_send_http_reject(conn); + return 0; + } + + return 1; +} + /** Called when data has arrived on a v1 control connection: Try to fetch * commands from conn->inbuf, and execute them. */ @@ -359,30 +413,7 @@ connection_control_process_inbuf(control_connection_t *conn) conn->incoming_cmd_cur_len = 0; } - if (conn->base_.state == CONTROL_CONN_STATE_NEEDAUTH && - peek_connection_has_control0_command(TO_CONN(conn))) { - /* Detect v0 commands and send a "no more v0" message. */ - size_t body_len; - char buf[128]; - set_uint16(buf+2, htons(0x0000)); /* type == error */ - set_uint16(buf+4, htons(0x0001)); /* code == internal error */ - strlcpy(buf+6, "The v0 control protocol is not supported by Tor 0.1.2.17 " - "and later; upgrade your controller.", - sizeof(buf)-6); - body_len = 2+strlen(buf+6)+2; /* code, msg, nul. */ - set_uint16(buf+0, htons(body_len)); - connection_buf_add(buf, 4+body_len, TO_CONN(conn)); - - connection_mark_and_flush(TO_CONN(conn)); - return 0; - } - - /* If the user has the HTTP proxy port and the control port confused. */ - if (conn->base_.state == CONTROL_CONN_STATE_NEEDAUTH && - peek_connection_has_http_command(TO_CONN(conn))) { - connection_write_str_to_buf(CONTROLPORT_IS_NOT_AN_HTTP_PROXY_MSG, conn); - log_notice(LD_CONTROL, "Received HTTP request on ControlPort"); - connection_mark_and_flush(TO_CONN(conn)); + if (!control_protocol_is_valid(conn)) { return 0; } From f0993d3831b6132e342fb192820fc04902c98b38 Mon Sep 17 00:00:00 2001 From: teor Date: Thu, 3 Oct 2019 16:31:20 +1000 Subject: [PATCH 1622/2557] configure: Give a more useful message when pkg-config fails When pkg-config is not installed, or a library that depends on pkg-config is not found, tell the user what to do to fix the problem. Fixes bug 31922; bugfix on 0.3.1.1-alpha. --- changes/bug31922 | 4 ++++ configure.ac | 19 +++++++++++++------ 2 files changed, 17 insertions(+), 6 deletions(-) create mode 100644 changes/bug31922 diff --git a/changes/bug31922 b/changes/bug31922 new file mode 100644 index 0000000000..e6f31ce66a --- /dev/null +++ b/changes/bug31922 @@ -0,0 +1,4 @@ + o Minor bugfixes (configuration): + - When pkg-config is not installed, or a library that depends on + pkg-config is not found, tell the user what to do to fix the + problem. Fixes bug 31922; bugfix on 0.3.1.1-alpha. diff --git a/configure.ac b/configure.ac index bd12e61671..dbcb51d47f 100644 --- a/configure.ac +++ b/configure.ac @@ -29,6 +29,13 @@ AC_USE_SYSTEM_EXTENSIONS AC_CANONICAL_HOST PKG_PROG_PKG_CONFIG +if test "x$PKG_CONFIG" = "x" ; then + pkg_config_user_action="install pkg-config, and check the PKG_CONFIG_PATH environment variable." + AC_MSG_NOTICE([Some libraries need pkg-config, including systemd, nss, lzma, zstd, and custom mallocs.]) + AC_MSG_NOTICE([To use those libraries, $pkg_config_user_action]) +else + pkg_config_user_action="check the PKG_CONFIG_PATH environment variable." +fi AC_ARG_ENABLE(openbsd-malloc, AS_HELP_STRING(--enable-openbsd-malloc, [use malloc code from OpenBSD. Linux only. Deprecated: see --with-malloc])) @@ -166,7 +173,7 @@ AC_SUBST(TOR_SYSTEMD_CFLAGS) AC_SUBST(TOR_SYSTEMD_LIBS) if test "x$enable_systemd" = "xyes" -a "x$have_systemd" != "xyes" ; then - AC_MSG_ERROR([Explicitly requested systemd support, but systemd not found]) + AC_MSG_ERROR([Explicitly requested systemd support, but systemd not found, $pkg_config_user_action]) fi case "$host" in @@ -870,7 +877,7 @@ if test "x$enable_nss" = "xyes"; then PKG_CHECK_MODULES(NSS, [nss], [have_nss=yes], - [have_nss=no; AC_MSG_ERROR([You asked for NSS but I can't find it.])]) + [have_nss=no; AC_MSG_ERROR([You asked for NSS but I can't find it, $pkg_config_user_action])]) AC_SUBST(NSS_CFLAGS) AC_SUBST(NSS_LIBS) fi @@ -1076,7 +1083,7 @@ else have_lzma=no) if test "x$have_lzma" = "xno" ; then - AC_MSG_WARN([Unable to find liblzma.]) + AC_MSG_WARN([Unable to find liblzma, $pkg_config_user_action]) fi fi @@ -1108,7 +1115,7 @@ else have_zstd=no) if test "x$have_zstd" = "xno" ; then - AC_MSG_WARN([Unable to find libzstd.]) + AC_MSG_WARN([Unable to find libzstd, $pkg_config_user_action]) fi fi @@ -1915,7 +1922,7 @@ AS_CASE([$malloc], have_tcmalloc=no) if test "x$have_tcmalloc" = "xno" ; then - AC_MSG_ERROR([Unable to find tcmalloc requested by --with-malloc.]) + AC_MSG_ERROR([Unable to find tcmalloc requested by --with-malloc, $pkg_config_user_action]) fi CFLAGS="$CFLAGS $TCMALLOC_CFLAGS" @@ -1929,7 +1936,7 @@ AS_CASE([$malloc], have_jemalloc=no) if test "x$have_tcmalloc" = "xno" ; then - AC_MSG_ERROR([Unable to find jemalloc requested by --with-malloc.]) + AC_MSG_ERROR([Unable to find jemalloc requested by --with-malloc, $pkg_config_user_action]) fi CFLAGS="$CFLAGS $JEMALLOC_CFLAGS" From 80bcd66213f59420e98a34232179814febe54e91 Mon Sep 17 00:00:00 2001 From: teor Date: Thu, 3 Oct 2019 16:32:30 +1000 Subject: [PATCH 1623/2557] configure: Fix a typo in the systemd version message --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index dbcb51d47f..31a9a7fcc5 100644 --- a/configure.ac +++ b/configure.ac @@ -167,7 +167,7 @@ if test "x$have_systemd" = "xyes"; then TOR_SYSTEMD_CFLAGS="${SYSTEMD_CFLAGS}" TOR_SYSTEMD_LIBS="${SYSTEMD_LIBS}" PKG_CHECK_MODULES(LIBSYSTEMD209, [libsystemd >= 209], - [AC_DEFINE(HAVE_SYSTEMD_209,1,[Have systemd v209 or more])], []) + [AC_DEFINE(HAVE_SYSTEMD_209,1,[Have systemd v209 or greater])], []) fi AC_SUBST(TOR_SYSTEMD_CFLAGS) AC_SUBST(TOR_SYSTEMD_LIBS) From 8c4c58cc8d9b2a1085bd64e725ec4f33c79745a7 Mon Sep 17 00:00:00 2001 From: teor Date: Thu, 3 Oct 2019 16:53:12 +1000 Subject: [PATCH 1624/2557] configure: List the env vars needed if pkg-config doesn't work Part of 31922. --- configure.ac | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/configure.ac b/configure.ac index 31a9a7fcc5..4745df0da5 100644 --- a/configure.ac +++ b/configure.ac @@ -30,11 +30,11 @@ AC_CANONICAL_HOST PKG_PROG_PKG_CONFIG if test "x$PKG_CONFIG" = "x" ; then - pkg_config_user_action="install pkg-config, and check the PKG_CONFIG_PATH environment variable." + pkg_config_user_action="install pkg-config, and check the PKG_CONFIG_PATH environment variable" AC_MSG_NOTICE([Some libraries need pkg-config, including systemd, nss, lzma, zstd, and custom mallocs.]) - AC_MSG_NOTICE([To use those libraries, $pkg_config_user_action]) + AC_MSG_NOTICE([To use those libraries, $pkg_config_user_action.]) else - pkg_config_user_action="check the PKG_CONFIG_PATH environment variable." + pkg_config_user_action="check the PKG_CONFIG_PATH environment variable" fi AC_ARG_ENABLE(openbsd-malloc, @@ -173,7 +173,7 @@ AC_SUBST(TOR_SYSTEMD_CFLAGS) AC_SUBST(TOR_SYSTEMD_LIBS) if test "x$enable_systemd" = "xyes" -a "x$have_systemd" != "xyes" ; then - AC_MSG_ERROR([Explicitly requested systemd support, but systemd not found, $pkg_config_user_action]) + AC_MSG_ERROR([Explicitly requested systemd support, but systemd not found, $pkg_config_user_action, or set SYSTEMD_CFLAGS and SYSTEMD_LIBS.]) fi case "$host" in @@ -877,7 +877,7 @@ if test "x$enable_nss" = "xyes"; then PKG_CHECK_MODULES(NSS, [nss], [have_nss=yes], - [have_nss=no; AC_MSG_ERROR([You asked for NSS but I can't find it, $pkg_config_user_action])]) + [have_nss=no; AC_MSG_ERROR([You asked for NSS but I can't find it, $pkg_config_user_action, or set NSS_CFLAGS and NSS_LIBS.])]) AC_SUBST(NSS_CFLAGS) AC_SUBST(NSS_LIBS) fi @@ -1083,7 +1083,7 @@ else have_lzma=no) if test "x$have_lzma" = "xno" ; then - AC_MSG_WARN([Unable to find liblzma, $pkg_config_user_action]) + AC_MSG_WARN([Unable to find liblzma, $pkg_config_user_action, or set LZMA_CFLAGS and LZMA_LIBS.]) fi fi @@ -1115,7 +1115,7 @@ else have_zstd=no) if test "x$have_zstd" = "xno" ; then - AC_MSG_WARN([Unable to find libzstd, $pkg_config_user_action]) + AC_MSG_WARN([Unable to find libzstd, $pkg_config_user_action, or set ZSTD_CFLAGS and ZSTD_LIBS.]) fi fi @@ -1922,7 +1922,7 @@ AS_CASE([$malloc], have_tcmalloc=no) if test "x$have_tcmalloc" = "xno" ; then - AC_MSG_ERROR([Unable to find tcmalloc requested by --with-malloc, $pkg_config_user_action]) + AC_MSG_ERROR([Unable to find tcmalloc requested by --with-malloc, $pkg_config_user_action, or set TCMALLOC_CFLAGS and TCMALLOC_LIBS.]) fi CFLAGS="$CFLAGS $TCMALLOC_CFLAGS" @@ -1936,7 +1936,7 @@ AS_CASE([$malloc], have_jemalloc=no) if test "x$have_tcmalloc" = "xno" ; then - AC_MSG_ERROR([Unable to find jemalloc requested by --with-malloc, $pkg_config_user_action]) + AC_MSG_ERROR([Unable to find jemalloc requested by --with-malloc, $pkg_config_user_action, or set JEMALLOC_CFLAGS and JEMALLOC_LIBS.]) fi CFLAGS="$CFLAGS $JEMALLOC_CFLAGS" From b2802ae3c31e4b803427bbce4f8de78558660738 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 3 Oct 2019 07:21:25 -0400 Subject: [PATCH 1625/2557] util/map_anon_nofork: Add a cast to avoid passing -48 to memset This fixes coverity CID 1454593, and bug 31948. Bug not in any released version of Tor. --- src/test/test_util.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/test_util.c b/src/test/test_util.c index 5024b23400..aebefe64c5 100644 --- a/src/test/test_util.c +++ b/src/test/test_util.c @@ -6197,7 +6197,7 @@ test_util_map_anon_nofork(void *arg) tor_munmap_anonymous(ptr, sz); ptr = tor_mmap_anonymous(sz, ANONMAP_NOINHERIT, &inherit); tt_ptr_op(ptr, OP_NE, 0); - memset(ptr, TEST_VALUE, sz); + memset(ptr, (uint8_t)TEST_VALUE, sz); tt_int_op(0, OP_EQ, pipe(pipefd)); pid_t child = fork(); From ac8f6d51f4200b27adc0da9fc1eeb44b89c8ebab Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 2 Oct 2019 21:22:53 -0400 Subject: [PATCH 1626/2557] Unify backend implementations for blocking hostname lookup We have a getaddrinfo() implementation that we prefer, and a gethostbyname*() implementation that we fall back on. Give them both the same interface, and let them be called by the same name. This is a preparatory step for making them both mockable. --- src/lib/net/resolve.c | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/src/lib/net/resolve.c b/src/lib/net/resolve.c index 78e72fba4f..c8fe338e52 100644 --- a/src/lib/net/resolve.c +++ b/src/lib/net/resolve.c @@ -71,9 +71,9 @@ tor_lookup_hostname,(const char *name, uint32_t *addr)) * See tor_addr_lookup() for details. */ static int -tor_addr_lookup_host_getaddrinfo(const char *name, - uint16_t family, - tor_addr_t *addr) +tor_addr_lookup_host_impl(const char *name, + uint16_t family, + tor_addr_t *addr) { int err; struct addrinfo *res=NULL, *res_p; @@ -120,15 +120,17 @@ tor_addr_lookup_host_getaddrinfo(const char *name, #else /* !defined(HAVE_GETADDRINFO) */ -/* Host lookup helper for tor_addr_lookup(), which calls getaddrinfo(). - * Used when gethostbyname() is not available on this system. +/* Host lookup helper for tor_addr_lookup(), which calls gethostbyname(). + * Used when getaddrinfo() is not available on this system. * * See tor_addr_lookup() for details. */ static int -tor_addr_lookup_host_gethostbyname(const char *name, - tor_addr_t *addr) +tor_addr_lookup_host_impl(const char *name, + uint16_t family, + tor_addr_t *addr) { + (void) family; struct hostent *ent; int err; #ifdef HAVE_GETHOSTBYNAME_R_6_ARG @@ -215,13 +217,8 @@ tor_addr_lookup,(const char *name, uint16_t family, tor_addr_t *addr)) } else { /* Clear the address after a failed tor_addr_parse(). */ memset(addr, 0, sizeof(tor_addr_t)); -#ifdef HAVE_GETADDRINFO - result = tor_addr_lookup_host_getaddrinfo(name, family, addr); + result = tor_addr_lookup_host_impl(name, family, addr); goto done; -#else /* !(defined(HAVE_GETADDRINFO)) */ - result = tor_addr_lookup_host_gethostbyname(name, addr); - goto done; -#endif /* defined(HAVE_GETADDRINFO) */ } /* If we weren't successful, and haven't already set the result, From 065e467e7cc78f25469f467d741c2d379fbbc246 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 3 Oct 2019 12:07:50 -0400 Subject: [PATCH 1627/2557] bump to 0.4.2.2-alpha --- configure.ac | 4 ++-- contrib/win32build/tor-mingw.nsi.in | 2 +- src/win32/orconfig.h | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/configure.ac b/configure.ac index bd12e61671..e623b33ad5 100644 --- a/configure.ac +++ b/configure.ac @@ -4,7 +4,7 @@ dnl Copyright (c) 2007-2019, The Tor Project, Inc. dnl See LICENSE for licensing information AC_PREREQ([2.63]) -AC_INIT([tor],[0.4.2.1-alpha-dev]) +AC_INIT([tor],[0.4.2.2-alpha]) AC_CONFIG_SRCDIR([src/app/main/tor_main.c]) AC_CONFIG_MACRO_DIR([m4]) @@ -14,7 +14,7 @@ AC_CONFIG_MACRO_DIR([m4]) # version number changes. Tor uses it to make sure that it # only shuts down for missing "required protocols" when those protocols # are listed as required by a consensus after this date. -AC_DEFINE(APPROX_RELEASE_DATE, ["2019-09-17"], # for 0.4.2.1-alpha-dev +AC_DEFINE(APPROX_RELEASE_DATE, ["2019-10-03"], # for 0.4.2.2-alpha [Approximate date when this software was released. (Updated when the version changes.)]) # "foreign" means we don't follow GNU package layout standards diff --git a/contrib/win32build/tor-mingw.nsi.in b/contrib/win32build/tor-mingw.nsi.in index 26387b03f6..bdd112a083 100644 --- a/contrib/win32build/tor-mingw.nsi.in +++ b/contrib/win32build/tor-mingw.nsi.in @@ -8,7 +8,7 @@ !include "LogicLib.nsh" !include "FileFunc.nsh" !insertmacro GetParameters -!define VERSION "0.4.2.1-alpha-dev" +!define VERSION "0.4.2.2-alpha" !define INSTALLER "tor-${VERSION}-win32.exe" !define WEBSITE "https://www.torproject.org/" !define LICENSE "LICENSE" diff --git a/src/win32/orconfig.h b/src/win32/orconfig.h index ac0513bc61..6bcd18ef6f 100644 --- a/src/win32/orconfig.h +++ b/src/win32/orconfig.h @@ -218,7 +218,7 @@ #define USING_TWOS_COMPLEMENT /* Version number of package */ -#define VERSION "0.4.2.1-alpha-dev" +#define VERSION "0.4.2.2-alpha" From 4d4e2abd2f961e735b9b8d93e9e09695515b8ac8 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 29 Aug 2019 11:43:43 -0400 Subject: [PATCH 1628/2557] Authorities reject relays running unsupported Tor release series. Our minimum version is now 0.2.9.5-alpha. Series 0.3.0, 0.3.1, 0.3.2, 0.3.3, and 0.3.4 are now rejected. Also, extract this version-checking code into a new function, so we can test it. Closes ticket 31549. Also reject 0.3.5.0 through 0.3.5.6-rc as unstable. --- changes/ticket31549 | 4 ++ src/feature/dirauth/process_descs.c | 59 +++++++++++++++++++++-------- src/feature/dirauth/process_descs.h | 2 + 3 files changed, 49 insertions(+), 16 deletions(-) create mode 100644 changes/ticket31549 diff --git a/changes/ticket31549 b/changes/ticket31549 new file mode 100644 index 0000000000..2c27aca4fb --- /dev/null +++ b/changes/ticket31549 @@ -0,0 +1,4 @@ + o Minor features (authority): + - Directory authorities now reject relays running all currently + deprecated release series. The currently supported release series + are: 0.2.9, 0.3.5, 0.4.0, 0.4.1, and 0.4.2. Closes ticket 31549. diff --git a/src/feature/dirauth/process_descs.c b/src/feature/dirauth/process_descs.c index e1a02179b0..74a2cde1bd 100644 --- a/src/feature/dirauth/process_descs.c +++ b/src/feature/dirauth/process_descs.c @@ -315,6 +315,47 @@ dirserv_would_reject_router(const routerstatus_t *rs) return (res & FP_REJECT) != 0; } +/** + * Check whether the platform string in platform describes a platform + * that, as a directory authority, we want to reject. If it does, return + * true, and set *msg (if present) to a rejection message. Otherwise + * return false. + */ +STATIC bool +dirserv_rejects_tor_version(const char *platform, + const char **msg) +{ + if (!platform) + return false; + + static const char please_upgrade_string[] = + "Tor version is insecure or unsupported. Please upgrade!"; + + /* Versions before Tor 0.2.9 are unsupported. Versions between 0.2.9.0 and + * 0.2.9.4 suffer from bug #20499, where relays don't keep their consensus + * up to date */ + if (!tor_version_as_new_as(platform,"0.2.9.5-alpha")) { + if (msg) + *msg = please_upgrade_string; + return true; + } + + /* Series between Tor 0.3.0 and 0.3.4 inclusive are unsupported, and some + * have bug #27841, which makes them broken as intro points. Reject them. + * + * Also reject unstable versions of 0.3.5, since (as of this writing) + * they are almost none of the network. */ + if (tor_version_as_new_as(platform,"0.3.0.0-alpha-dev") && + !tor_version_as_new_as(platform,"0.3.5.7")) { + if (msg) { + *msg = please_upgrade_string; + } + return true; + } + + return false; +} + /** Helper: As dirserv_router_get_status, but takes the router fingerprint * (hex, no spaces), nickname, address (used for logging only), IP address, OR * port and platform (logging only) as arguments. @@ -347,22 +388,8 @@ dirserv_get_status_impl(const char *id_digest, const char *nickname, } } - /* Versions before Tor 0.2.4.18-rc are too old to support, and are - * missing some important security fixes too. Disable them. */ - if (platform && !tor_version_as_new_as(platform,"0.2.4.18-rc")) { - if (msg) - *msg = "Tor version is insecure or unsupported. Please upgrade!"; - return FP_REJECT; - } - - /* Tor 0.2.9.x where x<5 suffers from bug #20499, where relays don't - * keep their consensus up to date so they make bad guards. - * The simple fix is to just drop them from the network. */ - if (platform && - tor_version_as_new_as(platform,"0.2.9.0-alpha") && - !tor_version_as_new_as(platform,"0.2.9.5-alpha")) { - if (msg) - *msg = "Tor version contains bug 20499. Please upgrade!"; + /* Check whether the version is obsolete, broken, insecure, etc... */ + if (platform && dirserv_rejects_tor_version(platform, msg)) { return FP_REJECT; } diff --git a/src/feature/dirauth/process_descs.h b/src/feature/dirauth/process_descs.h index 1d4085b091..0203cebfa9 100644 --- a/src/feature/dirauth/process_descs.h +++ b/src/feature/dirauth/process_descs.h @@ -38,6 +38,8 @@ int dirserv_would_reject_router(const routerstatus_t *rs); #ifdef TOR_UNIT_TESTS STATIC int dirserv_router_has_valid_address(routerinfo_t *ri); +STATIC bool dirserv_rejects_tor_version(const char *platform, + const char **msg); #endif /* defined(TOR_UNIT_TESTS) */ #endif /* !defined(TOR_RECV_UPLOADS_H) */ From 49d6990cae0f6e7e636e320d0efcaec31b3c0453 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 29 Aug 2019 11:45:41 -0400 Subject: [PATCH 1629/2557] Unit tests for dirserv_reject_tor_version(). --- src/test/include.am | 1 + src/test/test.c | 1 + src/test/test.h | 1 + src/test/test_process_descs.c | 67 +++++++++++++++++++++++++++++++++++ 4 files changed, 70 insertions(+) create mode 100644 src/test/test_process_descs.c diff --git a/src/test/include.am b/src/test/include.am index 1e20f3f53f..101e915cf4 100644 --- a/src/test/include.am +++ b/src/test/include.am @@ -175,6 +175,7 @@ src_test_test_SOURCES += \ src/test/test_periodic_event.c \ src/test/test_policy.c \ src/test/test_process.c \ + src/test/test_process_descs.c \ src/test/test_prob_distr.c \ src/test/test_procmon.c \ src/test/test_proto_http.c \ diff --git a/src/test/test.c b/src/test/test.c index b9a1da06f0..5f27f994f2 100644 --- a/src/test/test.c +++ b/src/test/test.c @@ -858,6 +858,7 @@ struct testgroup_t testgroups[] = { { "crypto/pem/", pem_tests }, { "crypto/rng/", crypto_rng_tests }, { "dir/", dir_tests }, + { "dir/auth/process_descs/", process_descs_tests }, { "dir/md/", microdesc_tests }, { "dir/voting/flags/", voting_flags_tests }, { "dir/voting/schedule/", voting_schedule_tests }, diff --git a/src/test/test.h b/src/test/test.h index f5c21bfe88..d0b045a80b 100644 --- a/src/test/test.h +++ b/src/test/test.h @@ -252,6 +252,7 @@ extern struct testcase_t prob_distr_tests[]; extern struct testcase_t slow_stochastic_prob_distr_tests[]; extern struct testcase_t procmon_tests[]; extern struct testcase_t process_tests[]; +extern struct testcase_t process_descs_tests[]; extern struct testcase_t proto_http_tests[]; extern struct testcase_t proto_misc_tests[]; extern struct testcase_t protover_tests[]; diff --git a/src/test/test_process_descs.c b/src/test/test_process_descs.c new file mode 100644 index 0000000000..7dc9abde31 --- /dev/null +++ b/src/test/test_process_descs.c @@ -0,0 +1,67 @@ +/* Copyright (c) 2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "orconfig.h" + +#include "core/or/or.h" +#include "feature/dirauth/process_descs.h" + +#include "test/test.h" + +static void +test_process_descs_versions(void *arg) +{ + (void)arg; + struct { + const char *version; + bool should_reject; + } cases[] = { + // a very old version: reject. + { "Tor 0.1.2.3-alpha", true }, + // a non-tor program: don't reject. + { "Wombat 0.1.2.3-alpha", false }, + // a slightly old version: reject + { "Tor 0.2.9.4-alpha", true }, + // a slightly old version: just new enough to support. + { "Tor 0.2.9.5-alpha", false }, + // a newer 0.2.9 version: supported. + { "Tor 0.2.9.100", false }, + // some unsupported versions: reject. + { "Tor 0.3.0.0-alpha-dev", true }, + { "Tor 0.3.0.2-alpha", true }, + { "Tor 0.3.0.5", true }, + { "Tor 0.3.1.4", true }, + { "Tor 0.3.2.4", true }, + { "Tor 0.3.3.4", true }, + { "Tor 0.3.4.1-alpha", true }, + { "Tor 0.3.4.100", true }, + { "Tor 0.3.5.1-alpha", true }, + { "Tor 0.3.5.6-rc", true}, + // new enough to be supported + { "Tor 0.3.5.7", false }, + { "Tor 0.3.5.8", false }, + { "Tor 0.4.0.1-alpha", false }, + { "Tor 0.4.1.5", false }, + // Very far in the future + { "Tor 100.100.1.5", false }, + }; + size_t n_cases = ARRAY_LENGTH(cases); + + for (unsigned i = 0; i < n_cases; ++i) { + const char *msg = NULL; + bool rejected = dirserv_rejects_tor_version(cases[i].version, &msg); + tt_int_op(rejected, OP_EQ, cases[i].should_reject); + tt_int_op(msg == NULL, OP_EQ, rejected == false); + } + + done: + ; +} + +#define T(name,flags) \ + { #name, test_process_descs_##name, (flags), NULL, NULL } + +struct testcase_t process_descs_tests[] = { + T(versions,0), + END_OF_TESTCASES +}; From 519afb0ece2485eb20450d0508637cf5632ce75d Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 3 Oct 2019 12:22:46 -0400 Subject: [PATCH 1630/2557] Sort changes entries into a changelog for 0.4.2.2-alpha --- ChangeLog | 122 +++++++++++++++++++++++++++++++++++ changes/bug28966 | 4 -- changes/bug30916 | 4 -- changes/bug31107 | 4 -- changes/bug31334 | 4 -- changes/bug31408 | 5 -- changes/bug31614 | 9 --- changes/bug31736 | 3 - changes/bug31825 | 3 - changes/bug31854 | 3 - changes/bug31884 | 3 - changes/bug31897 | 3 - changes/bug31898 | 4 -- changes/geoip-2019-10-01 | 4 -- changes/ticket30743 | 7 -- changes/ticket31338 | 4 -- changes/ticket31372_appveyor | 4 -- changes/ticket31372_travis | 4 -- changes/ticket31466 | 5 -- changes/ticket31549 | 4 -- changes/ticket31589 | 2 - changes/ticket31675 | 3 - changes/ticket31759 | 5 -- changes/ticket31772 | 4 -- changes/ticket31839 | 3 - changes/ticket31840 | 3 - changes/ticket31849 | 5 -- 27 files changed, 122 insertions(+), 106 deletions(-) delete mode 100644 changes/bug28966 delete mode 100644 changes/bug30916 delete mode 100644 changes/bug31107 delete mode 100644 changes/bug31334 delete mode 100644 changes/bug31408 delete mode 100644 changes/bug31614 delete mode 100644 changes/bug31736 delete mode 100644 changes/bug31825 delete mode 100644 changes/bug31854 delete mode 100644 changes/bug31884 delete mode 100644 changes/bug31897 delete mode 100644 changes/bug31898 delete mode 100644 changes/geoip-2019-10-01 delete mode 100644 changes/ticket30743 delete mode 100644 changes/ticket31338 delete mode 100644 changes/ticket31372_appveyor delete mode 100644 changes/ticket31372_travis delete mode 100644 changes/ticket31466 delete mode 100644 changes/ticket31549 delete mode 100644 changes/ticket31589 delete mode 100644 changes/ticket31675 delete mode 100644 changes/ticket31759 delete mode 100644 changes/ticket31772 delete mode 100644 changes/ticket31839 delete mode 100644 changes/ticket31840 delete mode 100644 changes/ticket31849 diff --git a/ChangeLog b/ChangeLog index 938e91545e..f69bc8c39c 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,125 @@ +Changes in version 0.4.2.2-alpha - 2019-10-0? + This release fixes several bugs from the previous alpha release. + + o Major bugfixes (embedded Tor): + - Avoid a possible crash when restarting Tor in embedded mode and + enabling a different set of publish/subscribe messages. Fixes bug + 31898; bugfix on 0.4.1.1-alpha. + + o Major bugfixes (torrc): + - Stop ignoring torrc options after an %include directive, when the + included directory ends with a file that does not contain any config + options. (But does contain comments or whitespace.) + Fixes bug 31408; bugfix on 0.3.1.1-alpha. + + o Minor features (authority): + - Directory authorities now reject relays running all currently + deprecated release series. The currently supported release series + are: 0.2.9, 0.3.5, 0.4.0, 0.4.1, and 0.4.2. Closes ticket 31549. + + o Minor features (auto-formatting scripts): + - When annotating C macros, never generate a line that our check-spaces + script would reject. Closes ticket 31759. + - When annotating C macros, try to remove cases of double-negation. + Closes ticket 31779. + + o Minor features (continuous integration): + - When building on Appveyor, pass the "-k" flag to make, so that + we are informed of all compilation failures, not just the first + one or two. Closes part of ticket 31372. + - When building on Travis, pass the "-k" flag to make, so that + we are informed of all compilation failures, not just the first + one or two. Closes part of ticket 31372. + + o Minor features (geoip): + - Update geoip and geoip6 to the October 1 2019 Maxmind GeoLite2 + Country database. Closes ticket 31931. + + o Minor features (maintenance scripts): + - Add a coccinelle script to detect bugs caused by incrementing or + decrementing a variable inside a call to log_debug(). Since + log_debug() is a macro whose arguments are conditionally evaluated, it + is usually an error to do this. One such bug was 30628, in which SENDME + cells were miscounted by a decrement operator inside a log_debug() + call. Closes ticket 30743. + + o Minor features (onion services v3): + - Assist users who try to setup v2 client authorization in v3 onion + services by pointing them to the right documentation. Closes ticket + 28966. + + o Minor bugfixes (Appveyor CI): + - Avoid spurious errors when Appveyor CI fails before the install step. + Fixes bug 31884; bugfix on 0.3.4.2-alpha. + + o Minor bugfixes (best practices tracker): + - When listing overbroad exceptions, do not also list problems, + and do not list insufficiently broad exceptions. Fixes bug 31338; + bugfix on 0.4.2.1-alpha. + + o Minor bugfixes (controller protocol): + - Fix the MAPADDRESS controller command to accept one or more + arguments. Previously, it required two or more arguments, and ignored + the first. Fixes bug 31772; bugfix on 0.4.1.1-alpha. + + o Minor bugfixes (logging): + - Add a missing check for HAVE_PTHREAD_H, because the backtrace code uses + mutexes. Fixes bug 31614; bugfix on 0.2.5.2-alpha. + - Disable backtrace signal handlers when shutting down tor. + Fixes bug 31614; bugfix on 0.2.5.2-alpha. + - Rate-limit our the logging message about the obsolete .exit notation. + Previously, there was no limit on this warning, which could potentially + be triggered many times by a hostile website. Fixes bug 31466; + bugfix on 0.2.2.1-alpha. + - When initialising log domain masks, only set known log domains. + Fixes bug 31854; bugfix on 0.2.1.1-alpha. + + o Minor bugfixes (logging, protocol violations): + - Do not log a nonfatal assertion failure when receiving a VERSIONS + cell on a connection using the obsolete v1 link protocol. Log a + protocol_warn instead. Fixes bug 31107; bugfix on 0.2.4.4-alpha. + + o Minor bugfixes (modules): + - Explain what the optional Directory Authority module is, and what + happens when it is disabled. Fixes bug 31825; bugfix on 0.3.4.1-alpha. + + o Minor bugfixes (multithreading): + - Avoid some undefined behaviour when freeing mutexes. + Fixes bug 31736; bugfix on 0.0.7. + + o Minor bugfixes (relay): + - Avoid crashing when starting with a corrupt keys directory where + the old ntor key and the new ntor key are identical. Fixes bug 30916; + bugfix on 0.2.4.8-alpha. + + o Minor bugfixes (tests, SunOS): + - Avoid a map_anon_nofork test failure due to a signed/unsigned integer + comparison. Fixes bug 31897; bugfix on 0.4.1.1-alpha. + + o Code simplification and refactoring (onion services): + - Interface for function `decrypt_desc_layer` cleaned up. Closes ticket 31589. + + o Code simplification and refactoring: + - Refactor connection_control_process_inbuf() to reduce the size of a + practracker exception. Closes ticket 31840. + - Refactor the microdescs_parse_from_string() function into smaller + pieces, for better comprehensibility. Closes ticket 31675. + - Use SEVERITY_MASK_IDX() to find the LOG_* mask indexes in the unit + tests and fuzzers, rather than using hard-coded values. + Closes ticket 31334. + + o Documentation: + - Document the signal-safe logging behaviour in the tor man page. Also + add some comments to the relevant functions. Closes ticket 31839. + - Explain why we can't destroy the backtrace buffer mutex. Explain why + we don't need to destroy the log mutex. + Closes ticket 31736. + - The Tor source code repository now includes a (somewhat dated) + description of Tor's modular architecture, in doc/HACKING/design. + This is based on the old "tor-guts.git" repository, which we are + adopting and superseding. Closes ticket 31849. + + Changes in version 0.4.1.6 - 2019-09-19 This release backports several bugfixes to improve stability and correctness. Anyone experiencing build problems or crashes with 0.4.1.5, diff --git a/changes/bug28966 b/changes/bug28966 deleted file mode 100644 index 61123a21eb..0000000000 --- a/changes/bug28966 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor features (onion services v3): - - Assist users who try to setup v2 client authorization in v3 onion - services by pointing them to the right documentation. Closes ticket - 28966. diff --git a/changes/bug30916 b/changes/bug30916 deleted file mode 100644 index b006bfc75d..0000000000 --- a/changes/bug30916 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (relay): - - Avoid crashing when starting with a corrupt keys directory where - the old ntor key and the new ntor key are identical. Fixes bug 30916; - bugfix on 0.2.4.8-alpha. diff --git a/changes/bug31107 b/changes/bug31107 deleted file mode 100644 index 9652927c30..0000000000 --- a/changes/bug31107 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (logging, protocol violations): - - Do not log a nonfatal assertion failure when receiving a VERSIONS - cell on a connection using the obsolete v1 link protocol. Log a - protocol_warn instead. Fixes bug 31107; bugfix on 0.2.4.4-alpha. diff --git a/changes/bug31334 b/changes/bug31334 deleted file mode 100644 index dfc9cc530e..0000000000 --- a/changes/bug31334 +++ /dev/null @@ -1,4 +0,0 @@ - o Code simplification and refactoring: - - Use SEVERITY_MASK_IDX() to find the LOG_* mask indexes in the unit - tests and fuzzers, rather than using hard-coded values. - Closes ticket 31334. diff --git a/changes/bug31408 b/changes/bug31408 deleted file mode 100644 index 3e4ffa927d..0000000000 --- a/changes/bug31408 +++ /dev/null @@ -1,5 +0,0 @@ - o Major bugfixes (torrc): - - Stop ignoring torrc options after an %include directive, when the - included directory ends with a file that does not contain any config - options. (But does contain comments or whitespace.) - Fixes bug 31408; bugfix on 0.3.1.1-alpha. diff --git a/changes/bug31614 b/changes/bug31614 deleted file mode 100644 index c425a9fcd4..0000000000 --- a/changes/bug31614 +++ /dev/null @@ -1,9 +0,0 @@ - o Minor bugfixes (logging): - - Disable backtrace signal handlers when shutting down tor. - Fixes bug 31614; bugfix on 0.2.5.2-alpha. - - Add a missing check for HAVE_PTHREAD_H, because the backtrace code uses - mutexes. Fixes bug 31614; bugfix on 0.2.5.2-alpha. - o Documentation: - - Explain why we can't destroy the backtrace buffer mutex. Explain why - we don't need to destroy the log mutex. - Closes ticket 31736. diff --git a/changes/bug31736 b/changes/bug31736 deleted file mode 100644 index beb09e5069..0000000000 --- a/changes/bug31736 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor bugfixes (multithreading): - - Avoid some undefined behaviour when freeing mutexes. - Fixes bug 31736; bugfix on 0.0.7. diff --git a/changes/bug31825 b/changes/bug31825 deleted file mode 100644 index fe90acf299..0000000000 --- a/changes/bug31825 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor bugfixes (modules): - - Explain what the optional Directory Authority module is, and what - happens when it is disabled. Fixes bug 31825; bugfix on 0.3.4.1-alpha. diff --git a/changes/bug31854 b/changes/bug31854 deleted file mode 100644 index 692a192fd9..0000000000 --- a/changes/bug31854 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor bugfixes (logging): - - When initialising log domain masks, only set known log domains. - Fixes bug 31854; bugfix on 0.2.1.1-alpha. diff --git a/changes/bug31884 b/changes/bug31884 deleted file mode 100644 index ddb6c50d74..0000000000 --- a/changes/bug31884 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor bugfixes (Appveyor CI): - - Avoid spurious errors when Appveyor CI fails before the install step. - Fixes bug 31884; bugfix on 0.3.4.2-alpha. diff --git a/changes/bug31897 b/changes/bug31897 deleted file mode 100644 index 81c63e704e..0000000000 --- a/changes/bug31897 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor bugfixes (tests, SunOS): - - Avoid a map_anon_nofork test failure due to a signed/unsigned integer - comparison. Fixes bug 31897; bugfix on 0.4.1.1-alpha. diff --git a/changes/bug31898 b/changes/bug31898 deleted file mode 100644 index 6f3e0a5465..0000000000 --- a/changes/bug31898 +++ /dev/null @@ -1,4 +0,0 @@ - o Major bugfixes (embedded Tor): - - Avoid a possible crash when restarting Tor in embedded mode and - enabling a different set of publish/subscribe messages. Fixes bug - 31898; bugfix on 0.4.1.1-alpha. diff --git a/changes/geoip-2019-10-01 b/changes/geoip-2019-10-01 deleted file mode 100644 index c7ed17b5c4..0000000000 --- a/changes/geoip-2019-10-01 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor features (geoip): - - Update geoip and geoip6 to the October 1 2019 Maxmind GeoLite2 - Country database. Closes ticket 31931. - diff --git a/changes/ticket30743 b/changes/ticket30743 deleted file mode 100644 index 4f029717db..0000000000 --- a/changes/ticket30743 +++ /dev/null @@ -1,7 +0,0 @@ - o Minor features (maintenance scripts): - - Add a coccinelle script to detect bugs caused by incrementing or - decrementing a variable inside a call to log_debug(). Since - log_debug() is a macro whose arguments are conditionally evaluated, it - is usually an error to do this. One such bug was 30628, in which SENDME - cells were miscounted by a decrement operator inside a log_debug() - call. Closes ticket 30743. diff --git a/changes/ticket31338 b/changes/ticket31338 deleted file mode 100644 index b76add635d..0000000000 --- a/changes/ticket31338 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (best practices tracker): - - When listing overbroad exceptions, do not also list problems, - and do not list insufficiently broad exceptions. Fixes bug 31338; - bugfix on 0.4.2.1-alpha. diff --git a/changes/ticket31372_appveyor b/changes/ticket31372_appveyor deleted file mode 100644 index e7bb03182e..0000000000 --- a/changes/ticket31372_appveyor +++ /dev/null @@ -1,4 +0,0 @@ - o Minor features (continuous integration): - - When building on Appveyor, pass the "-k" flag to make, so that - we are informed of all compilation failures, not just the first - one or two. Closes part of ticket 31372. diff --git a/changes/ticket31372_travis b/changes/ticket31372_travis deleted file mode 100644 index 403869b2ed..0000000000 --- a/changes/ticket31372_travis +++ /dev/null @@ -1,4 +0,0 @@ - o Minor features (continuous integration): - - When building on Travis, pass the "-k" flag to make, so that - we are informed of all compilation failures, not just the first - one or two. Closes part of ticket 31372. diff --git a/changes/ticket31466 b/changes/ticket31466 deleted file mode 100644 index e535b4502e..0000000000 --- a/changes/ticket31466 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor bugfixes (logging): - - Rate-limit our the logging message about the obsolete .exit notation. - Previously, there was no limit on this warning, which could potentially - be triggered many times by a hostile website. Fixes bug 31466; - bugfix on 0.2.2.1-alpha. diff --git a/changes/ticket31549 b/changes/ticket31549 deleted file mode 100644 index 2c27aca4fb..0000000000 --- a/changes/ticket31549 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor features (authority): - - Directory authorities now reject relays running all currently - deprecated release series. The currently supported release series - are: 0.2.9, 0.3.5, 0.4.0, 0.4.1, and 0.4.2. Closes ticket 31549. diff --git a/changes/ticket31589 b/changes/ticket31589 deleted file mode 100644 index 673ab653e2..0000000000 --- a/changes/ticket31589 +++ /dev/null @@ -1,2 +0,0 @@ - o Code simplification and refactoring (onion services): - - Interface for function `decrypt_desc_layer` cleaned up. Closes ticket 31589. diff --git a/changes/ticket31675 b/changes/ticket31675 deleted file mode 100644 index 2b426948f3..0000000000 --- a/changes/ticket31675 +++ /dev/null @@ -1,3 +0,0 @@ - o Code simplification and refactoring: - - Refactor the microdescs_parse_from_string() function into smaller - pieces, for better comprehensibility. Closes ticket 31675. diff --git a/changes/ticket31759 b/changes/ticket31759 deleted file mode 100644 index f7428f711c..0000000000 --- a/changes/ticket31759 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor features (auto-formatting scripts): - - When annotating C macros, never generate a line that our check-spaces - script would reject. Closes ticket 31759. - - When annotating C macros, try to remove cases of double-negation. - Closes ticket 31779. diff --git a/changes/ticket31772 b/changes/ticket31772 deleted file mode 100644 index 7847b3f746..0000000000 --- a/changes/ticket31772 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (controller protocol): - - Fix the MAPADDRESS controller command to accept one or more - arguments. Previously, it required two or more arguments, and ignored - the first. Fixes bug 31772; bugfix on 0.4.1.1-alpha. diff --git a/changes/ticket31839 b/changes/ticket31839 deleted file mode 100644 index d7da40f530..0000000000 --- a/changes/ticket31839 +++ /dev/null @@ -1,3 +0,0 @@ - o Documentation: - - Document the signal-safe logging behaviour in the tor man page. Also - add some comments to the relevant functions. Closes ticket 31839. diff --git a/changes/ticket31840 b/changes/ticket31840 deleted file mode 100644 index c75c5629f9..0000000000 --- a/changes/ticket31840 +++ /dev/null @@ -1,3 +0,0 @@ - o Code simplification and refactoring: - - Refactor connection_control_process_inbuf() to reduce the size of a - practracker exception. Closes ticket 31840. diff --git a/changes/ticket31849 b/changes/ticket31849 deleted file mode 100644 index 9d12d938c4..0000000000 --- a/changes/ticket31849 +++ /dev/null @@ -1,5 +0,0 @@ - o Documentation: - - The Tor source code repository now includes a (somewhat dated) - description of Tor's modular architecture, in doc/HACKING/design. - This is based on the old "tor-guts.git" repository, which we are - adopting and superseding. Closes ticket 31849. From 0ef59dd09bd51d2fab37246668c94646bf8df6f5 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 3 Oct 2019 12:23:05 -0400 Subject: [PATCH 1631/2557] Run format_changelog --- ChangeLog | 112 +++++++++++++++++++++++++++--------------------------- 1 file changed, 57 insertions(+), 55 deletions(-) diff --git a/ChangeLog b/ChangeLog index f69bc8c39c..915509cfa0 100644 --- a/ChangeLog +++ b/ChangeLog @@ -8,28 +8,28 @@ Changes in version 0.4.2.2-alpha - 2019-10-0? o Major bugfixes (torrc): - Stop ignoring torrc options after an %include directive, when the - included directory ends with a file that does not contain any config - options. (But does contain comments or whitespace.) - Fixes bug 31408; bugfix on 0.3.1.1-alpha. + included directory ends with a file that does not contain any + config options. (But does contain comments or whitespace.) Fixes + bug 31408; bugfix on 0.3.1.1-alpha. o Minor features (authority): - Directory authorities now reject relays running all currently - deprecated release series. The currently supported release series - are: 0.2.9, 0.3.5, 0.4.0, 0.4.1, and 0.4.2. Closes ticket 31549. + deprecated release series. The currently supported release series + are: 0.2.9, 0.3.5, 0.4.0, 0.4.1, and 0.4.2. Closes ticket 31549. o Minor features (auto-formatting scripts): - - When annotating C macros, never generate a line that our check-spaces - script would reject. Closes ticket 31759. + - When annotating C macros, never generate a line that our check- + spaces script would reject. Closes ticket 31759. - When annotating C macros, try to remove cases of double-negation. Closes ticket 31779. o Minor features (continuous integration): - - When building on Appveyor, pass the "-k" flag to make, so that - we are informed of all compilation failures, not just the first - one or two. Closes part of ticket 31372. - - When building on Travis, pass the "-k" flag to make, so that - we are informed of all compilation failures, not just the first - one or two. Closes part of ticket 31372. + - When building on Appveyor, pass the "-k" flag to make, so that we + are informed of all compilation failures, not just the first one + or two. Closes part of ticket 31372. + - When building on Travis, pass the "-k" flag to make, so that we + are informed of all compilation failures, not just the first one + or two. Closes part of ticket 31372. o Minor features (geoip): - Update geoip and geoip6 to the October 1 2019 Maxmind GeoLite2 @@ -37,40 +37,40 @@ Changes in version 0.4.2.2-alpha - 2019-10-0? o Minor features (maintenance scripts): - Add a coccinelle script to detect bugs caused by incrementing or - decrementing a variable inside a call to log_debug(). Since - log_debug() is a macro whose arguments are conditionally evaluated, it - is usually an error to do this. One such bug was 30628, in which SENDME - cells were miscounted by a decrement operator inside a log_debug() - call. Closes ticket 30743. + decrementing a variable inside a call to log_debug(). Since + log_debug() is a macro whose arguments are conditionally + evaluated, it is usually an error to do this. One such bug was + 30628, in which SENDME cells were miscounted by a decrement + operator inside a log_debug() call. Closes ticket 30743. o Minor features (onion services v3): - Assist users who try to setup v2 client authorization in v3 onion - services by pointing them to the right documentation. Closes ticket - 28966. + services by pointing them to the right documentation. Closes + ticket 28966. o Minor bugfixes (Appveyor CI): - - Avoid spurious errors when Appveyor CI fails before the install step. - Fixes bug 31884; bugfix on 0.3.4.2-alpha. + - Avoid spurious errors when Appveyor CI fails before the install + step. Fixes bug 31884; bugfix on 0.3.4.2-alpha. o Minor bugfixes (best practices tracker): - - When listing overbroad exceptions, do not also list problems, - and do not list insufficiently broad exceptions. Fixes bug 31338; + - When listing overbroad exceptions, do not also list problems, and + do not list insufficiently broad exceptions. Fixes bug 31338; bugfix on 0.4.2.1-alpha. o Minor bugfixes (controller protocol): - Fix the MAPADDRESS controller command to accept one or more - arguments. Previously, it required two or more arguments, and ignored - the first. Fixes bug 31772; bugfix on 0.4.1.1-alpha. + arguments. Previously, it required two or more arguments, and + ignored the first. Fixes bug 31772; bugfix on 0.4.1.1-alpha. o Minor bugfixes (logging): - - Add a missing check for HAVE_PTHREAD_H, because the backtrace code uses - mutexes. Fixes bug 31614; bugfix on 0.2.5.2-alpha. - - Disable backtrace signal handlers when shutting down tor. - Fixes bug 31614; bugfix on 0.2.5.2-alpha. - - Rate-limit our the logging message about the obsolete .exit notation. - Previously, there was no limit on this warning, which could potentially - be triggered many times by a hostile website. Fixes bug 31466; - bugfix on 0.2.2.1-alpha. + - Add a missing check for HAVE_PTHREAD_H, because the backtrace code + uses mutexes. Fixes bug 31614; bugfix on 0.2.5.2-alpha. + - Disable backtrace signal handlers when shutting down tor. Fixes + bug 31614; bugfix on 0.2.5.2-alpha. + - Rate-limit our the logging message about the obsolete .exit + notation. Previously, there was no limit on this warning, which + could potentially be triggered many times by a hostile website. + Fixes bug 31466; bugfix on 0.2.2.1-alpha. - When initialising log domain masks, only set known log domains. Fixes bug 31854; bugfix on 0.2.1.1-alpha. @@ -81,43 +81,45 @@ Changes in version 0.4.2.2-alpha - 2019-10-0? o Minor bugfixes (modules): - Explain what the optional Directory Authority module is, and what - happens when it is disabled. Fixes bug 31825; bugfix on 0.3.4.1-alpha. + happens when it is disabled. Fixes bug 31825; bugfix + on 0.3.4.1-alpha. o Minor bugfixes (multithreading): - - Avoid some undefined behaviour when freeing mutexes. - Fixes bug 31736; bugfix on 0.0.7. + - Avoid some undefined behaviour when freeing mutexes. Fixes bug + 31736; bugfix on 0.0.7. o Minor bugfixes (relay): - Avoid crashing when starting with a corrupt keys directory where - the old ntor key and the new ntor key are identical. Fixes bug 30916; - bugfix on 0.2.4.8-alpha. + the old ntor key and the new ntor key are identical. Fixes bug + 30916; bugfix on 0.2.4.8-alpha. o Minor bugfixes (tests, SunOS): - - Avoid a map_anon_nofork test failure due to a signed/unsigned integer - comparison. Fixes bug 31897; bugfix on 0.4.1.1-alpha. - - o Code simplification and refactoring (onion services): - - Interface for function `decrypt_desc_layer` cleaned up. Closes ticket 31589. + - Avoid a map_anon_nofork test failure due to a signed/unsigned + integer comparison. Fixes bug 31897; bugfix on 0.4.1.1-alpha. o Code simplification and refactoring: - - Refactor connection_control_process_inbuf() to reduce the size of a - practracker exception. Closes ticket 31840. + - Refactor connection_control_process_inbuf() to reduce the size of + a practracker exception. Closes ticket 31840. - Refactor the microdescs_parse_from_string() function into smaller - pieces, for better comprehensibility. Closes ticket 31675. + pieces, for better comprehensibility. Closes ticket 31675. - Use SEVERITY_MASK_IDX() to find the LOG_* mask indexes in the unit - tests and fuzzers, rather than using hard-coded values. - Closes ticket 31334. + tests and fuzzers, rather than using hard-coded values. Closes + ticket 31334. o Documentation: - - Document the signal-safe logging behaviour in the tor man page. Also - add some comments to the relevant functions. Closes ticket 31839. - - Explain why we can't destroy the backtrace buffer mutex. Explain why - we don't need to destroy the log mutex. - Closes ticket 31736. + - Document the signal-safe logging behaviour in the tor man page. + Also add some comments to the relevant functions. Closes + ticket 31839. + - Explain why we can't destroy the backtrace buffer mutex. Explain + why we don't need to destroy the log mutex. Closes ticket 31736. - The Tor source code repository now includes a (somewhat dated) description of Tor's modular architecture, in doc/HACKING/design. This is based on the old "tor-guts.git" repository, which we are - adopting and superseding. Closes ticket 31849. + adopting and superseding. Closes ticket 31849. + + o Code simplification and refactoring (onion services): + - Interface for function `decrypt_desc_layer` cleaned up. Closes + ticket 31589. Changes in version 0.4.1.6 - 2019-09-19 From 5ae1a574dc11035fcea915948943c9a0a2082eae Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 3 Oct 2019 12:24:47 -0400 Subject: [PATCH 1632/2557] lightly sort changelog entries --- ChangeLog | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/ChangeLog b/ChangeLog index 915509cfa0..6f1e09dd2f 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,22 +1,22 @@ Changes in version 0.4.2.2-alpha - 2019-10-0? This release fixes several bugs from the previous alpha release. + o Major features (directory authorities): + - Directory authorities now reject relays running all currently + deprecated release series. The currently supported release series + are: 0.2.9, 0.3.5, 0.4.0, 0.4.1, and 0.4.2. Closes ticket 31549. + o Major bugfixes (embedded Tor): - Avoid a possible crash when restarting Tor in embedded mode and enabling a different set of publish/subscribe messages. Fixes bug 31898; bugfix on 0.4.1.1-alpha. - o Major bugfixes (torrc): + o Major bugfixes (torrc parsing): - Stop ignoring torrc options after an %include directive, when the included directory ends with a file that does not contain any config options. (But does contain comments or whitespace.) Fixes bug 31408; bugfix on 0.3.1.1-alpha. - o Minor features (authority): - - Directory authorities now reject relays running all currently - deprecated release series. The currently supported release series - are: 0.2.9, 0.3.5, 0.4.0, 0.4.1, and 0.4.2. Closes ticket 31549. - o Minor features (auto-formatting scripts): - When annotating C macros, never generate a line that our check- spaces script would reject. Closes ticket 31759. @@ -48,7 +48,7 @@ Changes in version 0.4.2.2-alpha - 2019-10-0? services by pointing them to the right documentation. Closes ticket 28966. - o Minor bugfixes (Appveyor CI): + o Minor bugfixes (Appveyor continuous integration): - Avoid spurious errors when Appveyor CI fails before the install step. Fixes bug 31884; bugfix on 0.3.4.2-alpha. @@ -105,6 +105,8 @@ Changes in version 0.4.2.2-alpha - 2019-10-0? - Use SEVERITY_MASK_IDX() to find the LOG_* mask indexes in the unit tests and fuzzers, rather than using hard-coded values. Closes ticket 31334. + - Interface for function `decrypt_desc_layer` cleaned up. Closes + ticket 31589. o Documentation: - Document the signal-safe logging behaviour in the tor man page. @@ -117,10 +119,6 @@ Changes in version 0.4.2.2-alpha - 2019-10-0? This is based on the old "tor-guts.git" repository, which we are adopting and superseding. Closes ticket 31849. - o Code simplification and refactoring (onion services): - - Interface for function `decrypt_desc_layer` cleaned up. Closes - ticket 31589. - Changes in version 0.4.1.6 - 2019-09-19 This release backports several bugfixes to improve stability and From db976cd9277199664f7b55e2f9b105f6e96174c6 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 3 Oct 2019 12:34:21 -0400 Subject: [PATCH 1633/2557] Light edits on 0.4.2.2-alpha changelog. --- ChangeLog | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/ChangeLog b/ChangeLog index 6f1e09dd2f..bddac38254 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,7 @@ Changes in version 0.4.2.2-alpha - 2019-10-0? - This release fixes several bugs from the previous alpha release. + This release fixes several bugs from the previous alpha release, and + from earlier versions. It also includes a change in authorities, so + that they begin to reject the currently unsupported release series. o Major features (directory authorities): - Directory authorities now reject relays running all currently @@ -14,7 +16,7 @@ Changes in version 0.4.2.2-alpha - 2019-10-0? o Major bugfixes (torrc parsing): - Stop ignoring torrc options after an %include directive, when the included directory ends with a file that does not contain any - config options. (But does contain comments or whitespace.) Fixes + config options (but does contain comments or whitespace). Fixes bug 31408; bugfix on 0.3.1.1-alpha. o Minor features (auto-formatting scripts): @@ -24,19 +26,16 @@ Changes in version 0.4.2.2-alpha - 2019-10-0? Closes ticket 31779. o Minor features (continuous integration): - - When building on Appveyor, pass the "-k" flag to make, so that we - are informed of all compilation failures, not just the first one - or two. Closes part of ticket 31372. - - When building on Travis, pass the "-k" flag to make, so that we - are informed of all compilation failures, not just the first one - or two. Closes part of ticket 31372. + - When building on Appveyor and Travis, pass the "-k" flag to make, + so that we are informed of all compilation failures, not just the + first one or two. Closes ticket 31372. o Minor features (geoip): - Update geoip and geoip6 to the October 1 2019 Maxmind GeoLite2 Country database. Closes ticket 31931. o Minor features (maintenance scripts): - - Add a coccinelle script to detect bugs caused by incrementing or + - Add a Coccinelle script to detect bugs caused by incrementing or decrementing a variable inside a call to log_debug(). Since log_debug() is a macro whose arguments are conditionally evaluated, it is usually an error to do this. One such bug was From c8df2c720501fc8a3f96c34fcef0bced4498deb0 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 3 Oct 2019 14:58:51 -0400 Subject: [PATCH 1634/2557] Pick a release date for 0.4.2.2-alpha --- ChangeLog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index bddac38254..59ded0082e 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,4 +1,4 @@ -Changes in version 0.4.2.2-alpha - 2019-10-0? +Changes in version 0.4.2.2-alpha - 2019-10-07 This release fixes several bugs from the previous alpha release, and from earlier versions. It also includes a change in authorities, so that they begin to reject the currently unsupported release series. From 52b7ae71b31671c758a2798e8c98abb720ac22f5 Mon Sep 17 00:00:00 2001 From: David Goulet Date: Wed, 11 Sep 2019 08:46:31 -0400 Subject: [PATCH 1635/2557] hs: ADD_ONION NEW:BEST now defaults to ED25519-V3 From RSA1024 (v2) to v3 now. Closes #29669 Signed-off-by: David Goulet --- changes/ticket29669 | 3 +++ src/feature/control/control_cmd.c | 7 ++++--- src/test/test_controller.c | 28 +++++++++++++++------------- 3 files changed, 22 insertions(+), 16 deletions(-) create mode 100644 changes/ticket29669 diff --git a/changes/ticket29669 b/changes/ticket29669 new file mode 100644 index 0000000000..f7e98a16ce --- /dev/null +++ b/changes/ticket29669 @@ -0,0 +1,3 @@ + o Minor feature (hidden service, control port): + - The ADD_ONION key blob keyword "BEST" now defaults from RSA1024 (v2) to + ED25519-V3 (v3). Closes ticket 29669. diff --git a/src/feature/control/control_cmd.c b/src/feature/control/control_cmd.c index f804ceafbc..de1bef7e59 100644 --- a/src/feature/control/control_cmd.c +++ b/src/feature/control/control_cmd.c @@ -1982,8 +1982,7 @@ add_onion_helper_keyarg(const char *arg, int discard_pk, *hs_version = HS_VERSION_THREE; } else if (!strcasecmp(key_type_new, key_type)) { /* "NEW:" - Generating a new key, blob as algorithm. */ - if (!strcasecmp(key_type_rsa1024, key_blob) || - !strcasecmp(key_type_best, key_blob)) { + if (!strcasecmp(key_type_rsa1024, key_blob)) { /* "RSA1024", RSA 1024 bit, also currently "BEST" by default. */ pk = crypto_pk_new(); if (crypto_pk_generate_key(pk)) { @@ -2002,7 +2001,9 @@ add_onion_helper_keyarg(const char *arg, int discard_pk, } decoded_key->v2 = pk; *hs_version = HS_VERSION_TWO; - } else if (!strcasecmp(key_type_ed25519_v3, key_blob)) { + } else if (!strcasecmp(key_type_ed25519_v3, key_blob) || + !strcasecmp(key_type_best, key_blob)) { + /* "ED25519-V3", ed25519 key, also currently "BEST" by default. */ ed25519_secret_key_t *sk = tor_malloc_zero(sizeof(*sk)); if (ed25519_secret_key_generate(sk, 1) < 0) { tor_free(sk); diff --git a/src/test/test_controller.c b/src/test/test_controller.c index b9cbe0a14d..55eb79e448 100644 --- a/src/test/test_controller.c +++ b/src/test/test_controller.c @@ -243,8 +243,22 @@ test_add_onion_helper_keyarg_v3(void *arg) tor_free(pk.v3); pk.v3 = NULL; tor_free(key_new_blob); + /* Test "BEST" key generation (Assumes BEST = ED25519-V3). */ + tor_free(pk.v3); pk.v3 = NULL; + tor_free(key_new_blob); + ret = add_onion_helper_keyarg("NEW:BEST", 0, &key_new_alg, &key_new_blob, + &pk, &hs_version, NULL); + tt_int_op(ret, OP_EQ, 0); + tt_int_op(hs_version, OP_EQ, HS_VERSION_THREE); + tt_assert(pk.v3); + tt_str_op(key_new_alg, OP_EQ, "ED25519-V3"); + tt_assert(key_new_blob); + tt_ptr_op(reply_str, OP_EQ, NULL); + /* Test discarding the private key. */ tor_free(reply_str); + tor_free(pk.v3); pk.v3 = NULL; + tor_free(key_new_blob); ret = add_onion_helper_keyarg("NEW:ED25519-V3", 1, &key_new_alg, &key_new_blob, &pk, &hs_version, NULL); @@ -323,22 +337,10 @@ test_add_onion_helper_keyarg_v2(void *arg) tt_assert(key_new_blob); tt_ptr_op(reply_str, OP_EQ, NULL); - /* Test "BEST" key generation (Assumes BEST = RSA1024). */ - crypto_pk_free(pk.v2); pk.v2 = NULL; - tor_free(key_new_blob); - ret = add_onion_helper_keyarg("NEW:BEST", 0, &key_new_alg, &key_new_blob, - &pk, &hs_version, NULL); - tt_int_op(ret, OP_EQ, 0); - tt_int_op(hs_version, OP_EQ, HS_VERSION_TWO); - tt_assert(pk.v2); - tt_str_op(key_new_alg, OP_EQ, "RSA1024"); - tt_assert(key_new_blob); - tt_ptr_op(reply_str, OP_EQ, NULL); - /* Test discarding the private key. */ crypto_pk_free(pk.v2); pk.v2 = NULL; tor_free(key_new_blob); - ret = add_onion_helper_keyarg("NEW:BEST", 1, &key_new_alg, &key_new_blob, + ret = add_onion_helper_keyarg("NEW:RSA1024", 1, &key_new_alg, &key_new_blob, &pk, &hs_version, NULL); tt_int_op(ret, OP_EQ, 0); tt_int_op(hs_version, OP_EQ, HS_VERSION_TWO); From 9b73088c14fe24a1554950363fb80468c695937f Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 3 Oct 2019 07:21:25 -0400 Subject: [PATCH 1636/2557] util/map_anon_nofork: Add a cast to avoid passing -48 to memset This fixes coverity CID 1454593, and bug 31948. Bug not in any released version of Tor. --- src/test/test_util.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/test_util.c b/src/test/test_util.c index 6ecff6f1c3..b4d8a4d767 100644 --- a/src/test/test_util.c +++ b/src/test/test_util.c @@ -6190,7 +6190,7 @@ test_util_map_anon_nofork(void *arg) tor_munmap_anonymous(ptr, sz); ptr = tor_mmap_anonymous(sz, ANONMAP_NOINHERIT, &inherit); tt_ptr_op(ptr, OP_NE, 0); - memset(ptr, TEST_VALUE, sz); + memset(ptr, (uint8_t)TEST_VALUE, sz); tt_int_op(0, OP_EQ, pipe(pipefd)); pid_t child = fork(); From 9c24ceeb3f2082193bf76a77ad6d6d011561e239 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 3 Oct 2019 07:50:18 -0400 Subject: [PATCH 1637/2557] Make tor_addr_lookup_host_impl mockable. --- src/lib/net/resolve.c | 14 +++++++------- src/lib/net/resolve.h | 15 +++++++++++++++ 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/src/lib/net/resolve.c b/src/lib/net/resolve.c index c8fe338e52..442bc4a6b3 100644 --- a/src/lib/net/resolve.c +++ b/src/lib/net/resolve.c @@ -8,6 +8,7 @@ * \brief Use the libc DNS resolver to convert hostnames into addresses. **/ +#define RESOLVE_PRIVATE #include "lib/net/resolve.h" #include "lib/net/address.h" @@ -70,10 +71,10 @@ tor_lookup_hostname,(const char *name, uint32_t *addr)) * * See tor_addr_lookup() for details. */ -static int -tor_addr_lookup_host_impl(const char *name, +MOCK_IMPL(STATIC int, +tor_addr_lookup_host_impl,(const char *name, uint16_t family, - tor_addr_t *addr) + tor_addr_t *addr)) { int err; struct addrinfo *res=NULL, *res_p; @@ -125,10 +126,10 @@ tor_addr_lookup_host_impl(const char *name, * * See tor_addr_lookup() for details. */ -static int -tor_addr_lookup_host_impl(const char *name, +MOCK_IMPL(STATIC int, +tor_addr_lookup_host_impl,(const char *name, uint16_t family, - tor_addr_t *addr) + tor_addr_t *addr)) { (void) family; struct hostent *ent; @@ -172,7 +173,6 @@ tor_addr_lookup_host_impl(const char *name, return (err == TRY_AGAIN) ? 1 : -1; #endif } - #endif /* defined(HAVE_GETADDRINFO) */ /** Similar behavior to Unix gethostbyname: resolve name, and set diff --git a/src/lib/net/resolve.h b/src/lib/net/resolve.h index d7b60be917..b979b2fb41 100644 --- a/src/lib/net/resolve.h +++ b/src/lib/net/resolve.h @@ -24,12 +24,18 @@ struct tor_addr_t; +/* + * Primary lookup functions. + */ MOCK_DECL(int, tor_lookup_hostname,(const char *name, uint32_t *addr)); MOCK_DECL(int, tor_addr_lookup,(const char *name, uint16_t family, struct tor_addr_t *addr_out)); int tor_addr_port_lookup(const char *s, struct tor_addr_t *addr_out, uint16_t *port_out); +/* + * Sandbox helpers + */ struct addrinfo; #ifdef USE_SANDBOX_GETADDRINFO /** Pre-calls getaddrinfo in order to pre-record result. */ @@ -55,4 +61,13 @@ void tor_free_getaddrinfo_cache(void); void sandbox_disable_getaddrinfo_cache(void); void tor_make_getaddrinfo_cache_active(void); +/* + * Internal resolver wrapper; exposed for mocking. + */ +#ifdef RESOLVE_PRIVATE +MOCK_DECL(STATIC int, tor_addr_lookup_host_impl, (const char *name, + uint16_t family, + struct tor_addr_t *addr)); +#endif + #endif /* !defined(TOR_RESOLVE_H) */ From 4a0749596cc21198dd853bb0c631e9658db7fe2d Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 3 Oct 2019 08:54:17 -0400 Subject: [PATCH 1638/2557] Add a mock replacement for blocking hostname resolution Based on examination of our tests, this mock function accepts "localhost" and "torproject.org", and rejects everything else. --- src/test/include.am | 2 + src/test/resolve_test_helpers.c | 85 +++++++++++++++++++++++++++++++++ src/test/resolve_test_helpers.h | 18 +++++++ 3 files changed, 105 insertions(+) create mode 100644 src/test/resolve_test_helpers.c create mode 100644 src/test/resolve_test_helpers.h diff --git a/src/test/include.am b/src/test/include.am index 2dd4d8c583..acd0824d0d 100644 --- a/src/test/include.am +++ b/src/test/include.am @@ -102,6 +102,7 @@ src_test_test_SOURCES += \ src/test/log_test_helpers.c \ src/test/hs_test_helpers.c \ src/test/rend_test_helpers.c \ + src/test/resolve_test_helpers.c \ src/test/rng_test_helpers.c \ src/test/test.c \ src/test/test_accounting.c \ @@ -340,6 +341,7 @@ noinst_HEADERS+= \ src/test/hs_test_helpers.h \ src/test/log_test_helpers.h \ src/test/rend_test_helpers.h \ + src/test/resolve_test_helpers.h \ src/test/rng_test_helpers.h \ src/test/test.h \ src/test/ptr_helpers.h \ diff --git a/src/test/resolve_test_helpers.c b/src/test/resolve_test_helpers.c new file mode 100644 index 0000000000..73ea730149 --- /dev/null +++ b/src/test/resolve_test_helpers.c @@ -0,0 +1,85 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file resolve_test_helpers.c + * @brief Helper functions for mocking libc's blocking hostname lookup + * facilities. + **/ + +#define RESOLVE_PRIVATE +#include "orconfig.h" +#include "test/resolve_test_helpers.h" +#include "lib/net/address.h" +#include "lib/net/resolve.h" +#include "test/test.h" + +#include +#include + +/** + * Mock replacement for our getaddrinfo/gethostbyname wrapper. + **/ +static int +replacement_host_lookup(const char *name, uint16_t family, tor_addr_t *addr) +{ + static const struct lookup_table_ent { + const char *name; + const char *ipv4; + const char *ipv6; + } entries[] = { + { "localhost", "127.0.0.1", "::1" }, + { "torproject.org", "198.51.100.6", "2001:DB8::700" }, + { NULL, NULL, NULL }, + }; + + int r = -1; + + for (unsigned i = 0; entries[i].name != NULL; ++i) { + if (!strcasecmp(name, entries[i].name)) { + if (family == AF_INET6) { + int s = tor_addr_parse(addr, entries[i].ipv6); + tt_int_op(s, OP_EQ, AF_INET6); + } else { + int s = tor_addr_parse(addr, entries[i].ipv4); + tt_int_op(s, OP_EQ, AF_INET); + } + r = 0; + break; + } + } + + log_debug(LD_GENERAL, "resolve(%s,%d) => %s", + name, family, r == 0 ? fmt_addr(addr) : "-1"); + + return r; + done: + return -1; +} + +/** + * Set up a mock replacement for our wrapper on libc's resolver code. + * + * According to our replacement, only "localhost" and "torproject.org" + * are real addresses; everything else doesn't exist. + * + * Use this function to avoid using the DNS resolver during unit tests; + * call unmock_hostname_resolver() when you're done. + **/ +void +mock_hostname_resolver(void) +{ + MOCK(tor_addr_lookup_host_impl, replacement_host_lookup); +} + +/** + * Unmock our wrappers for libc's blocking hostname resolver code. + **/ +void +unmock_hostname_resolver(void) +{ + UNMOCK(tor_addr_lookup_host_impl); +} diff --git a/src/test/resolve_test_helpers.h b/src/test/resolve_test_helpers.h new file mode 100644 index 0000000000..e7d2e29373 --- /dev/null +++ b/src/test/resolve_test_helpers.h @@ -0,0 +1,18 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file resolve_test_helpers.h + * @brief Header for test/resolve_test_helpers.c + **/ + +#ifndef TOR_TEST_RESOLVE_TEST_HELPERS_H +#define TOR_TEST_RESOLVE_TEST_HELPERS_H + +void mock_hostname_resolver(void); +void unmock_hostname_resolver(void); + +#endif /* !defined(TOR_TEST_RESOLVE_TEST_HELPERS_H) */ From fdfb4b196b7c38f45f2d37f73fcc96d746816cc3 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 3 Oct 2019 08:57:14 -0400 Subject: [PATCH 1639/2557] Use {mock,unmock}_hostname_resolver() in relevant tests These tests all invoke the hostname resolver in one way or another, and therefore potentially block if our DNS server is missing, absent, or extremely slow. Closes ticket 31841. --- changes/ticket31841 | 5 +++++ src/test/test_addr.c | 6 +++++- src/test/test_config.c | 4 ++++ src/test/test_hs_config.c | 8 +++++--- src/test/test_options.c | 3 +++ 5 files changed, 22 insertions(+), 4 deletions(-) create mode 100644 changes/ticket31841 diff --git a/changes/ticket31841 b/changes/ticket31841 new file mode 100644 index 0000000000..6e7fbc1da1 --- /dev/null +++ b/changes/ticket31841 @@ -0,0 +1,5 @@ + o Minor features (testing): + - When running tests that attempt to look up hostname, replace the libc + name lookup functions with ones that do not actually touch the network. + This way, the tests complete more quickly in the presence of a slow or + missing DNS resolver. Closes ticket 31841. diff --git a/src/test/test_addr.c b/src/test/test_addr.c index f99e3be8f5..c89c6e78d4 100644 --- a/src/test/test_addr.c +++ b/src/test/test_addr.c @@ -12,6 +12,7 @@ #include "test/log_test_helpers.h" #include "lib/net/resolve.h" #include "test/rng_test_helpers.h" +#include "test/resolve_test_helpers.h" #ifdef HAVE_SYS_UN_H #include @@ -1160,6 +1161,7 @@ test_addr_parse_canonical(void *arg) static void test_addr_parse(void *arg) { + int r; tor_addr_t addr; uint16_t port; @@ -1169,6 +1171,8 @@ test_addr_parse(void *arg) (void)arg; + mock_hostname_resolver(); + /* IPv6-mapped IPv4 addresses. Tor doesn't really use these. */ TEST_ADDR_V6_PARSE("11:22:33:44:55:66:1.2.3.4", 0, "11:22:33:44:55:66:102:304"); @@ -1273,7 +1277,7 @@ test_addr_parse(void *arg) "11:22::88",99); done: - ; + unmock_hostname_resolver(); } static void diff --git a/src/test/test_config.c b/src/test/test_config.c index 1c6c913078..cbb84e4dcf 100644 --- a/src/test/test_config.c +++ b/src/test/test_config.c @@ -45,6 +45,7 @@ #include "app/config/statefile.h" #include "test/test_helpers.h" +#include "test/resolve_test_helpers.h" #include "feature/dirclient/dir_server_st.h" #include "core/or/port_cfg_st.h" @@ -4068,6 +4069,8 @@ test_config_parse_port_config__ports__ports_given(void *data) slout = smartlist_new(); + mock_hostname_resolver(); + // Test error when encounters an invalid Port specification config_port_invalid = mock_config_line("DNSPort", ""); ret = parse_port_config(NULL, config_port_invalid, "DNS", 0, NULL, @@ -4764,6 +4767,7 @@ test_config_parse_port_config__ports__ports_given(void *data) #endif /* defined(_WIN32) */ done: + unmock_hostname_resolver(); if (slout) SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf)); smartlist_free(slout); diff --git a/src/test/test_hs_config.c b/src/test/test_hs_config.c index 2b3afbb6e9..71e1529216 100644 --- a/src/test/test_hs_config.c +++ b/src/test/test_hs_config.c @@ -12,6 +12,7 @@ #include "test/test.h" #include "test/test_helpers.h" #include "test/log_test_helpers.h" +#include "test/resolve_test_helpers.h" #include "app/config/config.h" #include "feature/hs/hs_common.h" @@ -272,6 +273,7 @@ test_valid_service_v2(void *arg) int ret; (void) arg; + mock_hostname_resolver(); /* Valid complex configuration. Basic client authorization. */ { @@ -314,7 +316,7 @@ test_valid_service_v2(void *arg) } done: - ; + unmock_hostname_resolver(); } static void @@ -392,6 +394,7 @@ test_valid_service_v3(void *arg) int ret; (void) arg; + mock_hostname_resolver(); /* Valid complex configuration. */ { @@ -448,7 +451,7 @@ test_valid_service_v3(void *arg) } done: - ; + unmock_hostname_resolver(); } static void @@ -623,4 +626,3 @@ struct testcase_t hs_config_tests[] = { END_OF_TESTCASES }; - diff --git a/src/test/test_options.c b/src/test/test_options.c index 2d45ecd189..d8757491fa 100644 --- a/src/test/test_options.c +++ b/src/test/test_options.c @@ -14,6 +14,7 @@ #include "feature/nodelist/routerset.h" #include "core/mainloop/mainloop.h" #include "test/log_test_helpers.h" +#include "test/resolve_test_helpers.h" #include "lib/sandbox/sandbox.h" #include "lib/memarea/memarea.h" @@ -241,6 +242,7 @@ test_options_validate(void *arg) (void)arg; setup_log_callback(); sandbox_disable_getaddrinfo_cache(); + mock_hostname_resolver(); WANT_ERR("ExtORPort 500000", "Invalid ExtORPort", PH_VALIDATE); @@ -282,6 +284,7 @@ test_options_validate(void *arg) close_temp_logs(); clear_log_messages(); + unmock_hostname_resolver(); return; } From aa56465934da31eeab79376a2ab690c5f28b4aa0 Mon Sep 17 00:00:00 2001 From: Roger Dingledine Date: Mon, 7 Oct 2019 00:48:11 -0400 Subject: [PATCH 1640/2557] fix typo in how-to-review guidelines --- doc/HACKING/HowToReview.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/HACKING/HowToReview.md b/doc/HACKING/HowToReview.md index 2d1f3d1c9e..2325e70175 100644 --- a/doc/HACKING/HowToReview.md +++ b/doc/HACKING/HowToReview.md @@ -63,7 +63,7 @@ Let's look at the code! Let's look at the documentation! -------------------------------- -- Does the documentation confirm to CodingStandards.txt? +- Does the documentation conform to CodingStandards.txt? - Does it make sense? From 5fb5019a946d2ea97946e6865e001ce3a9b912b0 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 7 Oct 2019 09:32:26 -0400 Subject: [PATCH 1641/2557] bump to 0.4.2.2-alpha-dev --- configure.ac | 4 ++-- contrib/win32build/tor-mingw.nsi.in | 2 +- src/win32/orconfig.h | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/configure.ac b/configure.ac index e623b33ad5..8cf8fe8a48 100644 --- a/configure.ac +++ b/configure.ac @@ -4,7 +4,7 @@ dnl Copyright (c) 2007-2019, The Tor Project, Inc. dnl See LICENSE for licensing information AC_PREREQ([2.63]) -AC_INIT([tor],[0.4.2.2-alpha]) +AC_INIT([tor],[0.4.2.2-alpha-dev]) AC_CONFIG_SRCDIR([src/app/main/tor_main.c]) AC_CONFIG_MACRO_DIR([m4]) @@ -14,7 +14,7 @@ AC_CONFIG_MACRO_DIR([m4]) # version number changes. Tor uses it to make sure that it # only shuts down for missing "required protocols" when those protocols # are listed as required by a consensus after this date. -AC_DEFINE(APPROX_RELEASE_DATE, ["2019-10-03"], # for 0.4.2.2-alpha +AC_DEFINE(APPROX_RELEASE_DATE, ["2019-10-07"], # for 0.4.2.2-alpha-dev [Approximate date when this software was released. (Updated when the version changes.)]) # "foreign" means we don't follow GNU package layout standards diff --git a/contrib/win32build/tor-mingw.nsi.in b/contrib/win32build/tor-mingw.nsi.in index bdd112a083..a0867d10ae 100644 --- a/contrib/win32build/tor-mingw.nsi.in +++ b/contrib/win32build/tor-mingw.nsi.in @@ -8,7 +8,7 @@ !include "LogicLib.nsh" !include "FileFunc.nsh" !insertmacro GetParameters -!define VERSION "0.4.2.2-alpha" +!define VERSION "0.4.2.2-alpha-dev" !define INSTALLER "tor-${VERSION}-win32.exe" !define WEBSITE "https://www.torproject.org/" !define LICENSE "LICENSE" diff --git a/src/win32/orconfig.h b/src/win32/orconfig.h index 6bcd18ef6f..ddcca8ebb0 100644 --- a/src/win32/orconfig.h +++ b/src/win32/orconfig.h @@ -218,7 +218,7 @@ #define USING_TWOS_COMPLEMENT /* Version number of package */ -#define VERSION "0.4.2.2-alpha" +#define VERSION "0.4.2.2-alpha-dev" From 081bd37315a57616144a8e6272a2ea796ace989e Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Tue, 8 Oct 2019 18:59:27 +0300 Subject: [PATCH 1642/2557] Fix flapping of test_service_intro_point() unittest. --- changes/bug31995 | 3 +++ src/test/test_hs_service.c | 4 +++- 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 changes/bug31995 diff --git a/changes/bug31995 b/changes/bug31995 new file mode 100644 index 0000000000..c7ddd437a6 --- /dev/null +++ b/changes/bug31995 @@ -0,0 +1,3 @@ + o Minor bugfixes (testing): + - Avoid intermittent test failures due to a test that had relied on + inconsistent timing sources. Fixes bug 31995; bugfix on 0.3.1.3-alpha. diff --git a/src/test/test_hs_service.c b/src/test/test_hs_service.c index c5854f0ff8..8993ce3fe2 100644 --- a/src/test/test_hs_service.c +++ b/src/test/test_hs_service.c @@ -674,9 +674,11 @@ test_service_intro_point(void *arg) (void) arg; + update_approx_time(1481621834); + /* Test simple creation of an object. */ { - time_t now = time(NULL); + time_t now = approx_time(); ip = helper_create_service_ip(); tt_assert(ip); /* Make sure the authentication keypair is not zeroes. */ From c42a79499afe45304b9dcdd63357c851f9a6d115 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 8 Oct 2019 10:41:49 -0400 Subject: [PATCH 1643/2557] Extract the enumeration that tells Tor what command it is running. --- src/app/config/or_options_st.h | 8 ++------ src/app/config/tor_cmdline_mode.h | 32 +++++++++++++++++++++++++++++++ src/core/include.am | 1 + 3 files changed, 35 insertions(+), 6 deletions(-) create mode 100644 src/app/config/tor_cmdline_mode.h diff --git a/src/app/config/or_options_st.h b/src/app/config/or_options_st.h index 32dcd9fb18..21ddbb18b6 100644 --- a/src/app/config/or_options_st.h +++ b/src/app/config/or_options_st.h @@ -15,6 +15,7 @@ #include "lib/cc/torint.h" #include "lib/net/address.h" +#include "app/config/tor_cmdline_mode.h" struct smartlist_t; struct config_line_t; @@ -31,12 +32,7 @@ struct or_options_t { uint32_t magic_; /** What should the tor process actually do? */ - enum { - CMD_RUN_TOR=0, CMD_LIST_FINGERPRINT, CMD_HASH_PASSWORD, - CMD_VERIFY_CONFIG, CMD_RUN_UNITTESTS, CMD_DUMP_CONFIG, - CMD_KEYGEN, - CMD_KEY_EXPIRATION, - } command; + tor_cmdline_mode_t command; char *command_arg; /**< Argument for command-line option. */ struct config_line_t *Logs; /**< New-style list of configuration lines diff --git a/src/app/config/tor_cmdline_mode.h b/src/app/config/tor_cmdline_mode.h new file mode 100644 index 0000000000..9bc9a68d56 --- /dev/null +++ b/src/app/config/tor_cmdline_mode.h @@ -0,0 +1,32 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file tor_cmdline_mode.h + * \brief Declare the tor_cmdline_mode_t enumeration + **/ + +#ifndef TOR_CMDLINE_MODE_H +#define TOR_CMDLINE_MODE_H + +/** + * Enumeration to describe which command Tor is running. These commands + * are controlled by command-line options. + **/ +typedef enum tor_cmdline_mode_t { + CMD_RUN_TOR=0, /**< The default: run Tor as a daemon. */ + CMD_LIST_FINGERPRINT, /**< Running --list-fingerprint. */ + CMD_HASH_PASSWORD, /**< Running --hash-password. */ + CMD_VERIFY_CONFIG, /**< Running --verify-config. */ + CMD_DUMP_CONFIG, /**< Running --dump-config. */ + CMD_KEYGEN, /**< Running --keygen */ + CMD_KEY_EXPIRATION, /**< Running --key-expiration */ + CMD_RUN_UNITTESTS, /**< Special value: indicates that we have entered + * the Tor code from the unit tests, not from the + * regular Tor binary at all. */ +} tor_cmdline_mode_t; + +#endif /* !defined(TOR_CMDLINE_MODE_H) */ diff --git a/src/core/include.am b/src/core/include.am index 9b4b251c81..64004da342 100644 --- a/src/core/include.am +++ b/src/core/include.am @@ -216,6 +216,7 @@ noinst_HEADERS += \ src/app/config/or_options_st.h \ src/app/config/or_state_st.h \ src/app/config/statefile.h \ + src/app/config/tor_cmdline_mode.h \ src/app/main/main.h \ src/app/main/ntmain.h \ src/app/main/shutdown.h \ From 4c25ea6703296d068a133da9821643f87c5f5316 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 8 Oct 2019 10:46:16 -0400 Subject: [PATCH 1644/2557] Rename TAKES_NO_ARGUMENT to ARGUMENT_NONE. I'm doing this for consistency, so that all the values for this enum have the same prefix. This is an automated commit, generated by the following shell commands: for fn in $(git ls-tree --name-only -r HEAD src |grep '\.[ch]$'); do \ perl -i -pe 's!\bTAKES_NO_ARGUMENT\b!ARGUMENT_NONE!g;' "$fn"; \ done --- src/app/config/config.c | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/src/app/config/config.c b/src/app/config/config.c index deda2448b6..561fc65456 100644 --- a/src/app/config/config.c +++ b/src/app/config/config.c @@ -2448,7 +2448,7 @@ options_act(const or_options_t *old_options) } typedef enum { - TAKES_NO_ARGUMENT = 0, + ARGUMENT_NONE = 0, ARGUMENT_NECESSARY = 1, ARGUMENT_OPTIONAL = 2 } takes_argument_t; @@ -2458,29 +2458,29 @@ static const struct { takes_argument_t takes_argument; } CMDLINE_ONLY_OPTIONS[] = { { "-f", ARGUMENT_NECESSARY }, - { "--allow-missing-torrc", TAKES_NO_ARGUMENT }, + { "--allow-missing-torrc", ARGUMENT_NONE }, { "--defaults-torrc", ARGUMENT_NECESSARY }, { "--hash-password", ARGUMENT_NECESSARY }, { "--dump-config", ARGUMENT_OPTIONAL }, - { "--list-fingerprint", TAKES_NO_ARGUMENT }, - { "--keygen", TAKES_NO_ARGUMENT }, + { "--list-fingerprint", ARGUMENT_NONE }, + { "--keygen", ARGUMENT_NONE }, { "--key-expiration", ARGUMENT_OPTIONAL }, - { "--newpass", TAKES_NO_ARGUMENT }, - { "--no-passphrase", TAKES_NO_ARGUMENT }, + { "--newpass", ARGUMENT_NONE }, + { "--no-passphrase", ARGUMENT_NONE }, { "--passphrase-fd", ARGUMENT_NECESSARY }, - { "--verify-config", TAKES_NO_ARGUMENT }, - { "--ignore-missing-torrc", TAKES_NO_ARGUMENT }, - { "--quiet", TAKES_NO_ARGUMENT }, - { "--hush", TAKES_NO_ARGUMENT }, - { "--version", TAKES_NO_ARGUMENT }, - { "--list-modules", TAKES_NO_ARGUMENT }, - { "--library-versions", TAKES_NO_ARGUMENT }, - { "-h", TAKES_NO_ARGUMENT }, - { "--help", TAKES_NO_ARGUMENT }, - { "--list-torrc-options", TAKES_NO_ARGUMENT }, - { "--list-deprecated-options",TAKES_NO_ARGUMENT }, - { "--nt-service", TAKES_NO_ARGUMENT }, - { "-nt-service", TAKES_NO_ARGUMENT }, + { "--verify-config", ARGUMENT_NONE }, + { "--ignore-missing-torrc", ARGUMENT_NONE }, + { "--quiet", ARGUMENT_NONE }, + { "--hush", ARGUMENT_NONE }, + { "--version", ARGUMENT_NONE }, + { "--list-modules", ARGUMENT_NONE }, + { "--library-versions", ARGUMENT_NONE }, + { "-h", ARGUMENT_NONE }, + { "--help", ARGUMENT_NONE }, + { "--list-torrc-options", ARGUMENT_NONE }, + { "--list-deprecated-options",ARGUMENT_NONE }, + { "--nt-service", ARGUMENT_NONE }, + { "-nt-service", ARGUMENT_NONE }, { NULL, 0 }, }; @@ -2552,7 +2552,7 @@ config_parse_commandline(int argc, char **argv, int ignore_errors, } else if (want_arg == ARGUMENT_OPTIONAL && is_last) { arg = tor_strdup(""); } else { - arg = (want_arg != TAKES_NO_ARGUMENT) ? tor_strdup(argv[i+1]) : + arg = (want_arg != ARGUMENT_NONE) ? tor_strdup(argv[i+1]) : tor_strdup(""); } From 6cea2bd498b15aac14b884916b7dffa76f0547b7 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 8 Oct 2019 10:50:53 -0400 Subject: [PATCH 1645/2557] Document takes_argument_t and its members. --- src/app/config/config.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/app/config/config.c b/src/app/config/config.c index 561fc65456..44d6a82e56 100644 --- a/src/app/config/config.c +++ b/src/app/config/config.c @@ -2447,9 +2447,15 @@ options_act(const or_options_t *old_options) return 0; } +/** + * Enumeration to describe the syntax for a command-line option. + **/ typedef enum { + /** Describe an option that does not take an argument. */ ARGUMENT_NONE = 0, + /** Describes an option that takes a single argument. */ ARGUMENT_NECESSARY = 1, + /** Describes an option that takes a single optinal argument. */ ARGUMENT_OPTIONAL = 2 } takes_argument_t; From bd5ce112c92b81a98023513a53ecd64cf44122fe Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 8 Oct 2019 10:54:57 -0400 Subject: [PATCH 1646/2557] Use C99 struct-initializer syntax in COMMANDLINE_ONLY_OPTIONS I'm about to add more fields to this table, and this syntax change will make it easier to do so. --- src/app/config/config.c | 54 +++++++++++++++++++++++------------------ 1 file changed, 30 insertions(+), 24 deletions(-) diff --git a/src/app/config/config.c b/src/app/config/config.c index 44d6a82e56..26b1048022 100644 --- a/src/app/config/config.c +++ b/src/app/config/config.c @@ -2463,30 +2463,36 @@ static const struct { const char *name; takes_argument_t takes_argument; } CMDLINE_ONLY_OPTIONS[] = { - { "-f", ARGUMENT_NECESSARY }, - { "--allow-missing-torrc", ARGUMENT_NONE }, - { "--defaults-torrc", ARGUMENT_NECESSARY }, - { "--hash-password", ARGUMENT_NECESSARY }, - { "--dump-config", ARGUMENT_OPTIONAL }, - { "--list-fingerprint", ARGUMENT_NONE }, - { "--keygen", ARGUMENT_NONE }, - { "--key-expiration", ARGUMENT_OPTIONAL }, - { "--newpass", ARGUMENT_NONE }, - { "--no-passphrase", ARGUMENT_NONE }, - { "--passphrase-fd", ARGUMENT_NECESSARY }, - { "--verify-config", ARGUMENT_NONE }, - { "--ignore-missing-torrc", ARGUMENT_NONE }, - { "--quiet", ARGUMENT_NONE }, - { "--hush", ARGUMENT_NONE }, - { "--version", ARGUMENT_NONE }, - { "--list-modules", ARGUMENT_NONE }, - { "--library-versions", ARGUMENT_NONE }, - { "-h", ARGUMENT_NONE }, - { "--help", ARGUMENT_NONE }, - { "--list-torrc-options", ARGUMENT_NONE }, - { "--list-deprecated-options",ARGUMENT_NONE }, - { "--nt-service", ARGUMENT_NONE }, - { "-nt-service", ARGUMENT_NONE }, + { .name="-f", + .takes_argument=ARGUMENT_NECESSARY }, + { .name="--allow-missing-torrc" }, + { .name="--defaults-torrc", + .takes_argument=ARGUMENT_NECESSARY }, + { .name="--hash-password", + .takes_argument=ARGUMENT_NECESSARY }, + { .name="--dump-config", + .takes_argument=ARGUMENT_OPTIONAL }, + { .name="--list-fingerprint" }, + { .name="--keygen" }, + { .name="--key-expiration", + .takes_argument=ARGUMENT_OPTIONAL }, + { .name="--newpass" }, + { .name="--no-passphrase" }, + { .name="--passphrase-fd", + .takes_argument=ARGUMENT_NECESSARY }, + { .name="--verify-config" }, + { .name="--ignore-missing-torrc" }, + { .name="--quiet" }, + { .name="--hush" }, + { .name="--version" }, + { .name="--list-modules" }, + { .name="--library-versions" }, + { .name="-h" }, + { .name="--help" }, + { .name="--list-torrc-options" }, + { .name="--list-deprecated-options" }, + { .name="--nt-service" }, + { .name="-nt-service" }, { NULL, 0 }, }; From 911b16e6e7ba657024734e7a36fad56c22c740a9 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 8 Oct 2019 11:37:46 -0400 Subject: [PATCH 1647/2557] config.c: make a couple of arguments const. These functions do not modify their inputs, so they can take const arguments. --- src/app/config/config.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/app/config/config.c b/src/app/config/config.c index 26b1048022..1abe1e13a3 100644 --- a/src/app/config/config.c +++ b/src/app/config/config.c @@ -5060,12 +5060,12 @@ normalize_nickname_list(config_line_t **normalized_out, * filename if it doesn't exist. */ static char * -find_torrc_filename(config_line_t *cmd_arg, +find_torrc_filename(const config_line_t *cmd_arg, int defaults_file, int *using_default_fname, int *ignore_missing_torrc) { char *fname=NULL; - config_line_t *p_index; + const config_line_t *p_index; const char *fname_opt = defaults_file ? "--defaults-torrc" : "-f"; const char *ignore_opt = defaults_file ? NULL : "--ignore-missing-torrc"; @@ -5144,7 +5144,7 @@ load_torrc_from_stdin(void) * Return the contents of the file on success, and NULL on failure. */ static char * -load_torrc_from_disk(config_line_t *cmd_arg, int defaults_file) +load_torrc_from_disk(const config_line_t *cmd_arg, int defaults_file) { char *fname=NULL; char *cf = NULL; From f50de3a91872014f03856cf4c889f029ec5a1892 Mon Sep 17 00:00:00 2001 From: David Goulet Date: Tue, 10 Sep 2019 14:40:40 -0400 Subject: [PATCH 1648/2557] hs-v3: Do not remove intro point if circuit exists When considering introduction point of a service's descriptor, do not remove an intro point that has an established or pending circuit. Fixes #31652 Signed-off-by: David Goulet --- changes/bug31652 | 5 +++ src/feature/hs/hs_service.c | 87 +++++++++++++++++++++++++++---------- src/test/test_hs_service.c | 1 + 3 files changed, 70 insertions(+), 23 deletions(-) create mode 100644 changes/bug31652 diff --git a/changes/bug31652 b/changes/bug31652 new file mode 100644 index 0000000000..c4eca7994a --- /dev/null +++ b/changes/bug31652 @@ -0,0 +1,5 @@ + o Minor bugfixes (onion services): + - When we clean up intro circuits for a v3 onion service, don't remove + circuits that have an established or pending circuit even if ran out of + retries. This way, we don't cleanup the circuit of the last retry. Fixes + bug 31652; bugfix on 0.3.2.1-alpha. diff --git a/src/feature/hs/hs_service.c b/src/feature/hs/hs_service.c index f81987f69f..18c38ebc0a 100644 --- a/src/feature/hs/hs_service.c +++ b/src/feature/hs/hs_service.c @@ -2333,15 +2333,70 @@ intro_point_should_expire(const hs_service_intro_point_t *ip, return 1; } -/* Go over the given set of intro points for each service and remove any - * invalid ones. The conditions for removal are: +/* Return true iff we should remove the intro point ip from its service. * - * - The node doesn't exists anymore (not in consensus) - * OR - * - The intro point maximum circuit retry count has been reached and no - * circuit can be found associated with it. - * OR - * - The intro point has expired and we should pick a new one. + * We remove an intro point from the service descriptor list if one of + * these criteria is met: + * - It has expired (either in INTRO2 count or in time). + * - No node was found (fell off the consensus). + * - We are over the maximum amount of retries. + * + * If an established or pending circuit is found for the given ip object, this + * return false indicating it should not be removed. */ +static bool +should_remove_intro_point(hs_service_intro_point_t *ip, time_t now) +{ + bool ret = false; + + tor_assert(ip); + + /* Any one of the following needs to be True to furfill the criteria to + * remove an intro point. */ + bool has_no_retries = (ip->circuit_retries > + MAX_INTRO_POINT_CIRCUIT_RETRIES); + bool has_no_node = (get_node_from_intro_point(ip) == NULL); + bool has_expired = intro_point_should_expire(ip, now); + + /* If the node fell off the consensus or the IP has expired, we have to + * remove it now. */ + if (has_no_node || has_expired) { + ret = true; + goto end; + } + + /* Pass this point, even though we might be over the retry limit, we check + * if a circuit (established or pending) exists. In that case, we should not + * remove it because it might simply be valid and opened at the previous + * scheduled event for the last retry. */ + + /* Did we established already? */ + if (ip->circuit_established) { + goto end; + } + /* Do we simply have an existing circuit regardless of its state? */ + if (hs_circ_service_get_intro_circ(ip)) { + goto end; + } + + /* Getting here means we have _no_ circuits so then return if we have any + * remaining retries. */ + ret = has_no_retries; + + end: + /* Meaningful log in case we are about to remove the IP. */ + if (ret) { + log_info(LD_REND, "Intro point %s%s (retried: %u times). " + "Removing it.", + describe_intro_point(ip), + has_expired ? " has expired" : + (has_no_node) ? " fell off the consensus" : "", + ip->circuit_retries); + } + return ret; +} + +/* Go over the given set of intro points for each service and remove any + * invalid ones. * * If an intro point is removed, the circuit (if any) is immediately close. * If a circuit can't be found, the intro point is kept if it hasn't reached @@ -2366,21 +2421,7 @@ cleanup_intro_points(hs_service_t *service, time_t now) * valid and remove any of them that aren't. */ DIGEST256MAP_FOREACH_MODIFY(desc->intro_points.map, key, hs_service_intro_point_t *, ip) { - const node_t *node = get_node_from_intro_point(ip); - int has_expired = intro_point_should_expire(ip, now); - - /* We cleanup an intro point if it has expired or if we do not know the - * node_t anymore (removed from our latest consensus) or if we've - * reached the maximum number of retry with a non existing circuit. */ - if (has_expired || node == NULL || - ip->circuit_retries > MAX_INTRO_POINT_CIRCUIT_RETRIES) { - log_info(LD_REND, "Intro point %s%s (retried: %u times). " - "Removing it.", - describe_intro_point(ip), - has_expired ? " has expired" : - (node == NULL) ? " fell off the consensus" : "", - ip->circuit_retries); - + if (should_remove_intro_point(ip, now)) { /* We've retried too many times, remember it as a failed intro point * so we don't pick it up again for INTRO_CIRC_RETRY_PERIOD sec. */ if (ip->circuit_retries > MAX_INTRO_POINT_CIRCUIT_RETRIES) { diff --git a/src/test/test_hs_service.c b/src/test/test_hs_service.c index c5854f0ff8..a2594ed6af 100644 --- a/src/test/test_hs_service.c +++ b/src/test/test_hs_service.c @@ -1300,6 +1300,7 @@ test_service_event(void *arg) OP_EQ, 1); /* Remove the IP object at once for the next test. */ ip->circuit_retries = MAX_INTRO_POINT_CIRCUIT_RETRIES + 1; + ip->circuit_established = 0; run_housekeeping_event(now); tt_int_op(digest256map_size(service->desc_current->intro_points.map), OP_EQ, 0); From 7c1b2fceb7d46aed8945d86806dc791f97c47125 Mon Sep 17 00:00:00 2001 From: Neel Chauhan Date: Wed, 18 Sep 2019 11:40:10 -0400 Subject: [PATCH 1649/2557] test: New behavior on IP retry for HSv3 Unit test for #31652 where if we are over the retry limit for the IP but we have an established circuit, we don't remove the IP. Part of #31652 --- src/test/test_hs_service.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/test/test_hs_service.c b/src/test/test_hs_service.c index a2594ed6af..efe4166bf9 100644 --- a/src/test/test_hs_service.c +++ b/src/test/test_hs_service.c @@ -1296,6 +1296,11 @@ test_service_event(void *arg) service_intro_point_add(service->desc_current->intro_points.map, ip); ip->circuit_established = 1; /* We'll test that, it MUST be 0 after. */ run_housekeeping_event(now); + tt_int_op(digest256map_size(service->desc_current->intro_points.map), + OP_EQ, 1); + /* No removal if we have an established circuit after retries. */ + ip->circuit_retries = MAX_INTRO_POINT_CIRCUIT_RETRIES + 1; + run_housekeeping_event(now); tt_int_op(digest256map_size(service->desc_current->intro_points.map), OP_EQ, 1); /* Remove the IP object at once for the next test. */ From 4ab85f4928be7ed096643bd5f7d8d5f2b42859fe Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 9 Oct 2019 09:23:49 -0400 Subject: [PATCH 1650/2557] Document tor-coccinelle.h --- scripts/coccinelle/tor-coccinelle.h | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/scripts/coccinelle/tor-coccinelle.h b/scripts/coccinelle/tor-coccinelle.h index a94c3ef679..f597e912b4 100644 --- a/scripts/coccinelle/tor-coccinelle.h +++ b/scripts/coccinelle/tor-coccinelle.h @@ -1,3 +1,22 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/* + * This file looks like a C header, but its purpose is a bit different. + * + * We never include it from our real C files; we only tell Coccinelle + * about it in apply.sh. + * + * It tells the Coccinelle semantic patching tool how to understand + * things that would otherwise not be good C syntax, or which would + * otherwise not make sense to it as C. It doesn't need to produce + * semantically equivalent C, or even correct C: it only has to produce + * syntactically valid C. + */ + #define MOCK_IMPL(a, b, c) a b c #define CHECK_PRINTF(a, b) #define STATIC static From b356b3907a6424d9e1a14722e9729529862a698f Mon Sep 17 00:00:00 2001 From: David Goulet Date: Wed, 2 Oct 2019 13:19:51 -0400 Subject: [PATCH 1651/2557] hs-v3: Fix implicit ssize_t to size_t conversion Found by Coverity. Fixes #31682 Signed-off-by: David Goulet --- changes/ticket31682 | 3 +++ src/feature/hs/hs_cell.c | 26 +++++++++++++++++++++----- 2 files changed, 24 insertions(+), 5 deletions(-) create mode 100644 changes/ticket31682 diff --git a/changes/ticket31682 b/changes/ticket31682 new file mode 100644 index 0000000000..9777dec1f3 --- /dev/null +++ b/changes/ticket31682 @@ -0,0 +1,3 @@ + o Minor bugfixes (hidden service v3, coverity): + - Fix an implicit conversion from ssize_t to size_t discovered by Coverity. + Fixes bug 31682; bugfix on 0.4.2.1-alpha. diff --git a/src/feature/hs/hs_cell.c b/src/feature/hs/hs_cell.c index 547dda3e16..3147b898bc 100644 --- a/src/feature/hs/hs_cell.c +++ b/src/feature/hs/hs_cell.c @@ -495,11 +495,12 @@ build_establish_intro_dos_param(trn_cell_extension_dos_t *dos_ext, /* Build the DoS defense cell extension and put it in the given extensions * object. This can't fail. */ -static void +static int build_establish_intro_dos_extension(const hs_service_config_t *service_config, trn_cell_extension_t *extensions) { - ssize_t ret, dos_ext_encoded_len; + ssize_t ret; + size_t dos_ext_encoded_len; uint8_t *field_array; trn_cell_extension_field_t *field; trn_cell_extension_dos_t *dos_ext; @@ -526,7 +527,11 @@ build_establish_intro_dos_extension(const hs_service_config_t *service_config, service_config->intro_dos_burst_per_sec); /* Set the field with the encoded DoS extension. */ - dos_ext_encoded_len = trn_cell_extension_dos_encoded_len(dos_ext); + ret = trn_cell_extension_dos_encoded_len(dos_ext); + if (BUG(ret <= 0)) { + return -1; + } + dos_ext_encoded_len = ret; /* Set length field and the field array size length. */ trn_cell_extension_field_set_field_len(field, dos_ext_encoded_len); trn_cell_extension_field_setlen_field(field, dos_ext_encoded_len); @@ -534,7 +539,10 @@ build_establish_intro_dos_extension(const hs_service_config_t *service_config, field_array = trn_cell_extension_field_getarray_field(field); ret = trn_cell_extension_dos_encode(field_array, trn_cell_extension_field_getlen_field(field), dos_ext); - tor_assert(ret == dos_ext_encoded_len); + if (BUG(ret <= 0)) { + return -1; + } + tor_assert(ret == (ssize_t) dos_ext_encoded_len); /* Finally, encode field into the cell extension. */ trn_cell_extension_add_fields(extensions, field); @@ -546,6 +554,8 @@ build_establish_intro_dos_extension(const hs_service_config_t *service_config, /* Cleanup. DoS extension has been encoded at this point. */ trn_cell_extension_dos_free(dos_ext); + + return 0; } /* ========== */ @@ -558,6 +568,7 @@ STATIC trn_cell_extension_t * build_establish_intro_extensions(const hs_service_config_t *service_config, const hs_service_intro_point_t *ip) { + int ret; trn_cell_extension_t *extensions; tor_assert(service_config); @@ -571,9 +582,14 @@ build_establish_intro_extensions(const hs_service_config_t *service_config, if (service_config->has_dos_defense_enabled && ip->support_intro2_dos_defense) { /* This function takes care to increment the number of extensions. */ - build_establish_intro_dos_extension(service_config, extensions); + ret = build_establish_intro_dos_extension(service_config, extensions); + if (ret < 0) { + /* Return no extensions on error. */ + goto end; + } } + end: return extensions; } From 46efc0ff35a12206ee2d17bf46dade29877594ed Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 11 Oct 2019 10:21:45 -0400 Subject: [PATCH 1652/2557] Correct comment on build_establish_intro_dos_extension --- src/feature/hs/hs_cell.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/feature/hs/hs_cell.c b/src/feature/hs/hs_cell.c index 3147b898bc..d691a1b007 100644 --- a/src/feature/hs/hs_cell.c +++ b/src/feature/hs/hs_cell.c @@ -494,7 +494,8 @@ build_establish_intro_dos_param(trn_cell_extension_dos_t *dos_ext, } /* Build the DoS defense cell extension and put it in the given extensions - * object. This can't fail. */ + * object. Return 0 on success, -1 on failure. (Right now, failure is only + * possible if there is a bug.) */ static int build_establish_intro_dos_extension(const hs_service_config_t *service_config, trn_cell_extension_t *extensions) @@ -1075,4 +1076,3 @@ hs_cell_introduce1_data_clear(hs_cell_introduce1_data_t *data) /* The data object has no ownership of any members. */ memwipe(data, 0, sizeof(hs_cell_introduce1_data_t)); } - From 61fe3f448c1c60b6681797feebf2611e458adc47 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 11 Oct 2019 17:15:04 -0400 Subject: [PATCH 1653/2557] Bump master to 0.4.3.0-alpha-dev. --- configure.ac | 4 ++-- contrib/win32build/tor-mingw.nsi.in | 2 +- src/win32/orconfig.h | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/configure.ac b/configure.ac index f610fb0a07..bd961e74e1 100644 --- a/configure.ac +++ b/configure.ac @@ -4,7 +4,7 @@ dnl Copyright (c) 2007-2019, The Tor Project, Inc. dnl See LICENSE for licensing information AC_PREREQ([2.63]) -AC_INIT([tor],[0.4.2.2-alpha-dev]) +AC_INIT([tor],[0.4.3.0-alpha-dev]) AC_CONFIG_SRCDIR([src/app/main/tor_main.c]) AC_CONFIG_MACRO_DIR([m4]) @@ -14,7 +14,7 @@ AC_CONFIG_MACRO_DIR([m4]) # version number changes. Tor uses it to make sure that it # only shuts down for missing "required protocols" when those protocols # are listed as required by a consensus after this date. -AC_DEFINE(APPROX_RELEASE_DATE, ["2019-10-07"], # for 0.4.2.2-alpha-dev +AC_DEFINE(APPROX_RELEASE_DATE, ["2019-10-11"], # for 0.4.3.0-alpha-dev [Approximate date when this software was released. (Updated when the version changes.)]) # "foreign" means we don't follow GNU package layout standards diff --git a/contrib/win32build/tor-mingw.nsi.in b/contrib/win32build/tor-mingw.nsi.in index a0867d10ae..990f4933fe 100644 --- a/contrib/win32build/tor-mingw.nsi.in +++ b/contrib/win32build/tor-mingw.nsi.in @@ -8,7 +8,7 @@ !include "LogicLib.nsh" !include "FileFunc.nsh" !insertmacro GetParameters -!define VERSION "0.4.2.2-alpha-dev" +!define VERSION "0.4.3.0-alpha-dev" !define INSTALLER "tor-${VERSION}-win32.exe" !define WEBSITE "https://www.torproject.org/" !define LICENSE "LICENSE" diff --git a/src/win32/orconfig.h b/src/win32/orconfig.h index ddcca8ebb0..5fb1340465 100644 --- a/src/win32/orconfig.h +++ b/src/win32/orconfig.h @@ -218,7 +218,7 @@ #define USING_TWOS_COMPLEMENT /* Version number of package */ -#define VERSION "0.4.2.2-alpha-dev" +#define VERSION "0.4.3.0-alpha-dev" From a4790e7d4b6ef406bf8cd14d8fd29e89f4b334ac Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 11 Oct 2019 17:18:55 -0400 Subject: [PATCH 1654/2557] maint-0.4.2: remove ".enable_practracker_in_hooks". --- scripts/maint/practracker/.enable_practracker_in_hooks | 1 - 1 file changed, 1 deletion(-) delete mode 100644 scripts/maint/practracker/.enable_practracker_in_hooks diff --git a/scripts/maint/practracker/.enable_practracker_in_hooks b/scripts/maint/practracker/.enable_practracker_in_hooks deleted file mode 100644 index a9e707f5da..0000000000 --- a/scripts/maint/practracker/.enable_practracker_in_hooks +++ /dev/null @@ -1 +0,0 @@ -This file is present to tell our git hooks to run practracker on this branch. From 78f4e292192b3a5750b8801c836b512fe7b26ead Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 11 Oct 2019 17:26:34 -0400 Subject: [PATCH 1655/2557] git scripts: add maint/release-0.4.2. --- scripts/git/git-merge-forward.sh | 12 +++++++++++- scripts/git/git-pull-all.sh | 7 +++++++ scripts/git/git-push-all.sh | 5 +++++ 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/scripts/git/git-merge-forward.sh b/scripts/git/git-merge-forward.sh index e481b40975..d16519b3f5 100755 --- a/scripts/git/git-merge-forward.sh +++ b/scripts/git/git-merge-forward.sh @@ -98,13 +98,16 @@ MAINT_040=( "maint-0.4.0" "maint-0.3.5" "$GIT_PATH/$TOR_WKT_NAME/maint-0.4.0" \ "_040" "_035") MAINT_041=( "maint-0.4.1" "maint-0.4.0" "$GIT_PATH/$TOR_WKT_NAME/maint-0.4.1" \ "_041" "_040") -MAINT_MASTER=( "master" "maint-0.4.1" "$GIT_PATH/$TOR_MASTER_NAME" \ +MAINT_042=( "maint-0.4.2" "maint-0.4.1" "$GIT_PATH/$TOR_WKT_NAME/maint-0.4.2" \ + "_042" "_041") +MAINT_MASTER=( "master" "maint-0.4.2" "$GIT_PATH/$TOR_MASTER_NAME" \ "_master" "_041") RELEASE_029=( "release-0.2.9" "maint-0.2.9" "$GIT_PATH/$TOR_WKT_NAME/release-0.2.9" ) RELEASE_035=( "release-0.3.5" "maint-0.3.5" "$GIT_PATH/$TOR_WKT_NAME/release-0.3.5" ) RELEASE_040=( "release-0.4.0" "maint-0.4.0" "$GIT_PATH/$TOR_WKT_NAME/release-0.4.0" ) RELEASE_041=( "release-0.4.1" "maint-0.4.1" "$GIT_PATH/$TOR_WKT_NAME/release-0.4.1" ) +RELEASE_042=( "release-0.4.2" "maint-0.4.2" "$GIT_PATH/$TOR_WKT_NAME/release-0.4.2" ) # The master branch path has to be the main repository thus contains the # origin that will be used to fetch the updates. All the worktrees are created @@ -117,11 +120,13 @@ ${MAINT_029_TB[0]} ${MAINT_035[0]} ${MAINT_040[0]} ${MAINT_041[0]} +${MAINT_042[0]} ${MAINT_MASTER[0]} ${RELEASE_029[0]} ${RELEASE_035[0]} ${RELEASE_040[0]} ${RELEASE_041[0]} +${RELEASE_042[0]} EOF ####################### @@ -186,6 +191,9 @@ if [ -z "$TEST_BRANCH_PREFIX" ]; then MAINT_041[@] RELEASE_041[@] + MAINT_042[@] + RELEASE_042[@] + MAINT_MASTER[@] ) @@ -201,6 +209,8 @@ else MAINT_041[@] + MAINT_042[@] + MAINT_MASTER[@] ) diff --git a/scripts/git/git-pull-all.sh b/scripts/git/git-pull-all.sh index 0d6daf432d..773f97b42b 100755 --- a/scripts/git/git-pull-all.sh +++ b/scripts/git/git-pull-all.sh @@ -62,12 +62,14 @@ MAINT_029=( "maint-0.2.9" "$GIT_PATH/$TOR_WKT_NAME/maint-0.2.9" ) MAINT_035=( "maint-0.3.5" "$GIT_PATH/$TOR_WKT_NAME/maint-0.3.5" ) MAINT_040=( "maint-0.4.0" "$GIT_PATH/$TOR_WKT_NAME/maint-0.4.0" ) MAINT_041=( "maint-0.4.1" "$GIT_PATH/$TOR_WKT_NAME/maint-0.4.1" ) +MAINT_042=( "maint-0.4.2" "$GIT_PATH/$TOR_WKT_NAME/maint-0.4.2" ) MAINT_MASTER=( "master" "$GIT_PATH/$TOR_MASTER_NAME" ) RELEASE_029=( "release-0.2.9" "$GIT_PATH/$TOR_WKT_NAME/release-0.2.9" ) RELEASE_035=( "release-0.3.5" "$GIT_PATH/$TOR_WKT_NAME/release-0.3.5" ) RELEASE_040=( "release-0.4.0" "$GIT_PATH/$TOR_WKT_NAME/release-0.4.0" ) RELEASE_041=( "release-0.4.1" "$GIT_PATH/$TOR_WKT_NAME/release-0.4.1" ) +RELEASE_042=( "release-0.4.2" "$GIT_PATH/$TOR_WKT_NAME/release-0.4.2" ) # The master branch path has to be the main repository thus contains the # origin that will be used to fetch the updates. All the worktrees are created @@ -80,11 +82,13 @@ ${MAINT_029[0]} ${MAINT_035[0]} ${MAINT_040[0]} ${MAINT_041[0]} +${MAINT_042[0]} ${MAINT_MASTER[0]} ${RELEASE_029[0]} ${RELEASE_035[0]} ${RELEASE_040[0]} ${RELEASE_041[0]} +${RELEASE_042[0]} EOF ########################### @@ -106,6 +110,9 @@ WORKTREE=( MAINT_041[@] RELEASE_041[@] + MAINT_042[@] + RELEASE_042[@] + MAINT_MASTER[@] ) COUNT=${#WORKTREE[@]} diff --git a/scripts/git/git-push-all.sh b/scripts/git/git-push-all.sh index a388f01564..f848da7841 100755 --- a/scripts/git/git-push-all.sh +++ b/scripts/git/git-push-all.sh @@ -167,6 +167,7 @@ DEFAULT_UPSTREAM_BRANCHES= if [ "$DEFAULT_UPSTREAM_REMOTE" != "$UPSTREAM_REMOTE" ]; then DEFAULT_UPSTREAM_BRANCHES=$(echo \ "$DEFAULT_UPSTREAM_REMOTE"/master \ + "$DEFAULT_UPSTREAM_REMOTE"/{release,maint}-0.4.2 \ "$DEFAULT_UPSTREAM_REMOTE"/{release,maint}-0.4.1 \ "$DEFAULT_UPSTREAM_REMOTE"/{release,maint}-0.4.0 \ "$DEFAULT_UPSTREAM_REMOTE"/{release,maint}-0.3.5 \ @@ -176,6 +177,7 @@ fi UPSTREAM_BRANCHES=$(echo \ "$UPSTREAM_REMOTE"/master \ + "$UPSTREAM_REMOTE"/{release,maint}-0.4.2 \ "$UPSTREAM_REMOTE"/{release,maint}-0.4.1 \ "$UPSTREAM_REMOTE"/{release,maint}-0.4.0 \ "$UPSTREAM_REMOTE"/{release,maint}-0.3.5 \ @@ -188,6 +190,7 @@ UPSTREAM_BRANCHES=$(echo \ PUSH_BRANCHES=$(echo \ master \ + {release,maint}-0.4.2 \ {release,maint}-0.4.1 \ {release,maint}-0.4.0 \ {release,maint}-0.3.5 \ @@ -201,6 +204,7 @@ if [ -z "$TEST_BRANCH_PREFIX" ]; then # List of branches to push. Ordering is not important. PUSH_BRANCHES=$(echo \ master \ + {release,maint}-0.4.2 \ {release,maint}-0.4.1 \ {release,maint}-0.4.0 \ {release,maint}-0.3.5 \ @@ -213,6 +217,7 @@ else # List of branches to push. Ordering is not important. PUSH_BRANCHES=" \ ${TEST_BRANCH_PREFIX}_master \ + ${TEST_BRANCH_PREFIX}_042 \ ${TEST_BRANCH_PREFIX}_041 \ ${TEST_BRANCH_PREFIX}_040 \ ${TEST_BRANCH_PREFIX}_035 \ From 1b996b01f6cc5c9aec28ba7b962eeca727a36e9e Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 14 Oct 2019 10:57:15 -0400 Subject: [PATCH 1656/2557] 00-overview.md: Revise to describe intended document structure --- doc/HACKING/design/00-overview.md | 87 +++++++++++++++++++------------ 1 file changed, 53 insertions(+), 34 deletions(-) diff --git a/doc/HACKING/design/00-overview.md b/doc/HACKING/design/00-overview.md index 2103a9062a..fde476dfab 100644 --- a/doc/HACKING/design/00-overview.md +++ b/doc/HACKING/design/00-overview.md @@ -5,18 +5,20 @@ This document describes the general structure of the Tor codebase, how it fits together, what functionality is available for extending Tor, and gives some notes on how Tor got that way. -Tor remains a work in progress: We've been working on it for more than a -decade, and we've learned a lot about good coding since we first +Tor remains a work in progress: We've been working on it for nearly two +decades, and we've learned a lot about good coding since we first started. This means, however, that some of the older pieces of Tor will -have some "code smell" in them that could sure stand a brisk +have some "code smell" in them that could stand a brisk refactoring. So when I describe a piece of code, I'll sometimes give a note on how it got that way, and whether I still think that's a good idea. The first drafts of this document were written in the Summer and Fall of 2015, when Tor 0.2.6 was the most recent stable version, and Tor 0.2.7 -was under development. If you're reading this far in the future, some -things may have changed. Caveat haxxor! +was under development. There is a revision in progress (as of late +2019), to bring it up to pace with Tor as of version 0.4.2. If you're +reading this far in the future, some things may have changed. Caveat +haxxor! This document is not an overview of the Tor protocol. For that, see the design paper and the specifications at https://spec.torproject.org/ . @@ -24,8 +26,6 @@ design paper and the specifications at https://spec.torproject.org/ . For more information about Tor's coding standards and some helpful development tools, see doc/HACKING in the Tor repository. -For more information about writing tests, see doc/HACKING/WritingTests.txt -in the Tor repository. ### The very high level ### @@ -36,36 +36,60 @@ same codebase: the Tor process will run as a client, relay, or authority depending on its configuration. Tor has a few major dependencies, including Libevent (used to tell which -sockets are readable and writable), OpenSSL (used for many encryption +sockets are readable and writable), OpenSSL or NSS (used for many encryption functions, and to implement the TLS protocol), and zlib (used to compress and uncompress directory information). Most of Tor's work today is done in a single event-driven main thread. Tor also spawns one or more worker threads to handle CPU-intensive -tasks. (Right now, this only includes circuit encryption.) +tasks. (Right now, this only includes circuit encryption and the more +expensive compression algorithms.) On startup, Tor initializes its libraries, reads and responds to its configuration files, and launches a main event loop. At first, the only events that Tor listens for are a few signals (like TERM and HUP), and one or more listener sockets (for different kinds of incoming -connections). Tor also configures a timer function to run once per -second to handle periodic events. As Tor runs over time, other events -will open, and new events will be scheduled. +connections). Tor also configures several timers to handle periodic +events. As Tor runs over time, other events will open, and new events +will be scheduled. -The codebase is divided into a few main subdirectories: - - src/common -- utility functions, not necessarily tor-specific. - - src/or -- implements the Tor protocols. - - src/test -- unit and regression tests +The codebase is divided into a few top-level subdirectories, each of +which contains several sub-modules. src/ext -- Code maintained elsewhere that we include in the Tor source distribution. - src/trunnel -- automatically generated code (from the Trunnel) + src/lib -- Lower-level utility code, not necessarily tor-specific. + + src/trunnel -- Automatically generated code (from the Trunnel) tool: used to parse and encode binary formats. + src/core -- Networking code that is implements the central parts of + the Tor protocol and main loop. + + src/feature -- Aspects of Tor (like directory management, running a + relay, running a directory authorities, managing a list of nodes, + running and using onion services) that are built on top of the + mainloop code. + + src/app -- Highest-level functionality; responsible for setting up + and configuring the Tor project, making sure all the lower-level + modules start up when required, and so on. + + src/tools -- Binaries other than Tor that we produce. Currently this + is tor-resolve, tor-gencert, and the tor_runner.o helper module. + + src/test -- unit tests, regression tests, and a few integration + tests. + +In theory, the above parts of the codebase are sorted from highest-level +to lowest-level, where high-level code is only allowed to invoke +lower-level code, and lower-level code never includes or depends on code +of a higher level. In practice, this refactoring is incomplete: The +modules in src/lib are well-factored, but there are many "upward +dependencies" in src/core and src/feature. We aim to eliminate those +over time. + ### Some key high-level abstractions ### The most important abstractions at Tor's high-level are Connections, @@ -94,31 +118,26 @@ If we switch to other strategies in the future, we'll have more connection types. A 'Node' is a view of a Tor instance's current knowledge and opinions -about a Tor relay orbridge. +about a Tor relay or bridge. ### The rest of this document. ### > **Note**: This section describes the eventual organization of this > document, which is not yet complete. -We'll begin with an overview of the various utility functions available -in Tor's 'common' directory. Knowing about these is key to writing -portable, simple code in Tor. +We'll begin with an overview of the facilities provided by the modules +in src/lib. Knowing about these is key to writing portable, simple code +in Tor. + +Then we'll move on to a discussion of how parts of the Tor codebase are +initialized, finalized, configured, and managed. Then we'll go on and talk about the main data-flow of the Tor network: how Tor generates and responds to network traffic. This will occupy a chapter for the main overview, with other chapters for special topics. -After that, we'll mention the main modules in Tor, and describe the -function of each. - -We'll cover the directory subsystem next: how Tor learns about other -relays, and how relays advertise themselves. - -Then we'll cover a few specialized modules, such as hidden services, -sandboxing, hibernation, accounting, statistics, guards, path -generation, pluggable transports, and how they integrate with the rest of Tor. +After that, we'll mention the main modules in src/features and describe the +functions of each. We'll close with a meandering overview of important pending issues in the Tor codebase, and how they affect the future of the Tor software. - From 908070bbd5096efc09b251154dbc058559920f05 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 14 Oct 2019 10:59:32 -0400 Subject: [PATCH 1657/2557] Rename common-utils to lib-overview. --- doc/HACKING/design/{01-common-utils.md => 01.00-lib-overview.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename doc/HACKING/design/{01-common-utils.md => 01.00-lib-overview.md} (100%) diff --git a/doc/HACKING/design/01-common-utils.md b/doc/HACKING/design/01.00-lib-overview.md similarity index 100% rename from doc/HACKING/design/01-common-utils.md rename to doc/HACKING/design/01.00-lib-overview.md From 8ef5d96c2e7c026feff3a4dd20f0096f6d8cf901 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 14 Oct 2019 13:49:27 -0400 Subject: [PATCH 1658/2557] Rewrite "common" overview into a "lib" overview. --- doc/HACKING/design/01.00-lib-overview.md | 206 ++++++++++++++--------- 1 file changed, 128 insertions(+), 78 deletions(-) diff --git a/doc/HACKING/design/01.00-lib-overview.md b/doc/HACKING/design/01.00-lib-overview.md index 79a6a7b7d3..08dec51a00 100644 --- a/doc/HACKING/design/01.00-lib-overview.md +++ b/doc/HACKING/design/01.00-lib-overview.md @@ -1,121 +1,171 @@ -## Utility code in Tor +## Library code in Tor. -Most of Tor's utility code is in modules in the src/common subdirectory. +Most of Tor's utility code is in modules in the `src/lib` subdirectory. In +general, this code is not necessarily Tor-specific, but is instead possibly +useful for other applications. -These are divided, broadly, into _compatibility_ functions, _utility_ -functions, _containers_, and _cryptography_. (Someday in the future, it -would be great to split these modules into separate directories. Also, some -functions are probably put in the wrong modules) +This code includes: -### Compatibility code + * Compatibility wrappers, to provide a uniform API across different + platforms. -These functions live in src/common/compat\*.c; some corresponding macros live -in src/common/compat\*.h. They serve as wrappers around platform-specific or -compiler-specific logic functionality. + * Library wrappers, to provide a tor-like API over different libraries + that Tor uses for things like compression and cryptography. -In general, the rest of the Tor code *should not* be calling platform-specific -or otherwise non-portable functions. Instead, they should call wrappers from -compat.c, which implement a common cross-platform API. (If you don't know -whether a function is portable, it's usually good enough to see whether it -exists on OSX, Linux, and Windows.) + * Containers, to implement some general-purpose data container types. -Other compatibility modules include backtrace.c, which generates stack traces -for crash reporting; sandbox.c, which implements the Linux seccomp2 sandbox; -and procmon.c, which handles monitoring a child process. +The modules in `src/lib` are currently well-factored: each one depends +only on lower-level modules. You can see an up-to-date list of the +modules sorted from lowest to highest level by running +`./scripts/maint/practracker/includes.py --toposort`. -Parts of address.c are compatibility code for handling network addressing -issues; other parts are in util.c. +As of this writing, the library modules are (from lowest to highest +level): -Notable compatibility areas are: + * `lib/cc` -- Macros for managing the C compiler and + language. Includes macros for improving compatibility and clarity + across different C compilers. - * mmap support for mapping files into the address space (read-only) + * `lib/version` -- Holds the current version of Tor. - * Code to work around the intricacies + * `lib/testsupport` -- Helpers for making test-only code and test + mocking support. - * Workaround code for Windows's horrible winsock incompatibilities and - Linux's intricate socket extensions. + * `lib/defs` -- Lowest-level constants used in many places across the + code. - * Helpful string functions like memmem, memstr, asprintf, strlcpy, and - strlcat that not all platforms have. + * `lib/subsys` -- Types used for declaring a "subsystem". A subsystem + is a module with support for initialization, shutdown, + configuration, and so on. - * Locale-ignoring variants of the ctypes functions. + * `lib/conf` -- Types and macros used for declaring configuration + options. - * Time-manipulation functions + * `lib/arch` -- Compatibility functions and macros for handling + differences in CPU architecture. - * File locking function + * `lib/err` -- Lowest-level error handling code: responsible for + generating stack traces, handling raw assertion failures, and + otherwise reporting problems that might not be safe to report + via the regular logging module. - * IPv6 functions for platforms that don't have enough IPv6 support + * `lib/malloc` -- Wrappers and utilities for memory management. - * Endianness functions + * `lib/intmath` -- Utilities for integer mathematics. - * OS functions + * `lib/fdio` -- Utilities and compatibility code for reading and + writing data on file descriptors (and on sockets, for platforms + where a socket is not a kind of fd). - * Threading and locking functions. + * `lib/lock` -- Compatibility code for declaring and using locks. + Lower-level than the rest of the threading code. -=== Utility functions + * `lib/ctime` -- Constant-time implementations for data comparison + and table lookup, used to avoid timing side-channels from standard + implementations of memcmp() and so on. -General-purpose utilities are in util.c; they include higher-level wrappers -around many of the compatibility functions to provide things like -file-at-once access, memory management functions, math, string manipulation, -time manipulation, filesystem manipulation, etc. + * `lib/string` -- Low-level compatibility wrappers and utility + functions for string manipulation. -(Some functionality, like daemon-launching, would be better off in a -compatibility module.) + * `lib/wallclock` -- Compatibility and utility functions for + inspecting and manipulating the current (UTC) time. -In util_format.c, we have code to implement stuff like base-32 and base-64 -encoding. + * `lib/osinfo` -- Functions for inspecting the version and + capabilities of the operating system. -The address.c module interfaces with the system resolver and implements -address parsing and formatting functions. It converts sockaddrs to and from -a more compact tor_addr_t type. + * `lib/smartlist_core` -- The bare-bones pieces of our dynamic array + ("smartlist") implementation. There are higher-level pieces, but + these ones are used by (and therefore cannot use) the logging code. -The di_ops.c module provides constant-time comparison and associative-array -operations, for side-channel avoidance. + * `lib/log` -- Implements the logging system used by all higher-level + Tor code. You can think of this as the logical "midpoint" of the + library code: much of the higher-level code is higher-level + _because_ it uses the logging module, and much of the lower-level + code is specifically written to avoid having to log, because the + logging module depends on it. -The logging subsystem in log.c supports logging to files, to controllers, to -stdout/stderr, or to the system log. + * `lib/container` -- General purpose containers, including dynamic arrays, + hashtables, bit arrays, weak-reference-like "handles", bloom + filters, and a bit more. -The abstraction in memarea.c is used in cases when a large amount of -temporary objects need to be allocated, and they can all be freed at the same -time. + * `lib/trace` -- A general-purpose API for introducing + function-tracing functionality into Tor. Currently not much used. -The torgzip.c module wraps the zlib library to implement compression. + * `lib/thread` -- Threading compatibility and utility functionality, + other than low-level locks (which are in `lib/lock`) and + workqueue/threadpool code (which belongs in `lib/evloop`). -Workqueue.c provides a simple multithreaded work-queue implementation. + * `lib/term` -- Code for terminal manipulation functions (like + reading a password from the user). -### Containers + * `lib/memarea` -- A data structure for a fast "arena" style allocator, + where the data is freed all at once. Used for parsing. -The container.c module defines these container types, used throughout the Tor -codebase. + * `lib/encoding` -- Implementations for encoding data in various + formats, datatypes, and transformations. -There is a dynamic array called **smartlist**, used as our general resizeable -array type. It supports sorting, searching, common set operations, and so -on. It has specialized functions for smartlists of strings, and for -heap-based priority queues. + * `lib/dispatch` -- A general-purpose in-process message delivery + system. Used by `lib/pubsub` to implement our inter-module + publish/subscribe system. -There's a bit-array type. + * `lib/sandbox` -- Our Linux seccomp2 sandbox implementation. -A set of mapping types to map strings, 160-bit digests, and 256-bit digests -to void \*. These are what we generally use when we want O(1) lookup. + * `lib/pubsub` -- Code and macros to implement our publish/subscribe + message passing system. -Additionally, for containers, we use the ht.h and tor_queue.h headers, in -src/ext. These provide intrusive hashtable and linked-list macros. + * `lib/fs` -- Utility and compatibility code for manipulating files, + filenames, directories, and so on. -### Cryptography + * `lib/confmgt` -- Code to parse, encode, and manipulate our + configuration files, state files, and so forth. -Once, we tried to keep our cryptography code in a single "crypto.c" file, -with an "aes.c" module containing an AES implementation for use with older -OpenSSLs. + * `lib/crypt_ops` -- Cryptographic operations. This module contains + wrappers around the cryptographic libraries that we support, + and implementations for some higher-level cryptographic + constructions that we use. -Now, our practice has become to introduce crypto_\*.c modules when adding new -cryptography backend code. We have modules for Ed25519, Curve25519, -secret-to-key algorithms, and password-based boxed encryption. + * `lib/meminfo` -- Functions for inspecting our memory usage, if the + malloc implementation exposes that to us. -Our various TLS compatibility code, wrappers, and hacks are kept in -tortls.c, which is probably too full of Tor-specific kludges. I'm -hoping we can eliminate most of those kludges when we finally remove -support for older versions of our TLS handshake. + * `lib/time` -- Higher level time functions, including fine-gained and + monotonic timers. + * `lib/math` -- Floating-point mathematical utilities, including + compatibility code, and probability distributions. + * `lib/buf` -- A general purpose queued buffer implementation, + similar to the BSD kernel's "mbuf" structure. + * `lib/net` -- Networking code, including address manipulation, + compatibility wrappers, + + * `lib/compress` -- A compatibility wrapper around several + compression libraries, currently including zlib, zstd, and lzma. + + * `lib/geoip` -- Utilities to manage geoip (IP to country) lookups + and formats. + + * `lib/tls` -- Compatibility wrappers around the library (NSS or + OpenSSL, depending on configuration) that Tor uses to implement the + TLS link security protocol. + + * `lib/evloop` -- Tools to manage the event loop and related + functionality, in order to implement asynchronous networking, + timers, periodic events, and other scheduling tasks. + + * `lib/process` -- Utilities and compatibility code to launch and + manage subprocesses. + +### What belongs in lib? + +In general, if you can imagine some program wanting the functionality +you're writing, even if that program had nothing to do with Tor, your +functionality belongs in lib. + +If it falls into one of the existing "lib" categories, your +functionality belongs in lib. + +If you are using platform-specific `#ifdef`s to manage compatibility +issues among platforms, you should probably consider whether you can +put your code into lib. From 9201bbd96ce15889ddd7ccef8492cd22e2d26634 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 14 Oct 2019 14:05:50 -0400 Subject: [PATCH 1659/2557] Replace a unicode character which doxygen cannot handle. Merging without review, as this is an editorial fix in a comment. --- src/core/or/circuitpadding.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/or/circuitpadding.h b/src/core/or/circuitpadding.h index e9eb32c618..683ac95de7 100644 --- a/src/core/or/circuitpadding.h +++ b/src/core/or/circuitpadding.h @@ -301,7 +301,7 @@ typedef struct circpad_state_t { * | | | | | | | * | | | | | | | * 0 +----+----+-----+-----+---------+---------------+ - * 0 100 200 350 500 1000 ∞ microseconds + * 0 100 200 350 500 1000 inf microseconds * * would be specified the following way: * histogram_len = 6; From 77accf937e2272b6e46a1486301d87b4df8f9bab Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 14 Oct 2019 15:02:26 -0400 Subject: [PATCH 1660/2557] Edit 01a-memory.md --- doc/HACKING/design/01a-memory.md | 62 ++++++++++++++++++-------------- 1 file changed, 36 insertions(+), 26 deletions(-) diff --git a/doc/HACKING/design/01a-memory.md b/doc/HACKING/design/01a-memory.md index 9a20782962..4c6bb09018 100644 --- a/doc/HACKING/design/01a-memory.md +++ b/doc/HACKING/design/01a-memory.md @@ -1,7 +1,7 @@ ## Memory management -### Heap-allocation functions +### Heap-allocation functions: lib/malloc/malloc.h Tor imposes a few light wrappers over C's native malloc and free functions, to improve convenience, and to allow wholescale replacement @@ -12,63 +12,71 @@ own; always use the variants prefixed with 'tor_'. They are the same as the standard C functions, with the following exceptions: - * tor_free(NULL) is a no-op. - * tor_free() is a macro that takes an lvalue as an argument and sets it to - NULL after freeing it. To avoid this behavior, you can use tor_free_() + * `tor_free(NULL)` is a no-op. + * `tor_free()` is a macro that takes an lvalue as an argument and sets it to + NULL after freeing it. To avoid this behavior, you can use `tor_free_()` instead. * tor_malloc() and friends fail with an assertion if they are asked to allocate a value so large that it is probably an underflow. - * It is always safe to tor_malloc(0), regardless of whether your libc + * It is always safe to `tor_malloc(0)`, regardless of whether your libc allows it. - * tor_malloc(), tor_realloc(), and friends are never allowed to fail. + * `tor_malloc()`, `tor_realloc()`, and friends are never allowed to fail. Instead, Tor will die with an assertion. This means that you never need to check their return values. See the next subsection for information on why we think this is a good idea. We define additional general-purpose memory allocation functions as well: - * tor_malloc_zero(x) behaves as calloc(1, x), except the it makes clear + * `tor_malloc_zero(x)` behaves as `calloc(1, x)`, except the it makes clear the intent to allocate a single zeroed-out value. - * tor_reallocarray(x,y) behaves as the OpenBSD reallocarray function. + * `tor_reallocarray(x,y)` behaves as the OpenBSD reallocarray function. Use it for cases when you need to realloc() in a multiplication-safe way. And specific-purpose functions as well: - * tor_strdup() and tor_strndup() behaves as the underlying libc functions, - but use tor_malloc() instead of the underlying function. - * tor_memdup() copies a chunk of memory of a given size. - * tor_memdup_nulterm() copies a chunk of memory of a given size, then + * `tor_strdup()` and `tor_strndup()` behaves as the underlying libc + functions, but use `tor_malloc()` instead of the underlying function. + * `tor_memdup()` copies a chunk of memory of a given size. + * `tor_memdup_nulterm()` copies a chunk of memory of a given size, then NUL-terminates it just to be safe. -#### Why assert on failure? +#### Why assert on allocation failure? -Why don't we allow tor_malloc() and its allies to return NULL? +Why don't we allow `tor_malloc()` and its allies to return NULL? First, it's error-prone. Many programmers forget to check for NULL return -values, and testing for malloc() failures is a major pain. +values, and testing for `malloc()` failures is a major pain. Second, it's not necessarily a great way to handle OOM conditions. It's probably better (we think) to have a memory target where we dynamically free things ahead of time in order to stay under the target. Trying to respond to -an OOM at the point of tor_malloc() failure, on the other hand, would involve +an OOM at the point of `tor_malloc()` failure, on the other hand, would involve a rare operation invoked from deep in the call stack. (Again, that's error-prone and hard to debug.) Third, thanks to the rise of Linux and other operating systems that allow memory to be overcommitted, you can't actually ever rely on getting a NULL -from malloc() when you're out of memory; instead you have to use an approach +from `malloc()` when you're out of memory; instead you have to use an approach closer to tracking the total memory usage. #### Conventions for your own allocation functions. Whenever you create a new type, the convention is to give it a pair of -x_new() and x_free() functions, named after the type. +`x_new()` and `x_free_()` functions, named after the type. -Calling x_free(NULL) should always be a no-op. +Calling `x_free(NULL)` should always be a no-op. + +There should additionally be an `x_free()` macro, defined in terms of +`x_free_()`. This macro should set its lvalue to NULL. You can define it +using the FREE_AND_NULL macro, as follows: + +``` +#define x_free(ptr) FREE_AND_NULL(x_t, x_free_, (ptr)) +``` -### Grow-only memory allocation: memarea.c +### Grow-only memory allocation: lib/memarea It's often handy to allocate a large number of tiny objects, all of which need to disappear at the same time. You can do this in tor using the @@ -82,12 +90,14 @@ objects of similar size. But if you use tor_malloc() for the long-lived ones and a memarea for the temporary object, the malloc implementation is likelier to do better. -To create a new memarea, use memarea_new(). To drop all the storage from a -memarea, and invalidate its pointers, use memarea_drop_all(). +To create a new memarea, use `memarea_new()`. To drop all the storage from a +memarea, and invalidate its pointers, use `memarea_drop_all()`. -The allocation functions memarea_alloc(), memarea_alloc_zero(), -memarea_memdup(), memarea_strdup(), and memarea_strndup() are analogous to -the similarly-named malloc() functions. There is intentionally no -memarea_free() or memarea_realloc(). +The allocation functions `memarea_alloc()`, `memarea_alloc_zero()`, +`memarea_memdup()`, `memarea_strdup()`, and `memarea_strndup()` are analogous +to the similarly-named malloc() functions. There is intentionally no +`memarea_free()` or `memarea_realloc()`. +### Special allocation: lib/malloc/map_anon.h +TODO: WRITEME. From 358436592befbdeb3249c5301f3e2b802de61aca Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 14 Oct 2019 15:05:47 -0400 Subject: [PATCH 1661/2557] Edit 01b-collections.md a bit for md and missing content --- doc/HACKING/design/01b-collections.md | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/doc/HACKING/design/01b-collections.md b/doc/HACKING/design/01b-collections.md index def60b0f15..ed6fdc9071 100644 --- a/doc/HACKING/design/01b-collections.md +++ b/doc/HACKING/design/01b-collections.md @@ -4,27 +4,27 @@ ### Smartlists: Neither lists, nor especially smart. For historical reasons, we call our dynamic-allocated array type -"smartlist_t". It can grow or shrink as elements are added and removed. +`smartlist_t`. It can grow or shrink as elements are added and removed. -All smartlists hold an array of void \*. Whenever you expose a smartlist +All smartlists hold an array of `void *`. Whenever you expose a smartlist in an API you *must* document which types its pointers actually hold. -Smartlists are created empty with smartlist_new() and freed with -smartlist_free(). See the containers.h module documentation for more +Smartlists are created empty with `smartlist_new()` and freed with +`smartlist_free()`. See the `containers.h` module documentation for more information; there are many convenience functions for commonly needed operations. + ### Digest maps, string maps, and more. Tor makes frequent use of maps from 160-bit digests, 256-bit digests, -or nul-terminated strings to void \*. These types are digestmap_t, -digest256map_t, and strmap_t respectively. See the containers.h +or nul-terminated strings to `void *`. These types are `digestmap_t`, +`digest256map_t`, and `strmap_t` respectively. See the containers.h module documentation for more information. - ### Intrusive lists and hashtables For performance-sensitive cases, we sometimes want to use "intrusive" @@ -32,12 +32,14 @@ collections: ones where the bookkeeping pointers are stuck inside the structures that belong to the collection. If you've used the BSD-style sys/queue.h macros, you'll be familiar with these. -Unfortunately, the sys/queue.h macros vary significantly between the +Unfortunately, the `sys/queue.h` macros vary significantly between the platforms that have them, so we provide our own variants in -src/ext/tor_queue.h . +`src/ext/tor_queue.h`. -We also provide an intrusive hashtable implementation in src/ext/ht.h -. When you're using it, you'll need to define your own hash +We also provide an intrusive hashtable implementation in `src/ext/ht.h`. +When you're using it, you'll need to define your own hash functions. If attacker-induced collisions are a worry here, use the cryptographic siphash24g function to extract hashes. + From 1332d3b6fa4faa2ef961d1057044fb4cd2608257 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 14 Oct 2019 15:53:04 -0400 Subject: [PATCH 1662/2557] doc/HACKING/design: address comments from ahf --- doc/HACKING/design/00-overview.md | 34 ++++++++++++------------ doc/HACKING/design/01.00-lib-overview.md | 6 ++--- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/doc/HACKING/design/00-overview.md b/doc/HACKING/design/00-overview.md index fde476dfab..ff40a566be 100644 --- a/doc/HACKING/design/00-overview.md +++ b/doc/HACKING/design/00-overview.md @@ -56,38 +56,38 @@ will be scheduled. The codebase is divided into a few top-level subdirectories, each of which contains several sub-modules. - src/ext -- Code maintained elsewhere that we include in the Tor - source distribution. + * `src/ext` -- Code maintained elsewhere that we include in the Tor + source distribution. - src/lib -- Lower-level utility code, not necessarily tor-specific. + * src/lib` -- Lower-level utility code, not necessarily tor-specific. - src/trunnel -- Automatically generated code (from the Trunnel) - tool: used to parse and encode binary formats. + * `src/trunnel` -- Automatically generated code (from the Trunnel + tool): used to parse and encode binary formats. - src/core -- Networking code that is implements the central parts of + * `src/core` -- Networking code that is implements the central parts of the Tor protocol and main loop. - src/feature -- Aspects of Tor (like directory management, running a + * `src/feature` -- Aspects of Tor (like directory management, running a relay, running a directory authorities, managing a list of nodes, running and using onion services) that are built on top of the mainloop code. - src/app -- Highest-level functionality; responsible for setting up - and configuring the Tor project, making sure all the lower-level + * `src/app` -- Highest-level functionality; responsible for setting up + and configuring the Tor daemon, making sure all the lower-level modules start up when required, and so on. - src/tools -- Binaries other than Tor that we produce. Currently this + * `src/tools` -- Binaries other than Tor that we produce. Currently this is tor-resolve, tor-gencert, and the tor_runner.o helper module. - src/test -- unit tests, regression tests, and a few integration + * `src/test` -- unit tests, regression tests, and a few integration tests. -In theory, the above parts of the codebase are sorted from highest-level -to lowest-level, where high-level code is only allowed to invoke -lower-level code, and lower-level code never includes or depends on code -of a higher level. In practice, this refactoring is incomplete: The -modules in src/lib are well-factored, but there are many "upward -dependencies" in src/core and src/feature. We aim to eliminate those +In theory, the above parts of the codebase are sorted from highest-level to +lowest-level, where high-level code is only allowed to invoke lower-level +code, and lower-level code never includes or depends on code of a higher +level. In practice, this refactoring is incomplete: The modules in `src/lib` +are well-factored, but there are many layer violations ("upward +dependencies") in `src/core` and `src/feature`. We aim to eliminate those over time. ### Some key high-level abstractions ### diff --git a/doc/HACKING/design/01.00-lib-overview.md b/doc/HACKING/design/01.00-lib-overview.md index 08dec51a00..58a92f4062 100644 --- a/doc/HACKING/design/01.00-lib-overview.md +++ b/doc/HACKING/design/01.00-lib-overview.md @@ -85,9 +85,9 @@ level): code is specifically written to avoid having to log, because the logging module depends on it. - * `lib/container` -- General purpose containers, including dynamic arrays, - hashtables, bit arrays, weak-reference-like "handles", bloom - filters, and a bit more. + * `lib/container` -- General purpose containers, including dynamic arrays + ("smartlists"), hashtables, bit arrays, weak-reference-like "handles", + bloom filters, and a bit more. * `lib/trace` -- A general-purpose API for introducing function-tracing functionality into Tor. Currently not much used. From 028987a0a1e841d7faf3ac131b0bd2795a963aa5 Mon Sep 17 00:00:00 2001 From: teor Date: Tue, 15 Oct 2019 13:36:56 +1000 Subject: [PATCH 1663/2557] scripts: Merge forward test branch _042 into test branch _master Fixes an issue where test branch _041 was merged into _042 and _master. This issue only affects test branch mode (-t). --- scripts/git/git-merge-forward.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/git/git-merge-forward.sh b/scripts/git/git-merge-forward.sh index d16519b3f5..59adbab82d 100755 --- a/scripts/git/git-merge-forward.sh +++ b/scripts/git/git-merge-forward.sh @@ -101,7 +101,7 @@ MAINT_041=( "maint-0.4.1" "maint-0.4.0" "$GIT_PATH/$TOR_WKT_NAME/maint-0.4.1" \ MAINT_042=( "maint-0.4.2" "maint-0.4.1" "$GIT_PATH/$TOR_WKT_NAME/maint-0.4.2" \ "_042" "_041") MAINT_MASTER=( "master" "maint-0.4.2" "$GIT_PATH/$TOR_MASTER_NAME" \ - "_master" "_041") + "_master" "_042") RELEASE_029=( "release-0.2.9" "maint-0.2.9" "$GIT_PATH/$TOR_WKT_NAME/release-0.2.9" ) RELEASE_035=( "release-0.3.5" "maint-0.3.5" "$GIT_PATH/$TOR_WKT_NAME/release-0.3.5" ) From 11b5946e49bc2f92711ef0e3cec9bb6625c5bd14 Mon Sep 17 00:00:00 2001 From: teor Date: Tue, 15 Oct 2019 13:58:53 +1000 Subject: [PATCH 1664/2557] scripts: Fix a git script comment --- scripts/git/git-merge-forward.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/git/git-merge-forward.sh b/scripts/git/git-merge-forward.sh index 59adbab82d..339573b39c 100755 --- a/scripts/git/git-merge-forward.sh +++ b/scripts/git/git-merge-forward.sh @@ -333,7 +333,7 @@ function merge_branch fi } -# Pull the given branch name. +# Merge origin/(branch name) into the current branch. function merge_branch_origin { local cmd="git merge --ff-only 'origin/$1'" From 09e6c0f7c7b91a73c73df197e45072a96240ea8d Mon Sep 17 00:00:00 2001 From: David Goulet Date: Tue, 15 Oct 2019 08:54:11 -0400 Subject: [PATCH 1665/2557] hs-v3: Fix possible memory leak in error code path Found by coverity CID 1454769. There were a second possible leak that is also fixed in this commit. Fixes #32063 Signed-off-by: David Goulet --- changes/ticket32063 | 3 +++ src/feature/hs/hs_cell.c | 13 +++++++++---- 2 files changed, 12 insertions(+), 4 deletions(-) create mode 100644 changes/ticket32063 diff --git a/changes/ticket32063 b/changes/ticket32063 new file mode 100644 index 0000000000..2c0246917c --- /dev/null +++ b/changes/ticket32063 @@ -0,0 +1,3 @@ + o Minor bugfixes (hs-v3, memory leak): + - Fix memory leak in unlikely error code path when encoding HS DoS establish + intro extension cell. Fixes bug 32063; bugfix on 0.4.2.1-alpha. diff --git a/src/feature/hs/hs_cell.c b/src/feature/hs/hs_cell.c index d691a1b007..df59f73c1b 100644 --- a/src/feature/hs/hs_cell.c +++ b/src/feature/hs/hs_cell.c @@ -503,8 +503,8 @@ build_establish_intro_dos_extension(const hs_service_config_t *service_config, ssize_t ret; size_t dos_ext_encoded_len; uint8_t *field_array; - trn_cell_extension_field_t *field; - trn_cell_extension_dos_t *dos_ext; + trn_cell_extension_field_t *field = NULL; + trn_cell_extension_dos_t *dos_ext = NULL; tor_assert(service_config); tor_assert(extensions); @@ -530,7 +530,7 @@ build_establish_intro_dos_extension(const hs_service_config_t *service_config, /* Set the field with the encoded DoS extension. */ ret = trn_cell_extension_dos_encoded_len(dos_ext); if (BUG(ret <= 0)) { - return -1; + goto err; } dos_ext_encoded_len = ret; /* Set length field and the field array size length. */ @@ -541,7 +541,7 @@ build_establish_intro_dos_extension(const hs_service_config_t *service_config, ret = trn_cell_extension_dos_encode(field_array, trn_cell_extension_field_getlen_field(field), dos_ext); if (BUG(ret <= 0)) { - return -1; + goto err; } tor_assert(ret == (ssize_t) dos_ext_encoded_len); @@ -557,6 +557,11 @@ build_establish_intro_dos_extension(const hs_service_config_t *service_config, trn_cell_extension_dos_free(dos_ext); return 0; + + err: + trn_cell_extension_field_free(field); + trn_cell_extension_dos_free(dos_ext); + return -1; } /* ========== */ From 71daad1692bc3f2478a527da0570a6c13c3a99fd Mon Sep 17 00:00:00 2001 From: Roger Dingledine Date: Wed, 16 Oct 2019 03:31:23 -0400 Subject: [PATCH 1666/2557] never say XB, always XBytes fix three more instances in the man page that we forgot to fix in earlier attempts --- doc/tor.1.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/tor.1.txt b/doc/tor.1.txt index 6b81e19ba1..06bf022b15 100644 --- a/doc/tor.1.txt +++ b/doc/tor.1.txt @@ -715,7 +715,7 @@ GENERAL OPTIONS in separate files by hash, up to the specified size in total. Note that only files logged during the lifetime of this Tor process count toward the total; this is intended to be used to debug problems without opening live - servers to resource exhaustion attacks. (Default: 10 MB) + servers to resource exhaustion attacks. (Default: 10 MBytes) [[OutboundBindAddress]] **OutboundBindAddress** __IP__:: Make all outbound connections originate from the IP address specified. This @@ -2479,7 +2479,7 @@ is non-zero): option can create security issues; you should probably leave it off. (Default: 0) -[[MaxMemInQueues]] **MaxMemInQueues** __N__ **bytes**|**KB**|**MB**|**GB**:: +[[MaxMemInQueues]] **MaxMemInQueues** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**:: This option configures a threshold above which Tor will assume that it needs to stop queueing or buffering data because it's about to run out of memory. If it hits this threshold, it will begin killing circuits until @@ -3320,7 +3320,7 @@ The following options are used for running a testing Tor network. [[TestingMinExitFlagThreshold]] **TestingMinExitFlagThreshold** __N__ **KBytes**|**MBytes**|**GBytes**|**TBytes**|**KBits**|**MBits**|**GBits**|**TBits**:: Sets a lower-bound for assigning an exit flag when running as an authority on a testing network. Overrides the usual default lower bound - of 4 KB. (Default: 0) + of 4 KBytes. (Default: 0) [[TestingLinkCertLifetime]] **TestingLinkCertLifetime** __N__ **seconds**|**minutes**|**hours**|**days**|**weeks**|**months**:: Overrides the default lifetime for the certificates used to authenticate From 5e5e655e323c917ad0b5e5b48056bd5d49645182 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 16 Oct 2019 08:18:26 -0400 Subject: [PATCH 1667/2557] Doxygen: Fix mismatched filenames. In 8 places, our \file declarations didn't match the actual files they were in. --- src/app/config/or_state_st.h | 2 +- src/feature/dircache/consdiffmgr.c | 2 +- src/feature/rend/rendparse.h | 4 ++-- src/feature/stats/predict_ports.h | 2 +- src/lib/crypt_ops/crypto_dh_nss.c | 2 +- src/lib/crypt_ops/crypto_sys.h | 2 +- src/lib/net/network_sys.h | 2 +- src/lib/thread/thread_sys.h | 2 +- 8 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/app/config/or_state_st.h b/src/app/config/or_state_st.h index 225003bb7e..27cc936c7d 100644 --- a/src/app/config/or_state_st.h +++ b/src/app/config/or_state_st.h @@ -5,7 +5,7 @@ /* See LICENSE for licensing information */ /** - * \file or_state_t + * \file or_state_st.h * * \brief The or_state_t structure, which represents Tor's state file. */ diff --git a/src/feature/dircache/consdiffmgr.c b/src/feature/dircache/consdiffmgr.c index 397efa0341..bc02a33e30 100644 --- a/src/feature/dircache/consdiffmgr.c +++ b/src/feature/dircache/consdiffmgr.c @@ -2,7 +2,7 @@ /* See LICENSE for licensing information */ /** - * \file consdiffmsr.c + * \file consdiffmgr.c * * \brief consensus diff manager functions * diff --git a/src/feature/rend/rendparse.h b/src/feature/rend/rendparse.h index b1ccce9b6c..da2c2e4b7f 100644 --- a/src/feature/rend/rendparse.h +++ b/src/feature/rend/rendparse.h @@ -5,8 +5,8 @@ /* See LICENSE for licensing information */ /** - * \file rend_parse.h - * \brief Header file for rend_parse.c. + * \file rendparse.h + * \brief Header file for rendparse.c. **/ #ifndef TOR_REND_PARSE_H diff --git a/src/feature/stats/predict_ports.h b/src/feature/stats/predict_ports.h index 45b206c23a..a994db1d17 100644 --- a/src/feature/stats/predict_ports.h +++ b/src/feature/stats/predict_ports.h @@ -5,7 +5,7 @@ /* See LICENSE for licensing information */ /** - * \file predict_portst.h + * \file predict_ports.h * \brief Header file for predict_ports.c. **/ diff --git a/src/lib/crypt_ops/crypto_dh_nss.c b/src/lib/crypt_ops/crypto_dh_nss.c index 379eb84a4f..aa95fb508f 100644 --- a/src/lib/crypt_ops/crypto_dh_nss.c +++ b/src/lib/crypt_ops/crypto_dh_nss.c @@ -5,7 +5,7 @@ /* See LICENSE for licensing information */ /** - * \file crypto_dh_nss.h + * \file crypto_dh_nss.c * * \brief NSS implementation of Diffie-Hellman over Z_p. **/ diff --git a/src/lib/crypt_ops/crypto_sys.h b/src/lib/crypt_ops/crypto_sys.h index 894243b175..ff449d2e0b 100644 --- a/src/lib/crypt_ops/crypto_sys.h +++ b/src/lib/crypt_ops/crypto_sys.h @@ -2,7 +2,7 @@ /* See LICENSE for licensing information */ /** - * \file log_crypto.h + * \file crypto_sys.h * \brief Declare subsystem object for the crypto module. **/ diff --git a/src/lib/net/network_sys.h b/src/lib/net/network_sys.h index 43e62592ca..34ac3d120c 100644 --- a/src/lib/net/network_sys.h +++ b/src/lib/net/network_sys.h @@ -2,7 +2,7 @@ /* See LICENSE for licensing information */ /** - * \file log_network.h + * \file network_sys.h * \brief Declare subsystem object for the network module. **/ diff --git a/src/lib/thread/thread_sys.h b/src/lib/thread/thread_sys.h index c0daf2b5e9..ef27134a32 100644 --- a/src/lib/thread/thread_sys.h +++ b/src/lib/thread/thread_sys.h @@ -2,7 +2,7 @@ /* See LICENSE for licensing information */ /** - * \file threads_sys.h + * \file thread_sys.h * \brief Declare subsystem object for threads library **/ From 98735c40b6f503b296f7e6fefae36da24b33616e Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 16 Oct 2019 08:25:10 -0400 Subject: [PATCH 1668/2557] Doxygen: fix warnings about mismatched HTML tags. --- src/core/mainloop/periodic.c | 2 +- src/feature/client/entrynodes.c | 2 +- src/feature/dircache/consdiffmgr.c | 2 +- src/feature/dircommon/consdiff.c | 2 +- src/feature/dirparse/microdesc_parse.c | 2 +- src/feature/keymgt/loadkey.c | 2 +- src/lib/crypt_ops/crypto_dh_openssl.c | 2 +- src/lib/encoding/pem.c | 2 +- src/lib/evloop/compat_libevent.c | 2 +- src/lib/net/address.c | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/core/mainloop/periodic.c b/src/core/mainloop/periodic.c index dbc4553a73..51a0165d19 100644 --- a/src/core/mainloop/periodic.c +++ b/src/core/mainloop/periodic.c @@ -39,7 +39,7 @@ static const int MAX_INTERVAL = 10 * 365 * 86400; /** * Global list of periodic events that have been registered with - * periodic_event_register. + * periodic_event_register. **/ static smartlist_t *the_periodic_events = NULL; diff --git a/src/feature/client/entrynodes.c b/src/feature/client/entrynodes.c index 36b575ef20..4e61899ecd 100644 --- a/src/feature/client/entrynodes.c +++ b/src/feature/client/entrynodes.c @@ -1038,7 +1038,7 @@ get_max_sample_size(guard_selection_t *gs, * Return a smartlist of the all the guards that are not currently * members of the sample (GUARDS - SAMPLED_GUARDS). The elements of * this list are node_t pointers in the non-bridge case, and - * bridge_info_t pointers in the bridge case. Set *n_guards_out/b> + * bridge_info_t pointers in the bridge case. Set *n_guards_out * to the number of guards that we found in GUARDS, including those * that were already sampled. */ diff --git a/src/feature/dircache/consdiffmgr.c b/src/feature/dircache/consdiffmgr.c index bc02a33e30..058ff1f500 100644 --- a/src/feature/dircache/consdiffmgr.c +++ b/src/feature/dircache/consdiffmgr.c @@ -1293,7 +1293,7 @@ typedef struct compressed_result_t { /** * Compress the bytestring input of length len using the - * n_methods compression methods listed in the array methods. + * n_methods compression methods listed in the array methods. * * For each successful compression, set the fields in the results_out * array in the position corresponding to the compression method. Use diff --git a/src/feature/dircommon/consdiff.c b/src/feature/dircommon/consdiff.c index 8e93953f73..2c4d3a19c0 100644 --- a/src/feature/dircommon/consdiff.c +++ b/src/feature/dircommon/consdiff.c @@ -570,7 +570,7 @@ find_next_router_line(const smartlist_t *cons, /** Pre-process a consensus in cons (represented as a list of cdline_t) * to remove the signatures from it. If the footer is removed, return a * cdline_t containing a delete command to delete the footer, allocated in - * area. If no footer is removed, return NULL. + * area. If no footer is removed, return NULL. * * We remove the signatures here because they are not themselves signed, and * as such there might be different encodings for them. diff --git a/src/feature/dirparse/microdesc_parse.c b/src/feature/dirparse/microdesc_parse.c index 4bb4db7821..e526a355c1 100644 --- a/src/feature/dirparse/microdesc_parse.c +++ b/src/feature/dirparse/microdesc_parse.c @@ -164,7 +164,7 @@ microdesc_extract_body(microdesc_t *md, /** * Parse a microdescriptor which begins at s and ends at - * start_of_next_microdesc. Store its fields into md. Use + * start_of_next_microdesc. Store its fields into md. Use * where for generating log information. If allow_annotations * is true, then one or more annotations may precede the microdescriptor body * proper. Use area for memory management, clearing it when done. diff --git a/src/feature/keymgt/loadkey.c b/src/feature/keymgt/loadkey.c index a8cbf0e582..5c1e6353b8 100644 --- a/src/feature/keymgt/loadkey.c +++ b/src/feature/keymgt/loadkey.c @@ -33,7 +33,7 @@ /** Try to read an RSA key from fname. If fname doesn't exist * and generate is true, create a new RSA key and save it in * fname. Return the read/created key, or NULL on error. Log all - * errors at level severity. If created_out/b> is non-NULL and a + * errors at level severity. If created_out is non-NULL and a * new key was created, set *created_out to true. */ crypto_pk_t * diff --git a/src/lib/crypt_ops/crypto_dh_openssl.c b/src/lib/crypt_ops/crypto_dh_openssl.c index 8ae97373e8..e7f22d749b 100644 --- a/src/lib/crypt_ops/crypto_dh_openssl.c +++ b/src/lib/crypt_ops/crypto_dh_openssl.c @@ -103,7 +103,7 @@ crypto_validate_dh_params(const BIGNUM *p, const BIGNUM *g) #endif /* 0 */ /** - * Helper: convert hex to a bignum, and return it. Assert that the + * Helper: convert hex to a bignum, and return it. Assert that the * operation was successful. */ static BIGNUM * diff --git a/src/lib/encoding/pem.c b/src/lib/encoding/pem.c index 24b238b130..95f93ebeff 100644 --- a/src/lib/encoding/pem.c +++ b/src/lib/encoding/pem.c @@ -42,7 +42,7 @@ pem_encoded_size(size_t src_len, const char *objtype) /** * PEM-encode the srclen-byte object at src into the - * destlen<\b>-byte buffer at dest, tagging it with objtype. + * destlen-byte buffer at dest, tagging it with objtype. * Return 0 on success and -1 on failure. */ int diff --git a/src/lib/evloop/compat_libevent.c b/src/lib/evloop/compat_libevent.c index 91eacb9938..500c74831c 100644 --- a/src/lib/evloop/compat_libevent.c +++ b/src/lib/evloop/compat_libevent.c @@ -422,7 +422,7 @@ mainloop_event_activate(mainloop_event_t *event) * * If the event is scheduled for a different time, cancel it and run * after this delay instead. If the event is currently pending to run - * now, has no effect. + * now, has no effect. * * Do not call this function with tv == NULL -- use * mainloop_event_activate() instead. diff --git a/src/lib/net/address.c b/src/lib/net/address.c index 0a2c84caf2..dbff3bbb0d 100644 --- a/src/lib/net/address.c +++ b/src/lib/net/address.c @@ -2001,7 +2001,7 @@ tor_addr_port_new(const tor_addr_t *addr, uint16_t port) return ap; } -/** Return true iff a and b are the same address and port */ +/** Return true iff a and b are the same address and port */ int tor_addr_port_eq(const tor_addr_port_t *a, const tor_addr_port_t *b) From 591420df20211c19c687bc8abcb9234732bbc529 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 16 Oct 2019 08:39:41 -0400 Subject: [PATCH 1669/2557] Doxygen: fix unrecognized commands. Doxygen seems to interpret anything starting with \ as a command, even when we don't mean it that way. --- src/core/or/circuitpadding_machines.c | 2 - src/lib/math/prob_distr.c | 76 +++++++++++++-------------- src/lib/process/process.h | 4 +- 3 files changed, 40 insertions(+), 42 deletions(-) diff --git a/src/core/or/circuitpadding_machines.c b/src/core/or/circuitpadding_machines.c index 7220d657fc..98767f9e8f 100644 --- a/src/core/or/circuitpadding_machines.c +++ b/src/core/or/circuitpadding_machines.c @@ -5,8 +5,6 @@ * \file circuitpadding_machines.c * \brief Circuit padding state machines * - * \detail - * * Introduce circuit padding machines that will be used by Tor circuits, as * specified by proposal 302 "Hiding onion service clients using padding". * diff --git a/src/lib/math/prob_distr.c b/src/lib/math/prob_distr.c index d44dc28265..f9d65073ff 100644 --- a/src/lib/math/prob_distr.c +++ b/src/lib/math/prob_distr.c @@ -178,8 +178,8 @@ clz32(uint32_t x) /** * Compute the logistic function: f(x) = 1/(1 + e^{-x}) = e^x/(1 + e^x). - * Maps a log-odds-space probability in [-\infty, +\infty] into a direct-space - * probability in [0,1]. Inverse of logit. + * Maps a log-odds-space probability in [-infinity, +infinity] into a + * direct-space probability in [0,1]. Inverse of logit. * * Ill-conditioned for large x; the identity logistic(-x) = 1 - * logistic(x) and the function logistichalf(x) = logistic(x) - 1/2 may @@ -266,7 +266,7 @@ logistic(double x) /** * Compute the logit function: log p/(1 - p). Defined on [0,1]. Maps * a direct-space probability in [0,1] to a log-odds-space probability - * in [-\infty, +\infty]. Inverse of logistic. + * in [-infinity, +infinity]. Inverse of logistic. * * Ill-conditioned near 1/2 and 1; the identity logit(1 - p) = * -logit(p) and the function logithalf(p0) = logit(1/2 + p0) may help @@ -488,7 +488,7 @@ random_uniform_01(void) /* Functions for specific probability distributions start here: */ /* - * Logistic(mu, sigma) distribution, supported on (-\infty,+\infty) + * Logistic(mu, sigma) distribution, supported on (-infinity,+infinity) * * This is the uniform distribution on [0,1] mapped into log-odds * space, scaled by sigma and translated by mu. @@ -546,7 +546,7 @@ isf_logistic(double p, double mu, double sigma) } /* - * LogLogistic(alpha, beta) distribution, supported on (0, +\infty). + * LogLogistic(alpha, beta) distribution, supported on (0, +infinity). * * This is the uniform distribution on [0,1] mapped into odds space, * scaled by positive alpha and shaped by positive beta. @@ -687,7 +687,7 @@ isf_log_logistic(double p, double alpha, double beta) } /* - * Weibull(lambda, k) distribution, supported on (0, +\infty). + * Weibull(lambda, k) distribution, supported on (0, +infinity). * * pdf(x) = (k/lambda) (x/lambda)^{k - 1} e^{-(x/lambda)^k} * cdf(x) = 1 - e^{-(x/lambda)^k} @@ -753,7 +753,7 @@ isf_weibull(double p, double lambda, double k) } /* - * GeneralizedPareto(mu, sigma, xi), supported on (mu, +\infty) for + * GeneralizedPareto(mu, sigma, xi), supported on (mu, +infinity) for * nonnegative xi, or (mu, mu - sigma/xi) for negative xi. * * Samples: @@ -793,19 +793,19 @@ cdf_genpareto(double x, double mu, double sigma, double xi) /* * log(1 + xi x_0)/xi - * = (-1/xi) \sum_{n=1}^\infty (-xi x_0)^n/n - * = (-1/xi) (-xi x_0 + \sum_{n=2}^\infty (-xi x_0)^n/n) - * = x_0 - (1/xi) \sum_{n=2}^\infty (-xi x_0)^n/n - * = x_0 - x_0 \sum_{n=2}^\infty (-xi x_0)^{n-1}/n + * = (-1/xi) \sum_{n=1}^infinity (-xi x_0)^n/n + * = (-1/xi) (-xi x_0 + \sum_{n=2}^infinity (-xi x_0)^n/n) + * = x_0 - (1/xi) \sum_{n=2}^infinity (-xi x_0)^n/n + * = x_0 - x_0 \sum_{n=2}^infinity (-xi x_0)^{n-1}/n * = x_0 (1 - d), * - * where d = \sum_{n=2}^\infty (-xi x_0)^{n-1}/n. If |xi| < + * where d = \sum_{n=2}^infinity (-xi x_0)^{n-1}/n. If |xi| < * eps/4|x_0|, then * - * |d| <= \sum_{n=2}^\infty (eps/4)^{n-1}/n - * <= \sum_{n=2}^\infty (eps/4)^{n-1} - * = \sum_{n=1}^\infty (eps/4)^n - * = (eps/4) \sum_{n=0}^\infty (eps/4)^n + * |d| <= \sum_{n=2}^infinity (eps/4)^{n-1}/n + * <= \sum_{n=2}^infinity (eps/4)^{n-1} + * = \sum_{n=1}^infinity (eps/4)^n + * = (eps/4) \sum_{n=0}^infinity (eps/4)^n * = (eps/4)/(1 - eps/4) * < eps/2 * @@ -855,20 +855,20 @@ icdf_genpareto(double p, double mu, double sigma, double xi) * for xi near zero (note f(xi) --> -log U as xi --> 0), write * the absolutely convergent Taylor expansion * - * f(xi) = (1/xi)*(-xi log U + \sum_{n=2}^\infty (-xi log U)^n/n! - * = -log U + (1/xi)*\sum_{n=2}^\infty (-xi log U)^n/n! - * = -log U + \sum_{n=2}^\infty xi^{n-1} (-log U)^n/n! - * = -log U - log U \sum_{n=2}^\infty (-xi log U)^{n-1}/n! - * = -log U (1 + \sum_{n=2}^\infty (-xi log U)^{n-1}/n!). + * f(xi) = (1/xi)*(-xi log U + \sum_{n=2}^infinity (-xi log U)^n/n! + * = -log U + (1/xi)*\sum_{n=2}^infinity (-xi log U)^n/n! + * = -log U + \sum_{n=2}^infinity xi^{n-1} (-log U)^n/n! + * = -log U - log U \sum_{n=2}^infinity (-xi log U)^{n-1}/n! + * = -log U (1 + \sum_{n=2}^infinity (-xi log U)^{n-1}/n!). * - * Let d = \sum_{n=2}^\infty (-xi log U)^{n-1}/n!. What do we + * Let d = \sum_{n=2}^infinity (-xi log U)^{n-1}/n!. What do we * lose if we discard it and use -log U as an approximation to * f(xi)? If |xi| < eps/-4log U, then * - * |d| <= \sum_{n=2}^\infty |xi log U|^{n-1}/n! - * <= \sum_{n=2}^\infty (eps/4)^{n-1}/n! - * <= \sum_{n=1}^\infty (eps/4)^n - * = (eps/4) \sum_{n=0}^\infty (eps/4)^n + * |d| <= \sum_{n=2}^infinity |xi log U|^{n-1}/n! + * <= \sum_{n=2}^infinity (eps/4)^{n-1}/n! + * <= \sum_{n=1}^infinity (eps/4)^n + * = (eps/4) \sum_{n=0}^infinity (eps/4)^n * = (eps/4)/(1 - eps/4) * < eps/2, * @@ -1098,10 +1098,10 @@ sample_logistic(uint32_t s, double t, double p0) * We carve up the interval (0, 1) into subregions to compute * the inverse CDF precisely: * - * A = (0, 1/(1 + e)] ---> (-\infty, -1] + * A = (0, 1/(1 + e)] ---> (-infinity, -1] * B = [1/(1 + e), 1/2] ---> [-1, 0] * C = [1/2, 1 - 1/(1 + e)] ---> [0, 1] - * D = [1 - 1/(1 + e), 1) ---> [1, +\infty) + * D = [1 - 1/(1 + e), 1) ---> [1, +infinity) * * Cases D and C are mirror images of cases A and B, * respectively, so we choose between them by the sign chosen @@ -1234,19 +1234,19 @@ sample_genpareto(uint32_t s, double p0, double xi) * Write f(xi) = (e^{xi x} - 1)/xi for xi near zero as the * absolutely convergent Taylor series * - * f(x) = (1/xi) (xi x + \sum_{n=2}^\infty (xi x)^n/n!) - * = x + (1/xi) \sum_{n=2}^\inty (xi x)^n/n! - * = x + \sum_{n=2}^\infty xi^{n-1} x^n/n! - * = x + x \sum_{n=2}^\infty (xi x)^{n-1}/n! - * = x (1 + \sum_{n=2}^\infty (xi x)^{n-1}/n!). + * f(x) = (1/xi) (xi x + \sum_{n=2}^infinity (xi x)^n/n!) + * = x + (1/xi) \sum_{n=2}^infinity (xi x)^n/n! + * = x + \sum_{n=2}^infinity xi^{n-1} x^n/n! + * = x + x \sum_{n=2}^infinity (xi x)^{n-1}/n! + * = x (1 + \sum_{n=2}^infinity (xi x)^{n-1}/n!). * - * d = \sum_{n=2}^\infty (xi x)^{n-1}/n! is the relative error + * d = \sum_{n=2}^infinity (xi x)^{n-1}/n! is the relative error * of f(x) from x. If |xi| < eps/4x, then * - * |d| <= \sum_{n=2}^\infty |xi x|^{n-1}/n! - * <= \sum_{n=2}^\infty (eps/4)^{n-1}/n! - * <= \sum_{n=1}^\infty (eps/4) - * = (eps/4) \sum_{n=0}^\infty (eps/4)^n + * |d| <= \sum_{n=2}^infinity |xi x|^{n-1}/n! + * <= \sum_{n=2}^infinity (eps/4)^{n-1}/n! + * <= \sum_{n=1}^infinity (eps/4) + * = (eps/4) \sum_{n=0}^infinity (eps/4)^n * = (eps/4)/(1 - eps/4) * < eps/2, * diff --git a/src/lib/process/process.h b/src/lib/process/process.h index 05c091a5bf..b28f55e0a1 100644 --- a/src/lib/process/process.h +++ b/src/lib/process/process.h @@ -35,8 +35,8 @@ typedef enum { const char *process_status_to_string(process_status_t status); typedef enum { - /** Pass complete \n-terminated lines to the - * callback (with the \n or \r\n removed). */ + /** Pass complete newline-terminated lines to the + * callback (with the LF or CRLF removed). */ PROCESS_PROTOCOL_LINE, /** Pass the raw response from read() to the callback. */ From f9fac733b900091917b56f6517ef2b2ed775f6e0 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 16 Oct 2019 08:44:35 -0400 Subject: [PATCH 1670/2557] Doxygen: Fix things that looked like links. Doxygen thinks that saying #foo is linking to a "foo" anchor someplace. --- src/core/or/circuitpadding.h | 2 +- src/lib/math/prob_distr.h | 46 ++++++++++++++++++------------------ 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/src/core/or/circuitpadding.h b/src/core/or/circuitpadding.h index 683ac95de7..a5c46b4390 100644 --- a/src/core/or/circuitpadding.h +++ b/src/core/or/circuitpadding.h @@ -388,7 +388,7 @@ typedef struct circpad_state_t { * * States are allowed to transition to themselves, which means re-schedule * a new padding timer. They are also allowed to temporarily "transition" - * to the "IGNORE" and "CANCEL" pseudo-states. See #defines below + * to the "IGNORE" and "CANCEL" pseudo-states. See defines below * for details on state behavior and meaning. */ circpad_statenum_t next_state[CIRCPAD_NUM_EVENTS]; diff --git a/src/lib/math/prob_distr.h b/src/lib/math/prob_distr.h index 7254dc8623..a93d888950 100644 --- a/src/lib/math/prob_distr.h +++ b/src/lib/math/prob_distr.h @@ -66,41 +66,41 @@ struct dist { * type-specific macro built out of it -- but if you did use this * directly, it would be something like: * -* struct weibull mydist = { -* DIST_BASE_TYPED(&weibull_ops, mydist, struct weibull), -* .lambda = ..., -* .k = ..., -* }; +* struct weibull mydist = { +* DIST_BASE_TYPED(&weibull_ops, mydist, struct weibull), +* .lambda = ..., +* .k = ..., +* }; * * If you want to define a distribution type, define a canonical set of * operations and define a type-specific initializer element like so: * -* struct foo { -* struct dist base; -* int omega; -* double tau; -* double phi; -* }; +* struct foo { +* struct dist base; +* int omega; +* double tau; +* double phi; +* }; * -* struct dist_ops foo_ops = ...; +* struct dist_ops foo_ops = ...; * -* #define FOO(OBJ) DIST_BASE_TYPED(&foo_ops, OBJ, struct foo) +* #define FOO(OBJ) DIST_BASE_TYPED(&foo_ops, OBJ, struct foo) * * Then users can do: * -* struct foo mydist = { -* FOO(mydist), -* .omega = ..., -* .tau = ..., -* .phi = ..., -* }; +* struct foo mydist = { +* FOO(mydist), +* .omega = ..., +* .tau = ..., +* .phi = ..., +* }; * * If you accidentally write * -* struct bar mydist = { -* FOO(mydist), -* ... -* }; +* struct bar mydist = { +* FOO(mydist), +* ... +* }; * * then the compiler will report a type mismatch in the sizeof * expression, which otherwise evaporates at runtime. From 033d76b0e0ad255cec10f9ee048d5bb98c03ce7e Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 16 Oct 2019 09:07:21 -0400 Subject: [PATCH 1671/2557] Doxygen: update doxyfile to a more recent version. I have left the settings at their old values, except when they referred to nonexistent files. Closes ticket 32110 --- Doxyfile.in | 2542 +++++++++++++++++++++++++++++++------------ changes/ticket32110 | 4 + 2 files changed, 1826 insertions(+), 720 deletions(-) create mode 100644 changes/ticket32110 diff --git a/Doxyfile.in b/Doxyfile.in index 4caf421097..d242b2e151 100644 --- a/Doxyfile.in +++ b/Doxyfile.in @@ -1,96 +1,150 @@ -# Doxyfile 1.5.6 +# Doxyfile 1.8.15 + +# (Tor's Doxyfile is automatically generated from "Doxyfile.in". Don't +# edit Doxyfile; edit Doxyfile.in.) # This file describes the settings to be used by the documentation system -# doxygen (www.doxygen.org) for a project +# doxygen (www.doxygen.org) for a project. # -# All text after a hash (#) is considered a comment and will be ignored +# All text after a double hash (##) is considered a comment and is placed in +# front of the TAG it is preceding. +# +# All text after a single hash (#) is considered a comment and will be ignored. # The format is: -# TAG = value [value, ...] -# For lists items can also be appended using: -# TAG += value [value, ...] -# Values that contain spaces should be placed between quotes (" ") +# TAG = value [value, ...] +# For lists, items can also be appended using: +# TAG += value [value, ...] +# Values that contain spaces should be placed between quotes (\" \"). #--------------------------------------------------------------------------- # Project related configuration options #--------------------------------------------------------------------------- -# This tag specifies the encoding used for all characters in the config file -# that follow. The default is UTF-8 which is also the encoding used for all +# This tag specifies the encoding used for all characters in the configuration +# file that follow. The default is UTF-8 which is also the encoding used for all # text before the first occurrence of this tag. Doxygen uses libiconv (or the # iconv built into libc) for the transcoding. See -# http://www.gnu.org/software/libiconv for the list of possible encodings. +# https://www.gnu.org/software/libiconv/ for the list of possible encodings. +# The default value is: UTF-8. DOXYFILE_ENCODING = UTF-8 -# The PROJECT_NAME tag is a single word (or a sequence of words surrounded -# by quotes) that should identify the project. +# The PROJECT_NAME tag is a single word (or a sequence of words surrounded by +# double-quotes, unless you are using Doxywizard) that should identify the +# project for which the documentation is generated. This name is used in the +# title of most generated pages and in a few other places. +# The default value is: My Project. -PROJECT_NAME = tor +PROJECT_NAME = Tor -# The PROJECT_NUMBER tag can be used to enter a project or revision number. -# This could be handy for archiving the generated documentation or -# if some version control system is used. +# The PROJECT_NUMBER tag can be used to enter a project or revision number. This +# could be handy for archiving the generated documentation or if some version +# control system is used. PROJECT_NUMBER = @VERSION@ -# The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) -# base path where the generated documentation will be put. -# If a relative path is entered, it will be relative to the location -# where doxygen was started. If left blank the current directory will be used. +# Using the PROJECT_BRIEF tag one can provide an optional one line description +# for a project that appears at the top of each page and should give viewer a +# quick idea about the purpose of the project. Keep the description short. + +PROJECT_BRIEF = + +# With the PROJECT_LOGO tag one can specify a logo or an icon that is included +# in the documentation. The maximum height of the logo should not exceed 55 +# pixels and the maximum width should not exceed 200 pixels. Doxygen will copy +# the logo to the output directory. + +PROJECT_LOGO = + +# The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) path +# into which the generated documentation will be written. If a relative path is +# entered, it will be relative to the location where doxygen was started. If +# left blank the current directory will be used. OUTPUT_DIRECTORY = @top_builddir@/doc/doxygen -# If the CREATE_SUBDIRS tag is set to YES, then doxygen will create -# 4096 sub-directories (in 2 levels) under the output directory of each output -# format and will distribute the generated files over these directories. -# Enabling this option can be useful when feeding doxygen a huge amount of -# source files, where putting all generated files in the same directory would -# otherwise cause performance problems for the file system. +# If the CREATE_SUBDIRS tag is set to YES then doxygen will create 4096 sub- +# directories (in 2 levels) under the output directory of each output format and +# will distribute the generated files over these directories. Enabling this +# option can be useful when feeding doxygen a huge amount of source files, where +# putting all generated files in the same directory would otherwise causes +# performance problems for the file system. +# The default value is: NO. CREATE_SUBDIRS = NO +# If the ALLOW_UNICODE_NAMES tag is set to YES, doxygen will allow non-ASCII +# characters to appear in the names of generated files. If set to NO, non-ASCII +# characters will be escaped, for example _xE3_x81_x84 will be used for Unicode +# U+3044. +# The default value is: NO. + +ALLOW_UNICODE_NAMES = NO + # The OUTPUT_LANGUAGE tag is used to specify the language in which all # documentation generated by doxygen is written. Doxygen will use this # information to generate all constant output in the proper language. -# The default language is English, other supported languages are: -# Afrikaans, Arabic, Brazilian, Catalan, Chinese, Chinese-Traditional, -# Croatian, Czech, Danish, Dutch, Farsi, Finnish, French, German, Greek, -# Hungarian, Italian, Japanese, Japanese-en (Japanese with English messages), -# Korean, Korean-en, Lithuanian, Norwegian, Macedonian, Persian, Polish, -# Portuguese, Romanian, Russian, Serbian, Slovak, Slovene, Spanish, Swedish, -# and Ukrainian. +# Possible values are: Afrikaans, Arabic, Armenian, Brazilian, Catalan, Chinese, +# Chinese-Traditional, Croatian, Czech, Danish, Dutch, English (United States), +# Esperanto, Farsi (Persian), Finnish, French, German, Greek, Hungarian, +# Indonesian, Italian, Japanese, Japanese-en (Japanese with English messages), +# Korean, Korean-en (Korean with English messages), Latvian, Lithuanian, +# Macedonian, Norwegian, Persian (Farsi), Polish, Portuguese, Romanian, Russian, +# Serbian, Serbian-Cyrillic, Slovak, Slovene, Spanish, Swedish, Turkish, +# Ukrainian and Vietnamese. +# The default value is: English. OUTPUT_LANGUAGE = English -# If the BRIEF_MEMBER_DESC tag is set to YES (the default) Doxygen will -# include brief member descriptions after the members that are listed in -# the file and class documentation (similar to JavaDoc). -# Set to NO to disable this. +# The OUTPUT_TEXT_DIRECTION tag is used to specify the direction in which all +# documentation generated by doxygen is written. Doxygen will use this +# information to generate all generated output in the proper direction. +# Possible values are: None, LTR, RTL and Context. +# The default value is: None. -BRIEF_MEMBER_DESC = NO +OUTPUT_TEXT_DIRECTION = None -# If the REPEAT_BRIEF tag is set to YES (the default) Doxygen will prepend -# the brief description of a member or function before the detailed description. -# Note: if both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the +# If the BRIEF_MEMBER_DESC tag is set to YES, doxygen will include brief member +# descriptions after the members that are listed in the file and class +# documentation (similar to Javadoc). Set to NO to disable this. +# The default value is: YES. + +BRIEF_MEMBER_DESC = YES + +# If the REPEAT_BRIEF tag is set to YES, doxygen will prepend the brief +# description of a member or function before the detailed description +# +# Note: If both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the # brief descriptions will be completely suppressed. +# The default value is: YES. REPEAT_BRIEF = YES -# This tag implements a quasi-intelligent brief description abbreviator -# that is used to form the text in various listings. Each string -# in this list, if found as the leading text of the brief description, will be -# stripped from the text and the result after processing the whole list, is -# used as the annotated text. Otherwise, the brief description is used as-is. -# If left blank, the following values are used ("$name" is automatically -# replaced with the name of the entity): "The $name class" "The $name widget" -# "The $name file" "is" "provides" "specifies" "contains" -# "represents" "a" "an" "the" +# This tag implements a quasi-intelligent brief description abbreviator that is +# used to form the text in various listings. Each string in this list, if found +# as the leading text of the brief description, will be stripped from the text +# and the result, after processing the whole list, is used as the annotated +# text. Otherwise, the brief description is used as-is. If left blank, the +# following values are used ($name is automatically replaced with the name of +# the entity):The $name class, The $name widget, The $name file, is, provides, +# specifies, contains, represents, a, an and the. -ABBREVIATE_BRIEF = +ABBREVIATE_BRIEF = "The $name class" \ + "The $name widget" \ + "The $name file" \ + is \ + provides \ + specifies \ + contains \ + represents \ + a \ + an \ + the # If the ALWAYS_DETAILED_SEC and REPEAT_BRIEF tags are both set to YES then -# Doxygen will generate a detailed section even if there is only a brief +# doxygen will generate a detailed section even if there is only a brief # description. +# The default value is: NO. ALWAYS_DETAILED_SEC = NO @@ -98,482 +152,729 @@ ALWAYS_DETAILED_SEC = NO # inherited members of a class in the documentation of that class as if those # members were ordinary class members. Constructors, destructors and assignment # operators of the base classes will not be shown. +# The default value is: NO. INLINE_INHERITED_MEMB = NO -# If the FULL_PATH_NAMES tag is set to YES then Doxygen will prepend the full -# path before files name in the file list and in the header files. If set -# to NO the shortest path that makes the file name unique will be used. +# If the FULL_PATH_NAMES tag is set to YES, doxygen will prepend the full path +# before files name in the file list and in the header files. If set to NO the +# shortest path that makes the file name unique will be used +# The default value is: YES. FULL_PATH_NAMES = NO -# If the FULL_PATH_NAMES tag is set to YES then the STRIP_FROM_PATH tag -# can be used to strip a user-defined part of the path. Stripping is -# only done if one of the specified strings matches the left-hand part of -# the path. The tag can be used to show relative paths in the file list. -# If left blank the directory from which doxygen is run is used as the -# path to strip. +# The STRIP_FROM_PATH tag can be used to strip a user-defined part of the path. +# Stripping is only done if one of the specified strings matches the left-hand +# part of the path. The tag can be used to show relative paths in the file list. +# If left blank the directory from which doxygen is run is used as the path to +# strip. +# +# Note that you can specify absolute paths here, but also relative paths, which +# will be relative from the directory where doxygen is started. +# This tag requires that the tag FULL_PATH_NAMES is set to YES. STRIP_FROM_PATH = -# The STRIP_FROM_INC_PATH tag can be used to strip a user-defined part of -# the path mentioned in the documentation of a class, which tells -# the reader which header file to include in order to use a class. -# If left blank only the name of the header file containing the class -# definition is used. Otherwise one should specify the include paths that -# are normally passed to the compiler using the -I flag. +# The STRIP_FROM_INC_PATH tag can be used to strip a user-defined part of the +# path mentioned in the documentation of a class, which tells the reader which +# header file to include in order to use a class. If left blank only the name of +# the header file containing the class definition is used. Otherwise one should +# specify the list of include paths that are normally passed to the compiler +# using the -I flag. STRIP_FROM_INC_PATH = -# If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter -# (but less readable) file names. This can be useful is your file systems -# doesn't support long names like on DOS, Mac, or CD-ROM. +# If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter (but +# less readable) file names. This can be useful is your file systems doesn't +# support long names like on DOS, Mac, or CD-ROM. +# The default value is: NO. SHORT_NAMES = NO -# If the JAVADOC_AUTOBRIEF tag is set to YES then Doxygen -# will interpret the first line (until the first dot) of a JavaDoc-style -# comment as the brief description. If set to NO, the JavaDoc -# comments will behave just like regular Qt-style comments -# (thus requiring an explicit @brief command for a brief description.) +# If the JAVADOC_AUTOBRIEF tag is set to YES then doxygen will interpret the +# first line (until the first dot) of a Javadoc-style comment as the brief +# description. If set to NO, the Javadoc-style will behave just like regular Qt- +# style comments (thus requiring an explicit @brief command for a brief +# description.) +# The default value is: NO. JAVADOC_AUTOBRIEF = NO -# If the QT_AUTOBRIEF tag is set to YES then Doxygen will -# interpret the first line (until the first dot) of a Qt-style -# comment as the brief description. If set to NO, the comments -# will behave just like regular Qt-style comments (thus requiring -# an explicit \brief command for a brief description.) +# If the QT_AUTOBRIEF tag is set to YES then doxygen will interpret the first +# line (until the first dot) of a Qt-style comment as the brief description. If +# set to NO, the Qt-style will behave just like regular Qt-style comments (thus +# requiring an explicit \brief command for a brief description.) +# The default value is: NO. QT_AUTOBRIEF = NO -# The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make Doxygen -# treat a multi-line C++ special comment block (i.e. a block of //! or /// -# comments) as a brief description. This used to be the default behaviour. -# The new default is to treat a multi-line C++ comment block as a detailed -# description. Set this tag to YES if you prefer the old behaviour instead. +# The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make doxygen treat a +# multi-line C++ special comment block (i.e. a block of //! or /// comments) as +# a brief description. This used to be the default behavior. The new default is +# to treat a multi-line C++ comment block as a detailed description. Set this +# tag to YES if you prefer the old behavior instead. +# +# Note that setting this tag to YES also means that rational rose comments are +# not recognized any more. +# The default value is: NO. MULTILINE_CPP_IS_BRIEF = NO -# If the DETAILS_AT_TOP tag is set to YES then Doxygen -# will output the detailed description near the top, like JavaDoc. -# If set to NO, the detailed description appears after the member -# documentation. - -# DETAILS_AT_TOP = NO - -# If the INHERIT_DOCS tag is set to YES (the default) then an undocumented -# member inherits the documentation from any documented member that it -# re-implements. +# If the INHERIT_DOCS tag is set to YES then an undocumented member inherits the +# documentation from any documented member that it re-implements. +# The default value is: YES. INHERIT_DOCS = YES -# If the SEPARATE_MEMBER_PAGES tag is set to YES, then doxygen will produce -# a new page for each member. If set to NO, the documentation of a member will -# be part of the file/class/namespace that contains it. +# If the SEPARATE_MEMBER_PAGES tag is set to YES then doxygen will produce a new +# page for each member. If set to NO, the documentation of a member will be part +# of the file/class/namespace that contains it. +# The default value is: NO. SEPARATE_MEMBER_PAGES = NO -# The TAB_SIZE tag can be used to set the number of spaces in a tab. -# Doxygen uses this value to replace tabs by spaces in code fragments. +# The TAB_SIZE tag can be used to set the number of spaces in a tab. Doxygen +# uses this value to replace tabs by spaces in code fragments. +# Minimum value: 1, maximum value: 16, default value: 4. TAB_SIZE = 8 -# This tag can be used to specify a number of aliases that acts -# as commands in the documentation. An alias has the form "name=value". -# For example adding "sideeffect=\par Side Effects:\n" will allow you to -# put the command \sideeffect (or @sideeffect) in the documentation, which -# will result in a user-defined paragraph with heading "Side Effects:". -# You can put \n's in the value part of an alias to insert newlines. +# This tag can be used to specify a number of aliases that act as commands in +# the documentation. An alias has the form: +# name=value +# For example adding +# "sideeffect=@par Side Effects:\n" +# will allow you to put the command \sideeffect (or @sideeffect) in the +# documentation, which will result in a user-defined paragraph with heading +# "Side Effects:". You can put \n's in the value part of an alias to insert +# newlines (in the resulting output). You can put ^^ in the value part of an +# alias to insert a newline as if a physical newline was in the original file. +# When you need a literal { or } or , in the value part of an alias you have to +# escape them by means of a backslash (\), this can lead to conflicts with the +# commands \{ and \} for these it is advised to use the version @{ and @} or use +# a double escape (\\{ and \\}) ALIASES = -# Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C -# sources only. Doxygen will then generate output that is more tailored for C. -# For instance, some of the names that are used will be different. The list -# of all members will be omitted, etc. +# This tag can be used to specify a number of word-keyword mappings (TCL only). +# A mapping has the form "name=value". For example adding "class=itcl::class" +# will allow you to use the command class in the itcl::class meaning. + +TCL_SUBST = + +# Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C sources +# only. Doxygen will then generate output that is more tailored for C. For +# instance, some of the names that are used will be different. The list of all +# members will be omitted, etc. +# The default value is: NO. OPTIMIZE_OUTPUT_FOR_C = YES -# Set the OPTIMIZE_OUTPUT_JAVA tag to YES if your project consists of Java -# sources only. Doxygen will then generate output that is more tailored for -# Java. For instance, namespaces will be presented as packages, qualified -# scopes will look different, etc. +# Set the OPTIMIZE_OUTPUT_JAVA tag to YES if your project consists of Java or +# Python sources only. Doxygen will then generate output that is more tailored +# for that language. For instance, namespaces will be presented as packages, +# qualified scopes will look different, etc. +# The default value is: NO. OPTIMIZE_OUTPUT_JAVA = NO # Set the OPTIMIZE_FOR_FORTRAN tag to YES if your project consists of Fortran -# sources only. Doxygen will then generate output that is more tailored for -# Fortran. +# sources. Doxygen will then generate output that is tailored for Fortran. +# The default value is: NO. OPTIMIZE_FOR_FORTRAN = NO # Set the OPTIMIZE_OUTPUT_VHDL tag to YES if your project consists of VHDL -# sources. Doxygen will then generate output that is tailored for -# VHDL. +# sources. Doxygen will then generate output that is tailored for VHDL. +# The default value is: NO. OPTIMIZE_OUTPUT_VHDL = NO +# Set the OPTIMIZE_OUTPUT_SLICE tag to YES if your project consists of Slice +# sources only. Doxygen will then generate output that is more tailored for that +# language. For instance, namespaces will be presented as modules, types will be +# separated into more groups, etc. +# The default value is: NO. + +OPTIMIZE_OUTPUT_SLICE = NO + +# Doxygen selects the parser to use depending on the extension of the files it +# parses. With this tag you can assign which parser to use for a given +# extension. Doxygen has a built-in mapping, but you can override or extend it +# using this tag. The format is ext=language, where ext is a file extension, and +# language is one of the parsers supported by doxygen: IDL, Java, Javascript, +# Csharp (C#), C, C++, D, PHP, md (Markdown), Objective-C, Python, Slice, +# Fortran (fixed format Fortran: FortranFixed, free formatted Fortran: +# FortranFree, unknown formatted Fortran: Fortran. In the later case the parser +# tries to guess whether the code is fixed or free formatted code, this is the +# default for Fortran type files), VHDL, tcl. For instance to make doxygen treat +# .inc files as Fortran files (default is PHP), and .f files as C (default is +# Fortran), use: inc=Fortran f=C. +# +# Note: For files without extension you can use no_extension as a placeholder. +# +# Note that for custom extensions you also need to set FILE_PATTERNS otherwise +# the files are not read by doxygen. + +EXTENSION_MAPPING = + +# If the MARKDOWN_SUPPORT tag is enabled then doxygen pre-processes all comments +# according to the Markdown format, which allows for more readable +# documentation. See https://daringfireball.net/projects/markdown/ for details. +# The output of markdown processing is further processed by doxygen, so you can +# mix doxygen, HTML, and XML commands with Markdown formatting. Disable only in +# case of backward compatibilities issues. +# The default value is: YES. + +MARKDOWN_SUPPORT = YES + +# When the TOC_INCLUDE_HEADINGS tag is set to a non-zero value, all headings up +# to that level are automatically included in the table of contents, even if +# they do not have an id attribute. +# Note: This feature currently applies only to Markdown headings. +# Minimum value: 0, maximum value: 99, default value: 0. +# This tag requires that the tag MARKDOWN_SUPPORT is set to YES. + +TOC_INCLUDE_HEADINGS = 0 + +# When enabled doxygen tries to link words that correspond to documented +# classes, or namespaces to their corresponding documentation. Such a link can +# be prevented in individual cases by putting a % sign in front of the word or +# globally by setting AUTOLINK_SUPPORT to NO. +# The default value is: YES. + +AUTOLINK_SUPPORT = YES + # If you use STL classes (i.e. std::string, std::vector, etc.) but do not want -# to include (a tag file for) the STL sources as input, then you should -# set this tag to YES in order to let doxygen match functions declarations and -# definitions whose arguments contain STL classes (e.g. func(std::string); v.s. -# func(std::string) {}). This also make the inheritance and collaboration +# to include (a tag file for) the STL sources as input, then you should set this +# tag to YES in order to let doxygen match functions declarations and +# definitions whose arguments contain STL classes (e.g. func(std::string); +# versus func(std::string) {}). This also make the inheritance and collaboration # diagrams that involve STL classes more complete and accurate. +# The default value is: NO. BUILTIN_STL_SUPPORT = NO # If you use Microsoft's C++/CLI language, you should set this option to YES to # enable parsing support. +# The default value is: NO. CPP_CLI_SUPPORT = NO -# Set the SIP_SUPPORT tag to YES if your project consists of sip sources only. -# Doxygen will parse them like normal C++ but will assume all classes use public -# instead of private inheritance when no explicit protection keyword is present. +# Set the SIP_SUPPORT tag to YES if your project consists of sip (see: +# https://www.riverbankcomputing.com/software/sip/intro) sources only. Doxygen +# will parse them like normal C++ but will assume all classes use public instead +# of private inheritance when no explicit protection keyword is present. +# The default value is: NO. SIP_SUPPORT = NO -# For Microsoft's IDL there are propget and propput attributes to indicate getter -# and setter methods for a property. Setting this option to YES (the default) -# will make doxygen to replace the get and set methods by a property in the -# documentation. This will only work if the methods are indeed getting or -# setting a simple type. If this is not the case, or you want to show the -# methods anyway, you should set this option to NO. +# For Microsoft's IDL there are propget and propput attributes to indicate +# getter and setter methods for a property. Setting this option to YES will make +# doxygen to replace the get and set methods by a property in the documentation. +# This will only work if the methods are indeed getting or setting a simple +# type. If this is not the case, or you want to show the methods anyway, you +# should set this option to NO. +# The default value is: YES. IDL_PROPERTY_SUPPORT = NO # If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC -# tag is set to YES, then doxygen will reuse the documentation of the first +# tag is set to YES then doxygen will reuse the documentation of the first # member in the group (if any) for the other members of the group. By default # all members of a group must be documented explicitly. +# The default value is: NO. DISTRIBUTE_GROUP_DOC = NO -# Set the SUBGROUPING tag to YES (the default) to allow class member groups of -# the same type (for instance a group of public functions) to be put as a -# subgroup of that type (e.g. under the Public Functions section). Set it to -# NO to prevent subgrouping. Alternatively, this can be done per class using -# the \nosubgrouping command. +# If one adds a struct or class to a group and this option is enabled, then also +# any nested class or struct is added to the same group. By default this option +# is disabled and one has to add nested compounds explicitly via \ingroup. +# The default value is: NO. + +GROUP_NESTED_COMPOUNDS = NO + +# Set the SUBGROUPING tag to YES to allow class member groups of the same type +# (for instance a group of public functions) to be put as a subgroup of that +# type (e.g. under the Public Functions section). Set it to NO to prevent +# subgrouping. Alternatively, this can be done per class using the +# \nosubgrouping command. +# The default value is: YES. SUBGROUPING = YES -# When TYPEDEF_HIDES_STRUCT is enabled, a typedef of a struct, union, or enum -# is documented as struct, union, or enum with the name of the typedef. So +# When the INLINE_GROUPED_CLASSES tag is set to YES, classes, structs and unions +# are shown inside the group in which they are included (e.g. using \ingroup) +# instead of on a separate page (for HTML and Man pages) or section (for LaTeX +# and RTF). +# +# Note that this feature does not work in combination with +# SEPARATE_MEMBER_PAGES. +# The default value is: NO. + +INLINE_GROUPED_CLASSES = NO + +# When the INLINE_SIMPLE_STRUCTS tag is set to YES, structs, classes, and unions +# with only public data fields or simple typedef fields will be shown inline in +# the documentation of the scope in which they are defined (i.e. file, +# namespace, or group documentation), provided this scope is documented. If set +# to NO, structs, classes, and unions are shown on a separate page (for HTML and +# Man pages) or section (for LaTeX and RTF). +# The default value is: NO. + +INLINE_SIMPLE_STRUCTS = NO + +# When TYPEDEF_HIDES_STRUCT tag is enabled, a typedef of a struct, union, or +# enum is documented as struct, union, or enum with the name of the typedef. So # typedef struct TypeS {} TypeT, will appear in the documentation as a struct # with name TypeT. When disabled the typedef will appear as a member of a file, -# namespace, or class. And the struct will be named TypeS. This can typically -# be useful for C code in case the coding convention dictates that all compound +# namespace, or class. And the struct will be named TypeS. This can typically be +# useful for C code in case the coding convention dictates that all compound # types are typedef'ed and only the typedef is referenced, never the tag name. +# The default value is: NO. TYPEDEF_HIDES_STRUCT = NO +# The size of the symbol lookup cache can be set using LOOKUP_CACHE_SIZE. This +# cache is used to resolve symbols given their name and scope. Since this can be +# an expensive process and often the same symbol appears multiple times in the +# code, doxygen keeps a cache of pre-resolved symbols. If the cache is too small +# doxygen will become slower. If the cache is too large, memory is wasted. The +# cache size is given by this formula: 2^(16+LOOKUP_CACHE_SIZE). The valid range +# is 0..9, the default is 0, corresponding to a cache size of 2^16=65536 +# symbols. At the end of a run doxygen will report the cache usage and suggest +# the optimal cache size from a speed point of view. +# Minimum value: 0, maximum value: 9, default value: 0. + +LOOKUP_CACHE_SIZE = 0 + #--------------------------------------------------------------------------- # Build related configuration options #--------------------------------------------------------------------------- -# If the EXTRACT_ALL tag is set to YES doxygen will assume all entities in -# documentation are documented, even if no documentation was available. -# Private class members and static file members will be hidden unless -# the EXTRACT_PRIVATE and EXTRACT_STATIC tags are set to YES +# If the EXTRACT_ALL tag is set to YES, doxygen will assume all entities in +# documentation are documented, even if no documentation was available. Private +# class members and static file members will be hidden unless the +# EXTRACT_PRIVATE respectively EXTRACT_STATIC tags are set to YES. +# Note: This will also disable the warnings about undocumented members that are +# normally produced when WARNINGS is set to YES. +# The default value is: NO. EXTRACT_ALL = NO -# If the EXTRACT_PRIVATE tag is set to YES all private members of a class -# will be included in the documentation. +# If the EXTRACT_PRIVATE tag is set to YES, all private members of a class will +# be included in the documentation. +# The default value is: NO. EXTRACT_PRIVATE = NO -# If the EXTRACT_STATIC tag is set to YES all static members of a file -# will be included in the documentation. +# If the EXTRACT_PACKAGE tag is set to YES, all members with package or internal +# scope will be included in the documentation. +# The default value is: NO. + +EXTRACT_PACKAGE = NO + +# If the EXTRACT_STATIC tag is set to YES, all static members of a file will be +# included in the documentation. +# The default value is: NO. EXTRACT_STATIC = YES -# If the EXTRACT_LOCAL_CLASSES tag is set to YES classes (and structs) -# defined locally in source files will be included in the documentation. -# If set to NO only classes defined in header files are included. +# If the EXTRACT_LOCAL_CLASSES tag is set to YES, classes (and structs) defined +# locally in source files will be included in the documentation. If set to NO, +# only classes defined in header files are included. Does not have any effect +# for Java sources. +# The default value is: YES. EXTRACT_LOCAL_CLASSES = YES -# This flag is only useful for Objective-C code. When set to YES local -# methods, which are defined in the implementation section but not in -# the interface are included in the documentation. -# If set to NO (the default) only methods in the interface are included. +# This flag is only useful for Objective-C code. If set to YES, local methods, +# which are defined in the implementation section but not in the interface are +# included in the documentation. If set to NO, only methods in the interface are +# included. +# The default value is: NO. EXTRACT_LOCAL_METHODS = NO # If this flag is set to YES, the members of anonymous namespaces will be # extracted and appear in the documentation as a namespace called -# 'anonymous_namespace{file}', where file will be replaced with the base -# name of the file that contains the anonymous namespace. By default -# anonymous namespace are hidden. +# 'anonymous_namespace{file}', where file will be replaced with the base name of +# the file that contains the anonymous namespace. By default anonymous namespace +# are hidden. +# The default value is: NO. EXTRACT_ANON_NSPACES = NO -# If the HIDE_UNDOC_MEMBERS tag is set to YES, Doxygen will hide all -# undocumented members of documented classes, files or namespaces. -# If set to NO (the default) these members will be included in the -# various overviews, but no documentation section is generated. -# This option has no effect if EXTRACT_ALL is enabled. +# If the HIDE_UNDOC_MEMBERS tag is set to YES, doxygen will hide all +# undocumented members inside documented classes or files. If set to NO these +# members will be included in the various overviews, but no documentation +# section is generated. This option has no effect if EXTRACT_ALL is enabled. +# The default value is: NO. HIDE_UNDOC_MEMBERS = NO -# If the HIDE_UNDOC_CLASSES tag is set to YES, Doxygen will hide all -# undocumented classes that are normally visible in the class hierarchy. -# If set to NO (the default) these classes will be included in the various -# overviews. This option has no effect if EXTRACT_ALL is enabled. +# If the HIDE_UNDOC_CLASSES tag is set to YES, doxygen will hide all +# undocumented classes that are normally visible in the class hierarchy. If set +# to NO, these classes will be included in the various overviews. This option +# has no effect if EXTRACT_ALL is enabled. +# The default value is: NO. HIDE_UNDOC_CLASSES = NO -# If the HIDE_FRIEND_COMPOUNDS tag is set to YES, Doxygen will hide all -# friend (class|struct|union) declarations. -# If set to NO (the default) these declarations will be included in the -# documentation. +# If the HIDE_FRIEND_COMPOUNDS tag is set to YES, doxygen will hide all friend +# (class|struct|union) declarations. If set to NO, these declarations will be +# included in the documentation. +# The default value is: NO. HIDE_FRIEND_COMPOUNDS = NO -# If the HIDE_IN_BODY_DOCS tag is set to YES, Doxygen will hide any -# documentation blocks found inside the body of a function. -# If set to NO (the default) these blocks will be appended to the -# function's detailed documentation block. +# If the HIDE_IN_BODY_DOCS tag is set to YES, doxygen will hide any +# documentation blocks found inside the body of a function. If set to NO, these +# blocks will be appended to the function's detailed documentation block. +# The default value is: NO. HIDE_IN_BODY_DOCS = NO -# The INTERNAL_DOCS tag determines if documentation -# that is typed after a \internal command is included. If the tag is set -# to NO (the default) then the documentation will be excluded. -# Set it to YES to include the internal documentation. +# The INTERNAL_DOCS tag determines if documentation that is typed after a +# \internal command is included. If the tag is set to NO then the documentation +# will be excluded. Set it to YES to include the internal documentation. +# The default value is: NO. INTERNAL_DOCS = NO -# If the CASE_SENSE_NAMES tag is set to NO then Doxygen will only generate -# file names in lower-case letters. If set to YES upper-case letters are also +# If the CASE_SENSE_NAMES tag is set to NO then doxygen will only generate file +# names in lower-case letters. If set to YES, upper-case letters are also # allowed. This is useful if you have classes or files whose names only differ # in case and if your file system supports case sensitive file names. Windows # and Mac users are advised to set this option to NO. +# The default value is: system dependent. CASE_SENSE_NAMES = YES -# If the HIDE_SCOPE_NAMES tag is set to NO (the default) then Doxygen -# will show members with their full class and namespace scopes in the -# documentation. If set to YES the scope will be hidden. +# If the HIDE_SCOPE_NAMES tag is set to NO then doxygen will show members with +# their full class and namespace scopes in the documentation. If set to YES, the +# scope will be hidden. +# The default value is: NO. HIDE_SCOPE_NAMES = NO -# If the SHOW_INCLUDE_FILES tag is set to YES (the default) then Doxygen -# will put a list of the files that are included by a file in the documentation -# of that file. +# If the HIDE_COMPOUND_REFERENCE tag is set to NO (default) then doxygen will +# append additional text to a page's title, such as Class Reference. If set to +# YES the compound reference will be hidden. +# The default value is: NO. + +HIDE_COMPOUND_REFERENCE= NO + +# If the SHOW_INCLUDE_FILES tag is set to YES then doxygen will put a list of +# the files that are included by a file in the documentation of that file. +# The default value is: YES. SHOW_INCLUDE_FILES = YES -# If the INLINE_INFO tag is set to YES (the default) then a tag [inline] -# is inserted in the documentation for inline members. +# If the SHOW_GROUPED_MEMB_INC tag is set to YES then Doxygen will add for each +# grouped member an include statement to the documentation, telling the reader +# which file to include in order to use the member. +# The default value is: NO. + +SHOW_GROUPED_MEMB_INC = NO + +# If the FORCE_LOCAL_INCLUDES tag is set to YES then doxygen will list include +# files with double quotes in the documentation rather than with sharp brackets. +# The default value is: NO. + +FORCE_LOCAL_INCLUDES = NO + +# If the INLINE_INFO tag is set to YES then a tag [inline] is inserted in the +# documentation for inline members. +# The default value is: YES. INLINE_INFO = YES -# If the SORT_MEMBER_DOCS tag is set to YES (the default) then doxygen -# will sort the (detailed) documentation of file and class members -# alphabetically by member name. If set to NO the members will appear in -# declaration order. +# If the SORT_MEMBER_DOCS tag is set to YES then doxygen will sort the +# (detailed) documentation of file and class members alphabetically by member +# name. If set to NO, the members will appear in declaration order. +# The default value is: YES. SORT_MEMBER_DOCS = YES -# If the SORT_BRIEF_DOCS tag is set to YES then doxygen will sort the -# brief documentation of file, namespace and class members alphabetically -# by member name. If set to NO (the default) the members will appear in -# declaration order. +# If the SORT_BRIEF_DOCS tag is set to YES then doxygen will sort the brief +# descriptions of file, namespace and class members alphabetically by member +# name. If set to NO, the members will appear in declaration order. Note that +# this will also influence the order of the classes in the class list. +# The default value is: NO. SORT_BRIEF_DOCS = NO -# If the SORT_GROUP_NAMES tag is set to YES then doxygen will sort the -# hierarchy of group names into alphabetical order. If set to NO (the default) -# the group names will appear in their defined order. +# If the SORT_MEMBERS_CTORS_1ST tag is set to YES then doxygen will sort the +# (brief and detailed) documentation of class members so that constructors and +# destructors are listed first. If set to NO the constructors will appear in the +# respective orders defined by SORT_BRIEF_DOCS and SORT_MEMBER_DOCS. +# Note: If SORT_BRIEF_DOCS is set to NO this option is ignored for sorting brief +# member documentation. +# Note: If SORT_MEMBER_DOCS is set to NO this option is ignored for sorting +# detailed member documentation. +# The default value is: NO. + +SORT_MEMBERS_CTORS_1ST = NO + +# If the SORT_GROUP_NAMES tag is set to YES then doxygen will sort the hierarchy +# of group names into alphabetical order. If set to NO the group names will +# appear in their defined order. +# The default value is: NO. SORT_GROUP_NAMES = NO -# If the SORT_BY_SCOPE_NAME tag is set to YES, the class list will be -# sorted by fully-qualified names, including namespaces. If set to -# NO (the default), the class list will be sorted only by class name, -# not including the namespace part. +# If the SORT_BY_SCOPE_NAME tag is set to YES, the class list will be sorted by +# fully-qualified names, including namespaces. If set to NO, the class list will +# be sorted only by class name, not including the namespace part. # Note: This option is not very useful if HIDE_SCOPE_NAMES is set to YES. -# Note: This option applies only to the class list, not to the -# alphabetical list. +# Note: This option applies only to the class list, not to the alphabetical +# list. +# The default value is: NO. SORT_BY_SCOPE_NAME = NO -# The GENERATE_TODOLIST tag can be used to enable (YES) or -# disable (NO) the todo list. This list is created by putting \todo -# commands in the documentation. +# If the STRICT_PROTO_MATCHING option is enabled and doxygen fails to do proper +# type resolution of all parameters of a function it will reject a match between +# the prototype and the implementation of a member function even if there is +# only one candidate or it is obvious which candidate to choose by doing a +# simple string match. By disabling STRICT_PROTO_MATCHING doxygen will still +# accept a match between prototype and implementation in such cases. +# The default value is: NO. + +STRICT_PROTO_MATCHING = NO + +# The GENERATE_TODOLIST tag can be used to enable (YES) or disable (NO) the todo +# list. This list is created by putting \todo commands in the documentation. +# The default value is: YES. GENERATE_TODOLIST = YES -# The GENERATE_TESTLIST tag can be used to enable (YES) or -# disable (NO) the test list. This list is created by putting \test -# commands in the documentation. +# The GENERATE_TESTLIST tag can be used to enable (YES) or disable (NO) the test +# list. This list is created by putting \test commands in the documentation. +# The default value is: YES. GENERATE_TESTLIST = YES -# The GENERATE_BUGLIST tag can be used to enable (YES) or -# disable (NO) the bug list. This list is created by putting \bug -# commands in the documentation. +# The GENERATE_BUGLIST tag can be used to enable (YES) or disable (NO) the bug +# list. This list is created by putting \bug commands in the documentation. +# The default value is: YES. GENERATE_BUGLIST = YES -# The GENERATE_DEPRECATEDLIST tag can be used to enable (YES) or -# disable (NO) the deprecated list. This list is created by putting -# \deprecated commands in the documentation. +# The GENERATE_DEPRECATEDLIST tag can be used to enable (YES) or disable (NO) +# the deprecated list. This list is created by putting \deprecated commands in +# the documentation. +# The default value is: YES. GENERATE_DEPRECATEDLIST= YES -# The ENABLED_SECTIONS tag can be used to enable conditional -# documentation sections, marked by \if sectionname ... \endif. +# The ENABLED_SECTIONS tag can be used to enable conditional documentation +# sections, marked by \if ... \endif and \cond +# ... \endcond blocks. ENABLED_SECTIONS = -# The MAX_INITIALIZER_LINES tag determines the maximum number of lines -# the initial value of a variable or define consists of for it to appear in -# the documentation. If the initializer consists of more lines than specified -# here it will be hidden. Use a value of 0 to hide initializers completely. -# The appearance of the initializer of individual variables and defines in the -# documentation can be controlled using \showinitializer or \hideinitializer -# command in the documentation regardless of this setting. +# The MAX_INITIALIZER_LINES tag determines the maximum number of lines that the +# initial value of a variable or macro / define can have for it to appear in the +# documentation. If the initializer consists of more lines than specified here +# it will be hidden. Use a value of 0 to hide initializers completely. The +# appearance of the value of individual variables and macros / defines can be +# controlled using \showinitializer or \hideinitializer command in the +# documentation regardless of this setting. +# Minimum value: 0, maximum value: 10000, default value: 30. MAX_INITIALIZER_LINES = 30 -# Set the SHOW_USED_FILES tag to NO to disable the list of files generated -# at the bottom of the documentation of classes and structs. If set to YES the +# Set the SHOW_USED_FILES tag to NO to disable the list of files generated at +# the bottom of the documentation of classes and structs. If set to YES, the # list will mention the files that were used to generate the documentation. +# The default value is: YES. SHOW_USED_FILES = YES -# Set the SHOW_FILES tag to NO to disable the generation of the Files page. -# This will remove the Files entry from the Quick Index and from the -# Folder Tree View (if specified). The default is YES. +# Set the SHOW_FILES tag to NO to disable the generation of the Files page. This +# will remove the Files entry from the Quick Index and from the Folder Tree View +# (if specified). +# The default value is: YES. SHOW_FILES = YES -# Set the SHOW_NAMESPACES tag to NO to disable the generation of the -# Namespaces page. This will remove the Namespaces entry from the Quick Index -# and from the Folder Tree View (if specified). The default is YES. +# Set the SHOW_NAMESPACES tag to NO to disable the generation of the Namespaces +# page. This will remove the Namespaces entry from the Quick Index and from the +# Folder Tree View (if specified). +# The default value is: YES. SHOW_NAMESPACES = YES # The FILE_VERSION_FILTER tag can be used to specify a program or script that # doxygen should invoke to get the current version for each file (typically from # the version control system). Doxygen will invoke the program by executing (via -# popen()) the command , where is the value of -# the FILE_VERSION_FILTER tag, and is the name of an input file -# provided by doxygen. Whatever the program writes to standard output -# is used as the file version. See the manual for examples. +# popen()) the command command input-file, where command is the value of the +# FILE_VERSION_FILTER tag, and input-file is the name of an input file provided +# by doxygen. Whatever the program writes to standard output is used as the file +# version. For an example see the documentation. FILE_VERSION_FILTER = +# The LAYOUT_FILE tag can be used to specify a layout file which will be parsed +# by doxygen. The layout file controls the global structure of the generated +# output files in an output format independent way. To create the layout file +# that represents doxygen's defaults, run doxygen with the -l option. You can +# optionally specify a file name after the option, if omitted DoxygenLayout.xml +# will be used as the name of the layout file. +# +# Note that if you run doxygen from a directory containing a file called +# DoxygenLayout.xml, doxygen will parse it automatically even if the LAYOUT_FILE +# tag is left empty. + +LAYOUT_FILE = + +# The CITE_BIB_FILES tag can be used to specify one or more bib files containing +# the reference definitions. This must be a list of .bib files. The .bib +# extension is automatically appended if omitted. This requires the bibtex tool +# to be installed. See also https://en.wikipedia.org/wiki/BibTeX for more info. +# For LaTeX the style of the bibliography can be controlled using +# LATEX_BIB_STYLE. To use this feature you need bibtex and perl available in the +# search path. See also \cite for info how to create references. + +CITE_BIB_FILES = + #--------------------------------------------------------------------------- -# configuration options related to warning and progress messages +# Configuration options related to warning and progress messages #--------------------------------------------------------------------------- -# The QUIET tag can be used to turn on/off the messages that are generated -# by doxygen. Possible values are YES and NO. If left blank NO is used. +# The QUIET tag can be used to turn on/off the messages that are generated to +# standard output by doxygen. If QUIET is set to YES this implies that the +# messages are off. +# The default value is: NO. QUIET = NO # The WARNINGS tag can be used to turn on/off the warning messages that are -# generated by doxygen. Possible values are YES and NO. If left blank -# NO is used. +# generated to standard error (stderr) by doxygen. If WARNINGS is set to YES +# this implies that the warnings are on. +# +# Tip: Turn warnings on while writing the documentation. +# The default value is: YES. WARNINGS = YES -# If WARN_IF_UNDOCUMENTED is set to YES, then doxygen will generate warnings -# for undocumented members. If EXTRACT_ALL is set to YES then this flag will -# automatically be disabled. +# If the WARN_IF_UNDOCUMENTED tag is set to YES then doxygen will generate +# warnings for undocumented members. If EXTRACT_ALL is set to YES then this flag +# will automatically be disabled. +# The default value is: YES. WARN_IF_UNDOCUMENTED = YES -# If WARN_IF_DOC_ERROR is set to YES, doxygen will generate warnings for -# potential errors in the documentation, such as not documenting some -# parameters in a documented function, or documenting parameters that -# don't exist or using markup commands wrongly. +# If the WARN_IF_DOC_ERROR tag is set to YES, doxygen will generate warnings for +# potential errors in the documentation, such as not documenting some parameters +# in a documented function, or documenting parameters that don't exist or using +# markup commands wrongly. +# The default value is: YES. WARN_IF_DOC_ERROR = YES -# This WARN_NO_PARAMDOC option can be abled to get warnings for -# functions that are documented, but have no documentation for their parameters -# or return value. If set to NO (the default) doxygen will only warn about -# wrong or incomplete parameter documentation, but not about the absence of -# documentation. +# This WARN_NO_PARAMDOC option can be enabled to get warnings for functions that +# are documented, but have no documentation for their parameters or return +# value. If set to NO, doxygen will only warn about wrong or incomplete +# parameter documentation, but not about the absence of documentation. If +# EXTRACT_ALL is set to YES then this flag will automatically be disabled. +# The default value is: NO. WARN_NO_PARAMDOC = NO -# The WARN_FORMAT tag determines the format of the warning messages that -# doxygen can produce. The string should contain the $file, $line, and $text -# tags, which will be replaced by the file and line number from which the -# warning originated and the warning text. Optionally the format may contain -# $version, which will be replaced by the version of the file (if it could -# be obtained via FILE_VERSION_FILTER) +# If the WARN_AS_ERROR tag is set to YES then doxygen will immediately stop when +# a warning is encountered. +# The default value is: NO. + +WARN_AS_ERROR = NO + +# The WARN_FORMAT tag determines the format of the warning messages that doxygen +# can produce. The string should contain the $file, $line, and $text tags, which +# will be replaced by the file and line number from which the warning originated +# and the warning text. Optionally the format may contain $version, which will +# be replaced by the version of the file (if it could be obtained via +# FILE_VERSION_FILTER) +# The default value is: $file:$line: $text. WARN_FORMAT = "$file:$line: $text" -# The WARN_LOGFILE tag can be used to specify a file to which warning -# and error messages should be written. If left blank the output is written -# to stderr. +# The WARN_LOGFILE tag can be used to specify a file to which warning and error +# messages should be written. If left blank the output is written to standard +# error (stderr). WARN_LOGFILE = #--------------------------------------------------------------------------- -# configuration options related to the input files +# Configuration options related to the input files #--------------------------------------------------------------------------- -# The INPUT tag can be used to specify the files and/or directories that contain -# documented source files. You may enter file names like "myfile.cpp" or -# directories like "/usr/src/myproject". Separate the files or directories -# with spaces. +# The INPUT tag is used to specify the files and/or directories that contain +# documented source files. You may enter file names like myfile.cpp or +# directories like /usr/src/myproject. Separate the files or directories with +# spaces. See also FILE_PATTERNS and EXTENSION_MAPPING +# Note: If this tag is empty the current directory is searched. INPUT = @top_srcdir@/src/lib \ - @top_srcdir@/src/core \ - @top_srcdir@/src/feature \ - @top_srcdir@/src/app + @top_srcdir@/src/core \ + @top_srcdir@/src/feature \ + @top_srcdir@/src/app # This tag can be used to specify the character encoding of the source files -# that doxygen parses. Internally doxygen uses the UTF-8 encoding, which is -# also the default input encoding. Doxygen uses libiconv (or the iconv built -# into libc) for the transcoding. See http://www.gnu.org/software/libiconv for -# the list of possible encodings. +# that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses +# libiconv (or the iconv built into libc) for the transcoding. See the libiconv +# documentation (see: https://www.gnu.org/software/libiconv/) for the list of +# possible encodings. +# The default value is: UTF-8. INPUT_ENCODING = UTF-8 # If the value of the INPUT tag contains directories, you can use the -# FILE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp -# and *.h) to filter out the source-files in the directories. If left -# blank the following patterns are tested: -# *.c *.cc *.cxx *.cpp *.c++ *.java *.ii *.ixx *.ipp *.i++ *.inl *.h *.hh *.hxx -# *.hpp *.h++ *.idl *.odl *.cs *.php *.php3 *.inc *.m *.mm *.py *.f90 +# FILE_PATTERNS tag to specify one or more wildcard patterns (like *.cpp and +# *.h) to filter out the source-files in the directories. +# +# Note that for custom extensions or not directly supported extensions you also +# need to set EXTENSION_MAPPING for the extension otherwise the files are not +# read by doxygen. +# +# If left blank the following patterns are tested:*.c, *.cc, *.cxx, *.cpp, +# *.c++, *.java, *.ii, *.ixx, *.ipp, *.i++, *.inl, *.idl, *.ddl, *.odl, *.h, +# *.hh, *.hxx, *.hpp, *.h++, *.cs, *.d, *.php, *.php4, *.php5, *.phtml, *.inc, +# *.m, *.markdown, *.md, *.mm, *.dox, *.py, *.pyw, *.f90, *.f95, *.f03, *.f08, +# *.f, *.for, *.tcl, *.vhd, *.vhdl, *.ucf, *.qsf and *.ice. FILE_PATTERNS = *.c \ *.h -# The RECURSIVE tag can be used to turn specify whether or not subdirectories -# should be searched for input files as well. Possible values are YES and NO. -# If left blank NO is used. +# The RECURSIVE tag can be used to specify whether or not subdirectories should +# be searched for input files as well. +# The default value is: NO. RECURSIVE = YES -# The EXCLUDE tag can be used to specify files and/or directories that should +# The EXCLUDE tag can be used to specify files and/or directories that should be # excluded from the INPUT source files. This way you can easily exclude a # subdirectory from a directory tree whose root is specified with the INPUT tag. +# +# Note that relative paths are relative to the directory from which doxygen is +# run. -EXCLUDE = tree.h +EXCLUDE = -# The EXCLUDE_SYMLINKS tag can be used select whether or not files or -# directories that are symbolic links (a Unix filesystem feature) are excluded +# The EXCLUDE_SYMLINKS tag can be used to select whether or not files or +# directories that are symbolic links (a Unix file system feature) are excluded # from the input. +# The default value is: NO. EXCLUDE_SYMLINKS = NO # If the value of the INPUT tag contains directories, you can use the # EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude -# certain files from those directories. Note that the wildcards are matched -# against the file with absolute path, so to exclude all test directories -# for example use the pattern */test/* +# certain files from those directories. +# +# Note that the wildcards are matched against the file with absolute path, so to +# exclude all test directories for example use the pattern */test/* EXCLUDE_PATTERNS = @@ -582,510 +883,1169 @@ EXCLUDE_PATTERNS = # output. The symbol name can be a fully qualified name, a word, or if the # wildcard * is used, a substring. Examples: ANamespace, AClass, # AClass::ANamespace, ANamespace::*Test +# +# Note that the wildcards are matched against the file with absolute path, so to +# exclude all test directories use the pattern */test/* EXCLUDE_SYMBOLS = -# The EXAMPLE_PATH tag can be used to specify one or more files or -# directories that contain example code fragments that are included (see -# the \include command). +# The EXAMPLE_PATH tag can be used to specify one or more files or directories +# that contain example code fragments that are included (see the \include +# command). EXAMPLE_PATH = # If the value of the EXAMPLE_PATH tag contains directories, you can use the -# EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp -# and *.h) to filter out the source-files in the directories. If left -# blank all files are included. +# EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp and +# *.h) to filter out the source-files in the directories. If left blank all +# files are included. -EXAMPLE_PATTERNS = +EXAMPLE_PATTERNS = * # If the EXAMPLE_RECURSIVE tag is set to YES then subdirectories will be -# searched for input files to be used with the \include or \dontinclude -# commands irrespective of the value of the RECURSIVE tag. -# Possible values are YES and NO. If left blank NO is used. +# searched for input files to be used with the \include or \dontinclude commands +# irrespective of the value of the RECURSIVE tag. +# The default value is: NO. EXAMPLE_RECURSIVE = NO -# The IMAGE_PATH tag can be used to specify one or more files or -# directories that contain image that are included in the documentation (see -# the \image command). +# The IMAGE_PATH tag can be used to specify one or more files or directories +# that contain images that are to be included in the documentation (see the +# \image command). IMAGE_PATH = # The INPUT_FILTER tag can be used to specify a program that doxygen should # invoke to filter for each input file. Doxygen will invoke the filter program -# by executing (via popen()) the command , where -# is the value of the INPUT_FILTER tag, and is the name of an -# input file. Doxygen will then use the output that the filter program writes -# to standard output. If FILTER_PATTERNS is specified, this tag will be -# ignored. +# by executing (via popen()) the command: +# +# +# +# where is the value of the INPUT_FILTER tag, and is the +# name of an input file. Doxygen will then use the output that the filter +# program writes to standard output. If FILTER_PATTERNS is specified, this tag +# will be ignored. +# +# Note that the filter must not add or remove lines; it is applied before the +# code is scanned, but not when the output code is generated. If lines are added +# or removed, the anchors will not be placed correctly. +# +# Note that for custom extensions or not directly supported extensions you also +# need to set EXTENSION_MAPPING for the extension otherwise the files are not +# properly processed by doxygen. INPUT_FILTER = # The FILTER_PATTERNS tag can be used to specify filters on a per file pattern -# basis. Doxygen will compare the file name with each pattern and apply the -# filter if there is a match. The filters are a list of the form: -# pattern=filter (like *.cpp=my_cpp_filter). See INPUT_FILTER for further -# info on how filters are used. If FILTER_PATTERNS is empty, INPUT_FILTER -# is applied to all files. +# basis. Doxygen will compare the file name with each pattern and apply the +# filter if there is a match. The filters are a list of the form: pattern=filter +# (like *.cpp=my_cpp_filter). See INPUT_FILTER for further information on how +# filters are used. If the FILTER_PATTERNS tag is empty or if none of the +# patterns match the file name, INPUT_FILTER is applied. +# +# Note that for custom extensions or not directly supported extensions you also +# need to set EXTENSION_MAPPING for the extension otherwise the files are not +# properly processed by doxygen. FILTER_PATTERNS = # If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using -# INPUT_FILTER) will be used to filter the input files when producing source -# files to browse (i.e. when SOURCE_BROWSER is set to YES). +# INPUT_FILTER) will also be used to filter the input files that are used for +# producing the source files to browse (i.e. when SOURCE_BROWSER is set to YES). +# The default value is: NO. FILTER_SOURCE_FILES = NO +# The FILTER_SOURCE_PATTERNS tag can be used to specify source filters per file +# pattern. A pattern will override the setting for FILTER_PATTERN (if any) and +# it is also possible to disable source filtering for a specific pattern using +# *.ext= (so without naming a filter). +# This tag requires that the tag FILTER_SOURCE_FILES is set to YES. + +FILTER_SOURCE_PATTERNS = + +# If the USE_MDFILE_AS_MAINPAGE tag refers to the name of a markdown file that +# is part of the input, its contents will be placed on the main page +# (index.html). This can be useful if you have a project on for instance GitHub +# and want to reuse the introduction page also for the doxygen output. + +USE_MDFILE_AS_MAINPAGE = + #--------------------------------------------------------------------------- -# configuration options related to source browsing +# Configuration options related to source browsing #--------------------------------------------------------------------------- -# If the SOURCE_BROWSER tag is set to YES then a list of source files will -# be generated. Documented entities will be cross-referenced with these sources. -# Note: To get rid of all source code in the generated output, make sure also -# VERBATIM_HEADERS is set to NO. +# If the SOURCE_BROWSER tag is set to YES then a list of source files will be +# generated. Documented entities will be cross-referenced with these sources. +# +# Note: To get rid of all source code in the generated output, make sure that +# also VERBATIM_HEADERS is set to NO. +# The default value is: NO. -SOURCE_BROWSER = YES +SOURCE_BROWSER = NO -# Setting the INLINE_SOURCES tag to YES will include the body -# of functions and classes directly in the documentation. +# Setting the INLINE_SOURCES tag to YES will include the body of functions, +# classes and enums directly into the documentation. +# The default value is: NO. INLINE_SOURCES = NO -# Setting the STRIP_CODE_COMMENTS tag to YES (the default) will instruct -# doxygen to hide any special comment blocks from generated source code -# fragments. Normal C and C++ comments will always remain visible. +# Setting the STRIP_CODE_COMMENTS tag to YES will instruct doxygen to hide any +# special comment blocks from generated source code fragments. Normal C, C++ and +# Fortran comments will always remain visible. +# The default value is: YES. STRIP_CODE_COMMENTS = YES -# If the REFERENCED_BY_RELATION tag is set to YES -# then for each documented function all documented -# functions referencing it will be listed. +# If the REFERENCED_BY_RELATION tag is set to YES then for each documented +# entity all documented functions referencing it will be listed. +# The default value is: NO. -REFERENCED_BY_RELATION = YES +REFERENCED_BY_RELATION = NO -# If the REFERENCES_RELATION tag is set to YES -# then for each documented function all documented entities -# called/used by that function will be listed. +# If the REFERENCES_RELATION tag is set to YES then for each documented function +# all documented entities called/used by that function will be listed. +# The default value is: NO. -REFERENCES_RELATION = YES +REFERENCES_RELATION = NO -# If the REFERENCES_LINK_SOURCE tag is set to YES (the default) -# and SOURCE_BROWSER tag is set to YES, then the hyperlinks from -# functions in REFERENCES_RELATION and REFERENCED_BY_RELATION lists will -# link to the source code. Otherwise they will link to the documentstion. +# If the REFERENCES_LINK_SOURCE tag is set to YES and SOURCE_BROWSER tag is set +# to YES then the hyperlinks from functions in REFERENCES_RELATION and +# REFERENCED_BY_RELATION lists will link to the source code. Otherwise they will +# link to the documentation. +# The default value is: YES. REFERENCES_LINK_SOURCE = YES -# If the USE_HTAGS tag is set to YES then the references to source code -# will point to the HTML generated by the htags(1) tool instead of doxygen -# built-in source browser. The htags tool is part of GNU's global source -# tagging system (see http://www.gnu.org/software/global/global.html). You -# will need version 4.8.6 or higher. +# If SOURCE_TOOLTIPS is enabled (the default) then hovering a hyperlink in the +# source code will show a tooltip with additional information such as prototype, +# brief description and links to the definition and documentation. Since this +# will make the HTML file larger and loading of large files a bit slower, you +# can opt to disable this feature. +# The default value is: YES. +# This tag requires that the tag SOURCE_BROWSER is set to YES. + +SOURCE_TOOLTIPS = YES + +# If the USE_HTAGS tag is set to YES then the references to source code will +# point to the HTML generated by the htags(1) tool instead of doxygen built-in +# source browser. The htags tool is part of GNU's global source tagging system +# (see https://www.gnu.org/software/global/global.html). You will need version +# 4.8.6 or higher. +# +# To use it do the following: +# - Install the latest version of global +# - Enable SOURCE_BROWSER and USE_HTAGS in the configuration file +# - Make sure the INPUT points to the root of the source tree +# - Run doxygen as normal +# +# Doxygen will invoke htags (and that will in turn invoke gtags), so these +# tools must be available from the command line (i.e. in the search path). +# +# The result: instead of the source browser generated by doxygen, the links to +# source code will now point to the output of htags. +# The default value is: NO. +# This tag requires that the tag SOURCE_BROWSER is set to YES. USE_HTAGS = NO -# If the VERBATIM_HEADERS tag is set to YES (the default) then Doxygen -# will generate a verbatim copy of the header file for each class for -# which an include is specified. Set to NO to disable this. +# If the VERBATIM_HEADERS tag is set the YES then doxygen will generate a +# verbatim copy of the header file for each class for which an include is +# specified. Set to NO to disable this. +# See also: Section \class. +# The default value is: YES. VERBATIM_HEADERS = YES #--------------------------------------------------------------------------- -# configuration options related to the alphabetical class index +# Configuration options related to the alphabetical class index #--------------------------------------------------------------------------- -# If the ALPHABETICAL_INDEX tag is set to YES, an alphabetical index -# of all compounds will be generated. Enable this if the project -# contains a lot of classes, structs, unions or interfaces. +# If the ALPHABETICAL_INDEX tag is set to YES, an alphabetical index of all +# compounds will be generated. Enable this if the project contains a lot of +# classes, structs, unions or interfaces. +# The default value is: YES. -ALPHABETICAL_INDEX = NO +ALPHABETICAL_INDEX = YES -# If the alphabetical index is enabled (see ALPHABETICAL_INDEX) then -# the COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns -# in which this list will be split (can be a number in the range [1..20]) +# The COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns in +# which the alphabetical index list will be split. +# Minimum value: 1, maximum value: 20, default value: 5. +# This tag requires that the tag ALPHABETICAL_INDEX is set to YES. COLS_IN_ALPHA_INDEX = 5 -# In case all classes in a project start with a common prefix, all -# classes will be put under the same header in the alphabetical index. -# The IGNORE_PREFIX tag can be used to specify one or more prefixes that -# should be ignored while generating the index headers. +# In case all classes in a project start with a common prefix, all classes will +# be put under the same header in the alphabetical index. The IGNORE_PREFIX tag +# can be used to specify a prefix (or a list of prefixes) that should be ignored +# while generating the index headers. +# This tag requires that the tag ALPHABETICAL_INDEX is set to YES. IGNORE_PREFIX = #--------------------------------------------------------------------------- -# configuration options related to the HTML output +# Configuration options related to the HTML output #--------------------------------------------------------------------------- -# If the GENERATE_HTML tag is set to YES (the default) Doxygen will -# generate HTML output. +# If the GENERATE_HTML tag is set to YES, doxygen will generate HTML output +# The default value is: YES. GENERATE_HTML = YES -# The HTML_OUTPUT tag is used to specify where the HTML docs will be put. -# If a relative path is entered the value of OUTPUT_DIRECTORY will be -# put in front of it. If left blank `html' will be used as the default path. +# The HTML_OUTPUT tag is used to specify where the HTML docs will be put. If a +# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of +# it. +# The default directory is: html. +# This tag requires that the tag GENERATE_HTML is set to YES. HTML_OUTPUT = html -# The HTML_FILE_EXTENSION tag can be used to specify the file extension for -# each generated HTML page (for example: .htm,.php,.asp). If it is left blank -# doxygen will generate files with .html extension. +# The HTML_FILE_EXTENSION tag can be used to specify the file extension for each +# generated HTML page (for example: .htm, .php, .asp). +# The default value is: .html. +# This tag requires that the tag GENERATE_HTML is set to YES. HTML_FILE_EXTENSION = .html -# The HTML_HEADER tag can be used to specify a personal HTML header for -# each generated HTML page. If it is left blank doxygen will generate a +# The HTML_HEADER tag can be used to specify a user-defined HTML header file for +# each generated HTML page. If the tag is left blank doxygen will generate a # standard header. +# +# To get valid HTML the header file that includes any scripts and style sheets +# that doxygen needs, which is dependent on the configuration options used (e.g. +# the setting GENERATE_TREEVIEW). It is highly recommended to start with a +# default header using +# doxygen -w html new_header.html new_footer.html new_stylesheet.css +# YourConfigFile +# and then modify the file new_header.html. See also section "Doxygen usage" +# for information on how to generate the default header that doxygen normally +# uses. +# Note: The header is subject to change so you typically have to regenerate the +# default header when upgrading to a newer version of doxygen. For a description +# of the possible markers and block names see the documentation. +# This tag requires that the tag GENERATE_HTML is set to YES. HTML_HEADER = -# The HTML_FOOTER tag can be used to specify a personal HTML footer for -# each generated HTML page. If it is left blank doxygen will generate a -# standard footer. +# The HTML_FOOTER tag can be used to specify a user-defined HTML footer for each +# generated HTML page. If the tag is left blank doxygen will generate a standard +# footer. See HTML_HEADER for more information on how to generate a default +# footer and what special commands can be used inside the footer. See also +# section "Doxygen usage" for information on how to generate the default footer +# that doxygen normally uses. +# This tag requires that the tag GENERATE_HTML is set to YES. HTML_FOOTER = -# The HTML_STYLESHEET tag can be used to specify a user-defined cascading -# style sheet that is used by each HTML page. It can be used to -# fine-tune the look of the HTML output. If the tag is left blank doxygen -# will generate a default style sheet. Note that doxygen will try to copy -# the style sheet file to the HTML output directory, so don't put your own -# stylesheet in the HTML output directory as well, or it will be erased! +# The HTML_STYLESHEET tag can be used to specify a user-defined cascading style +# sheet that is used by each HTML page. It can be used to fine-tune the look of +# the HTML output. If left blank doxygen will generate a default style sheet. +# See also section "Doxygen usage" for information on how to generate the style +# sheet that doxygen normally uses. +# Note: It is recommended to use HTML_EXTRA_STYLESHEET instead of this tag, as +# it is more robust and this tag (HTML_STYLESHEET) will in the future become +# obsolete. +# This tag requires that the tag GENERATE_HTML is set to YES. HTML_STYLESHEET = -# If the GENERATE_HTMLHELP tag is set to YES, additional index files -# will be generated that can be used as input for tools like the -# Microsoft HTML help workshop to generate a compiled HTML help file (.chm) -# of the generated HTML documentation. +# The HTML_EXTRA_STYLESHEET tag can be used to specify additional user-defined +# cascading style sheets that are included after the standard style sheets +# created by doxygen. Using this option one can overrule certain style aspects. +# This is preferred over using HTML_STYLESHEET since it does not replace the +# standard style sheet and is therefore more robust against future updates. +# Doxygen will copy the style sheet files to the output directory. +# Note: The order of the extra style sheet files is of importance (e.g. the last +# style sheet in the list overrules the setting of the previous ones in the +# list). For an example see the documentation. +# This tag requires that the tag GENERATE_HTML is set to YES. -GENERATE_HTMLHELP = NO +HTML_EXTRA_STYLESHEET = -# If the GENERATE_DOCSET tag is set to YES, additional index files -# will be generated that can be used as input for Apple's Xcode 3 -# integrated development environment, introduced with OSX 10.5 (Leopard). -# To create a documentation set, doxygen will generate a Makefile in the -# HTML output directory. Running make will produce the docset in that -# directory and running "make install" will install the docset in -# ~/Library/Developer/Shared/Documentation/DocSets so that Xcode will find -# it at startup. +# The HTML_EXTRA_FILES tag can be used to specify one or more extra images or +# other source files which should be copied to the HTML output directory. Note +# that these files will be copied to the base HTML output directory. Use the +# $relpath^ marker in the HTML_HEADER and/or HTML_FOOTER files to load these +# files. In the HTML_STYLESHEET file, use the file name only. Also note that the +# files will be copied as-is; there are no commands or markers available. +# This tag requires that the tag GENERATE_HTML is set to YES. -GENERATE_DOCSET = NO +HTML_EXTRA_FILES = -# When GENERATE_DOCSET tag is set to YES, this tag determines the name of the -# feed. A documentation feed provides an umbrella under which multiple -# documentation sets from a single provider (such as a company or product suite) -# can be grouped. +# The HTML_COLORSTYLE_HUE tag controls the color of the HTML output. Doxygen +# will adjust the colors in the style sheet and background images according to +# this color. Hue is specified as an angle on a colorwheel, see +# https://en.wikipedia.org/wiki/Hue for more information. For instance the value +# 0 represents red, 60 is yellow, 120 is green, 180 is cyan, 240 is blue, 300 +# purple, and 360 is red again. +# Minimum value: 0, maximum value: 359, default value: 220. +# This tag requires that the tag GENERATE_HTML is set to YES. -DOCSET_FEEDNAME = "Doxygen generated docs for Tor" +HTML_COLORSTYLE_HUE = 220 -# When GENERATE_DOCSET tag is set to YES, this tag specifies a string that -# should uniquely identify the documentation set bundle. This should be a -# reverse domain-name style string, e.g. com.mycompany.MyDocSet. Doxygen -# will append .docset to the name. +# The HTML_COLORSTYLE_SAT tag controls the purity (or saturation) of the colors +# in the HTML output. For a value of 0 the output will use grayscales only. A +# value of 255 will produce the most vivid colors. +# Minimum value: 0, maximum value: 255, default value: 100. +# This tag requires that the tag GENERATE_HTML is set to YES. -DOCSET_BUNDLE_ID = org.torproject.Tor +HTML_COLORSTYLE_SAT = 100 + +# The HTML_COLORSTYLE_GAMMA tag controls the gamma correction applied to the +# luminance component of the colors in the HTML output. Values below 100 +# gradually make the output lighter, whereas values above 100 make the output +# darker. The value divided by 100 is the actual gamma applied, so 80 represents +# a gamma of 0.8, The value 220 represents a gamma of 2.2, and 100 does not +# change the gamma. +# Minimum value: 40, maximum value: 240, default value: 80. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_COLORSTYLE_GAMMA = 80 + +# If the HTML_TIMESTAMP tag is set to YES then the footer of each generated HTML +# page will contain the date and time when the page was generated. Setting this +# to YES can help to show when doxygen was last run and thus if the +# documentation is up to date. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_TIMESTAMP = NO + +# If the HTML_DYNAMIC_MENUS tag is set to YES then the generated HTML +# documentation will contain a main index with vertical navigation menus that +# are dynamically created via Javascript. If disabled, the navigation index will +# consists of multiple levels of tabs that are statically embedded in every HTML +# page. Disable this option to support browsers that do not have Javascript, +# like the Qt help browser. +# The default value is: YES. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_DYNAMIC_MENUS = YES # If the HTML_DYNAMIC_SECTIONS tag is set to YES then the generated HTML # documentation will contain sections that can be hidden and shown after the -# page has loaded. For this to work a browser that supports -# JavaScript and DHTML is required (for instance Mozilla 1.0+, Firefox -# Netscape 6.0+, Internet explorer 5.0+, Konqueror, or Safari). +# page has loaded. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. HTML_DYNAMIC_SECTIONS = NO -# If the GENERATE_HTMLHELP tag is set to YES, the CHM_FILE tag can -# be used to specify the file name of the resulting .chm file. You -# can add a path in front of the file if the result should not be +# With HTML_INDEX_NUM_ENTRIES one can control the preferred number of entries +# shown in the various tree structured indices initially; the user can expand +# and collapse entries dynamically later on. Doxygen will expand the tree to +# such a level that at most the specified number of entries are visible (unless +# a fully collapsed tree already exceeds this amount). So setting the number of +# entries 1 will produce a full collapsed tree by default. 0 is a special value +# representing an infinite number of entries and will result in a full expanded +# tree by default. +# Minimum value: 0, maximum value: 9999, default value: 100. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_INDEX_NUM_ENTRIES = 100 + +# If the GENERATE_DOCSET tag is set to YES, additional index files will be +# generated that can be used as input for Apple's Xcode 3 integrated development +# environment (see: https://developer.apple.com/xcode/), introduced with OSX +# 10.5 (Leopard). To create a documentation set, doxygen will generate a +# Makefile in the HTML output directory. Running make will produce the docset in +# that directory and running make install will install the docset in +# ~/Library/Developer/Shared/Documentation/DocSets so that Xcode will find it at +# startup. See https://developer.apple.com/library/archive/featuredarticles/Doxy +# genXcode/_index.html for more information. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +GENERATE_DOCSET = NO + +# This tag determines the name of the docset feed. A documentation feed provides +# an umbrella under which multiple documentation sets from a single provider +# (such as a company or product suite) can be grouped. +# The default value is: Doxygen generated docs. +# This tag requires that the tag GENERATE_DOCSET is set to YES. + +DOCSET_FEEDNAME = "Doxygen generated docs" + +# This tag specifies a string that should uniquely identify the documentation +# set bundle. This should be a reverse domain-name style string, e.g. +# com.mycompany.MyDocSet. Doxygen will append .docset to the name. +# The default value is: org.doxygen.Project. +# This tag requires that the tag GENERATE_DOCSET is set to YES. + +DOCSET_BUNDLE_ID = org.doxygen.Project + +# The DOCSET_PUBLISHER_ID tag specifies a string that should uniquely identify +# the documentation publisher. This should be a reverse domain-name style +# string, e.g. com.mycompany.MyDocSet.documentation. +# The default value is: org.doxygen.Publisher. +# This tag requires that the tag GENERATE_DOCSET is set to YES. + +DOCSET_PUBLISHER_ID = org.doxygen.Publisher + +# The DOCSET_PUBLISHER_NAME tag identifies the documentation publisher. +# The default value is: Publisher. +# This tag requires that the tag GENERATE_DOCSET is set to YES. + +DOCSET_PUBLISHER_NAME = Publisher + +# If the GENERATE_HTMLHELP tag is set to YES then doxygen generates three +# additional HTML index files: index.hhp, index.hhc, and index.hhk. The +# index.hhp is a project file that can be read by Microsoft's HTML Help Workshop +# (see: https://www.microsoft.com/en-us/download/details.aspx?id=21138) on +# Windows. +# +# The HTML Help Workshop contains a compiler that can convert all HTML output +# generated by doxygen into a single compiled HTML file (.chm). Compiled HTML +# files are now used as the Windows 98 help format, and will replace the old +# Windows help format (.hlp) on all Windows platforms in the future. Compressed +# HTML files also contain an index, a table of contents, and you can search for +# words in the documentation. The HTML workshop also contains a viewer for +# compressed HTML files. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +GENERATE_HTMLHELP = NO + +# The CHM_FILE tag can be used to specify the file name of the resulting .chm +# file. You can add a path in front of the file if the result should not be # written to the html output directory. +# This tag requires that the tag GENERATE_HTMLHELP is set to YES. CHM_FILE = -# If the GENERATE_HTMLHELP tag is set to YES, the HHC_LOCATION tag can -# be used to specify the location (absolute path including file name) of -# the HTML help compiler (hhc.exe). If non-empty doxygen will try to run -# the HTML help compiler on the generated index.hhp. +# The HHC_LOCATION tag can be used to specify the location (absolute path +# including file name) of the HTML help compiler (hhc.exe). If non-empty, +# doxygen will try to run the HTML help compiler on the generated index.hhp. +# The file has to be specified with full path. +# This tag requires that the tag GENERATE_HTMLHELP is set to YES. HHC_LOCATION = -# If the GENERATE_HTMLHELP tag is set to YES, the GENERATE_CHI flag -# controls if a separate .chi index file is generated (YES) or that -# it should be included in the master .chm file (NO). +# The GENERATE_CHI flag controls if a separate .chi index file is generated +# (YES) or that it should be included in the master .chm file (NO). +# The default value is: NO. +# This tag requires that the tag GENERATE_HTMLHELP is set to YES. GENERATE_CHI = NO -# If the GENERATE_HTMLHELP tag is set to YES, the CHM_INDEX_ENCODING -# is used to encode HtmlHelp index (hhk), content (hhc) and project file -# content. +# The CHM_INDEX_ENCODING is used to encode HtmlHelp index (hhk), content (hhc) +# and project file content. +# This tag requires that the tag GENERATE_HTMLHELP is set to YES. CHM_INDEX_ENCODING = -# If the GENERATE_HTMLHELP tag is set to YES, the BINARY_TOC flag -# controls whether a binary table of contents is generated (YES) or a -# normal table of contents (NO) in the .chm file. +# The BINARY_TOC flag controls whether a binary table of contents is generated +# (YES) or a normal table of contents (NO) in the .chm file. Furthermore it +# enables the Previous and Next buttons. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTMLHELP is set to YES. BINARY_TOC = NO -# The TOC_EXPAND flag can be set to YES to add extra items for group members -# to the contents of the HTML help documentation and to the tree view. +# The TOC_EXPAND flag can be set to YES to add extra items for group members to +# the table of contents of the HTML help documentation and to the tree view. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTMLHELP is set to YES. TOC_EXPAND = NO -# The DISABLE_INDEX tag can be used to turn on/off the condensed index at -# top of each HTML page. The value NO (the default) enables the index and -# the value YES disables it. +# If the GENERATE_QHP tag is set to YES and both QHP_NAMESPACE and +# QHP_VIRTUAL_FOLDER are set, an additional index file will be generated that +# can be used as input for Qt's qhelpgenerator to generate a Qt Compressed Help +# (.qch) of the generated HTML documentation. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +GENERATE_QHP = NO + +# If the QHG_LOCATION tag is specified, the QCH_FILE tag can be used to specify +# the file name of the resulting .qch file. The path specified is relative to +# the HTML output folder. +# This tag requires that the tag GENERATE_QHP is set to YES. + +QCH_FILE = + +# The QHP_NAMESPACE tag specifies the namespace to use when generating Qt Help +# Project output. For more information please see Qt Help Project / Namespace +# (see: http://doc.qt.io/archives/qt-4.8/qthelpproject.html#namespace). +# The default value is: org.doxygen.Project. +# This tag requires that the tag GENERATE_QHP is set to YES. + +QHP_NAMESPACE = org.doxygen.Project + +# The QHP_VIRTUAL_FOLDER tag specifies the namespace to use when generating Qt +# Help Project output. For more information please see Qt Help Project / Virtual +# Folders (see: http://doc.qt.io/archives/qt-4.8/qthelpproject.html#virtual- +# folders). +# The default value is: doc. +# This tag requires that the tag GENERATE_QHP is set to YES. + +QHP_VIRTUAL_FOLDER = doc + +# If the QHP_CUST_FILTER_NAME tag is set, it specifies the name of a custom +# filter to add. For more information please see Qt Help Project / Custom +# Filters (see: http://doc.qt.io/archives/qt-4.8/qthelpproject.html#custom- +# filters). +# This tag requires that the tag GENERATE_QHP is set to YES. + +QHP_CUST_FILTER_NAME = + +# The QHP_CUST_FILTER_ATTRS tag specifies the list of the attributes of the +# custom filter to add. For more information please see Qt Help Project / Custom +# Filters (see: http://doc.qt.io/archives/qt-4.8/qthelpproject.html#custom- +# filters). +# This tag requires that the tag GENERATE_QHP is set to YES. + +QHP_CUST_FILTER_ATTRS = + +# The QHP_SECT_FILTER_ATTRS tag specifies the list of the attributes this +# project's filter section matches. Qt Help Project / Filter Attributes (see: +# http://doc.qt.io/archives/qt-4.8/qthelpproject.html#filter-attributes). +# This tag requires that the tag GENERATE_QHP is set to YES. + +QHP_SECT_FILTER_ATTRS = + +# The QHG_LOCATION tag can be used to specify the location of Qt's +# qhelpgenerator. If non-empty doxygen will try to run qhelpgenerator on the +# generated .qhp file. +# This tag requires that the tag GENERATE_QHP is set to YES. + +QHG_LOCATION = + +# If the GENERATE_ECLIPSEHELP tag is set to YES, additional index files will be +# generated, together with the HTML files, they form an Eclipse help plugin. To +# install this plugin and make it available under the help contents menu in +# Eclipse, the contents of the directory containing the HTML and XML files needs +# to be copied into the plugins directory of eclipse. The name of the directory +# within the plugins directory should be the same as the ECLIPSE_DOC_ID value. +# After copying Eclipse needs to be restarted before the help appears. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +GENERATE_ECLIPSEHELP = NO + +# A unique identifier for the Eclipse help plugin. When installing the plugin +# the directory name containing the HTML and XML files should also have this +# name. Each documentation set should have its own identifier. +# The default value is: org.doxygen.Project. +# This tag requires that the tag GENERATE_ECLIPSEHELP is set to YES. + +ECLIPSE_DOC_ID = org.doxygen.Project + +# If you want full control over the layout of the generated HTML pages it might +# be necessary to disable the index and replace it with your own. The +# DISABLE_INDEX tag can be used to turn on/off the condensed index (tabs) at top +# of each HTML page. A value of NO enables the index and the value YES disables +# it. Since the tabs in the index contain the same information as the navigation +# tree, you can set this option to YES if you also set GENERATE_TREEVIEW to YES. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. DISABLE_INDEX = NO -# This tag can be used to set the number of enum values (range [1..20]) -# that doxygen will group on one line in the generated HTML documentation. - -ENUM_VALUES_PER_LINE = 4 - # The GENERATE_TREEVIEW tag is used to specify whether a tree-like index -# structure should be generated to display hierarchical information. -# If the tag value is set to FRAME, a side panel will be generated -# containing a tree-like index structure (just like the one that -# is generated for HTML Help). For this to work a browser that supports -# JavaScript, DHTML, CSS and frames is required (for instance Mozilla 1.0+, -# Netscape 6.0+, Internet explorer 5.0+, or Konqueror). Windows users are -# probably better off using the HTML help feature. Other possible values -# for this tag are: HIERARCHIES, which will generate the Groups, Directories, -# and Class Hiererachy pages using a tree view instead of an ordered list; -# ALL, which combines the behavior of FRAME and HIERARCHIES; and NONE, which -# disables this behavior completely. For backwards compatibility with previous -# releases of Doxygen, the values YES and NO are equivalent to FRAME and NONE -# respectively. +# structure should be generated to display hierarchical information. If the tag +# value is set to YES, a side panel will be generated containing a tree-like +# index structure (just like the one that is generated for HTML Help). For this +# to work a browser that supports JavaScript, DHTML, CSS and frames is required +# (i.e. any modern browser). Windows users are probably better off using the +# HTML help feature. Via custom style sheets (see HTML_EXTRA_STYLESHEET) one can +# further fine-tune the look of the index. As an example, the default style +# sheet generated by doxygen has an example that shows how to put an image at +# the root of the tree instead of the PROJECT_NAME. Since the tree basically has +# the same information as the tab index, you could consider setting +# DISABLE_INDEX to YES when enabling this option. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. GENERATE_TREEVIEW = NO -# If the treeview is enabled (see GENERATE_TREEVIEW) then this tag can be -# used to set the initial width (in pixels) of the frame in which the tree -# is shown. +# The ENUM_VALUES_PER_LINE tag can be used to set the number of enum values that +# doxygen will group on one line in the generated HTML documentation. +# +# Note that a value of 0 will completely suppress the enum values from appearing +# in the overview section. +# Minimum value: 0, maximum value: 20, default value: 4. +# This tag requires that the tag GENERATE_HTML is set to YES. + +ENUM_VALUES_PER_LINE = 4 + +# If the treeview is enabled (see GENERATE_TREEVIEW) then this tag can be used +# to set the initial width (in pixels) of the frame in which the tree is shown. +# Minimum value: 0, maximum value: 1500, default value: 250. +# This tag requires that the tag GENERATE_HTML is set to YES. TREEVIEW_WIDTH = 250 -# Use this tag to change the font size of Latex formulas included -# as images in the HTML documentation. The default is 10. Note that -# when you change the font size after a successful doxygen run you need -# to manually remove any form_*.png images from the HTML output directory -# to force them to be regenerated. +# If the EXT_LINKS_IN_WINDOW option is set to YES, doxygen will open links to +# external symbols imported via tag files in a separate window. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +EXT_LINKS_IN_WINDOW = NO + +# Use this tag to change the font size of LaTeX formulas included as images in +# the HTML documentation. When you change the font size after a successful +# doxygen run you need to manually remove any form_*.png images from the HTML +# output directory to force them to be regenerated. +# Minimum value: 8, maximum value: 50, default value: 10. +# This tag requires that the tag GENERATE_HTML is set to YES. FORMULA_FONTSIZE = 10 +# Use the FORMULA_TRANSPARENT tag to determine whether or not the images +# generated for formulas are transparent PNGs. Transparent PNGs are not +# supported properly for IE 6.0, but are supported on all modern browsers. +# +# Note that when changing this option you need to delete any form_*.png files in +# the HTML output directory before the changes have effect. +# The default value is: YES. +# This tag requires that the tag GENERATE_HTML is set to YES. + +FORMULA_TRANSPARENT = YES + +# Enable the USE_MATHJAX option to render LaTeX formulas using MathJax (see +# https://www.mathjax.org) which uses client side Javascript for the rendering +# instead of using pre-rendered bitmaps. Use this if you do not have LaTeX +# installed or if you want to formulas look prettier in the HTML output. When +# enabled you may also need to install MathJax separately and configure the path +# to it using the MATHJAX_RELPATH option. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +USE_MATHJAX = NO + +# When MathJax is enabled you can set the default output format to be used for +# the MathJax output. See the MathJax site (see: +# http://docs.mathjax.org/en/latest/output.html) for more details. +# Possible values are: HTML-CSS (which is slower, but has the best +# compatibility), NativeMML (i.e. MathML) and SVG. +# The default value is: HTML-CSS. +# This tag requires that the tag USE_MATHJAX is set to YES. + +MATHJAX_FORMAT = HTML-CSS + +# When MathJax is enabled you need to specify the location relative to the HTML +# output directory using the MATHJAX_RELPATH option. The destination directory +# should contain the MathJax.js script. For instance, if the mathjax directory +# is located at the same level as the HTML output directory, then +# MATHJAX_RELPATH should be ../mathjax. The default value points to the MathJax +# Content Delivery Network so you can quickly see the result without installing +# MathJax. However, it is strongly recommended to install a local copy of +# MathJax from https://www.mathjax.org before deployment. +# The default value is: https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/. +# This tag requires that the tag USE_MATHJAX is set to YES. + +MATHJAX_RELPATH = https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/ + +# The MATHJAX_EXTENSIONS tag can be used to specify one or more MathJax +# extension names that should be enabled during MathJax rendering. For example +# MATHJAX_EXTENSIONS = TeX/AMSmath TeX/AMSsymbols +# This tag requires that the tag USE_MATHJAX is set to YES. + +MATHJAX_EXTENSIONS = + +# The MATHJAX_CODEFILE tag can be used to specify a file with javascript pieces +# of code that will be used on startup of the MathJax code. See the MathJax site +# (see: http://docs.mathjax.org/en/latest/output.html) for more details. For an +# example see the documentation. +# This tag requires that the tag USE_MATHJAX is set to YES. + +MATHJAX_CODEFILE = + +# When the SEARCHENGINE tag is enabled doxygen will generate a search box for +# the HTML output. The underlying search engine uses javascript and DHTML and +# should work on any modern browser. Note that when using HTML help +# (GENERATE_HTMLHELP), Qt help (GENERATE_QHP), or docsets (GENERATE_DOCSET) +# there is already a search function so this one should typically be disabled. +# For large projects the javascript based search engine can be slow, then +# enabling SERVER_BASED_SEARCH may provide a better solution. It is possible to +# search using the keyboard; to jump to the search box use + S +# (what the is depends on the OS and browser, but it is typically +# , /